From b65d97792a53f78cb14b226231063209d22c4602 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jan 2019 19:14:21 -0800 Subject: [PATCH 0001/2303] Switch to using PYPI_PASSWORD env var in Travis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bc9361f..d292f194 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,8 +50,7 @@ jobs: - provider: pypi user: simonw distributions: bdist_wheel - password: - secure: ikY6iF+/2AxFlwcun35iAEz7Wbrnp0Fp7b32Z7uob5RsGpIDCYYDa3T/qNhiyNsg5fecGp01rBf1F+dL357b0rWEeaoONtsDrTKFeBUB6bMnzNtr3QbHq/TeQ1f+Vn6FWpY9360Ihbz0pKPzWWYHJxOjXPM793rGWmPu7siCc4oQZOpwLK28GZeFP803wq/QG81hFRWR8IiMlqljkecEGhaM6ftxzizk1LBoTZCw3DdL2xDwzrLvF3Hg1jXX08pJm9WrLJNS6i+LRhiLv9IJ5KxRDwNHJhrvblRrZn0CKVbiWR/8985r4R7CXaeG68uJ505RcVoXYRsq9D7mAcYAB3U5AzU9TsPQlvUgwJlFpKOsZlCw+5AHFIYUXvlP2Qo5kfXUYC9KKdJwbWjcW7isUqWpzInGfnmRNMJGzmIm2q65ua0FNLV4vK1wl98O60HFSyhXJi84YmCAPgidJ22sEA6sHAdMEFiWglhap12zPc6i7trXKC8aLjW/3qGXhUq3BK8vQcbutGer+Q3z9UM7kthB76FtCUVc8eqljtdAcGt5kXoHIEPwGb50ikOxo/qpXsXVH36Z6plbELbayXb3ocUrMdhWP5+kfnrdLEmx+X4IpUauxGUqLGliYSQpfjp8XyA87ASCjsSqUEjr/4t4YztW5a3mWBYZ+IjjYXW6PQg= + password: ${PYPI_PASSWORD} on: branch: master tags: true From 8da2db4b71096b19e7a9ef1929369b8483d448bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 4 Jan 2019 17:34:22 -0800 Subject: [PATCH 0002/2303] WIP --- datasette/app.py | 3 +++ datasette/cli.py | 2 ++ datasette/templates/database.html | 12 ++++++------ datasette/templates/index.html | 4 ++-- datasette/templates/query.html | 4 ++-- datasette/templates/row.html | 4 ++-- datasette/templates/table.html | 4 ++-- datasette/views/base.py | 3 ++- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index cdfcb11d..d9b8ac72 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -99,6 +99,9 @@ CONFIG_OPTIONS = ( ConfigOption("force_https_urls", False, """ Force URLs in API output to always use https:// protocol """.strip()), + ConfigOption("url_prefix", "", """ + URLs within Datasette itself should start with this (defaults to auto) + """.strip()), ) DEFAULT_CONFIG = { option.name: option.default diff --git a/datasette/cli.py b/datasette/cli.py index 2cadabbf..90ae137e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -49,6 +49,8 @@ class Config(click.ParamType): ) return return name, int(value) + elif isinstance(default, str): + return name, value else: # Should never happen: self.fail('Invalid option') diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 05acff84..db6eb0dc 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -10,7 +10,7 @@ {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} {% block content %} -
home
+
home

{{ metadata.title or database }}

@@ -27,7 +27,7 @@ {% for table in tables %} {% if show_hidden or not table.hidden %}
-

{{ table.name }}{% if table.hidden %} (hidden){% endif %}

+

{{ table.name }}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns[:9] %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}{% if table.columns|length > 9 %}...{% endif %}

{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}

@@ -35,14 +35,14 @@ {% endfor %} {% if hidden_count and not show_hidden %} -

... and {{ "{:,}".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}

+

... and {{ "{:,}".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}

{% endif %} {% if views %}

Views

{% endif %} @@ -51,13 +51,13 @@

Queries

{% endif %} {% if config.allow_download %} -

Download SQLite DB: {{ database }}.db

+

Download SQLite DB: {{ database }}.db

{% endif %} {% include "_codemirror_foot.html" %} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index cb52740a..bea9c893 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -10,7 +10,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% for database in databases %} -

{{ database.name }}

+

{{ database.name }}

{{ "{:,}".format(database.table_rows_sum) }} rows in {{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif %} {% if database.hidden_tables_count %} @@ -21,7 +21,7 @@ {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %} {% endif %}

-

{% for table in database.tables_truncated %}{{ table.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_more %}, ...{% endif %}

+

{% for table in database.tables_truncated %}{{ table.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_more %}, ...{% endif %}

{% endfor %} {% endblock %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b23c67d8..97d3dd58 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -19,13 +19,13 @@ {% block body_class %}query db-{{ database|to_css_class }}{% endblock %} {% block content %} -
home / {{ database }}
+
home / {{ database }}

{{ metadata.title or database }}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -
+

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}

{% if editable and config.allow_sql %}

diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 4ef0b758..1aa6f80a 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -16,7 +16,7 @@ {% block body_class %}row db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %} {% block content %} -
home / {{ database }} / {{ table }}
+
home / {{ database }} / {{ table }}

{{ table }}: {{ ', '.join(primary_key_values) }}

@@ -31,7 +31,7 @@
    {% for other in foreign_key_tables %}
  • - + {{ "{:,}".format(other.count) }} row{% if other.count == 1 %}{% else %}s{% endif %} from {{ other.other_column }} in {{ other.other_table }}
    • diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 6177163a..8d4e68f7 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -17,7 +17,7 @@ {% block body_class %}table db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %} {% block content %} -
    home / {{ database }}
    +
    home / {{ database }}

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}

    @@ -29,7 +29,7 @@ {% endif %} - + {% if supports_search %}
    {% endif %} diff --git a/datasette/views/base.py b/datasette/views/base.py index 4b89c975..e38b0525 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -102,6 +102,7 @@ class RenderMixin(HTTPMethodView): "extra_js_urls": self._asset_urls( "extra_js_urls", template, context ), + "prefix": self.ds.config("url_prefix") or "/", } } ) @@ -190,7 +191,7 @@ class BaseView(RenderMixin): return name, expected, None def absolute_url(self, request, path): - url = urllib.parse.urljoin(request.url, path) + url = urllib.parse.urljoin(self.ds.config("url_prefix") or request.url, path) if url.startswith("http://") and self.ds.config("force_https_urls"): url = "https://" + url[len("http://"):] return url From 788f218e267ad9e29df904c6c6a61e78bcc8e744 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jan 2019 20:44:34 -0800 Subject: [PATCH 0003/2303] Supress pytest warnings from 3rd party modules --- pytest.ini | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..92b08b4d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +filterwarnings= + # https://github.com/pallets/jinja/issues/927 + ignore:Using or importing the ABCs::jinja2 + # https://bugs.launchpad.net/beautifulsoup/+bug/1778909 + ignore:Using or importing the ABCs::bs4.element + # Sanic verify_ssl=True + ignore:verify_ssl is deprecated::sanic From 5b026115126bedbb66457767e169139146d1c9fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Jan 2019 16:25:29 -0800 Subject: [PATCH 0004/2303] Corrected import path in plugin docs --- docs/plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 3380bbc4..0ec30434 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -380,7 +380,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ .. code-block:: python from datasette import hookimpl - from datasette.common import add_common_publish_arguments_and_options + from datasette.publish.common import add_common_publish_arguments_and_options import click From 8ce7866312f9b7c873541d6e3662ada3e850fa85 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Jan 2019 16:44:37 -0800 Subject: [PATCH 0005/2303] compile_options output in /-/versions, closes #396 --- datasette/app.py | 3 +++ docs/introspection.rst | 12 +++++++++++- tests/test_api.py | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index cdfcb11d..8252184c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -363,6 +363,9 @@ class Datasette: "version": sqlite_version, "fts_versions": fts_versions, "extensions": sqlite_extensions, + "compile_options": [ + r[0] for r in conn.execute("pragma compile_options;").fetchall() + ], }, } diff --git a/docs/introspection.rst b/docs/introspection.rst index 1a0ab952..b4dbfc6e 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -61,7 +61,7 @@ This is an internal implementation detail of Datasette and the format should not /-/versions ----------- -Shows the version of Datasette, Python and SQLite. `Versions example `_:: +Shows the version of Datasette, Python and SQLite. `Versions example `_:: { "datasette": { @@ -76,9 +76,19 @@ Shows the version of Datasette, Python and SQLite. `Versions example Date: Thu, 10 Jan 2019 16:47:15 -0800 Subject: [PATCH 0006/2303] Bump aiohttp to 3.5.3 to fix a warning --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7edd78a1..92b92c3f 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ setup( extras_require={ 'test': [ 'pytest==4.0.2', - 'aiohttp==3.3.2', + 'aiohttp==3.5.3', 'beautifulsoup4==4.6.1', ] }, From 50d8d8216dcc5557e6b581a13bb871f033722117 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Jan 2019 16:47:54 -0800 Subject: [PATCH 0007/2303] Dockerfile now builds SQLite 3.26.0, closes #397 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb3d6621..cd9b642c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ RUN apt update \ && apt clean -RUN wget "https://www.sqlite.org/2018/sqlite-autoconf-3230100.tar.gz" && tar xzf sqlite-autoconf-3230100.tar.gz \ - && cd sqlite-autoconf-3230100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ +RUN wget "https://www.sqlite.org/2018/sqlite-autoconf-3260000.tar.gz" && tar xzf sqlite-autoconf-3260000.tar.gz \ + && cd sqlite-autoconf-3260000 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ && make && make install RUN wget "https://www.gaia-gis.it/gaia-sins/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \ From 4722acc73ce761556b18f5dcbe36b7fef2ee2c69 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Jan 2019 16:51:38 -0800 Subject: [PATCH 0008/2303] Release 0.26.1 --- README.md | 1 + docs/changelog.rst | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index c2acab1c..9eb6dafe 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 10th January 2019: [Datasette 0.26.1](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. * 2nd January 2019: [Datasette 0.26](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. * 18th December 2018: [Fast Autocomplete Search for Your Website](https://24ways.org/2018/fast-autocomplete-search-for-your-website/) - a new tutorial on using Datasette to build a JavaScript autocomplete search engine. * 3rd October 2018: [The interesting ideas in Datasette](https://simonwillison.net/2018/Oct/4/datasette-ideas/) - a write-up of some of the less obvious interesting ideas embedded in the Datasette project. diff --git a/docs/changelog.rst b/docs/changelog.rst index e59142e2..b6c46c69 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_26_1: + +0.26.1 (2019-01-10) +------------------- + +- ``/-/versions`` now includes SQLite ``compile_options`` (`#396 `__) +- `datasetteproject/datasette `__ Docker image now uses SQLite 3.26.0 (`#397 `__) +- Cleaned up some deprecation warnings under Python 3.7 + .. _v0_26: 0.26 (2019-01-02) From c3a78eb05c0b146b2eb37b50c1668b9430b8f50a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Jan 2019 14:23:44 -0800 Subject: [PATCH 0009/2303] app_client() fixture doesn't need to take **kwargs --- tests/fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index a77d141d..16395553 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -61,8 +61,8 @@ def make_app_client( @pytest.fixture(scope="session") -def app_client(**kwargs): - yield from make_app_client(**kwargs) +def app_client(): + yield from make_app_client() @pytest.fixture(scope='session') From a418c8b44f82d456be523c8690cf7236bb648c22 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Jan 2019 15:09:48 -0800 Subject: [PATCH 0010/2303] Expose current git tag to Docker build, closes #399 --- .dockerignore | 1 - Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2c34db66..938173e9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,6 @@ .DS_Store .cache .eggs -.git .gitignore .ipynb_checkpoints .travis.yml diff --git a/Dockerfile b/Dockerfile index cd9b642c..7001e799 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.6-slim-stretch as build # Setup build dependencies RUN apt update \ -&& apt install -y python3-dev build-essential wget libxml2-dev libproj-dev libgeos-dev libsqlite3-dev zlib1g-dev pkg-config \ +&& apt install -y python3-dev build-essential wget libxml2-dev libproj-dev libgeos-dev libsqlite3-dev zlib1g-dev pkg-config git \ && apt clean From 7950105c278b140e6cb665c68b59df219870f9bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Jan 2019 15:33:50 -0800 Subject: [PATCH 0011/2303] Python 3.7.2 as base for Docker image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7001e799..08639e52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim-stretch as build +FROM python:3.7.2-slim-stretch as build # Setup build dependencies RUN apt update \ @@ -27,7 +27,7 @@ COPY . /datasette RUN pip install /datasette -FROM python:3.6-slim-stretch +FROM python:3.7.2-slim-stretch # Copy python dependencies and spatialite libraries COPY --from=build /usr/local/lib/ /usr/local/lib/ From 909cc8fbdfc9c05e447f40e9a73489809602c3cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 26 Jan 2019 12:01:16 -0800 Subject: [PATCH 0012/2303] New 'datasette plugins' command to list installed plugins --- datasette/app.py | 7 +++++-- datasette/cli.py | 13 +++++++++++++ docs/plugins.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8252184c..7bbdef3e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -369,7 +369,10 @@ class Datasette: }, } - def plugins(self): + def plugins(self, show_all=False): + ps = list(get_plugins(pm)) + if not show_all: + ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] return [ { "name": p["name"], @@ -377,7 +380,7 @@ class Datasette: "templates": p["templates_path"] is not None, "version": p.get("version"), } - for p in get_plugins(pm) if p["name"] not in DEFAULT_PLUGINS + for p in ps ] async def execute( diff --git a/datasette/cli.py b/datasette/cli.py index 2cadabbf..34ed0020 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -159,6 +159,19 @@ def skeleton(files, metadata, sqlite_extensions): click.echo("Wrote skeleton to {}".format(metadata)) +@cli.command() +@click.option("--all", help="Include built-in default plugins", is_flag=True) +@click.option( + "--plugins-dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Path to directory containing custom plugins", +) +def plugins(all, plugins_dir): + "List currently available plugins" + app = Datasette([], plugins_dir=plugins_dir) + click.echo(json.dumps(app.plugins(all), indent=4)) + + @cli.command() @click.argument("files", type=click.Path(exists=True), nargs=-1, required=True) @click.option( diff --git a/docs/plugins.rst b/docs/plugins.rst index 0ec30434..6efa131b 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -58,6 +58,49 @@ Now you can navigate to http://localhost:8001/mydb and run this SQL:: To see the output of your plugin. +Seeing what plugins are installed +--------------------------------- + +You can see a list of installed plugins by navigating to the ``/-/plugins`` page of your Datasette instance - for example: https://fivethirtyeight.datasettes.com/-/plugins + +You can also use the ``datasette plugins`` command:: + + $ datasette plugins + [ + { + "name": "datasette_json_html", + "static": false, + "templates": false, + "version": "0.4.0" + } + ] + +If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:: + + $ datasette plugins --all + [ + { + "name": "datasette_json_html", + "static": false, + "templates": false, + "version": "0.4.0" + }, + { + "name": "datasette.publish.heroku", + "static": false, + "templates": false, + "version": null + }, + { + "name": "datasette.publish.now", + "static": false, + "templates": false, + "version": null + } + ] + +You can add the ``--plugins-dir=`` option to include any plugins found in that directory. + Packaging a plugin ------------------ From b5dd83981a7dbff571284d4d90a950c740245b05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 27 Jan 2019 17:40:23 -0800 Subject: [PATCH 0013/2303] Export option: _shape=array&_nl=on for newline-delimited JSON --- datasette/templates/table.html | 8 +++++++- datasette/views/base.py | 12 ++++++++++-- docs/advanced_export.png | Bin 7662 -> 24148 bytes docs/json_api.rst | 24 +++++++++++++++++------- tests/test_api.py | 22 ++++++++++++++++++++++ tests/test_html.py | 3 ++- 6 files changed, 58 insertions(+), 11 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 6177163a..7a9f9115 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -140,7 +140,13 @@ {% if display_rows %}

    Advanced export

    -

    JSON shape: default, array{% if primary_keys %}, object{% endif %}

    +

    JSON shape: + default, + array, + newline-delimited{% if primary_keys %}, + object + {% endif %} +

    CSV options: diff --git a/datasette/views/base.py b/datasette/views/base.py index 4b89c975..f4a8afaf 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -432,10 +432,18 @@ class BaseView(RenderMixin): headers = {} if self.ds.cors: headers["Access-Control-Allow-Origin"] = "*" + # Handle _nl option for _shape=array + nl = request.args.get("_nl", "") + if nl and shape == "array": + body = "\n".join(json.dumps(item) for item in data) + content_type = "text/plain" + else: + body = json.dumps(data, cls=CustomJSONEncoder) + content_type = "application/json" r = response.HTTPResponse( - json.dumps(data, cls=CustomJSONEncoder), + body, status=status_code, - content_type="application/json", + content_type=content_type, headers=headers, ) else: diff --git a/docs/advanced_export.png b/docs/advanced_export.png index 90fecbde34ceaf86a5088d95db93fe465ca48081..d4349fac5c8625d9e9640c5f66ac6732aab13ae1 100644 GIT binary patch literal 24148 zcmd42cT`jB+bxQsA|N6qqN1Qk4IQMHpn&ue2p~0dr1xF|TM!5;O7Cb82)%b9ND=AM zJE2MMy(Qd*d;j*i_kQ<&=btmiISj^#t`$~!->1$wpKx_mIWm$5Bm@KmWD4>!8UzH_ z9ug2-xp(s_aKuU7WuAb*h*m-7Im~N(qn;=Mq?L5B)wS8RzTUOo)3X_jvp?Pv&R~7~ zoDu!`FW{H>Iw3MF@G3)?DFKE0omP&|#!UpwA0J+87gSCH>AkLWX^WGZS}fZ2SgKypyK`P1sxmX?t1ZC&2?%E1&2Y$E|MSns;6EP(&?|pF z!$|%)j=K8qkv}J-V1Iw_KgR>n|KrG?Gj~9LKK?!a82rx>^+42|3^L`~<>|ly#ZPxY zY{cY$-=*GoE>To-%Y7NM$q&2VoBcVXUM)j$&-)kXlZ@;O*-v?P!cj39 zf1e0j0Q2wWv-cob;Vv4~s!xI~nrFX0{PXI8FBrqSgk=psMWAmM#ozo%oPDRMbXHyC zHsR%;N7{C6b!CMyy~-pBg+>GwbPecZc{%f2fIyG9)nPQpg} z&%#7L60(_f3*jd5vxRV@7-nmzRH6TDRS&^r3OcY*-ny9KI&_vh^H&+g-%Ahiu&XF5 zVn0_~&DsrLhwPriY<<4F9=nAc_UNZ~UJ(O0D-jI;~~6kPRtfO+go%{9|d2i7a5dVum5I z&nP6aaq>UUln4e#=6!+A%I&4JDC<&E!vWR+m+?Ad` zYlE`?j}x-pYmQEfdS!RX&xR+G1Z<-X(Dzi+dCTOAs`RU@&$PEZztYE~6r(>bwJ1cV zPL!Bg%}Dwlt~+B?lO>;Suf2Uwi}l`m)6?s2zvHwPS1|6jJbF@3e)`sG$Z@K&d&>ME zNx(8(HBoe{C07+e%uq+=o^CZl2FCSHn90eFe<>WI-n4qIZTR zE<=GqZ$v#d`wzoCa1flqg-V0%aN$&4+dJIi2~90z{IFf^6p_1~%?hrxn-aCzPlz}2 zE*N?mc@VHvFXFnyHWb986wkT02M2LHww_c2tLzQZ>(25YZY5!igG%cUGVEg^W^M1W zsc~`QredBO4}=0ti>l7h4y*{tv$@Xeq%^t*H;esFkWy3Xbh*W%Za&)Ah0K0%oJ8U* zQ{bGjLwAh?LOrSeY>p}h{i@V0(Xp^yzCf@3?U`d04ZLHe$$O6`22-tQ?36K!Q=wkk zmT%P(DD;#xtgXLS9Seb>#Dp&@Y)7U%&J2m=pbML|*(Q2=wWJ)hQ@FtfX_myT5v-Hz zOjL)CL5=bECMP{uF<(b>p;cs(YMr5>bj?E0zBI@W9)!em=(-Pc?dKiBBM5d-Ad-zPMF z5|5wX>l~Fkt{vB{UHfoKW}vkr+bTh})w{x`R)Jt{@ta*+k-%+ZcnUL=Kjzwm zYBH!HiGNmIwB!FiIj#mBwY7CRZ+3rbRls&?dCNOfDQ#`M`>34D!!!Ka%N$bn>U0|; zMDswe+iL%YEH)u-w!vinrDr(bcTDo!p7CCsVpu==BVG+E6GF1TsJ~XD1lDSpv}9tt z!0G$*8XIdrrx($f2iD@m+g(MA2fo54t6ydBI@@ffu9)i9CtW}XEj-1WVeT)rsqi&l zXzjo(EA57|nwYfwF1>Lz7=_veJ$FCC6VZG4?l8fr+a>VkV%{D8XB(E&Du77@qJ_Ir zrpG^_J?dgrUTi|rQ$kV}ayJTh-}g~wc6KL-bmGc=x|0GWR!^gFsyjxGN6zuL&TWeK z;&tuPkl~K@2!4mZ=!1apMCRE}B-8rnfVP?<+#c({bc7gjbV{#_MQH74(CE=L@VX3+ z;K}S$UpC&1<8d0xgN&E^GnjDWH$122Y4?$8+7$2Ztvu6`fzGy_050Orr|gISO>m@n z@L=Rb7UOq9g1DBmL)eWASCf8w`-q-`wZ(J#tPxl4I#Os`SnxZ7>}VjRX743P zWuisWllmUYdqKXZ9tGTBM#l5ob~N~)MRE~D)ji|2N_&6EX^R11RoA0tdTCbA4Mu`d zrtSR}cqV)bmjd(BmYSbWU9VVB`C#qN-J`rNgA=>){Q}4dxOmrVW85A8xveL4M?o_P znza2cbvUw8MFD2uqbor3YG((b;m4JuF*If0HR^Z{$FAa3n-pZos{)>F--N^DtF8qY z)^bdUrT_sh5wjUxF?aDrJ17r`p{v{*ufCaEzkv3YY9(jfS8DeElRJQngF=~}9X`Up zZY-XCI;lo0=y1!fay~$V?idoXl+HRHKvP}z1=NrEb<2UNQf9?PwVu^6I_7(iVO^F= z`p$9m#`zf|H7JuS|4y~X&tJ2fBN1RDn>28jQ;Z+ARE>F0%+PXCo7Yo1GMr&-msXK; zDk+cY;PR`(fsRa*rj;ad3mZ4+7v%EQ_#&9d!=OdZGQ0iTkHpSKa(9Di8kA7iWXz;s z#{=kx`2AvTB0r0?k>LimY-{LSi0S;@-eVN%)s6=E_jbsivIj_;Q0VNDerJMysoNNk z>{9bQ+*J^z$U2+8IVSwOoM54&r3WXE_VDGIYLK%RlJh^*+$71LAy}R0Et;0#w_gt& zh9`|+XUgRonW#af>x5NF&%BYCuiwRS_VFEB)r^V~UbybG_K^=~n;B}SD^b&1-%?f7Rmnzl51hYIx5y}4Qjtp`%A#&%;}>dUyn=LKe4wv*3{ zEl#DGOxK0xx$>Qyg_(E1<&SH%%eImUVIPF`FR%?X0bUyR39k{#de9i@RJa*CG96}- z?hU%DTJnw}VHst*{mZ29squvOi9FX0L0^I2(K4{(h}=gTVs5+fDs6lpX0{z^CEv@q ztsJ6Af31FM145ZY$e;36rZrL;)&(zqTx$;=!Tr|RXlmrhYE-a!i^+)L23PtYr6y~^ z^JT1mXX>XxftpCMh_uFT%msT0q|G0lMGcZR`}cICJFZ+Phig)RNbhpErDb+%z0)A~ zIr1J68iQQ8NKsl@nfd|otf`gp1BhKmmIj1Z@TSrF_RRa*Ra-EuJBB|fO5)C^)U)v{Z&y*toQ}R zp%4X$eoC^Cqe3ay7?yPy`>@ixb2xy<`reUdYOUur?M=ziZx#(OCgb-K$ug^7f3$<+nY2hC}TEjY<%7)8)6fJ5wi7Caiw%gs&h9> zt=+7SEC!kwI8DdL?bVwI1Pfw~B@|;n_!A0exdZz0hT?y0`U|0>UI*LZx!DrzJ5n*g zVpG;#>duFKJekD_Cbq54f@NFf()V2$j(^DFZ6QJO1i*i zMxYsO)i;$BgxgQJb!)reQdpokfj_+Av9l{1vM()FtDG#^%smHWw?xq zYwV2Cz)$@zOrRyUN1Kh#jxJ1{F`7|0W4vTp0lodê|oZE_dY`TC^rY-0`{M6~e zMc3)N7Ks{Upq<{tiFc3+x1#K@`H?Z0TDEC`=PHpN0*L*-$POVMb81=Gv6y=MX}Y*Y z?nxf?+uO9x`0Eyid{Yj3iXR|~kMC#b?YxXMYG~B2e(n4MYSPauA+dKW8mJ~uYz;T9 zy)bn)6P{Q@^+g8E%w+Mq)dYlOEyjZZiB_-MUen58CeJkj>{wKM{jw1SJe3hT8%cOs zx+s)d@SR3)#ro`F@Nbj-GxmTQ9j&|eBbfGCNTkJf`sN4bo53r6Yxa=9XFq)D_x)F5 zN18?|{ZUgnVI)sT1wK33OnF5>qNvhm`Ck{47U`+)^mqE*charM^NXIcbGwh-j7k;r zyb&k2#@4|LF9V!nuHlCYF7N9VYAeLT^UZ>3U1v%w z_N1wC^87mKdDsDNIa82;pzD)5(P{@W<=l(c=+Ry+mM1Vr^$h0nU>YuA( zY2ZFg3Z*#NA3OfWkcw##2P>m5AX`%tu#$v8cVAX}cIlVY2JL@cZ+Zx4fcvX}U zh2HLUmiyC>qUN4OkRbY%RB7LywjZ8h91A?N&lLJ>2M&qvFXK&UHqtiH)I)W9n~av& zGzs5x-#DN5Xw@3XiVchQqM85`UIytHQF%>rR!wn5 zh@14V_Vi+YN&Ir;G52uHRi`NpvM1`#Zq_+TqZU=ufJ4{(dnn1 zE&ad>HoZTugLi29C!3|4VidCGox=tFgr!kS-SC;L1Dc26 z=`q*E+m8faG(Dc1_t>1wxp$jZ@&)fC(EHeW{qFxOB;w_}2b@DO+_D+My{SS7AOQ?m zbxi_IM#F-%6f7Z|&-^?%Tr*omEnvGwx?OCKRzwuvN z^#60gga16)|5_yCAH?oI?K{E0u*SdG$^RQo`FG9!3)uHJ-1A>yjsNC}=iX6*e9F`E zt}&gvPo0thZSvOHY#aR6Jz=xO{3&Z|=w=RMSBk(HP~fPfJQLyMlsL+!7m4EN>dPQ; zeHMBzE7(PHq%Bo}wYT9y(gQ3~<+;^O&rEUpmZ` z*yRZbn36oNhQD94xhQx=e39FkYdDlS5kc(U>0aTe9k|EMT4yr< zBV}`J8_!I#r@XYJS}y}Rn^T%v1+pk!#>Oe*B~h;^`$kFIuBm#8Foo~-_cs^Ek=C_u zCXaWnqrhnar@m9?J!0hBwQQ%?NR+nj<dlQ7;NFveTb}ksr`#O)E@c{ZiUrkEG>U!Cf<> zwIC>TP_~tj{9(*Q$^1eh?4;R!H#7b=TcT$6;)Jo`@muLox*_Ra3C~_N85s48ZtsOp zULt}H?|^tA)4Dh)uAz&F&Ka>02vQzC>73KhQeCaCkN ztn7d#=fc2D!w=$Sb~(HDmYAFY1Ho9JCHDh*aL)EOH^?7ZKBbI^=O7D1j$qeyfj0Uy z0~$@F28Dgi%ysT7g6P%^Pv-=p?o|6;4&_Kw(j)ZCB``I_U&@fGR}*@zF3_ni03dO!H# z4oVE`RNpet4`@xZ>fGIpceUOnt2S$!!-&$pZbed^Zhiljbgky)?)I$Rrx@O6HiLab zd1?Uj*?yogwJccuf{dMCXl688-`sfgNN98PaN~RSu~0e5-b%LI{x8ghf276AY;#WQ znAKD*6(JGbBZ<=lt)_EIkj-<&EQ4yc5ZRD9i0|Rf6X_2Q<1xnvAhbx3pC;c@FC^`} zJNp)NKr_5I!P%f{7ojm9tG2heL-OnM_bV8HD0mDe(SnJOHN5frO64+6Y4iQB5WXVM z^-=4KjT3Fh5J;6*4r7)#huKmlP(iaVyy@1zArrD$Y&^>_0G1AY|1%k_4z2K6;FWEL zKMm-#uR441BOIxfr-OZ!T}Zz>{Za+UvWXiR?)$Ifm*A}KCf6aa+WCG@FZXql51-T& z?jmw={GB@a`F^djTni#g@Xx-RKvDE?C}$uGuDt=KDRrwscI#J8FLjOH(i5K?4=}89 z@GlHOb9)Zup5bTd`DnB{V>xPc1hrq(Dc*R0jyTBc`UpAG)yvWFweYSZz?>Nl`*-2a zFxhqRvwac2!!h4;_{9$WfJ5CzpkwJ^5kd;TG0s$MieE{ei~l)EPU}=EN!S}(hd;%6 zzpjNcoHUV!E?jIk6Is}ozjofH!Gsa5jqPpc>$V-pCGzczZESsWJtjxUk09I&3@e=H z!)x8is}I%|@n@KUQ1V_o4#FEPbtzhRiROrVZ6uy|irbC1=G3R#hqvR& z(*1VNoGP}aaq5cBg415}+lX_y+6x1%Edg8A6cqa7Xx#Wt#!(e!le@x`w0c6 zVZG>@i}T)na)>zPof6^^nEYR--G^8_r=dOPuMCsj#UHds?Cx9(MTk^p;@O2^rucPI zs~oV1AlapH>BFj@gaE+Nt4cfXLeJNS>g?AWikt@wq#Jg}X-T}jEApCAFA$x7Rk_my z8NSW{TFIR7Yxi-;-ZH~99i%e(@>?KejnRMv3iJE{d>0wZ>OUeu#&jds&~Tt%AM;## ztIV^%pUKm`TQ3qaR_Gt9I%1&T=_iRh?+iqL^!ap)UHe-MwNhQS*l>MD1E*x)8|@<9 zdAW*_34efjsJ5n^Dg)w|{rB)WOz#@@0~gq`M;zY<`N0{E#S3oa<{b*y`xKkD)}Nv; z_8{4@>{UX>IsnoaZrVDRz5%%qF?{PX|AWZla{|TrPFo^9e)soL2V?(l%R9@C{$Dq; z=_GE8@$1ER`k=Kr2Uug?yCYGitAl1f!`@34dZ4AWI=PGPX3@^DPi`obNi$*3?nmgd zObEx5B}bYLo08@Brn=)f|5CHf`lCiTKFZ<7WT}NSse7}a7`5hyQ>Z+|9M07w@!QgGr`;cELGYzzEFJp4)xSHvGSRHKO-WHXlaG#Zr zQ|jcqxJ=#!Ht&zW#(NZC3ubn%OjvdVfHH~4_<8F4vnE>}4_#D>bY9*ne&n6il|Hthbo+F4<}=kEgDIo}IaPSYy? zJUKd7+_H{=_j@~R(G7CvEbqXi3I`a^*l>+xr(oOK-o}f=i+$HUc!^6a?fX82QVds>`3fK0dUUdw znY{9nTvCkX_KYXBCZW+%G;_aIm*DR5Tb`bQU>Qn_#dPBY7hxi8oJC$+~vIBd~{%G?6fh z(Hm{bYW_`znVrdK9#v-**?svWG^?R#ny*&wdi2*4abDsw%|NrS)VhnA zKbp0+W~sOF=&#i=6uO{NYoTJYzZ<4MQn=G;!6~cuVXSCah{LtxX8>mJvgmBg9%kGg~;)@11=i zwao8GR`7Zat&pWrrcTn=m=m3X+TI@?d`8)cW9<4O*|a}gj5H~Dd!*k!6p+Tom$rRq zWqgss1Mo5llDLHEYYvloh-J*;R@QYY)CqZ-=0=gZLnT6rTkQ#7F)xepA} z5Z}`mzY?{}46FG=&V@&;0JH_WS!Q#vT8zHKZ`o5y06?0`3a|u&n$KObp{2c@bfR`L zxEo|}pWhXM=mcTwQhMbRIMDB&kkgBF_)Tx_sY;0-Hw)_+x!S*i0cF3~9WMYY4N3?hD;Oo$FB3AUBD8VDX^un(x7CA$VPUf)q?3QVdy0qx_kb4ORh%JwrA4YgGlHp>tk?>=0^IbUM8*F+Nn%b z7}BuLb0X?p-Io1V;U(%WnUH=;yaLUS%s#pFV;{CVFRu@Nv#_PoxdMgmozg~`Zvnhi zFHga3eMYJ@Ixir7JhjYFi?~bVdb%31*NdH>&{C6#o!9XI%r5A|d2k(MWkjlQ8B_Gl zMffqHY6d!tPJs%%GQwpxTqD*HRui2=37FSw2-yD1hLUV>Ij4tHqL@o`j6}9bk zyp+$?Qc|BIBL9(1<1p%Ush9y6OjdAdvL;JBm;PZtdy$Cy&WaWMxaspiS&bUvf$-~M z6@+Ng>r+ht!<-4E4iTS_nBuKPhgoop7FUo|%Gm>axzdw-Z1EEgutW99@b>ScXWt-p z1MExj-0;4l3oAuKz^Uw?J^h0AS}19w`k~l2T%?lRDklz*wGMSV)39u#QA5~{25$Vz zKpEQ*bWLnziGQ}}VD-`VMlP-6q~oLQSnvi}H7R#j%E`uEG3GH158Npg9}!T^_)h&| zE}>=vRwjxWystU)f(2}z!T7Dr?z2Mlhx+r~FK1g2ubtMjNfJG*-^pA=h9K$B6Sw7U ze`8B;Hz-A#0+Y?)=+-;Voil9q#9+SD>^OLWlWwN9nW8gvm=NpyU*0} z_xVT!{VU=-(91+=!4#@9*)eugAtF%RDMsR zH9>pse5VqKMILmXXwt8MaS0YI8&ckq8L#Df^l1>PoEDI!39woKB~1Bp0;9#IUsig?@iikBls^*r=yqh96^hVdtkK;Z~~PqFrY2dS!o?$aNl(0^@W^SRhvNuN)P`o5hy{W;Yx>ee)1(@-1SaF)qX1JS7vQiQ|;z?#T^Av zTl+q%WI?RsxElk@lHj~rb=Hfw5u5^aBD-jP$2)OXM8ZjQ^8|apNR9kiWhAY`XgYv) ztl!2?t~=_67B%?1`jMiqobt=hdO~n>?Ayun5gK2>%r#V0W%vz8jZi?h{46`_fILF| zExXg!bm&3a@jU&&lrr7KbNQFsA0DwV<#oh$A5#A0hJLcKBdyt`d2|_FwCfg{l(&en|Vmcy5Ndj&NpMwy03E7)il4NZkH1#vPCCn=e;sy- zP{=+HM*;;=yzE*WhnSs!M{=dwC;~*LTfdvgJ(G_%p0QS z#m*#wo!{epV@#QLV5qTl?l}mQOL*#tZIGj^|EUUI0wI>nLal#V8rD8b8zrLq`R;^> zzJ-3IV(m)+ZVutL>4y%KkyR|=Tb-nOQZf9Hys#1tmQ-G@rkRf2;bzeq!$6`=Ndr*(a@d${%v5DfnsRe-`_) zqFl6_CrUa?10)Rm$!CDDNVs>+F^u+9Xeh_?*vXU@?H_tf{dncd(%v+Y|HgUUbLqE> zzg8-W4MLQtv)q#_8xX3f7q@67o9-Hq{>X0!h^%)_;(ePdff$mG56xwq22YVBpkRy6 zcZp?x)x#qt($rzqa>w$LV``iG&ic2f0bJ39LaWj*bGs5&W4b2r+p_1@=64ah(Z0qT z!j;DRRTvdg`W@T49awI;P&m3+M3&@!^&##k@*1!OGX0q3zQU=a>zqK{!>0x1by*Tlf9f!=K>f*gf(%e*@OwvYJkXZm z@JB0^pXc^^`N^K^x_f*MCeytZSG<&gmS?LBF*u3%0Hkm${ceM@MSO8}>E7MF%CqPV zY?69qB8Qu8Rw`PZ@D@?SN|s&n>Ra0n1@rNWw+`=ah+4O)j0Tui!d8|bYQ@<2*0x6) z;eS^C3K9wMKCK_T(Xvz%NMVlQ`ZFCef)^&ro2<`q%4g zQ)+6xr_aE$a56g6i5~55`=wGW{*=%sj6sv8Zx+}Sp;?qd1*mfod~-|^O)ZAQ zou$fqFK%IIu&d(Yw8$+vbf$|!j6{@2UrMmt{}^YlVc9F`p!CS+Kr}@LHtDk#P-gbQ z%l{JqI#m##HiXZJ%C)`kH-tHPaTNi6Z37yT7XCn^adul1aFP{Yl=W9ZWCsuk= zSvJChVE5AfJz7-xe__ZTn4eVyyg7@xi%W*1UWk6bEd~IAfnsl7R47>3c)Xg~<}Tao zk@SsJY)QxNR$*FuCC)qH{mnm|C6P#&nSMg-MR^#>gtCIk)r)j5Bg$y}< z76t%r()%dDRFShVCgwQpEOzllA^Odx3+~H>9&e3`JI)onS-Td9J>`J>{t?-`lR`2j zcL1O?KXac5UNxK3)@q%OsNF_I%oXH#?MPq)l4GAZgtA|xIQkYYbhJm`;|G|TpE}Sl zGJs}_gjNmhxinQUQSQ$789)UHgCLtDSlvAUdzf1Ie%F3nvJZepX<}-AiZ}a8un|W> zhNcbOe+!IA4z7M07aHRAXYf2TJX)Wu;djz#A+C91D;!~T=4d~*(v@_IcKRLN)-#JZoV-knz z?+9JVh|AN?>9l`RO)2TV$_rN?lQ_sjO0_P;l|lB^E1!qsas0N`3{Q0_aS0xmDQ&Y3 z(*7V1J0poGdZJs^>%I}^lr(^h%`^W|qoS9Z=PQSsefo0obwAldHvs*ds16nx8wME5 zl>;GPIyJYK4lYoNy$^Yv?nPP_hDY)63W^&`g*BDKEI0_+Bqu`e^>G-WN+LRQWfU!RJqkG-*Kh?ay0= zbhgU!`Ll1S33vPVrbQz$`ih&yL)64e)W3`Vgl63nH9(Ug;qwr`20wcB<@nI2{eqWX5gu2ySf1Ajlr!KKijSIqlhaY z>H=JUi_-##s?SX3z5;mBCs3XnQ}0zK|CLgUjF*wnrwgo={uc5aSC}ipHl^0SP~YGU&nWD_NlN%<&^rLDJ1HW($F0pr3_J7goNs-jG9dFU1>h@2Gns!0uFk23Z#aI zkRX;URS8Sl*1OKT?3E$vCu-aj^BfeY(?qAfvfldlN&A2(7-m$!J9Gk|EA;Q37|u(~ zN&rSOPnoLlX@>&r-l=IZ zLNPLGXLBg-uxclqVYdF=;A&Bx{~jRO$b2>XY&n`!9~ockG*&n`JCM4_L4?VpDw2y6kUPspI@Y}0(`?N9dsI{t6k9*7G6E4B9jn;Gu^GBfiJvHmZO{_otv zztaK#lP8`j-&U_0h3B;d9K?_6TFtt6cA(3+BPzVF|059yd}Y^t2~JCR3}ae)azSEi zYzF3@hU@v*R=m1+r3K4B13TLTi`brsRIR(8KC+AmdJJDLfxv_Rnc+DPwjO8m7)CrL zSMA2J4ObEY0+(aQZTedLj>* z#y_&>4uuAMUS`*)(W>}9aF5LPcFwSr?^!j*P%kCB&c4t3wYpTn3e2W-bDe91Y7bZc z03EljDec~`^#*mg$!T*wQBzXpAbT%mRUD0pk%LheqDO%V0f10djL$nBaqT&p0%W;Q zb3SfIDMV)v)@KAf_A8;x6m#?O05qjsud=P`IxYZqI|j)J?(>o`Xp$IcJphd1#hI*(~Axtx5RVR*kVCEu+yK(oujm}7yK=HIJAgTx4;yor^ zMAU5(76S>h zAx$aLgHWb`4j8YUAFSsBC?klbJ(H{Kyg#2v5BFdmZpZGyYiuf3$gIf`#~P9Mcx~K( zy*U`bsU(YB?w%C}1rKuyZ z?koZFRr6W$=*_puSMz$68{3uuNx%&a7Jmj1*Lw3J& zhk#D36aW{V-e22r5qJZLNdz6vWeiL0#&)-_nu3BdB_8{5M!$Djp(U9wRUM7e6ev6) zy$gu+w#2#>lBnzIh8U++W(y}6Ose@YrkS?br?0gNu4?3Z!hAiJMi`SI&|jE(He z?}HlosV^x&+OHzT535698J)`RFX`wGcH45@3Q;fO0ozy3N#^&Q&dFckS^!(c%8<$N z>P(UDyRkx!yswo`olN}Y6Z+!o$J`rI+fSIF6++0d52 z_xmXsZA@lIy@c-}W4Ll5lP$eW^RP5~;~bNcR$Cl5e3ocD|Qsnu4fK^Ir1Cr zy3E@FFHm#4Hac~&sS3Z6Wx3gMNYA#SH^i260p?#cs+o(-$8nW^9kFNLvK)0Odm#!q zZ$F;Q1}Nt+)8fO;y+SfuVd+a@Q=vZQJ9*Ytc|h%MlC73r71lixGWG3;d&Ly&jx$iZ z6WvYf)C~-_1v9*yDdTP!R4btKRu=fZ=}SAdtLqAYuasWR9sBt}gBWnFN95u!OHMjf zpsn}%Sz3}#XLA4m>y?NL{HakciSCfHkuhsNMwyjK}Ka8a5PM03$2J+*`j^beFblEI(_ir%PkSqbPqZ9iEeRh^D(n#A_V0 zX1&tWJ8dqY7pPcINm83pvfN+0NNkbvA}16mZ}hlsv#TgyGM8yk3`o5yfY}mO6vH=_ zziO(^zELSTf2lO$1H@ILBN1=$+Cyaegt7%4?P=9(X>S8|xdC7Sa5S7mgkAnnN-nxg z(T7&NLDR+V+8LUgPJ1)%Lz2s9v); zuBSmd`yrf_X-(%=jzxWtQY`0|UkA`WA6ET!;8Pd51qT$tlyP27qIWVyQjqBF_&rEaRaOd7}C+D4<26! zCZx#ysQ1$%5vOHG@Jk6Igc$loOrQVQaxyr^ zk3n|U&yJF%VPIaRUhx6smqF8y2Q?S|e9$kRI6!|V_+;l;bAwhOppovY;MVjpA5~GV zbN$I084B~;?zX}!0ErCr2k+K$hbM|Ko|%lD72AF`2Um zLKO#KA6e1nWtdEBJUk#VEiJ8|*`Fj|$<0KO+~-m`jg11b!RB<|75?SEN;T&5Nj-<^ zL~#ct0+IV2o4c3O8|T2pBH>cet!HZf{DaDVduWS1!65R$8bF%MBGd{%q(|(P5VHw;VbgQl-9&SDC(UD>RPcaC)|yaq-sMu*Ew`BImFVNi>V4}zWI6T{zxr9cK+n?5F06lkw^;z)Vx8Ic$1wX*dg+x+Iu;(yTqnXAaI^A-spjMGu2e8vQwZ3%Bk;v)lv0^?uqW2{ew zhX!OxvO6|Zm0ZonAD&`etFd>BQfc{SFmTsocR)ueEMqUaa2z8|>uF4b{`;I$BWX*ki;GDK8k)jTZDT0kz zRIjW0A6WJ#=JhHzeON$g5@yKgbGr&CU$a@Ah7#}I0jbg{@t|E+>rHGa3ar;E zc$|J24_(W4cMoZwk5L>hx_3CH(caNg1jy!s0^WZ}t^0QPk{sT$D4Oxa$Mq)-bk>t~ z+1~;uAo#win^awE{j84#9~|Pq-C%xn|5BX-dtUwB6+&;|8Ef>VXW$mW#fAh>09jN| z?-c=8s_vmlLRVkpjn0~tShZQXUhKV?4M$qjnmzilcx-!;R+36&<|^2xcRNBtm{HKj zeZ8f9w7I$Ls$1e6?B|sY-4Zd$h&I2@>Cu*h5=56eTmQWxRkCTPjm_$9VD9V*puugC z&%@>PBs`lfdLf3Kd*DdxApiYaTMuj4UV#+Qok3GUMy|NQtQS0pC@ z?j8=9uL3Yoj#H3pa5`WaN&ulWoUrqk;g8$E`v})Cn+SFYl#gokV^kr08Tnol(56lx z`iQ!oj#)W82_BE>#+8k6D=72&>!f63h?3svSxEuIEK>}hh1b{?vs9BeuSPyRsHh7s z;LxD;)Wlp49j3oFXHR9|8uY-~^JNig-UWnhG|Jf57*~_sn4C=)0W`y_-*kYWIbJp( zdnSUre(oLr2tRmoj!Hzx=?VW7LiZ%=0!a7nTgqDRfKtZE5aiJg%$FPLVopz!@tqx0 zUjeCiv*)2bGbZ16-$#tRfUOp-{z2F7;P$>%z!^WW{ctAM2F8275)u$>F|Ri!V#~Mf zj*KLqEz~Ua|WLTqZ=M!gDPPnx)WmGS?xb zGr5onv%H}-Pi^iJC2G4rfb$_UMXp9N?qGMz6K$*$r z2c^LNC0Gcy#8!B#$y(tSWJ~ozJ-6vlN@-Y`BBDU%Es;-g^6C zyWpSuNmUgq*#9>==HJsgYzX*?Gl97c{f(m_~_9{hClL^cNW41L_P$~?K zind<0tS+QJsYIV}by4zn?#e+{|fEK9Tj{qcS5wx;aLOJ=#x`!$15 zI7PmT>b+}=vp?rN^AhZs6Zj*W20&LsmyQpSWmQeMq=w2{7|OGN;}X)b&6N+cVrN>d z=?=5Xg)w@XZqQEy_e!y}_IzEA0zn0h2RSaLqo}3Gyla*}ms`^+q`YTQK%&KJ2vvW* z958-?Z;yXU6Y?NIqdX%FX={sJ=$#o6YFe3YWSGw$hs@V5MW-mH6$1w2tk!&n+ulr_ ze=Bf{b~>J2XEsXw)lK3*X?Mi@-L(I-JLLMJO11a>26B@6#KV$C}v3v>Q_Rx5pl~KR?4TT{i}iO|pTAxrl-1S%dEkq1F!=aT zqOjeltA>P%*aY9U5r;)p%v{I!-*Sqc91%xuVsdnLi{x%UsMi%BMrb(zYE z20>0i_oudCWCGf$tv|IxlC}W^sPFl7-Bk^*#VxlxloiGUo!MFa7a#sbQJ86cDD z-PW1Ng}b!UfcPF$-lz<})6hyU3;Rlaa}^Tc!M^m!I$JR2f3C;8Hm!*KGbFessJoKmvnzrgs&gZ;XRJmdA}d;=i~9XGv2I-L3#~`Zh^J{FB7-h z{LeuYN$9$wU@d$O<(;im-hM%9P2U9rij!&WHm`uD-dkM}4NWUd6yS#W(bg>8wsb@h%)p&& zJarsh#?59}dd2Ze^18}TPL-}(^-v;2Xk9H^zy5fVJ}JpfZ|Y}(F`ap62!@${I*S-! zZmVp$zxpe(?{jK_7DpX)A$F zpCrUZ#J$gKJNIGAZx)i$ZA%wAGSEOBDj>@9dO<_%hgErZb!NLz9+8DF^iW-o%ZwWJ zDQj>=l1wD4<=FR&0}2vnKr}9LSznU+)kKRQI#hn|73jSk=l24lS`b!q2eKbNl%VAI zv*-Ke!Zfe0BH$xllyQ}g=N8RS*sR`GoH`6lE`J=!KO%lVL&r3*ronkO20X&9I7Mhh z`dFdYiQOO3E78{`^Yklbi^MsH2j+NX7I)O09YG`E@5R9j(~u$-G{^p4HFx|J$Zj|> zvF0WfLgz3eE!1Ne3X@b&;2Kn zv1@jD@#E}tk1qMqfoBHAMh1hLSW3xucH`@} zI%jUlVzm)pc8UO4phFT}S0p-$i39J$RA3?p$xrDdex=n{Yy=EV><< z=A0h|H8iQO#I7x#8B4+uPH)SWx23+m0$S_Gg?W)sBmEsqJuDd7fiaci_S?84@KHnk zBOc_Q2=awYf`^KiM2D})brk|31ktF|BsjT$`n}03pAe(HB3GJq%z-yOcBYi#2&|;` zM1s94gfmP`&4$_vTJf2s8PhmiU2V=@)R?{Wo}~o_x%vs znorH@oqp9WQF08aAdCq358c#KLruGJp2M)M444pX+}==uu12*As!!OwiJ0ayw8!zJ zPre5XtD(rD*PECalQY$w&VySUu2sjNJt*Y}U*e#H;*lXGgR*oL(-V$kX-!sHiwp7g zHF$fax{=xfl1qof0X4XvX0?$Js^#HAQmPAz__+|3(6Pd1Ztc==AXVidn}6;3byr1Y zjs}uER$AwVgB~MOiD78qg4Ah2h`b#KcjojR>BKXt!AQ9eB_)XMls=85F9+UssQHyT z8$m-}nBRiC_j*NLL@T{((h;mh_M?97m|nPo0d6eKS6CgzcJP*6KkL~rZT~(M1vdzv zBZj=uv!Ke#b?MiXYaD0s%cgJHGdW4m25aFxBVMt4ZOWcP4HrdCA>Y=p0||thg12}P zC=l;8o+8BT$EMBM5w{)OwOeJ1HFX-6U=e%lry*7^_7k!FVx!Y=$$XPe9GNo^72`@e z*EeJ8W1Ad&Kg!YIhiHUzN#m2nGGR2TY_3Z$Zd;~#=R@*-we6GU?&m&gbKIO}>2}lO z_D~#84}~3$xGFUCX6R9Y{0VFX;eFUF9mNX_gzAtsMV@tP2Ulb5L*GRCm3+u2bTZjb zX7@WdPPJ{!y)is@KGyD`l9j32*L9emyf`$7U>`#;rput4(_PeDR4=h$(nO?nUtqkC zSOpIW3H+38>+)d}EpL{q%(e^|Hr`S~HmKL3)S1!ggsAESQbTcfAQtA*zQysQg_q!I zMA-B6TngVz%K%`j(5XV(&&9VC5xdvSW3pRzESQ9a^OfoXqXYvPtJX!0b>APUGYM+Q zS;QOMvFpHi6-bGQuV26zA6y3J28J=6?9XIzBtQTcd!iipq&~GXHa8*=9-0(oFd@js z-`ed0&k7|Scuqxd84rqGvQ(tasA~{=cza--9fjnVeAEweb@?S`NTq{2;4Z4h5-0+- zqEvo8b(xVCd&4o>OOqzR{fq&~*14}wsZR&bCG9=4e6RL7L}{*WL0=Xu+BX@aqPGrjy!_K}MvDcj@Nb9NhU_)Zt} zFMQYL-e;>#N^5Y2>Qu++D#BR$zK9mM=Q`_d8Dl6p)bISFKw!j;Tk9hcH{J66B384` zqAdMpS28rH+bma2W)%(=W|dn(kwnKz161+l^*WHG+?$mbBiJyEJwj+YC>kC>Jh4{d zkm|N}{^qJ*!XVR+*J2#~jV_N6o7k7zf&&%e*4JliLw6u0hq~1Z@CbJJSRh&Tx!Fh6 zt%ol8uleB-g~tTI4o)58g&xkZEkA-(4@)my+U=R>*~|)q9SIkF`mr-f^3yvBlM_rgs^#daa(E3+R{6Bd)(N(WAF}JNMxL4SBwA36smtj z1K?E}wLCTxz`O`7konjYp}Y#pf)r|{ZN8DWSF6Hp<&42ghf&Rbo+qP%+(VTn#8ja6 zUi1j{E8lt{vb(mn;d@B^%4VAgGGw`WI7*`B8qNE%9HX(LE|}f9w_u=Uj3C%n-Z<@% zYvZ@szBuww0Tmx2^EwL%5I`X6<*){qc@$}@wQYdzB)Yl;fNNIdEF0jYOxq<<0bmW3DZpLjDo#Oq zJ|h$fFa-d4k?GJ5<^l2rChaClK8{cX-D-wTNeppl(9i^W|^&;5;FRmOx zAkx+oKM*?UdJH-whde*t5y8Y}*VZ%NbiEvt_sF38@N!f7LDtB!*{bZ=V^t-;>0i4{ z`_i|DO*~cg+LO%9BaZ1n`(Lv`1SJA!SRn=T9%MBq)uS8JYo$rgM?)r!(f3T>aH0ia zFMm14phPQ0-*H_|;pxwCtx~rGRx=|KF)zo zX@N%=;29#G3~$%zJZ(pLiF%Y->m>K5F{9xHtC}z2nqf%o~;M`DeU&+<7&kCEv z3+t~VB{=2gq6FHwtR^Vl5_bKMmuTj^x?51q-Pr~+>mudAGofhdg7JR$r!`#h%|!je z2d3}Jv259mLit_{WE_t>EiRZb?ii&BAJD?(RlE>UC}^wUSKY9fwxTNyujv z@}1Hf9DQXbe*cz}=11l=x1y$U9}|L2p10jdB=$7qtWFCDoTa62UjkpN+oQbet%PQ7 zvk`dya`1b3PnlI6DwKuV}ke=ivGs6IM57cc!C!7yxM3(Id-jjE8_Il34Q)iZ|t z(r2v-Q`Un?`lv1RW~klvfcaMo(NK-S%oV$k4}gZRqL#$k+D2SjdURMJZhvc3U>Hfv z5~>r$az}R`j%x-nl-E>ORv)`y?_N~FAPahzjjOFQHy4dGTDu+{O;zz2bfrpH$>Z5d zK!Ng>KibN$hPe4nXmsSKk$UNOaFK6I`L5(AcFjH`ZE4ZAn^%@^2FUSkvmuY#1^RG% zp9uC6YVT?L3DI0u6Ol*sIGzXTWnyi6TZZBu8|c9ssJ6jmAEmktm2m+>yXI-`J`X91 z=ijAKU80)rxVc$c9<2<&Frq{kGN?eCt2wEYYnwsYd#N&x5b5imJ=I6MvJ7nc)G>vQc}H;xOn_qS$}2r&)Z<)~R! z--)rf$w1;z?cqAmc}zGAQXZPFq&L2(+>U&CjtO9sfJ6ph{bkOw|1`2V^yP6(x?YsD zCfy-2(ql$Q$RS?9RM7-<{6FU@fM&pJ8 zvvlZXi!ER?olw$=|BTFgjvocx<-s_T&oQdqns z%Qxrh0OM--ITy;;wk^J-{j{s-djt%|^;j1e$#KGST^&LOK1-iAeAN24Ru5KJ?tX;d zH>+zWV!@09^1Hedk37A!4 zhny*>=Jp!Xd~a_&xoFynP%pjfipGpwVAMoB%J3S>BgtqC*`nwD1CN$pqs>9>$ZyFp zjnb%TQBw((nmUroIUD|W`nR4sDRV$n6J(Y@+csIfVc>_hu7q~*^)>c9)EvwxU`bm0 zh<9XIMvK0GbJp<`fi&7BS3F zU-e6rwl0+q!GG7D>_FzU-KL)n*cJRAtZ_5@Zvmr!Y7}I z8;f|#@4u^zDUD?8G$j7>o=iB3Cw9-F$*ZNO;ZMlC{C1pq-M>4+`<-?NL zogD9%WC7xW`hsLf!;bR3C?h>%v7K(JJWw$X4xk8Adl5J06oe6f1@vc7D)Y#Xrie9S zH}}q!WK-Y~YSv}bW(|cFG*1jLFm~U4MNYd5tAojnJH9_met}R8Pl?t%*^h~|HHkH2 z0qKd!aYp&iAfJ+aoMzWasZOi$GB88|QU#>JYP6!0_8jN4DTLR$jyX@{cTR!?SD1eN zyYpyN{k>bCLjSsS8>EniK)uVsXPo}XUz$t@COj?=FZzDYtT=Cu77qD0)rELIBH=3F3PBX z!`6UU(BBOE=L37i zEmP6%rSK1o_%6$7$hzo1=l5OUDwO6pdOwg6|L+#$U09)*T^7jxacxZm@6aO`D)W4s zsn+osmcHL4qrd##^)Fb~VsVYnV|m6XT7rR?niGKMoR8kvzq|V7_u<|(8&f_8Hsi|# zJ!CbnBtJlGzJX}!pY?$U2L}DYRo5CU9f6H9>!#h@5R`z2*+2E0VZ4$pg5y18NB;E} zzz5`%))o$_hz*JmYN%GVnD)wn8osd-6-kvvYf5p|Fe$(MXmt!r661*9o)=;C!Z5x z$`z@~gf#a(rkp$T=QzLTH-vH9JCJ*6zCvJ(IG&1Jd-SG~7W41Tv)@!vkQ%&Gfs+D- z%uCmqn^l=M0<^MJe-ox(y&nHNFrSvb!@XG#uEYwdG@|UY8uO~SQPnOjp6mCYUZvIK9kXzG5~g|& z;p+^0d1FJ6|DVr(Hd-~0Tt=Kjx8{O^_Dzwl=TE~kzj&=prI VULW{Y_U!NtRhXtyu7c_7{{dsBmbL%@ literal 7662 zcmbt(_ct6~&^IC?>NkiQeHTlDtP(vsyBmEGy$hn(=+S#ytBblsCx{ZgTfGa>qC_XU z)%MBz{s-^-{4n>-%$a-U%$d*Uo|$u_)K!&8iRg)NaBxT=%JP~xICzMEdKuxPe79i;I^78if_T=OQ#~XWneSLgHJt$V z4vQT+xXsDU!~Rnru2=zbGcz+^qCft*!*bciAS-?@Ri&Oz*DfqBp59~S*||`YM@^aG z*H|pAs7suk4EA=11t7e+wS~R?`#CryDmv!jaEcEA(hIrUyMAa%y?nUXof&9pYHC_I zdpNziPfkjxO!nJ7xmsRbJ=J&FG4etBrx}z7j3i+b616Kh(qY;oaKu-R{EJ_wLc7xsL3Dir*Ix-`ego0xqx5 zkMB-a7svYClPkZZrx*wTwzlsS1+Q}kc7joz#l^)CzZ7Ky52jZz7LK>=>lbrdM+3$2 zU^Qb~Rk4=AhveiNHuIa)qa9T9;Bb32gOG(N8%Q`Gd$V`rBz)Dfa5TKIKUSY5=9mx| z9^X+IiHddiv3d^@RR}ke*t^FjZ)1NAV*GqCVglDzHXoU!_29C6#gh-3@>l&+Yv0i; z-95d9)s26K>ytA}Z2ePbN|4O(fIw|4ub)=jPH`Mz&=5c-ULond&rbV57&emW|lr z)Z5x$XDZIABHB*Xu?|z?Sn5yM?umzjriZ%1+xp2}?A2y&alKhorB=`vuxVIHkeRmE z?VlD5iz3ES?y4gEQr|bvJQtfhh5a;wRgk@tbiU1Vy?XUhFsKQ;yzu%h4i0?XtZ=Qk+M3~ zGr~0LXH=Dj1kwNDQ%b8Rs&_g+T~GRKng~rGToBg$d`1-;gm7)- zRIWwDVp)J8mHf6`QHuYJ)qnhchd|`CGLn#N@C6@{kzHCk5+N!88*g6%pwEcFL*rnKkL1OPy^u;&S|Vq6fKkhr4opYC4`vX{B+@440L_dB?I|hZrXU zNF=3LR&_f;Ii9Z5u`EiaNfjj`-*NF(<2YNV<+^k=Rc}mfQ0uHM7o0|iHl5FSf9|!k zIJ~wj)V06q=EQkA+&?7(gA{7-+1=fHczZL&=}I-LX?Pnq97vx#ePL?hPaxV-I0|h7 zDHw&mH{2xP`1@BIUBf32|G*kB`K{HoKsrYjm`i2;nI!LOk8=I-vkU65amV2IkDY7M zSbpTIR1XsLtm4@$gh7WlRDhK_5$@Yy)_@6?iDX|ziV{^c#PvPk$2BEYqkS@PLJE)fikR#ZhH+r6bW7a>bYV#hZU? z=%&#EwO)g}#kJR&MF6sEy31wW2F1jiU|tB?qF;+(BUkpCaN+914WY(H^8T42*!e^Q zT^m(jZ_Cyj!<(t?NMNgMLGeLZPl7<^^V-8!r7r|n)MPdu^Tfmc$77s+1*HMgoh%BP z2s3xCA?V?7MX#iGWRZ)Di;W#yXLYb(DpbnCUg&$^-21)UcqR01b#U4n+&CvFB5*yq z;{joitc50UGeHz5T_s730c6sl4NshO3w0%d?;d)R_a05q#q*WpSxW&sQ zY`heTJdYJgPa(1K)1siP$3@HJkH;e1+5OsRbCfwo{aYLsdU;X=AMN2QO)je_^h+Y5 z1EUdgvA*aD3eM#8`ARNGNCq?~#|AnEB@k=}MNyN&Ttr?}h7s}^uV=u9=<)?CGFt6i zwUNEbH_fVU=vcaA$mF8Oo*)xnbv7&>=`tz$eOg>>>(HToE!_GPWac}Gjc`@C{^~j; zI-{wf2>ZtX6*Rz5FWo?9JtN)F?bSO9sTWRnOi>zNQJ;iR+@Lrsh;*3(JpY$k;3A{{~EE zNpejc-DO9>GN{T6zyYw8kFMkzWeJNY9DFP<&OLd>|Bczy1w41|CDQ*0eAImY6B#{5 z7SWQ>^P1df)YC8T(fM$EpOcTf^o?ot4AI6fhL>B1#qpu`&&7*sm`ru|kZaBr`5B#| zKSd5q{e43eex!VuL81P(TS^xkVABA=pyo$rK)OD49rW`4yc5A6GI>2Jfw@WyK=!Co5%{=4*yTbWU*p7Dw5TyiryuN)y)3Y`);j7$nUI^QQF&Tqz%t%k(&nwfOCu};^zknKV~1iOD9jvu2NgOaZY3$Sm22YE0r zET+dzN3kqSmzcNHkdKUZxMaF&d~LWh+3;Kv+)KcbOteS~U3@~l_R1js(wflNo+}UT zK%URg;Daix4AU_rmq=H!XPx-g$yD&`NdrRrq9)lsyzQmEU<16C-J>9@pE5ZvkbGfO zCo^NjY)bd8VzWjK^uf!Rl(f(EtabeFdPJsY#4X`!&m!*1w6vH1Xkz4Ks$`W&a zN5O*{Mm2xqiN7BK|IR2d-n#QoyT=vVJ}Fa2XLG-F2-Wl(k4+aP40IS$qGVX>v3@fU z>?^F}U1U)AG`!)9UWTZ?L&{~u3%31UR0Ry>GY2iP;Jh%R3xbXnsB&_-WU0LO76i(OUPAckTCW8!44q_ta!jp(_{h%ZoO}?@-N;R26|I26BtA= z`%XmuCW~h6B$~d`lVrwBNG5k|_D`OTMGT+i1Kb(B|6;}_swDri*s7Xi!1(Fx&WnaY z_hiIvEm>>^CPfF2CT=`2u=e90G-c;?q)CdqXEE2C%C=!u(30VW#Aj$hcy1EHde-`L zDBO$ETYv0a;RYdk%JrlFKRO}2zD0(dbosFK$ryK7Ma83MpIzYvG~zhfG-FDp>){Y1 zSdE;{84nW4M_dj8l@W+8a=dXr(1#JHL9bQzNiU7I*`tZn zW-`G&*$(yBOw9->eG7D@FQ;$&+7Xgcf;t0YdKWTo>sMM^PsH0nqGtIpZoO16U1Dls zAq6uHI|*~~*R5p4--BL@<4ov!m&j^C3MLid?>7CX*Z&=Kg_y7*=EVvG+*7#uUfNC> zU4@`40XX0@rVDvqdLZjF^OSh$BmRW8)T`9>la&E(J?ke_YF;e)r2_Wz}v{h^ZOu;F*v{>r@=jLNE!j01{Q=S&7i0+)X+N#rPB^HQ~x_R>4fA zwZRc-c9_{kwf8^2I-q)u*gsw+EPSm*4OL|~-G?1Rg{iWvY)BXwzf&{t4*E}qJ#i#c zRxJIop=rPkdE%c5tzvipORMn#*#)BWSNBmU5e8iUMkkL8st82B(Gj0dE>bA`RZIPA z(~gqKyJ~OzDi<)x$5o>Tqq^&W83Y>D(IxgdpBQam2A>H-6P@+uhBY`;{oxW(b2sh2!; zwFBo$I$A2LUPsF9)3H@8iYQ`fmf80xM720zVn`{vJ0atkvhJY9(Jbo_jmrijjmt0Ue%kR zqff#+A#a0~*@tsw{ErSO&*N%h9P<(+oqg3O_EyO88GVI!^u8iya>Jq)yR>_YHaErz z7=3w5$e*)yp{rDXc9Fe1wvcg>pmD!tZQm%q8MM$r3}7r|8p$KR-bQ{wRryg4mVYvV zT=Vn3X_%|3@=RwC7bA_F7;0KNcpma;e?y}auKz@A8{y&t=bHQxb5*4wd#brUCVILN z?nS~ZDpmDMFegN(xVJKIoxPGl#%g<8AtB)GLCV= z2GvY}KBH_R&8SbFy%*MMf9l@1l-Ti9zVVH*1;(UnjqXr~fW>-+d;-4$)vHXah|c4A zOy=}siCsW^6oSKX7LBLOo5rAr?nz$zaXS2E9Qm2khendy$uc(V?zwSPS0WTj5)Rii z=SLwZLReFfv?QgdUQ=ZtkU)+b^qN)^^1lm78=v>OwDoqlOR_~kp{@PHuZ~}vuwT{t z^?!th$NM{kK*vG;Brr}jvQf~{TdFUN_;+z< zO47W;@a&ttIZxd{N&U>WPRU$fCA9B{)M^MrvTkH`w|+LbrEk!?iDqsC&Z6m3@HGL4 zd(@a9Eo%JF7=4yMJvUu|1GHg)EV?1+vB2EeDhu*_37-!{cQhB>|FG<*0WADjuQMZW z0w|{qVCTu$eGYs@T+mKwy`KCZhy1%2HBfIn-)L<`rWkiJ*#?{B_e%uDzotH=WX2EN zQ~llWm)hMm68+2<3t>i;46l%^XWskCe5iA9{%l&~P-j5tu;QFYVR3s?d<(5~=(Y$=0oBIhC;43qlI- z?^&e7INh#)Cg*m`;iyM4OFJnO@LS^96>Nw8}~mEtrXJd^-XCCci=P z&)3d@ygGV=10%uk5u)&j>&(?Sk#EFRh4))o@By&M`92j|mKF4e+78r~?Ix>@;3aH~ zlKig^wjCwxYZDw`efFk{vUPn%E7Eu8lhj{jywEX)6Z!>?v735Eiq)cR!#BWX;*{LH zN|QqzwN?-{4h6K|`Ci0|MdX{pt=;I~xQ3w5<8s;RoR*88y(wpZn~dX_*zgJ)rgWO7{rWf()i{~7BPWBg&SI%Uutl~_=;j1wsq2?B9(MCE`k1-+s-{b(|)Mtd& zTVxya^O^!Jq&+2@ssgXM8~@V=S~O6bdtZVtrFkASJy{!wE`Nq5dMOm?t%g-7NxY#{ z<9WkAyTW|JPVK&REjyz3lj&IkQ(ig5yAzLEP4~20cBvI&@747c9;A+Vm!9m3!`Xs# zHwvOw1zK>d^(+tG#4QNC=JeH50>tmn?y#~YdNQs(fi5zcp2G#z5j)OKt;$X_6~u_l z;=atqP(%z)UL&R;TZ|_s;E^~yeINGDh|z$lYm0MQAULGThTZVwM#$i1}r!aWyq2~zJ~Nq^FDWoTG@O!tU? zsTL;W-&Je~PC{w%gs(!(SH4SO}}nniJ3O66Jg2%POJPa*75j8xBFi zX02UJC4IwRWzAXR&whq(DRatSs)kSDMz-1jK+QC=^A_csJ?f)O;5x9cP16G=#s zV{pgr9?jW7EBTx>#qbT6DMM=1xoSI0sZT{QvNhqht7AM}ZI-d(CM#$3i7riY{8t@a zMtY5rFYHuVwjN+Uk}sP|t3cZr*l|u05r!z%Y${osn=DPcsfDaYqlxWTW>=t=rJgp36~WV(X`eT{Y`zKCrgAjb!v_Re(Z>q-4bhSPJlivsAFg2azcE)wCsc6j6;9>>ryulZgA zEF~30q~z#Xj0^b(8zxQWinm=?h~<`3+hJ+q`Q<{Hed%uN0*iTU=rsh zyR0(6k63KAe^_=0pGuvPP3LQ&<7eS(bK!ZrVX~nneUzt zG5Q(3L2ddXmD8jIur*s1pc|F0J&`NCqizF5^Y@Z^JxiA4&PWi+h25i^Eth(5hhQQ+ zpN6#n)02lsP|c(@0_YE-Bqfvx2(mF^H1g7s2;i^<`i)dpgUYg9JnAi11&TKQ!vdky zD{aV7Y#Azt$Ot0&47tKQ>(Jt>dGTHaWLx{Iqu7zZHvK?wo~Q9UNSOJ+VdYz<<(>Yq zBBX!yg>xX|VR_@~AFF>9M3Pd4xJhJu8>;fke)hAD*oaFpb{%JxJkR&^^f!(>qoWHq z=6-~8Vg0cc@fRqEXAVK)Dl<0?yh z^0AsZ6|{PztezHTn`fYd@dHnBF-K6ACT5L`xUDi(Xek*{r1P;my*1;yAPdnbRu0jU zGc<%qK956g+S#lqSa>w~Y8ri$AX(}qqfer#EcRY^a4>#7pEY*C#Q`)C7WYMSw?8Rc zlsU?jYFQojS~}1Tawaqh%ROO#;^gXTi&wCE4Nx7-==&IPke@6`@stj>Yv*NHRD;-@ zrLK9=&)ikIS$SA2!eZ$UcU5eBm6PI!ccMHhjLP;87OA=rUsp)20OZOGW4h<+m?fT}muOvUqNeJiHGiqll38=!}j+Dg>_lAqI-4vG7X8D5>X4l{pNH;L<$u)nsD^jp0r1FFAtMU3%0f7->}!XdfUFvnOu=vR zZ7E|sdz|Z!Lkj5LEdCxVJx}S^f^P`6HXefr7q;d9!vohf>tCv(!8{`|irJzzWxq7c z9u8gVU}$a+-ew#51SU?NL8EEAY96}NIvyS-RnU<(?Za!G#sKx+myRj2=r|(B zJivt^zJhtsW0MIgxrwfo_Yfw{zR2BgPu1!8xL@hPo~}i6YKU;y(Vb5>AUuGg~2d9_;Z$>tSUIK=HT(CF^3X**Js1h+Fl z){sp{aFHG{B6oY-0FEH1&!q4s_k(OY3sF@YW2pUAoOJJdOpU}o{o|MZ-e+?3w%{E; z1zKhvwy@2Y^(Xo!F)L2)j>c91RirrRGf!WTTQf1V9X;j^Xyv_I1 zFlV(gru`|Wl<7_*z7@|+B^h#$!uB#__cZMlpYl6;Q{eL^Za@16u=AgFG3)!`;-m=3|tFr{1dCI zZ>8Cq+pj5uNRgXp)F0HpX_An=4?cx|vp@@=aYc_nq{gWvBF2+=>=-)ps4n#p!zDm|K_ Date: Thu, 31 Jan 2019 19:36:07 -0800 Subject: [PATCH 0014/2303] Added documentation on the Datasette Ecosystem https://datasette.readthedocs.io/en/latest/ecosystem.html --- docs/Makefile | 5 ++- docs/ecosystem.rst | 76 ++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 5 +-- 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 docs/ecosystem.rst diff --git a/docs/Makefile b/docs/Makefile index dbb89483..2b092179 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,7 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +livehtml: + sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst new file mode 100644 index 00000000..d235ab7b --- /dev/null +++ b/docs/ecosystem.rst @@ -0,0 +1,76 @@ +.. _ecosystem: + +======================= +The Datasette Ecosystem +======================= + +Datasette sits at the center of a growing ecosystem of open source tools aimed at making it as easy as possible to gather, analyze and publish interesting data. + +These tools are divided into two main groups: tools for building SQLite databases (for use with Datasette) and plugins that extend Datasette's functionality. + +Tools for creating SQLite databases +=================================== + +csvs-to-sqlite +-------------- + +`csvs-to-sqlite `__ lets you take one or more CSV files and load them into a SQLite database. It can also extract repeated columns out into a separate table and configure SQLite full-text search against the contents of specific columns. + +sqlite-utils +------------ + +`sqlite-utils `__ is a Python library and CLI tool that provides shortcuts for loading data into SQLite. It can be used programmatically (e.g. in a `Jupyter notebook `__) to load data, and will automatically create SQLite tables with the necessary schema. + +The CLI tool can consume JSON streams directly and use them to create tables. It can also be used to query SQLite databases and output the results as CSV or JSON. + +db-to-sqlite +------------ + +`db-to-sqlite `__ is a CLI tool that builds on top of `SQLAlchemy `__ and allows you to connect to any database supported by that library (including MySQL, oracle and PostgreSQL), run a SQL query and save the results to a new table in a SQLite database. + +dbf-to-sqlite +------------- + +`dbf-to-sqlite `__ works with `dBase files `__ such as those produced by Visual FoxPro. It is a command-line tool that can convert one or more ``.dbf`` file to tables in a SQLite database. + +markdown-to-sqlite +------------------ + +`markdown-to-sqlite `__ reads Markdown files with embedded YAML metadata (e.g. for `Jekyll Front Matter `__) and creates a SQLite table with a schema matching the metadata. This is useful if you want to keep structured data in text form in a GitHub repository and use that to build a SQLite database. + +Datasette Plugins +================= + +Datasette's :ref:`plugin system ` makes it easy to enhance Datasette with additional functionality. + +datasette-cluster-map +--------------------- + +`datasette-cluster-map `__ is the original Datasette plugin, described in `Datasette plugins, and building a clustered map visualization `__. + +The plugin works against any table with latitude and longitude columns. It can load over 100,000 points onto a map to visualize the geographical distribution of the underlying data. + +datasette-vega +-------------- + +`datasette-vega `__ exposes the powerful `Vega `__ charting library, allowing you to construct line, bar and scatter charts against your data and share links to your visualizations. + +datasette-json-html +------------------- + +`datasette-json-html `__ renders HTML in Datasette's table view driven by JSON returned from your SQL queries. This provides a way to embed images, links and lists of links directly in Datasette's main interface, defined using custom SQL statements. + +datasette-render-images +----------------------- + +`datasette-render-images `__ works with SQLite tables that contain binary image data in BLOB columns. It converts any images it finds into ``data-uri`` image elements, allowing you to view them directly in the Datasette interface. + +datasette-pretty-json +--------------------- + +`datasette-pretty-json `__ seeks out JSON values in Datasette's table browsing interface and pretty-prints them, making them easier to read. + +datasette-sqlite-fts4 +--------------------- + +`datasette-sqlite-fts4 `__ provides search relevance ranking algorithms that can be used with SQLite's FTS4 search module. You can read more about it in `Exploring search relevance algorithms with SQLite `__. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 686aa5de..d18e7ed4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,9 +5,9 @@ Datasette Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API. -Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world. +Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world. It is part of a :ref:`wider ecosystem of tools ` dedicated to make working with structured data as productive as possible. -`Explore a demo `__, watch `a video about the project `__ or try it out by `uploading and publishing your own CSV data `__. +`Explore a demo `__, watch `a presentation about the project `__ or try it out in your browser by `uploading and publishing your own CSV data `__. More examples: https://github.com/simonw/datasette/wiki/Datasettes @@ -17,6 +17,7 @@ Contents .. toctree:: :maxdepth: 2 + ecosystem installation getting_started pages From 436b8bc1d17c2ab415800ab209204f94e7f7929e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Jan 2019 19:47:05 -0800 Subject: [PATCH 0015/2303] Datasette 0.27 --- README.md | 1 + docs/changelog.rst | 10 ++++++++++ docs/plugins.rst | 2 ++ 3 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 9eb6dafe..6ad43ca6 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 31st Janary 2019: [Datasette 0.27](https://datasette.readthedocs.io/en/latest/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://datasette.readthedocs.io/en/latest/ecosystem.html). * 10th January 2019: [Datasette 0.26.1](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. * 2nd January 2019: [Datasette 0.26](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. * 18th December 2018: [Fast Autocomplete Search for Your Website](https://24ways.org/2018/fast-autocomplete-search-for-your-website/) - a new tutorial on using Datasette to build a JavaScript autocomplete search engine. diff --git a/docs/changelog.rst b/docs/changelog.rst index b6c46c69..db1fdd91 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v0_27: + +0.27 (2019-01-31) +----------------- + +- New command: ``datasette plugins`` (:ref:`documentation `) shows you the currently installed list of plugins. +- Datasette can now output `newline-delimited JSON `__ using the new ``?_shape=array&_nl=on`` querystring option. +- Added documentation on :ref:`ecosystem`. +- Now using Python 3.7.2 as the base for the official `Datasette Docker image `__. + .. _v0_26_1: 0.26.1 (2019-01-10) diff --git a/docs/plugins.rst b/docs/plugins.rst index 6efa131b..984e5c95 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -58,6 +58,8 @@ Now you can navigate to http://localhost:8001/mydb and run this SQL:: To see the output of your plugin. +.. _plugins_installed: + Seeing what plugins are installed --------------------------------- From 195a5b36349d0d24a6bbb758cebb719b6de303b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Feb 2019 20:15:46 -0800 Subject: [PATCH 0016/2303] Heroku --include-vcs-ignore (#407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Means `datasette publish heroku` can work under Travis, unlike this failure: https://travis-ci.org/simonw/fivethirtyeight-datasette/builds/488047550 ``` 2.25s$ datasette publish heroku fivethirtyeight.db -m metadata.json -n fivethirtyeight-datasette tar: unrecognized option '--exclude-vcs-ignores' Try 'tar --help' or 'tar --usage' for more information. ▸ Command failed: tar cz -C /tmp/tmpuaxm7i8f --exclude-vcs-ignores --exclude ▸ .git --exclude .gitmodules . > ▸ /tmp/f49440e0-1bf3-4d3f-9eb0-fbc2967d1fd4.tar.gz ▸ tar: unrecognized option '--exclude-vcs-ignores' ▸ Try 'tar --help' or 'tar --usage' for more information. ▸ The command "datasette publish heroku fivethirtyeight.db -m metadata.json -n fivethirtyeight-datasette" exited with 0. ``` The fix for that issue is to call the heroku command like this: heroku builds:create -a app_name --include-vcs-ignore --- datasette/publish/heroku.py | 2 +- tests/test_publish_heroku.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index af53a37a..d42f4e83 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -96,4 +96,4 @@ def publish_subcommand(publish): create_output = check_output(cmd).decode("utf8") app_name = json.loads(create_output)["name"] - call(["heroku", "builds:create", "-a", app_name]) + call(["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"]) diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 3d8fb6e6..852403ca 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -57,7 +57,7 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): open("test.db", "w").write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) assert 0 == result.exit_code, result.output - mock_call.assert_called_once_with(["heroku", "builds:create", "-a", "f"]) + mock_call.assert_called_once_with(["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"]) @mock.patch("shutil.which") From 4462a5ab2817ac0d9ffe20dafbbf27c5c5b81466 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Feb 2019 20:53:44 -0800 Subject: [PATCH 0017/2303] Show size of database file next to download link, closes #172 --- datasette/app.py | 1 + datasette/static/app.css | 5 +++++ datasette/templates/database.html | 2 +- datasette/utils.py | 12 ++++++++++++ datasette/views/base.py | 2 ++ datasette/views/database.py | 1 + tests/test_utils.py | 15 +++++++++++++++ 7 files changed, 37 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 7bbdef3e..3a74a5e5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -305,6 +305,7 @@ class Datasette: self._inspect[name] = { "hash": inspect_hash(path), "file": str(path), + "size": path.stat().st_size, "views": inspect_views(conn), "tables": inspect_tables(conn, (self.metadata("databases") or {}).get(name, {})) } diff --git a/datasette/static/app.css b/datasette/static/app.css index 2fc16940..f21d3ea4 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -286,3 +286,8 @@ a.not-underlined { display: inline-block; box-shadow: 1px 2px 8px 2px rgba(0,0,0,0.08); } + +.download-sqlite em { + font-style: normal; + font-size: 0.8em; +} diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 05acff84..f64d5c90 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -57,7 +57,7 @@ {% endif %} {% if config.allow_download %} -

    Download SQLite DB: {{ database }}.db

    +

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    {% endif %} {% include "_codemirror_foot.html" %} diff --git a/datasette/utils.py b/datasette/utils.py index b0f74f0d..30fc4231 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -901,3 +901,15 @@ class StaticMount(click.ParamType): if not os.path.exists(dirpath) or not os.path.isdir(dirpath): self.fail("%s is not a valid directory path" % value, param, ctx) return path, dirpath + + +def format_bytes(bytes): + current = float(bytes) + for unit in ("bytes", "KB", "MB", "GB", "TB"): + if current < 1024: + break + current = current / 1024 + if unit == "bytes": + return "{} {}".format(int(current), unit) + else: + return "{:.1f} {}".format(current, unit) diff --git a/datasette/views/base.py b/datasette/views/base.py index f4a8afaf..e98762b7 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -19,6 +19,7 @@ from datasette.utils import ( InterruptedError, InvalidSql, LimitedWriter, + format_bytes, is_url, path_from_row_pks, path_with_added_args, @@ -102,6 +103,7 @@ class RenderMixin(HTTPMethodView): "extra_js_urls": self._asset_urls( "extra_js_urls", template, context ), + "format_bytes": format_bytes, } } ) diff --git a/datasette/views/database.py b/datasette/views/database.py index fb6b242b..9c44a800 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -24,6 +24,7 @@ class DatabaseView(BaseView): tables.sort(key=lambda t: (t["hidden"], t["name"])) return { "database": database, + "size": info["size"], "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": info["views"], diff --git a/tests/test_utils.py b/tests/test_utils.py index 43e2f44b..b406d70b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -374,3 +374,18 @@ def test_path_with_format(path, format, extra_qs, expected): ) actual = utils.path_with_format(request, format, extra_qs) assert expected == actual + + +@pytest.mark.parametrize( + "bytes,expected", + [ + (120, '120 bytes'), + (1024, '1.0 KB'), + (1024 * 1024, '1.0 MB'), + (1024 * 1024 * 1024, '1.0 GB'), + (1024 * 1024 * 1024 * 1.3, '1.3 GB'), + (1024 * 1024 * 1024 * 1024, '1.0 TB'), + ] +) +def test_format_bytes(bytes, expected): + assert expected == utils.format_bytes(bytes) From d5d39da12b26c06a426e97872bce4e94248db42d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Feb 2019 22:23:27 -0800 Subject: [PATCH 0018/2303] Expanded section on db-to-sqlite --- docs/ecosystem.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index d235ab7b..401cd22a 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -28,6 +28,10 @@ db-to-sqlite `db-to-sqlite `__ is a CLI tool that builds on top of `SQLAlchemy `__ and allows you to connect to any database supported by that library (including MySQL, oracle and PostgreSQL), run a SQL query and save the results to a new table in a SQLite database. +You can mirror an entire database (including copying foreign key relationships) with the ``--all`` option:: + + $ db-to-sqlite --connection="postgresql://simonw@localhost/myblog" --all blog.db + dbf-to-sqlite ------------- @@ -73,4 +77,4 @@ datasette-pretty-json datasette-sqlite-fts4 --------------------- -`datasette-sqlite-fts4 `__ provides search relevance ranking algorithms that can be used with SQLite's FTS4 search module. You can read more about it in `Exploring search relevance algorithms with SQLite `__. \ No newline at end of file +`datasette-sqlite-fts4 `__ provides search relevance ranking algorithms that can be used with SQLite's FTS4 search module. You can read more about it in `Exploring search relevance algorithms with SQLite `__. From 1f91065b20cbc691f464bccfd8eef7d1ce4b14a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 Feb 2019 08:06:10 -0800 Subject: [PATCH 0019/2303] Added socrata2sql to the ecosystem page A fantastic new tool created by @achavez at the Dallas Morning News. --- docs/ecosystem.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 401cd22a..18390122 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -42,6 +42,15 @@ markdown-to-sqlite `markdown-to-sqlite `__ reads Markdown files with embedded YAML metadata (e.g. for `Jekyll Front Matter `__) and creates a SQLite table with a schema matching the metadata. This is useful if you want to keep structured data in text form in a GitHub repository and use that to build a SQLite database. +socrata2sql +----------- + +`socrata2sql `__ is a tool by Andrew Chavez at the Dallas Morning News. It works with Socrata, a widely used platform for local and national government open data portals. It uses the Socrata API to pull down government datasets and store them in a local SQLite database (it can also export data to PostgreSQL, MySQL and other SQLAlchemy-supported databases). + +For example, to create a SQLite database of the `City of Dallas Payment Register `__ you would run the following command:: + + $ socrata2sql insert www.dallasopendata.com 64pp-jeba + Datasette Plugins ================= From 5b3af3d015a6048a3977234bf46d20905ce720c0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Feb 2019 19:47:11 -0800 Subject: [PATCH 0020/2303] Added sqlite-utils blog entry to news section --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6ad43ca6..7fe15b0e 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 24th February 2019: [ +sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) - a partner tool for easily creating SQLite databases for use with Datasette. * 31st Janary 2019: [Datasette 0.27](https://datasette.readthedocs.io/en/latest/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://datasette.readthedocs.io/en/latest/ecosystem.html). * 10th January 2019: [Datasette 0.26.1](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. * 2nd January 2019: [Datasette 0.26](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. From 4147c388cbb9b14d137dd5229ae3f574ee6a25fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Mar 2019 20:30:21 -0800 Subject: [PATCH 0021/2303] Link to sqlite-utils blog entry --- docs/ecosystem.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 18390122..29b80707 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -23,6 +23,8 @@ sqlite-utils The CLI tool can consume JSON streams directly and use them to create tables. It can also be used to query SQLite databases and output the results as CSV or JSON. +See `sqlite-utils: a Python library and CLI tool for building SQLite databases `__ for more. + db-to-sqlite ------------ From 41744d76908bbef0edc038a5050a709cdd26e529 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 9 Mar 2019 10:43:12 -0800 Subject: [PATCH 0022/2303] Added datasette-jellyfish --- docs/ecosystem.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 29b80707..69f935d0 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -75,6 +75,11 @@ datasette-json-html `datasette-json-html `__ renders HTML in Datasette's table view driven by JSON returned from your SQL queries. This provides a way to embed images, links and lists of links directly in Datasette's main interface, defined using custom SQL statements. +datasette-jellyfish +------------------- + +`datasette-jellyfish `__ exposes custom SQL functions for a range of common fuzzy string matching functions, including soundex, porter stemming and levenshtein distance. It builds on top of the `Jellyfish Python library `__. + datasette-render-images ----------------------- From bf6b0f918de4aeee7c1036ac975ce2fb23237da7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 Mar 2019 14:37:11 -0700 Subject: [PATCH 0023/2303] about and about_url metadata options --- datasette/app.py | 3 +++ datasette/cli.py | 2 ++ datasette/publish/common.py | 2 ++ datasette/publish/heroku.py | 4 ++++ datasette/publish/now.py | 4 ++++ datasette/templates/_description_source_license.html | 5 +++++ datasette/templates/base.html | 5 +++++ docs/datasette-package-help.txt | 2 ++ docs/datasette-publish-heroku-help.txt | 2 ++ docs/datasette-publish-now-help.txt | 2 ++ tests/fixtures.py | 2 ++ tests/test_html.py | 6 ++++-- 12 files changed, 37 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3a74a5e5..453ce5c0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -267,6 +267,9 @@ class Datasette: "license": metadata.get("license") or self.metadata("license"), "license_url": metadata.get("license_url") or self.metadata("license_url"), + "about": metadata.get("about") or self.metadata("about"), + "about_url": metadata.get("about_url") + or self.metadata("about_url"), } ) diff --git a/datasette/cli.py b/datasette/cli.py index 34ed0020..446456f4 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -217,6 +217,8 @@ def plugins(all, plugins_dir): @click.option("--license_url", help="License URL for metadata") @click.option("--source", help="Source label for metadata") @click.option("--source_url", help="Source URL for metadata") +@click.option("--about", help="About label for metadata") +@click.option("--about_url", help="About URL for metadata") def package( files, tag, diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 9dd2ae9e..4cc69b4a 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -42,6 +42,8 @@ def add_common_publish_arguments_and_options(subcommand): click.option("--license_url", help="License URL for metadata"), click.option("--source", help="Source label for metadata"), click.option("--source_url", help="Source URL for metadata"), + click.option("--about", help="About label for metadata"), + click.option("--about_url", help="About URL for metadata"), )): subcommand = decorator(subcommand) return subcommand diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index d42f4e83..32c861cd 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -35,6 +35,8 @@ def publish_subcommand(publish): license_url, source, source_url, + about, + about_url, name, ): fail_if_publish_binary_not_installed( @@ -72,6 +74,8 @@ def publish_subcommand(publish): "license_url": license_url, "source": source, "source_url": source_url, + "about": about, + "about_url": about_url, }, ): app_name = None diff --git a/datasette/publish/now.py b/datasette/publish/now.py index bd2b051b..64a73279 100644 --- a/datasette/publish/now.py +++ b/datasette/publish/now.py @@ -39,6 +39,8 @@ def publish_subcommand(publish): license_url, source, source_url, + about, + about_url, name, force, token, @@ -70,6 +72,8 @@ def publish_subcommand(publish): "license_url": license_url, "source": source, "source_url": source_url, + "about": about, + "about_url": about_url, }, ): now_json = { diff --git a/datasette/templates/_description_source_license.html b/datasette/templates/_description_source_license.html index eba4eb1a..3327706e 100644 --- a/datasette/templates/_description_source_license.html +++ b/datasette/templates/_description_source_license.html @@ -21,5 +21,10 @@ {% endif %}{{ metadata.source or metadata.source_url }}{% if metadata.source_url %}{% endif %} {% endif %} + {% if metadata.about or metadata.about_url %}{% if metadata.license or metadata.license_url or metadata.source or metadat.source_url %}·{% endif %} + About: {% if metadata.about_url %} + + {% endif %}{{ metadata.about or metadata.about_url }}{% if metadata.about_url %}{% endif %} + {% endif %}

    {% endif %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c766a44e..0ea41d7e 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -33,6 +33,11 @@ {% endif %}{{ metadata.source or metadata.source_url }}{% if metadata.source_url %}{% endif %} {% endif %} + {% if metadata.about or metadata.about_url %}· + About: {% if metadata.about_url %} + + {% endif %}{{ metadata.about or metadata.about_url }}{% if metadata.about_url %}{% endif %} + {% endif %} {% endif %}
    diff --git a/docs/datasette-package-help.txt b/docs/datasette-package-help.txt index d0ad2d3e..39be04e9 100644 --- a/docs/datasette-package-help.txt +++ b/docs/datasette-package-help.txt @@ -21,4 +21,6 @@ Options: --license_url TEXT License URL for metadata --source TEXT Source label for metadata --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata --help Show this message and exit. diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt index f82eaf3e..cd9af09b 100644 --- a/docs/datasette-publish-heroku-help.txt +++ b/docs/datasette-publish-heroku-help.txt @@ -16,5 +16,7 @@ Options: --license_url TEXT License URL for metadata --source TEXT Source label for metadata --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata -n, --name TEXT Application name to use when deploying --help Show this message and exit. diff --git a/docs/datasette-publish-now-help.txt b/docs/datasette-publish-now-help.txt index a9c01f39..445c1bc7 100644 --- a/docs/datasette-publish-now-help.txt +++ b/docs/datasette-publish-now-help.txt @@ -16,6 +16,8 @@ Options: --license_url TEXT License URL for metadata --source TEXT Source label for metadata --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata -n, --name TEXT Application name to use when deploying --force Pass --force option to now --token TEXT Auth token to use for deploy diff --git a/tests/fixtures.py b/tests/fixtures.py index 16395553..a77a3f4a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -133,6 +133,8 @@ METADATA = { 'license_url': 'https://github.com/simonw/datasette/blob/master/LICENSE', 'source': 'tests/fixtures.py', 'source_url': 'https://github.com/simonw/datasette/blob/master/tests/fixtures.py', + 'about': 'About Datasette', + 'about_url': 'https://github.com/simonw/datasette', "plugins": { "name-of-plugin": { "depth": "root" diff --git a/tests/test_html.py b/tests/test_html.py index 9adfe8f2..ca6d62aa 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -744,14 +744,16 @@ def assert_querystring_equal(expected, actual): def assert_footer_links(soup): footer_links = soup.find('div', {'class': 'ft'}).findAll('a') - assert 3 == len(footer_links) - datasette_link, license_link, source_link = footer_links + assert 4 == len(footer_links) + datasette_link, license_link, source_link, about_link = footer_links assert 'Datasette' == datasette_link.text.strip() assert 'tests/fixtures.py' == source_link.text.strip() assert 'Apache License 2.0' == license_link.text.strip() + assert 'About Datasette' == about_link.text.strip() assert 'https://github.com/simonw/datasette' == datasette_link['href'] assert 'https://github.com/simonw/datasette/blob/master/tests/fixtures.py' == source_link['href'] assert 'https://github.com/simonw/datasette/blob/master/LICENSE' == license_link['href'] + assert 'https://github.com/simonw/datasette' == about_link['href'] def inner_html(soup): From 9743e1d91b5f0a2b3c1c0bd6ffce8739341f43c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Mar 2019 16:42:38 -0700 Subject: [PATCH 0024/2303] Support for :memory: databases If you start Datasette with no files, it will connect to :memory: instead. When starting it with files you can add --memory to also get a :memory: database. --- datasette/app.py | 79 +++++++++++++++++++------------ datasette/cli.py | 5 ++ datasette/templates/database.html | 4 +- docs/datasette-serve-help.txt | 1 + tests/fixtures.py | 8 ++++ tests/test_api.py | 26 ++++++++++ 6 files changed, 90 insertions(+), 33 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 453ce5c0..37b199a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -42,7 +42,7 @@ from .version import __version__ app_root = Path(__file__).parent.parent connections = threading.local() - +MEMORY = object() ConfigOption = collections.namedtuple( "ConfigOption", ("name", "default", "help") @@ -123,10 +123,15 @@ class Datasette: template_dir=None, plugins_dir=None, static_mounts=None, + memory=False, config=None, version_note=None, ): self.files = files + if not self.files: + self.files = [MEMORY] + elif memory: + self.files = (MEMORY,) + self.files self.cache_headers = cache_headers self.cors = cors self._inspect = inspect_data @@ -296,31 +301,40 @@ class Datasette: self._inspect = {} for filename in self.files: - path = Path(filename) - name = path.stem - if name in self._inspect: - raise Exception("Multiple files with same stem %s" % name) - try: - with sqlite3.connect( - "file:{}?immutable=1".format(path), uri=True - ) as conn: - self.prepare_connection(conn) - self._inspect[name] = { - "hash": inspect_hash(path), - "file": str(path), - "size": path.stat().st_size, - "views": inspect_views(conn), - "tables": inspect_tables(conn, (self.metadata("databases") or {}).get(name, {})) - } - except sqlite3.OperationalError as e: - if (e.args[0] == 'no such module: VirtualSpatialIndex'): - raise click.UsageError( - "It looks like you're trying to load a SpatiaLite" - " database without first loading the SpatiaLite module." - "\n\nRead more: https://datasette.readthedocs.io/en/latest/spatialite.html" - ) - else: - raise + if filename is MEMORY: + self._inspect[":memory:"] = { + "hash": "000", + "file": ":memory:", + "size": 0, + "views": {}, + "tables": {}, + } + else: + path = Path(filename) + name = path.stem + if name in self._inspect: + raise Exception("Multiple files with same stem %s" % name) + try: + with sqlite3.connect( + "file:{}?immutable=1".format(path), uri=True + ) as conn: + self.prepare_connection(conn) + self._inspect[name] = { + "hash": inspect_hash(path), + "file": str(path), + "size": path.stat().st_size, + "views": inspect_views(conn), + "tables": inspect_tables(conn, (self.metadata("databases") or {}).get(name, {})) + } + except sqlite3.OperationalError as e: + if (e.args[0] == 'no such module: VirtualSpatialIndex'): + raise click.UsageError( + "It looks like you're trying to load a SpatiaLite" + " database without first loading the SpatiaLite module." + "\n\nRead more: https://datasette.readthedocs.io/en/latest/spatialite.html" + ) + else: + raise return self._inspect def register_custom_units(self): @@ -403,11 +417,14 @@ class Datasette: conn = getattr(connections, db_name, None) if not conn: info = self.inspect()[db_name] - conn = sqlite3.connect( - "file:{}?immutable=1".format(info["file"]), - uri=True, - check_same_thread=False, - ) + if info["file"] == ":memory:": + conn = sqlite3.connect(":memory:") + else: + conn = sqlite3.connect( + "file:{}?immutable=1".format(info["file"]), + uri=True, + check_same_thread=False, + ) self.prepare_connection(conn) setattr(connections, db_name, conn) diff --git a/datasette/cli.py b/datasette/cli.py index 446456f4..6fbc9908 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -315,6 +315,9 @@ def package( help="mountpoint:path-to-directory for serving static files", multiple=True, ) +@click.option( + "--memory", is_flag=True, help="Make :memory: database available" +) @click.option( "--config", type=Config(), @@ -340,6 +343,7 @@ def serve( template_dir, plugins_dir, static, + memory, config, version_note, help_config, @@ -384,6 +388,7 @@ def serve( plugins_dir=plugins_dir, static_mounts=static, config=dict(config), + memory=memory, version_note=version_note, ) # Force initial hashing/table counting diff --git a/datasette/templates/database.html b/datasette/templates/database.html index f64d5c90..f827e584 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -19,7 +19,7 @@ {% if config.allow_sql %}

    Custom SQL query

    -

    +

    {% endif %} @@ -56,7 +56,7 @@
{% endif %} -{% if config.allow_download %} +{% if config.allow_download and database != ":memory:" %}

Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

{% endif %} diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index caa00e33..65b9aceb 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -17,6 +17,7 @@ Options: --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --memory Make :memory: database available --config CONFIG Set config option using configname:value datasette.readthedocs.io/en/latest/config.html --version-note TEXT Additional note to show on /-/versions diff --git a/tests/fixtures.py b/tests/fixtures.py index a77a3f4a..efd85fab 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -65,6 +65,14 @@ def app_client(): yield from make_app_client() +@pytest.fixture(scope="session") +def app_client_no_files(): + ds = Datasette([]) + client = TestClient(ds.app().test_client) + client.ds = ds + yield client + + @pytest.fixture(scope='session') def app_client_shorter_time_limit(): yield from make_app_client(20) diff --git a/tests/test_api.py b/tests/test_api.py index 8cd1e94e..a6ba3f37 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,6 @@ from .fixtures import ( # noqa app_client, + app_client_no_files, app_client_shorter_time_limit, app_client_larger_cache_size, app_client_returned_rows_matches_page_size, @@ -368,6 +369,31 @@ def test_database_page(app_client): }] == data['tables'] +def test_no_files_uses_memory_database(app_client_no_files): + response = app_client_no_files.get("/.json") + assert response.status == 200 + assert { + ":memory:": { + "hash": "000", + "hidden_table_rows_sum": 0, + "hidden_tables_count": 0, + "name": ":memory:", + "path": ":memory:-000", + "table_rows_sum": 0, + "tables_count": 0, + "tables_more": False, + "tables_truncated": [], + "views_count": 0, + } + } == response.json + # Try that SQL query + response = app_client_no_files.get( + "/:memory:-0.json?sql=select+sqlite_version()&_shape=array" + ) + assert 1 == len(response.json) + assert ["sqlite_version()"] == list(response.json[0].keys()) + + def test_database_page_for_database_with_dot_in_name(app_client_with_dot): response = app_client_with_dot.get("/fixtures.dot.json") assert 200 == response.status From b3e739332624c2d4f2668a105afd727af774100b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Mar 2019 21:41:43 -0700 Subject: [PATCH 0025/2303] Allow more recent versions of Click Closes #414 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92b92c3f..fb00f2d0 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( package_data={'datasette': ['templates/*.html']}, include_package_data=True, install_requires=[ - 'click==6.7', + 'click>=6.7', 'click-default-group==1.2', 'Sanic==0.7.0', 'Jinja2==2.10', From 285566790879b31d2fdd2a8c6f56825162eb71b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Mar 2019 22:00:13 -0700 Subject: [PATCH 0026/2303] Fix for test failure with Click 7.0 --- tests/test_publish_heroku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 852403ca..da4e213a 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -24,7 +24,7 @@ def test_publish_heroku_installs_plugin(mock_call, mock_check_output, mock_which with runner.isolated_filesystem(): open("t.db", "w").write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "t.db"], input="y\n") - assert -1 == result.exit_code + assert 0 != result.exit_code mock_check_output.assert_has_calls( [mock.call(["heroku", "plugins"]), mock.call(["heroku", "apps:list", "--json"])] ) From 9e8c36793bfbb17c2f67371cc7f9aa8b9202fdc4 Mon Sep 17 00:00:00 2001 From: joelondon Date: Fri, 15 Mar 2019 05:06:45 +0000 Subject: [PATCH 0027/2303] Update spatialite.rst (#413) a line of sql added to create the idx_ in the python recipe --- docs/spatialite.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 5a8a31b1..58179e70 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -68,6 +68,8 @@ Here's a recipe for taking a table with existing latitude and longitude columns, UPDATE events SET point_geom = GeomFromText('POINT('||"longitude"||' '||"latitude"||')',4326); ''') + # Now add a spatial index to that column + conn.execute('select CreateSpatialIndex("museums", "point_geom");') # If you don't commit your changes will not be persisted: conn.commit() conn.close() From afe9aa3ae03c485c5d6652741438d09445a486c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Mar 2019 22:22:24 -0700 Subject: [PATCH 0028/2303] show/hide link for SQL on custom query page Closes #415 --- datasette/templates/query.html | 12 +++++++----- datasette/views/base.py | 5 +++++ tests/test_html.py | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b23c67d8..06651689 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -26,11 +26,13 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
-

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}

- {% if editable and config.allow_sql %} -

- {% else %} - 
{% if query %}{{ query.sql }}{% endif %}
+

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %} {% if hide_sql %}(show){% else %}(hide){% endif %}

+ {% if not hide_sql %} + {% if editable and config.allow_sql %} +

+ {% else %} + 
{% if query %}{{ query.sql }}{% endif %}
+ {% endif %} {% endif %} {% if named_parameter_values %}

Query parameters

diff --git a/datasette/views/base.py b/datasette/views/base.py index e98762b7..4db1b654 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -23,6 +23,7 @@ from datasette.utils import ( is_url, path_from_row_pks, path_with_added_args, + path_with_removed_args, path_with_format, remove_infinites, resolve_table_and_format, @@ -578,6 +579,10 @@ class BaseView(RenderMixin): "canned_query": canned_query, "metadata": metadata, "config": self.ds.config_dict(), + "request": request, + "path_with_added_args": path_with_added_args, + "path_with_removed_args": path_with_removed_args, + "hide_sql": "_hide_sql" in params, } return { diff --git a/tests/test_html.py b/tests/test_html.py index ca6d62aa..36335201 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -824,3 +824,22 @@ def test_urlify_custom_queries(app_client): https://twitter.com/simonw ''' == soup.find("td", {"class": "col-user_url"}).prettify().strip() + + +def test_show_hide_sql_query(app_client): + path = "/fixtures?" + urllib.parse.urlencode({ + "sql": "select ('https://twitter.com/' || 'simonw') as user_url;" + }) + response = app_client.get(path) + soup = Soup(response.body, "html.parser") + span = soup.select(".show-hide-sql")[0] + assert span.find("a")["href"].endswith("&_hide_sql=1") + assert "(hide)" == span.getText() + assert soup.find("textarea") is not None + # Now follow the link to hide it + response = app_client.get(span.find("a")["href"]) + soup = Soup(response.body, "html.parser") + span = soup.select(".show-hide-sql")[0] + assert not span.find("a")["href"].endswith("&_hide_sql=1") + assert "(show)" == span.getText() + assert soup.find("textarea") is None From 6f6d0ff2b41f1cacaf42287b1b230b646bcba9ee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Mar 2019 15:55:04 -0700 Subject: [PATCH 0029/2303] URL hashing is now off by default - closes #418 Prior to this commit Datasette would calculate the content hash of every database and redirect to a URL containing that hash, like so: https://v0-27.datasette.io/fixtures => https://v0-27.datasette.io/fixtures-dd88475 This assumed that all databases were opened in immutable mode and were not expected to change. This will be changing as a result of #419 - so this commit takes the first step in implementing that change by changing this default behaviour. Datasette will now only redirect hash-free URLs under two circumstances: * The new `hash_urls` config option is set to true (it defaults to false). * The user passes `?_hash=1` in the URL --- datasette/app.py | 8 +++++- datasette/templates/database.html | 14 ++++----- datasette/templates/query.html | 6 ++-- datasette/templates/row.html | 6 ++-- datasette/templates/table.html | 8 +++--- datasette/utils.py | 10 +++++-- datasette/views/base.py | 48 ++++++++++++++++++++++--------- datasette/views/database.py | 3 +- datasette/views/index.py | 2 +- datasette/views/table.py | 2 -- docs/config.rst | 30 +++++++++++++++++-- tests/fixtures.py | 7 +++++ tests/test_api.py | 32 +++++++++++++++------ tests/test_html.py | 36 +++++++++-------------- tests/test_utils.py | 7 +++++ 15 files changed, 149 insertions(+), 70 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 37b199a4..c292542f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -69,6 +69,9 @@ CONFIG_OPTIONS = ( ConfigOption("facet_suggest_time_limit_ms", 50, """ Time limit for calculating a suggested facet """.strip()), + ConfigOption("hash_urls", False, """ + Include DB file contents hash in URLs, for far-future caching + """.strip()), ConfigOption("allow_facet", True, """ Allow users to specify columns to facet using ?_facet= parameter """.strip()), @@ -81,9 +84,12 @@ CONFIG_OPTIONS = ( ConfigOption("allow_sql", True, """ Allow arbitrary SQL queries via ?sql= parameter """.strip()), - ConfigOption("default_cache_ttl", 365 * 24 * 60 * 60, """ + ConfigOption("default_cache_ttl", 5, """ Default HTTP cache TTL (used in Cache-Control: max-age= header) """.strip()), + ConfigOption("default_cache_ttl_hashed", 365 * 24 * 60 * 60, """ + Default HTTP cache TTL for hashed URL pages + """.strip()), ConfigOption("cache_size_kb", 0, """ SQLite cache size in KB (0 == use SQLite default) """.strip()), diff --git a/datasette/templates/database.html b/datasette/templates/database.html index f827e584..0e80c8b6 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -12,12 +12,12 @@ {% block content %}
home
-

{{ metadata.title or database }}

+

{{ metadata.title or database }}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if config.allow_sql %} - +

Custom SQL query

@@ -27,7 +27,7 @@ {% for table in tables %} {% if show_hidden or not table.hidden %}
-

{{ table.name }}{% if table.hidden %} (hidden){% endif %}

+

{{ table.name }}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns[:9] %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}{% if table.columns|length > 9 %}...{% endif %}

{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}

@@ -35,14 +35,14 @@ {% endfor %} {% if hidden_count and not show_hidden %} -

... and {{ "{:,}".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}

+

... and {{ "{:,}".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}

{% endif %} {% if views %}

Views

{% endif %} @@ -51,13 +51,13 @@

Queries

{% endif %} {% if config.allow_download and database != ":memory:" %} -

Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

+

Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

{% endif %} {% include "_codemirror_foot.html" %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 06651689..46aac76f 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -19,13 +19,13 @@ {% block body_class %}query db-{{ database|to_css_class }}{% endblock %} {% block content %} -
home / {{ database }}
+
home / {{ database }}
-

{{ metadata.title or database }}

+

{{ metadata.title or database }}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %} {% if hide_sql %}(show){% else %}(hide){% endif %}

{% if not hide_sql %} {% if editable and config.allow_sql %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 4ef0b758..389b16b2 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -16,9 +16,9 @@ {% block body_class %}row db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %} {% block content %} -
home / {{ database }} / {{ table }}
+
home / {{ database }} / {{ table }}
-

{{ table }}: {{ ', '.join(primary_key_values) }}

+

{{ table }}: {{ ', '.join(primary_key_values) }}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} @@ -31,7 +31,7 @@
    {% for other in foreign_key_tables %}
  • - + {{ "{:,}".format(other.count) }} row{% if other.count == 1 %}{% else %}s{% endif %} from {{ other.other_column }} in {{ other.other_table }}
    • diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 7a9f9115..c9ec4aa1 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -17,9 +17,9 @@ {% block body_class %}table db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %} {% block content %} -
    home / {{ database }}
    +
    home / {{ database }}
    -

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}

    +

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}

    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} @@ -29,7 +29,7 @@ {% endif %} - + {% if supports_search %}
    {% endif %} @@ -89,7 +89,7 @@ {% if query.sql and config.allow_sql %} -

    ✎ View and edit SQL

    +

    ✎ View and edit SQL

    {% endif %}

    This data as JSON{% if display_rows %}, CSV (advanced){% endif %}

    diff --git a/datasette/utils.py b/datasette/utils.py index 30fc4231..8bcaefc2 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -208,8 +208,14 @@ def path_with_added_args(request, args, path=None): def path_with_removed_args(request, args, path=None): + query_string = request.query_string + if path is None: + path = request.path + else: + if "?" in path: + bits = path.split("?", 1) + path, query_string = bits # args can be a dict or a set - path = path or request.path current = [] if isinstance(args, set): def should_remove(key, value): @@ -218,7 +224,7 @@ def path_with_removed_args(request, args, path=None): # Must match key AND value def should_remove(key, value): return args.get(key) == value - for key, value in urllib.parse.parse_qsl(request.query_string): + for key, value in urllib.parse.parse_qsl(query_string): if not should_remove(key, value): current.append((key, value)) query_string = urllib.parse.urlencode(current) diff --git a/datasette/views/base.py b/datasette/views/base.py index 4db1b654..33b7524e 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -75,6 +75,17 @@ class RenderMixin(HTTPMethodView): else: yield {"url": url} + def database_url(self, database): + if not self.ds.config("hash_urls"): + return "/{}".format(database) + else: + return "/{}-{}".format( + database, self.ds.inspect()[database]["hash"][:HASH_LENGTH] + ) + + def database_color(self, database): + return 'ff0000' + def render(self, templates, **context): template = self.ds.jinja_env.select_template(templates) select_templates = [ @@ -105,6 +116,8 @@ class RenderMixin(HTTPMethodView): "extra_js_urls", template, context ), "format_bytes": format_bytes, + "database_url": self.database_url, + "database_color": self.database_color, } } ) @@ -131,16 +144,18 @@ class BaseView(RenderMixin): r.headers["Access-Control-Allow-Origin"] = "*" return r - def redirect(self, request, path, forward_querystring=True): + def redirect(self, request, path, forward_querystring=True, remove_args=None): if request.query_string and "?" not in path and forward_querystring: path = "{}?{}".format(path, request.query_string) + if remove_args: + path = path_with_removed_args(request, remove_args, path=path) r = response.redirect(path) r.headers["Link"] = "<{}>; rel=preload".format(path) if self.ds.cors: r.headers["Access-Control-Allow-Origin"] = "*" return r - def resolve_db_name(self, db_name, **kwargs): + def resolve_db_name(self, request, db_name, **kwargs): databases = self.ds.inspect() hash = None name = None @@ -161,7 +176,9 @@ class BaseView(RenderMixin): raise NotFound("Database not found: {}".format(name)) expected = info["hash"][:HASH_LENGTH] - if expected != hash: + correct_hash_provided = (expected == hash) + + if not correct_hash_provided: if "table_and_format" in kwargs: table, _format = resolve_table_and_format( table_and_format=urllib.parse.unquote_plus( @@ -188,9 +205,11 @@ class BaseView(RenderMixin): should_redirect += kwargs["as_format"] if "as_db" in kwargs: should_redirect += kwargs["as_db"] - return name, expected, should_redirect - return name, expected, None + if self.ds.config("hash_urls") or "_hash" in request.args: + return name, expected, correct_hash_provided, should_redirect + + return name, expected, correct_hash_provided, None def absolute_url(self, request, path): url = urllib.parse.urljoin(request.url, path) @@ -202,11 +221,13 @@ class BaseView(RenderMixin): assert NotImplemented async def get(self, request, db_name, **kwargs): - database, hash, should_redirect = self.resolve_db_name(db_name, **kwargs) + database, hash, correct_hash_provided, should_redirect = self.resolve_db_name( + request, db_name, **kwargs + ) if should_redirect: - return self.redirect(request, should_redirect) + return self.redirect(request, should_redirect, remove_args={"_hash"}) - return await self.view_get(request, database, hash, **kwargs) + return await self.view_get(request, database, hash, correct_hash_provided, **kwargs) async def as_csv(self, request, database, hash, **kwargs): stream = request.args.get("_stream") @@ -301,7 +322,7 @@ class BaseView(RenderMixin): content_type=content_type ) - async def view_get(self, request, database, hash, **kwargs): + async def view_get(self, request, database, hash, correct_hash_provided, **kwargs): # If ?_format= is provided, use that as the format _format = request.args.get("_format", None) if not _format: @@ -418,7 +439,6 @@ class BaseView(RenderMixin): "ok": False, "error": error, "database": database, - "database_hash": hash, } elif shape == "array": data = data["rows"] @@ -489,10 +509,13 @@ class BaseView(RenderMixin): r = self.render(templates, **context) r.status = status_code # Set far-future cache expiry - if self.ds.cache_headers: + if self.ds.cache_headers and r.status == 200: ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): - ttl = self.ds.config("default_cache_ttl") + if correct_hash_provided: + ttl = self.ds.config("default_cache_ttl_hashed") + else: + ttl = self.ds.config("default_cache_ttl") else: ttl = int(ttl) if ttl == 0: @@ -572,7 +595,6 @@ class BaseView(RenderMixin): display_rows.append(display_row) return { "display_rows": display_rows, - "database_hash": hash, "custom_sql": True, "named_parameter_values": named_parameter_values, "editable": editable, diff --git a/datasette/views/database.py b/datasette/views/database.py index 9c44a800..0cbb432b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -30,7 +30,6 @@ class DatabaseView(BaseView): "views": info["views"], "queries": self.ds.get_canned_queries(database), }, { - "database_hash": hash, "show_hidden": request.args.get("_show_hidden"), "editable": True, "metadata": metadata, @@ -41,7 +40,7 @@ class DatabaseView(BaseView): class DatabaseDownload(BaseView): - async def view_get(self, request, database, hash, **kwargs): + async def view_get(self, request, database, hash, correct_hash_present, **kwargs): if not self.ds.config("allow_download"): raise DatasetteError("Database download is forbidden", status=403) filepath = self.ds.inspect()[database]["file"] diff --git a/datasette/views/index.py b/datasette/views/index.py index 32c04585..70f7e943 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -21,7 +21,7 @@ class IndexView(RenderMixin): database = { "name": key, "hash": info["hash"], - "path": "{}-{}".format(key, info["hash"][:HASH_LENGTH]), + "path": self.database_url(key), "tables_truncated": sorted( tables, key=lambda t: t["count"], reverse=True )[ diff --git a/datasette/views/table.py b/datasette/views/table.py index cb744708..14f3be6f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -750,7 +750,6 @@ class TableView(RowTableShared): ) self.ds.update_with_inherited_metadata(metadata) return { - "database_hash": hash, "supports_search": bool(fts_table), "search": search or "", "use_rowid": use_rowid, @@ -851,7 +850,6 @@ class RowView(RowTableShared): for column in display_columns: column["sortable"] = False return { - "database_hash": hash, "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values ), diff --git a/docs/config.rst b/docs/config.rst index b934ef2a..dac4e044 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -115,11 +115,21 @@ Enable/disable the ability for users to run custom SQL directly against a databa default_cache_ttl ----------------- -Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` querystring parameter. Set this to ``0`` to disable HTTP caching entirely. Defaults to 365 days (31536000 seconds). +Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` querystring parameter. Set this to ``0`` to disable HTTP caching entirely. Defaults to 5 seconds. :: - datasette mydatabase.db --config default_cache_ttl:10 + datasette mydatabase.db --config default_cache_ttl:60 + +default_cache_ttl_hashed +------------------------ + +Default HTTP caching max-age for responses served using using the :ref:`hashed-urls mechanism `. Defaults to 365 days (31536000 seconds). + +:: + + datasette mydatabase.db --config default_cache_ttl_hashed:10000 + cache_size_kb ------------- @@ -179,3 +189,19 @@ HTTP but is served to the outside world via a proxy that enables HTTPS. :: datasette mydatabase.db --config force_https_urls:1 + +.. _config_hash_urls: + +hash_urls +--------- + +When enabled, this setting causes Datasette to append a content hash of the +database file to the URL path for every table and query within that database. + +When combined with far-future expire headers this ensures that queries can be +cached forever, safe in the knowledge that any modifications to the database +itself will result in new, uncachcacheed URL paths. + +:: + + datasette mydatabase.db --config hash_urls:1 diff --git a/tests/fixtures.py b/tests/fixtures.py index efd85fab..81432e30 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -73,6 +73,13 @@ def app_client_no_files(): yield client +@pytest.fixture(scope="session") +def app_client_with_hash(): + yield from make_app_client(config={ + 'hash_urls': True + }) + + @pytest.fixture(scope='session') def app_client_shorter_time_limit(): yield from make_app_client(20) diff --git a/tests/test_api.py b/tests/test_api.py index a6ba3f37..b92b9ffb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ from .fixtures import ( # noqa app_client, app_client_no_files, + app_client_with_hash, app_client_shorter_time_limit, app_client_larger_cache_size, app_client_returned_rows_matches_page_size, @@ -378,7 +379,7 @@ def test_no_files_uses_memory_database(app_client_no_files): "hidden_table_rows_sum": 0, "hidden_tables_count": 0, "name": ":memory:", - "path": ":memory:-000", + "path": "/:memory:", "table_rows_sum": 0, "tables_count": 0, "tables_more": False, @@ -388,7 +389,7 @@ def test_no_files_uses_memory_database(app_client_no_files): } == response.json # Try that SQL query response = app_client_no_files.get( - "/:memory:-0.json?sql=select+sqlite_version()&_shape=array" + "/:memory:.json?sql=select+sqlite_version()&_shape=array" ) assert 1 == len(response.json) assert ["sqlite_version()"] == list(response.json[0].keys()) @@ -501,12 +502,12 @@ def test_table_not_exists_json(app_client): } == app_client.get('/fixtures/blah.json').json -def test_jsono_redirects_to_shape_objects(app_client): - response_1 = app_client.get( +def test_jsono_redirects_to_shape_objects(app_client_with_hash): + response_1 = app_client_with_hash.get( '/fixtures/simple_primary_key.jsono', allow_redirects=False ) - response = app_client.get( + response = app_client_with_hash.get( response_1.headers['Location'], allow_redirects=False ) @@ -1049,13 +1050,15 @@ def test_config_json(app_client): "allow_facet": True, "suggest_facets": True, "allow_sql": True, - "default_cache_ttl": 365 * 24 * 60 * 60, + "default_cache_ttl": 5, + "default_cache_ttl_hashed": 365 * 24 * 60 * 60, "num_sql_threads": 3, "cache_size_kb": 0, "allow_csv_stream": True, "max_csv_mb": 100, "truncate_cells_html": 2048, "force_https_urls": False, + "hash_urls": False, } == response.json @@ -1300,8 +1303,8 @@ def test_expand_label(app_client): @pytest.mark.parametrize('path,expected_cache_control', [ - ("/fixtures/facetable.json", "max-age=31536000"), - ("/fixtures/facetable.json?_ttl=invalid", "max-age=31536000"), + ("/fixtures/facetable.json", "max-age=5"), + ("/fixtures/facetable.json?_ttl=invalid", "max-age=5"), ("/fixtures/facetable.json?_ttl=10", "max-age=10"), ("/fixtures/facetable.json?_ttl=0", "no-cache"), ]) @@ -1310,6 +1313,19 @@ def test_ttl_parameter(app_client, path, expected_cache_control): assert expected_cache_control == response.headers['Cache-Control'] +@pytest.mark.parametrize("path,expected_redirect", [ + ("/fixtures/facetable.json?_hash=1", "/fixtures-HASH/facetable.json"), + ("/fixtures/facetable.json?city_id=1&_hash=1", "/fixtures-HASH/facetable.json?city_id=1"), +]) +def test_hash_parameter(app_client, path, expected_redirect): + # First get the current hash for the fixtures database + current_hash = app_client.get("/-/inspect.json").json["fixtures"]["hash"][:7] + response = app_client.get(path, allow_redirects=False) + assert response.status == 302 + location = response.headers["Location"] + assert expected_redirect.replace("HASH", current_hash) == location + + test_json_columns_default_expected = [{ "intval": 1, "strval": "s", diff --git a/tests/test_html.py b/tests/test_html.py index 36335201..18e6bb04 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -2,6 +2,7 @@ from bs4 import BeautifulSoup as Soup from .fixtures import ( # noqa app_client, app_client_shorter_time_limit, + app_client_with_hash, make_app_client, ) import pytest @@ -15,10 +16,10 @@ def test_homepage(app_client): assert 'fixtures' in response.text -def test_database_page(app_client): - response = app_client.get('/fixtures', allow_redirects=False) +def test_database_page_redirects_with_url_hash(app_client_with_hash): + response = app_client_with_hash.get('/fixtures', allow_redirects=False) assert response.status == 302 - response = app_client.get('/fixtures') + response = app_client_with_hash.get('/fixtures') assert 'fixtures' in response.text @@ -41,19 +42,19 @@ def test_sql_time_limit(app_client_shorter_time_limit): assert expected_html_fragment in response.text -def test_row(app_client): - response = app_client.get( +def test_row_redirects_with_url_hash(app_client_with_hash): + response = app_client_with_hash.get( '/fixtures/simple_primary_key/1', allow_redirects=False ) assert response.status == 302 assert response.headers['Location'].endswith('/1') - response = app_client.get('/fixtures/simple_primary_key/1') + response = app_client_with_hash.get('/fixtures/simple_primary_key/1') assert response.status == 200 -def test_row_strange_table_name(app_client): - response = app_client.get( +def test_row_strange_table_name_with_url_hash(app_client_with_hash): + response = app_client_with_hash.get( '/fixtures/table%2Fwith%2Fslashes.csv/3', allow_redirects=False ) @@ -61,7 +62,7 @@ def test_row_strange_table_name(app_client): assert response.headers['Location'].endswith( '/table%2Fwith%2Fslashes.csv/3' ) - response = app_client.get('/fixtures/table%2Fwith%2Fslashes.csv/3') + response = app_client_with_hash.get('/fixtures/table%2Fwith%2Fslashes.csv/3') assert response.status == 200 @@ -105,10 +106,7 @@ def test_add_filter_redirects(app_client): '_filter_op': 'startswith', '_filter_value': 'x' }) - # First we need to resolve the correct path before testing more redirects - path_base = app_client.get( - '/fixtures/simple_primary_key', allow_redirects=False - ).headers['Location'] + path_base = '/fixtures/simple_primary_key' path = path_base + '?' + filter_args response = app_client.get(path, allow_redirects=False) assert response.status == 302 @@ -146,9 +144,7 @@ def test_existing_filter_redirects(app_client): '_filter_op_4': 'contains', '_filter_value_4': 'world', } - path_base = app_client.get( - '/fixtures/simple_primary_key', allow_redirects=False - ).headers['Location'] + path_base = '/fixtures/simple_primary_key' path = path_base + '?' + urllib.parse.urlencode(filter_args) response = app_client.get(path, allow_redirects=False) assert response.status == 302 @@ -174,9 +170,7 @@ def test_existing_filter_redirects(app_client): def test_empty_search_parameter_gets_removed(app_client): - path_base = app_client.get( - '/fixtures/simple_primary_key', allow_redirects=False - ).headers['Location'] + path_base = '/fixtures/simple_primary_key' path = path_base + '?' + urllib.parse.urlencode({ '_search': '', '_filter_column': 'name', @@ -191,9 +185,7 @@ def test_empty_search_parameter_gets_removed(app_client): def test_sort_by_desc_redirects(app_client): - path_base = app_client.get( - '/fixtures/sortable', allow_redirects=False - ).headers['Location'] + path_base = '/fixtures/sortable' path = path_base + '?' + urllib.parse.urlencode({ '_sort': 'sortable', '_sort_by_desc': '1', diff --git a/tests/test_utils.py b/tests/test_utils.py index b406d70b..1f0079c9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -59,6 +59,13 @@ def test_path_with_removed_args(path, args, expected): ) actual = utils.path_with_removed_args(request, args) assert expected == actual + # Run the test again but this time use the path= argument + request = Request( + "/".encode('utf8'), + {}, '1.1', 'GET', None + ) + actual = utils.path_with_removed_args(request, args, path=path) + assert expected == actual @pytest.mark.parametrize('path,args,expected', [ From 47032636b55718dd711670f19b753f4ada9e8f2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Mar 2019 16:25:15 -0700 Subject: [PATCH 0030/2303] 'datasette serve -i immutable.db' option, refs #419 --- datasette/app.py | 14 +++++++++++--- datasette/cli.py | 11 ++++++++++- docs/datasette-serve-help.txt | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c292542f..5e656e5f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -121,6 +121,7 @@ class Datasette: def __init__( self, files, + immutables=None, cache_headers=True, cors=False, inspect_data=None, @@ -133,7 +134,9 @@ class Datasette: config=None, version_note=None, ): - self.files = files + immutables = immutables or [] + self.files = files + immutables + self.immutables = set(immutables) if not self.files: self.files = [MEMORY] elif memory: @@ -322,7 +325,7 @@ class Datasette: raise Exception("Multiple files with same stem %s" % name) try: with sqlite3.connect( - "file:{}?immutable=1".format(path), uri=True + "file:{}?mode=ro".format(path), uri=True ) as conn: self.prepare_connection(conn) self._inspect[name] = { @@ -426,8 +429,13 @@ class Datasette: if info["file"] == ":memory:": conn = sqlite3.connect(":memory:") else: + # mode=ro or immutable=1? + if info["file"] in self.immutables: + qs = "immutable=1" + else: + qs = "mode=ro" conn = sqlite3.connect( - "file:{}?immutable=1".format(info["file"]), + "file:{}?{}".format(info["file"], qs), uri=True, check_same_thread=False, ) diff --git a/datasette/cli.py b/datasette/cli.py index 6fbc9908..8ae54f8f 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -267,6 +267,13 @@ def package( @cli.command() @click.argument("files", type=click.Path(exists=True), nargs=-1) +@click.option( + "-i", + "--immutable", + type=click.Path(exists=True), + help="Database files to open in immutable mode", + multiple=True, +) @click.option( "-h", "--host", default="127.0.0.1", help="host for server, defaults to 127.0.0.1" ) @@ -332,6 +339,7 @@ def package( ) def serve( files, + immutable, host, port, debug, @@ -376,9 +384,10 @@ def serve( if metadata: metadata_data = json.loads(metadata.read()) - click.echo("Serve! files={} on port {}".format(files, port)) + click.echo("Serve! files={} (immutables={}) on port {}".format(files, immutable, port)) ds = Datasette( files, + immutables=immutable, cache_headers=not debug and not reload, cors=cors, inspect_data=inspect_data, diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 65b9aceb..7b7c3b09 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -5,6 +5,7 @@ Usage: datasette serve [OPTIONS] [FILES]... Serve up specified SQLite database files with a web UI Options: + -i, --immutable PATH Database files to open in immutable mode -h, --host TEXT host for server, defaults to 127.0.0.1 -p, --port INTEGER port for server, defaults to 8001 --debug Enable debug mode - useful for development From 82fec6048148b58748040a7e2caa163387e982a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Mar 2019 16:36:35 -0700 Subject: [PATCH 0031/2303] Fix for TypeError File "../datasette/app.py", line 138, in __init__ self.files = files + immutables TypeError: can only concatenate tuple (not "list") to tuple --- datasette/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 5e656e5f..fcffb36a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -135,7 +135,7 @@ class Datasette: version_note=None, ): immutables = immutables or [] - self.files = files + immutables + self.files = tuple(files) + tuple(immutables) self.immutables = set(immutables) if not self.files: self.files = [MEMORY] From 0209a0a344503157351e625f0629b686961763c9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 Mar 2019 11:02:22 -0700 Subject: [PATCH 0032/2303] table_exists() now uses async SQL, refs #420 --- datasette/app.py | 46 ++++++++++++++++++++++++++--------------- datasette/utils.py | 5 +++-- datasette/views/base.py | 16 ++++++++------ setup.py | 1 + tests/test_utils.py | 7 ++++--- 5 files changed, 47 insertions(+), 28 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index fcffb36a..d5994e5e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -300,8 +300,13 @@ class Datasette: conn.execute('PRAGMA cache_size=-{}'.format(self.config("cache_size_kb"))) pm.hook.prepare_connection(conn=conn) - def table_exists(self, database, table): - return table in self.inspect().get(database, {}).get("tables") + async def table_exists(self, database, table): + results = await self.execute( + database, + "select 1 from sqlite_master where type='table' and name=?", + params=(table,) + ) + return bool(results.rows) def inspect(self): " Inspect the database and return a dictionary of table metadata " @@ -410,19 +415,8 @@ class Datasette: for p in ps ] - async def execute( - self, - db_name, - sql, - params=None, - truncate=False, - custom_time_limit=None, - page_size=None, - ): - """Executes sql against db_name in a thread""" - page_size = page_size or self.page_size - - def sql_operation_in_thread(): + async def execute_against_connection_in_thread(self, db_name, fn): + def in_thread(): conn = getattr(connections, db_name, None) if not conn: info = self.inspect()[db_name] @@ -441,7 +435,25 @@ class Datasette: ) self.prepare_connection(conn) setattr(connections, db_name, conn) + return fn(conn) + return await asyncio.get_event_loop().run_in_executor( + self.executor, in_thread + ) + + async def execute( + self, + db_name, + sql, + params=None, + truncate=False, + custom_time_limit=None, + page_size=None, + ): + """Executes sql against db_name in a thread""" + page_size = page_size or self.page_size + + def sql_operation_in_thread(conn): time_limit_ms = self.sql_time_limit_ms if custom_time_limit and custom_time_limit < time_limit_ms: time_limit_ms = custom_time_limit @@ -476,8 +488,8 @@ class Datasette: else: return Results(rows, False, cursor.description) - return await asyncio.get_event_loop().run_in_executor( - self.executor, sql_operation_in_thread + return await self.execute_against_connection_in_thread( + db_name, sql_operation_in_thread ) def app(self): diff --git a/datasette/utils.py b/datasette/utils.py index 8bcaefc2..3a7e90c4 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -801,10 +801,11 @@ def get_plugins(pm): FORMATS = ('csv', 'json', 'jsono') -def resolve_table_and_format(table_and_format, table_exists): +async def resolve_table_and_format(table_and_format, table_exists): if '.' in table_and_format: # Check if a table exists with this exact name - if table_exists(table_and_format): + it_exists = await table_exists(table_and_format) + if it_exists: return table_and_format, None # Check if table ends with a known format for _format in FORMATS: diff --git a/datasette/views/base.py b/datasette/views/base.py index 33b7524e..e79a46ec 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -155,7 +155,7 @@ class BaseView(RenderMixin): r.headers["Access-Control-Allow-Origin"] = "*" return r - def resolve_db_name(self, request, db_name, **kwargs): + async def resolve_db_name(self, request, db_name, **kwargs): databases = self.ds.inspect() hash = None name = None @@ -180,11 +180,13 @@ class BaseView(RenderMixin): if not correct_hash_provided: if "table_and_format" in kwargs: - table, _format = resolve_table_and_format( + async def async_table_exists(t): + return await self.ds.table_exists(name, t) + table, _format = await resolve_table_and_format( table_and_format=urllib.parse.unquote_plus( kwargs["table_and_format"] ), - table_exists=lambda t: self.ds.table_exists(name, t) + table_exists=async_table_exists ) kwargs["table"] = table if _format: @@ -221,7 +223,7 @@ class BaseView(RenderMixin): assert NotImplemented async def get(self, request, db_name, **kwargs): - database, hash, correct_hash_provided, should_redirect = self.resolve_db_name( + database, hash, correct_hash_provided, should_redirect = await self.resolve_db_name( request, db_name, **kwargs ) if should_redirect: @@ -328,11 +330,13 @@ class BaseView(RenderMixin): if not _format: _format = (kwargs.pop("as_format", None) or "").lstrip(".") if "table_and_format" in kwargs: - table, _ext_format = resolve_table_and_format( + async def async_table_exists(t): + return await self.ds.table_exists(database, t) + table, _ext_format = await resolve_table_and_format( table_and_format=urllib.parse.unquote_plus( kwargs["table_and_format"] ), - table_exists=lambda t: self.ds.table_exists(database, t) + table_exists=async_table_exists ) _format = _format or _ext_format kwargs["table"] = table diff --git a/setup.py b/setup.py index fb00f2d0..524f5243 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ setup( extras_require={ 'test': [ 'pytest==4.0.2', + 'pytest-asyncio==0.10.0', 'aiohttp==3.5.3', 'beautifulsoup4==4.6.1', ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 1f0079c9..fad1ac84 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -332,10 +332,11 @@ def test_compound_keys_after_sql(): '''.strip() == utils.compound_keys_after_sql(['a', 'b', 'c']) -def table_exists(table): +async def table_exists(table): return table == "exists.csv" +@pytest.mark.asyncio @pytest.mark.parametrize( "table_and_format,expected_table,expected_format", [ @@ -346,10 +347,10 @@ def table_exists(table): ("exists.csv", "exists.csv", None), ], ) -def test_resolve_table_and_format( +async def test_resolve_table_and_format( table_and_format, expected_table, expected_format ): - actual_table, actual_format = utils.resolve_table_and_format( + actual_table, actual_format = await utils.resolve_table_and_format( table_and_format, table_exists ) assert expected_table == actual_table From 7d0f668556b6d9a6c3d70a7b8a07a542857bbb56 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 Mar 2019 16:51:52 -0700 Subject: [PATCH 0033/2303] .resolve_db_name() and .execute() work without inspect Refs #420 --- datasette/app.py | 61 +++++++++++++++++++++++++++++++++++++---- datasette/views/base.py | 9 +++--- tests/fixtures.py | 8 ++++-- tests/test_api.py | 6 ++-- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d5994e5e..bc5aaa74 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -116,6 +116,43 @@ async def favicon(request): return response.text("") +class ConnectedDatabase: + def __init__(self, path=None, is_mutable=False, is_memory=False): + self.path = path + self.is_mutable = is_mutable + self.is_memory = is_memory + self.hash = None + self.size = None + if not self.is_mutable: + p = Path(path) + self.hash = inspect_hash(p) + self.size = p.stat().st_size + + @property + def name(self): + if self.is_memory: + return ":memory:" + else: + return Path(self.path).stem + + def __repr__(self): + tags = [] + if self.is_mutable: + tags.append("mutable") + if self.is_memory: + tags.append("memory") + if self.hash: + tags.append("hash={}".format(self.hash)) + if self.size is not None: + tags.append("size={}".format(self.size)) + tags_str = "" + if tags: + tags_str = " ({})".format(", ".join(tags)) + return "".format( + self.name, tags_str + ) + + class Datasette: def __init__( @@ -141,6 +178,18 @@ class Datasette: self.files = [MEMORY] elif memory: self.files = (MEMORY,) + self.files + self.databases = {} + for file in self.files: + path = file + is_memory = False + if file is MEMORY: + path = None + is_memory = True + db = ConnectedDatabase(path, is_mutable=path not in self.immutables, is_memory=is_memory) + if db.name in self.databases: + raise Exception("Multiple files with same stem: {}".format(db.name)) + self.databases[db.name] = db + print(self.databases) self.cache_headers = cache_headers self.cors = cors self._inspect = inspect_data @@ -419,17 +468,17 @@ class Datasette: def in_thread(): conn = getattr(connections, db_name, None) if not conn: - info = self.inspect()[db_name] - if info["file"] == ":memory:": + db = self.databases[db_name] + if db.is_memory: conn = sqlite3.connect(":memory:") else: # mode=ro or immutable=1? - if info["file"] in self.immutables: - qs = "immutable=1" - else: + if db.is_mutable: qs = "mode=ro" + else: + qs = "immutable=1" conn = sqlite3.connect( - "file:{}?{}".format(info["file"], qs), + "file:{}?{}".format(db.path, qs), uri=True, check_same_thread=False, ) diff --git a/datasette/views/base.py b/datasette/views/base.py index e79a46ec..8da51d65 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -156,14 +156,13 @@ class BaseView(RenderMixin): return r async def resolve_db_name(self, request, db_name, **kwargs): - databases = self.ds.inspect() hash = None name = None if "-" in db_name: # Might be name-and-hash, or might just be # a name with a hyphen in it name, hash = db_name.rsplit("-", 1) - if name not in databases: + if name not in self.ds.databases: # Try the whole name name = db_name hash = None @@ -171,11 +170,13 @@ class BaseView(RenderMixin): name = db_name # Verify the hash try: - info = databases[name] + db = self.ds.databases[name] except KeyError: raise NotFound("Database not found: {}".format(name)) - expected = info["hash"][:HASH_LENGTH] + expected = "000" + if db.hash is not None: + expected = db.hash[:HASH_LENGTH] correct_hash_provided = (expected == hash) if not correct_hash_provided: diff --git a/tests/fixtures.py b/tests/fixtures.py index 81432e30..c3883fe7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -29,6 +29,7 @@ def make_app_client( cors=False, config=None, filename="fixtures.db", + is_immutable=False, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) @@ -48,7 +49,8 @@ def make_app_client( } ) ds = Datasette( - [filepath], + [] if is_immutable else [filepath], + immutables=[filepath] if is_immutable else [], cors=cors, metadata=METADATA, plugins_dir=plugins_dir, @@ -76,8 +78,8 @@ def app_client_no_files(): @pytest.fixture(scope="session") def app_client_with_hash(): yield from make_app_client(config={ - 'hash_urls': True - }) + 'hash_urls': True, + }, is_immutable=True) @pytest.fixture(scope='session') diff --git a/tests/test_api.py b/tests/test_api.py index b92b9ffb..6fdcb115 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1317,10 +1317,10 @@ def test_ttl_parameter(app_client, path, expected_cache_control): ("/fixtures/facetable.json?_hash=1", "/fixtures-HASH/facetable.json"), ("/fixtures/facetable.json?city_id=1&_hash=1", "/fixtures-HASH/facetable.json?city_id=1"), ]) -def test_hash_parameter(app_client, path, expected_redirect): +def test_hash_parameter(app_client_with_hash, path, expected_redirect): # First get the current hash for the fixtures database - current_hash = app_client.get("/-/inspect.json").json["fixtures"]["hash"][:7] - response = app_client.get(path, allow_redirects=False) + current_hash = app_client_with_hash.get("/-/inspect.json").json["fixtures"]["hash"][:7] + response = app_client_with_hash.get(path, allow_redirects=False) assert response.status == 302 location = response.headers["Location"] assert expected_redirect.replace("HASH", current_hash) == location From 29a3896fe1e09d5c35a90d55690c9f71fae81e24 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 Mar 2019 16:55:38 -0700 Subject: [PATCH 0034/2303] .database_url(database) no longer needs inspect, refs #420 --- datasette/views/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 8da51d65..99bb175f 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -76,12 +76,13 @@ class RenderMixin(HTTPMethodView): yield {"url": url} def database_url(self, database): - if not self.ds.config("hash_urls"): - return "/{}".format(database) - else: + db = self.ds.databases[database] + if self.ds.config("hash_urls") and db.hash: return "/{}-{}".format( - database, self.ds.inspect()[database]["hash"][:HASH_LENGTH] + database, db.hash[:HASH_LENGTH] ) + else: + return "/{}".format(database) def database_color(self, database): return 'ff0000' From 468c6fd9532f08859c77ae670307fd5f4c6632f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 Mar 2019 19:04:17 -0700 Subject: [PATCH 0035/2303] DatabaseDownload no longer uses .inspect(), refs #420 --- datasette/views/database.py | 9 ++++++++- tests/fixtures.py | 5 +++++ tests/test_html.py | 20 ++++++++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0cbb432b..9f43980b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -43,7 +43,14 @@ class DatabaseDownload(BaseView): async def view_get(self, request, database, hash, correct_hash_present, **kwargs): if not self.ds.config("allow_download"): raise DatasetteError("Database download is forbidden", status=403) - filepath = self.ds.inspect()[database]["file"] + if database not in self.ds.databases: + raise DatasetteError("Invalid database", status=404) + db = self.ds.databases[database] + if db.is_memory: + raise DatasetteError("Cannot download :memory: database", status=404) + if not db.path: + raise DatasetteError("Cannot download database", status=404) + filepath = db.path return await response.file_stream( filepath, filename=os.path.basename(filepath), diff --git a/tests/fixtures.py b/tests/fixtures.py index c3883fe7..8f5f0b68 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -27,6 +27,7 @@ def make_app_client( sql_time_limit_ms=None, max_returned_rows=None, cors=False, + memory=False, config=None, filename="fixtures.db", is_immutable=False, @@ -51,6 +52,7 @@ def make_app_client( ds = Datasette( [] if is_immutable else [filepath], immutables=[filepath] if is_immutable else [], + memory=memory, cors=cors, metadata=METADATA, plugins_dir=plugins_dir, @@ -74,6 +76,9 @@ def app_client_no_files(): client.ds = ds yield client +@pytest.fixture(scope="session") +def app_client_with_memory(): + yield from make_app_client(memory=True) @pytest.fixture(scope="session") def app_client_with_hash(): diff --git a/tests/test_html.py b/tests/test_html.py index 18e6bb04..1babaa60 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -3,6 +3,7 @@ from .fixtures import ( # noqa app_client, app_client_shorter_time_limit, app_client_with_hash, + app_client_with_memory, make_app_client, ) import pytest @@ -676,12 +677,25 @@ def test_table_metadata(app_client): assert_footer_links(soup) -def test_allow_download_on(app_client): - response = app_client.get( +def test_database_download(app_client_with_memory): + # Regular page should have a download link + response = app_client_with_memory.get( "/fixtures" ) soup = Soup(response.body, 'html.parser') assert len(soup.findAll('a', {'href': re.compile(r'\.db$')})) + # Check we can actually download it + assert 200 == app_client_with_memory.get( + "/fixtures.db", + ).status + # Memory page should NOT have a download link + response2 = app_client_with_memory.get("/:memory:") + soup2 = Soup(response2.body, 'html.parser') + assert 0 == len(soup2.findAll('a', {'href': re.compile(r'\.db$')})) + # The URL itself should 404 + assert 404 == app_client_with_memory.get( + "/:memory:.db", + ).status def test_allow_download_off(): @@ -690,14 +704,12 @@ def test_allow_download_off(): }): response = client.get( "/fixtures", - ) soup = Soup(response.body, 'html.parser') assert not len(soup.findAll('a', {'href': re.compile(r'\.db$')})) # Accessing URL directly should 403 response = client.get( "/fixtures.db", - ) assert 403 == response.status From d8851e0011f256ce52775aa776e0e4323db50c8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 18:54:51 -0700 Subject: [PATCH 0036/2303] Removed rogue print() --- datasette/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index bc5aaa74..21d17194 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -189,7 +189,6 @@ class Datasette: if db.name in self.databases: raise Exception("Multiple files with same stem: {}".format(db.name)) self.databases[db.name] = db - print(self.databases) self.cache_headers = cache_headers self.cors = cors self._inspect = inspect_data From 97331f3435ba1583a0f9dbcaffc25de8894cf1f8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 18:58:51 -0700 Subject: [PATCH 0037/2303] sortable_columns_for_table() no longer uses inspect() Refs #420 --- datasette/app.py | 6 ++++++ datasette/inspect.py | 8 ++------ datasette/utils.py | 9 +++++++++ datasette/views/table.py | 10 +++++----- tests/test_utils.py | 9 +++++++++ 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 21d17194..29dd91af 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,6 +33,7 @@ from .utils import ( module_from_path, sqlite3, sqlite_timelimit, + table_columns, to_css_class ) from .inspect import inspect_hash, inspect_views, inspect_tables @@ -463,6 +464,11 @@ class Datasette: for p in ps ] + async def table_columns(self, db_name, table): + return await self.execute_against_connection_in_thread( + db_name, lambda conn: table_columns(conn, table) + ) + async def execute_against_connection_in_thread(self, db_name, fn): def in_thread(): conn = getattr(connections, db_name, None) diff --git a/datasette/inspect.py b/datasette/inspect.py index 42e7aeff..12f67f0a 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -5,6 +5,7 @@ from .utils import ( detect_fts, escape_sqlite, get_all_foreign_keys, + table_columns, sqlite3 ) @@ -78,12 +79,7 @@ def inspect_tables(conn, database_metadata): # e.g. "select count(*) from some_fts;" count = 0 - column_names = [ - r[1] - for r in conn.execute( - "PRAGMA table_info({});".format(escape_sqlite(table)) - ).fetchall() - ] + column_names = table_columns(conn, table) tables[table] = { "name": table, diff --git a/datasette/utils.py b/datasette/utils.py index 3a7e90c4..98f70592 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -536,6 +536,15 @@ def detect_fts_sql(table): '''.format(table=table) +def table_columns(conn, table): + return [ + r[1] + for r in conn.execute( + "PRAGMA table_info({});".format(escape_sqlite(table)) + ).fetchall() + ] + + class Filter: def __init__(self, key, display, sql_template, human_template, format='{}', numeric=False, no_argument=False): self.key = key diff --git a/datasette/views/table.py b/datasette/views/table.py index 14f3be6f..84ebec05 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -19,6 +19,7 @@ from datasette.utils import ( path_with_removed_args, path_with_replaced_args, sqlite3, + table_columns, to_css_class, urlsafe_components, value_as_boolean, @@ -31,13 +32,12 @@ LINK_WITH_VALUE = '{id}' class RowTableShared(BaseView): - def sortable_columns_for_table(self, database, table, use_rowid): + async def sortable_columns_for_table(self, database, table, use_rowid): table_metadata = self.table_metadata(database, table) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: - table_info = self.ds.inspect()[database]["tables"].get(table) or {} - sortable_columns = set(table_info.get("columns", [])) + sortable_columns = set(await self.ds.table_columns(database, table)) if use_rowid: sortable_columns.add("rowid") return sortable_columns @@ -121,7 +121,7 @@ class RowTableShared(BaseView): "Returns columns, rows for specified table - including fancy foreign key treatment" table_metadata = self.table_metadata(database, table) info = self.ds.inspect()[database] - sortable_columns = self.sortable_columns_for_table(database, table, True) + sortable_columns = await self.sortable_columns_for_table(database, table, True) columns = [ {"name": r[0], "sortable": r[0] in sortable_columns} for r in description ] @@ -363,7 +363,7 @@ class TableView(RowTableShared): if not is_view: table_rows_count = table_info["count"] - sortable_columns = self.sortable_columns_for_table(database, table, use_rowid) + sortable_columns = await self.sortable_columns_for_table(database, table, use_rowid) # Allow for custom sort order sort = special_args.get("_sort") diff --git a/tests/test_utils.py b/tests/test_utils.py index fad1ac84..9a00b4b4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,6 +7,7 @@ import json import os import pytest from sanic.request import Request +import sqlite3 import tempfile from unittest.mock import patch @@ -357,6 +358,14 @@ async def test_resolve_table_and_format( assert expected_format == actual_format +def test_table_columns(): + conn = sqlite3.connect(":memory:") + conn.executescript(""" + create table places (id integer primary key, name text, bob integer) + """) + assert ["id", "name", "bob"] == utils.table_columns(conn, "places") + + @pytest.mark.parametrize( "path,format,extra_qs,expected", [ From 53bf875483d98861314db3a0cdcec8f5ce22ee96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 19:56:07 -0700 Subject: [PATCH 0038/2303] expand_foreign_keys() no longer uses inspect, refs #420 --- datasette/app.py | 29 +++++++++++++++++++++++++++++ datasette/inspect.py | 12 ------------ datasette/utils.py | 16 ++++++++++++++++ datasette/views/base.py | 8 -------- datasette/views/table.py | 24 +++++++----------------- docs/introspection.rst | 1 - tests/test_api.py | 25 ------------------------- 7 files changed, 52 insertions(+), 63 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 29dd91af..0b992fbe 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -29,6 +29,7 @@ from .utils import ( Results, escape_css_string, escape_sqlite, + get_outbound_foreign_keys, get_plugins, module_from_path, sqlite3, @@ -464,11 +465,39 @@ class Datasette: for p in ps ] + def table_metadata(self, database, table): + "Fetch table-specific metadata." + return (self.metadata("databases") or {}).get(database, {}).get( + "tables", {} + ).get( + table, {} + ) + async def table_columns(self, db_name, table): return await self.execute_against_connection_in_thread( db_name, lambda conn: table_columns(conn, table) ) + async def foreign_keys_for_table(self, database, table): + return await self.execute_against_connection_in_thread( + database, lambda conn: get_outbound_foreign_keys(conn, table) + ) + + async def label_column_for_table(self, db_name, table): + explicit_label_column = ( + self.table_metadata( + db_name, table + ).get("label_column") + ) + if explicit_label_column: + return explicit_label_column + # If a table has two columns, one of which is ID, then label_column is the other one + column_names = await self.table_columns(db_name, table) + if (column_names and len(column_names) == 2 and "id" in column_names): + return [c for c in column_names if c != "id"][0] + # Couldn't find a label: + return None + async def execute_against_connection_in_thread(self, db_name, fn): def in_thread(): conn = getattr(connections, db_name, None) diff --git a/datasette/inspect.py b/datasette/inspect.py index 12f67f0a..3ff8b367 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -31,17 +31,6 @@ def inspect_views(conn): return [v[0] for v in conn.execute('select name from sqlite_master where type = "view"')] -def detect_label_column(column_names): - """ Detect the label column - which we display as the label for a joined column. - - If a table has two columns, one of which is ID, then label_column is the other one. - """ - if (column_names and len(column_names) == 2 and "id" in column_names): - return [c for c in column_names if c != "id"][0] - - return None - - def detect_primary_keys(conn, table): " Figure out primary keys for a table. " table_info_rows = [ @@ -86,7 +75,6 @@ def inspect_tables(conn, database_metadata): "columns": column_names, "primary_keys": detect_primary_keys(conn, table), "count": count, - "label_column": detect_label_column(column_names), "hidden": table_metadata.get("hidden") or False, "fts_table": detect_fts(conn, table), } diff --git a/datasette/utils.py b/datasette/utils.py index 98f70592..50299ab7 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -475,6 +475,22 @@ def temporary_heroku_directory( os.chdir(saved_cwd) +def get_outbound_foreign_keys(conn, table): + infos = conn.execute( + 'PRAGMA foreign_key_list([{}])'.format(table) + ).fetchall() + fks = [] + for info in infos: + if info is not None: + id, seq, table_name, from_, to_, on_update, on_delete, match = info + fks.append({ + 'other_table': table_name, + 'column': from_, + 'other_column': to_ + }) + return fks + + def get_all_foreign_keys(conn): tables = [r[0] for r in conn.execute('select name from sqlite_master where type="table"')] table_to_foreign_keys = {} diff --git a/datasette/views/base.py b/datasette/views/base.py index 99bb175f..a0a16679 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -131,14 +131,6 @@ class BaseView(RenderMixin): def __init__(self, datasette): self.ds = datasette - def table_metadata(self, database, table): - "Fetch table-specific metadata." - return (self.ds.metadata("databases") or {}).get(database, {}).get( - "tables", {} - ).get( - table, {} - ) - def options(self, request, *args, **kwargs): r = response.text("ok") if self.ds.cors: diff --git a/datasette/views/table.py b/datasette/views/table.py index 84ebec05..49b7ac4b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -33,7 +33,7 @@ LINK_WITH_VALUE = '{id}' class RowTableShared(BaseView): async def sortable_columns_for_table(self, database, table, use_rowid): - table_metadata = self.table_metadata(database, table) + table_metadata = self.ds.table_metadata(database, table) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: @@ -51,7 +51,7 @@ class RowTableShared(BaseView): expandables = [] for fk in table_info["foreign_keys"]["outgoing"]: label_column = ( - self.table_metadata( + self.ds.table_metadata( database, fk["other_table"] ).get("label_column") or tables.get(fk["other_table"], {}).get("label_column") @@ -62,11 +62,7 @@ class RowTableShared(BaseView): async def expand_foreign_keys(self, database, table, column, values): "Returns dict mapping (column, value) -> label" labeled_fks = {} - tables_info = self.ds.inspect()[database]["tables"] - table_info = tables_info.get(table) or {} - if not table_info: - return {} - foreign_keys = table_info["foreign_keys"]["outgoing"] + foreign_keys = await self.ds.foreign_keys_for_table(database, table) # Find the foreign_key for this column try: fk = [ @@ -75,13 +71,7 @@ class RowTableShared(BaseView): ][0] except IndexError: return {} - label_column = ( - # First look in metadata.json for this foreign key table: - self.table_metadata( - database, fk["other_table"] - ).get("label_column") - or tables_info.get(fk["other_table"], {}).get("label_column") - ) + label_column = await self.ds.label_column_for_table(database, fk["other_table"]) if not label_column: return { (fk["column"], value): str(value) @@ -119,7 +109,7 @@ class RowTableShared(BaseView): truncate_cells=0, ): "Returns columns, rows for specified table - including fancy foreign key treatment" - table_metadata = self.table_metadata(database, table) + table_metadata = self.ds.table_metadata(database, table) info = self.ds.inspect()[database] sortable_columns = await self.sortable_columns_for_table(database, table, True) columns = [ @@ -309,7 +299,7 @@ class TableView(RowTableShared): forward_querystring=False, ) - table_metadata = self.table_metadata(database, table) + table_metadata = self.ds.table_metadata(database, table) units = table_metadata.get("units", {}) filters = Filters(sorted(other_args.items()), units, ureg) where_clauses, params = filters.build_where_clauses() @@ -880,7 +870,7 @@ class RowView(RowTableShared): "columns": columns, "primary_keys": pks, "primary_key_values": pk_values, - "units": self.table_metadata(database, table).get("units", {}), + "units": self.ds.table_metadata(database, table).get("units", {}), } if "foreign_key_tables" in (request.raw_args.get("_extras") or "").split(","): diff --git a/docs/introspection.rst b/docs/introspection.rst index b4dbfc6e..6aeedc61 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -50,7 +50,6 @@ This is an internal implementation detail of Datasette and the format should not }, "fts_table": null, "hidden": false, - "label_column": null, "name": "./index", "primary_keys": [] }, diff --git a/tests/test_api.py b/tests/test_api.py index 6fdcb115..a50148c1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -35,7 +35,6 @@ def test_database_page(app_client): 'count': 0, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': [], }, { @@ -44,7 +43,6 @@ def test_database_page(app_client): 'count': 0, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk'], }, { @@ -68,7 +66,6 @@ def test_database_page(app_client): }], }, 'hidden': False, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk'], }, { @@ -77,7 +74,6 @@ def test_database_page(app_client): 'count': 1, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk1', 'pk2'], }, { @@ -86,7 +82,6 @@ def test_database_page(app_client): 'count': 1001, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk1', 'pk2', 'pk3'], }, { @@ -102,7 +97,6 @@ def test_database_page(app_client): 'other_table': 'primary_key_multiple_columns_explicit_label' }], }, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk'], }, { @@ -119,7 +113,6 @@ def test_database_page(app_client): }, 'fts_table': None, 'hidden': False, - 'label_column': 'name', 'primary_keys': ['id'], }, { 'columns': ['pk', 'planet_int', 'on_earth', 'state', 'city_id', 'neighborhood'], @@ -135,7 +128,6 @@ def test_database_page(app_client): }, 'fts_table': None, 'hidden': False, - 'label_column': None, 'primary_keys': ['pk'], }, { 'columns': ['pk', 'foreign_key_with_label', 'foreign_key_with_no_label'], @@ -154,7 +146,6 @@ def test_database_page(app_client): 'other_table': 'simple_primary_key', }], }, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk'], }, { @@ -162,7 +153,6 @@ def test_database_page(app_client): "columns": ["value"], "count": 3, "primary_keys": [], - "label_column": None, "hidden": False, "fts_table": None, "foreign_keys": {"incoming": [], "outgoing": []} @@ -179,7 +169,6 @@ def test_database_page(app_client): 'outgoing': [] }, 'hidden': False, - 'label_column': None, 'fts_table': None, 'primary_keys': ['id'] }, { @@ -195,7 +184,6 @@ def test_database_page(app_client): 'outgoing': [] }, 'hidden': False, - 'label_column': None, 'fts_table': None, 'primary_keys': ['id'] }, { @@ -209,14 +197,12 @@ def test_database_page(app_client): }], 'outgoing': []}, 'fts_table': 'searchable_fts', 'hidden': False, - 'label_column': None, 'primary_keys': ['pk'], }, { "name": "searchable_tags", "columns": ["searchable_id", "tag"], "primary_keys": ["searchable_id", "tag"], "count": 2, - "label_column": None, "hidden": False, "fts_table": None, "foreign_keys": { @@ -240,7 +226,6 @@ def test_database_page(app_client): 'count': 1, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': [], }, { @@ -268,7 +253,6 @@ def test_database_page(app_client): }], 'outgoing': [], }, - 'label_column': 'content', 'fts_table': None, 'primary_keys': ['id'], }, { @@ -280,7 +264,6 @@ def test_database_page(app_client): 'count': 201, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk1', 'pk2'], }, { @@ -289,7 +272,6 @@ def test_database_page(app_client): 'count': 1, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk'], }, { @@ -297,7 +279,6 @@ def test_database_page(app_client): "columns": ["tag"], "primary_keys": ["tag"], "count": 2, - "label_column": None, "hidden": False, "fts_table": None, "foreign_keys": { @@ -316,7 +297,6 @@ def test_database_page(app_client): 'count': 3, 'hidden': False, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': ['pk'], }, { @@ -325,7 +305,6 @@ def test_database_page(app_client): 'count': 201, 'hidden': True, 'foreign_keys': {'incoming': [], 'outgoing': []}, - 'label_column': None, 'fts_table': None, 'primary_keys': [], }, { @@ -334,7 +313,6 @@ def test_database_page(app_client): 'foreign_keys': {'incoming': [], 'outgoing': []}, 'fts_table': 'searchable_fts', 'hidden': True, - 'label_column': None, 'name': 'searchable_fts', 'primary_keys': [] }, { @@ -343,7 +321,6 @@ def test_database_page(app_client): 'foreign_keys': {'incoming': [], 'outgoing': []}, 'fts_table': None, 'hidden': True, - 'label_column': None, 'name': 'searchable_fts_content', 'primary_keys': ['docid'] }, { @@ -355,7 +332,6 @@ def test_database_page(app_client): 'foreign_keys': {'incoming': [], 'outgoing': []}, 'fts_table': None, 'hidden': True, - 'label_column': None, 'name': 'searchable_fts_segdir', 'primary_keys': ['level', 'idx'] }, { @@ -364,7 +340,6 @@ def test_database_page(app_client): 'foreign_keys': {'incoming': [], 'outgoing': []}, 'fts_table': None, 'hidden': True, - 'label_column': None, 'name': 'searchable_fts_segments', 'primary_keys': ['blockid'] }] == data['tables'] From c882e9262f64e0fc7c2fc89c611bdb768c069af7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 20:11:08 -0700 Subject: [PATCH 0039/2303] display_columns_and_rows() no longer uses inspect, refs #420 --- datasette/inspect.py | 14 +------------- datasette/utils.py | 13 +++++++++++++ datasette/views/table.py | 14 ++++++-------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/datasette/inspect.py b/datasette/inspect.py index 3ff8b367..4c498623 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -3,6 +3,7 @@ import hashlib from .utils import ( detect_spatialite, detect_fts, + detect_primary_keys, escape_sqlite, get_all_foreign_keys, table_columns, @@ -31,19 +32,6 @@ def inspect_views(conn): return [v[0] for v in conn.execute('select name from sqlite_master where type = "view"')] -def detect_primary_keys(conn, table): - " Figure out primary keys for a table. " - table_info_rows = [ - row - for row in conn.execute( - 'PRAGMA table_info("{}")'.format(table) - ).fetchall() - if row[-1] - ] - table_info_rows.sort(key=lambda row: row[-1]) - return [str(r[1]) for r in table_info_rows] - - def inspect_tables(conn, database_metadata): " List tables and their row counts, excluding uninteresting tables. " tables = {} diff --git a/datasette/utils.py b/datasette/utils.py index 50299ab7..c62e0713 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -475,6 +475,19 @@ def temporary_heroku_directory( os.chdir(saved_cwd) +def detect_primary_keys(conn, table): + " Figure out primary keys for a table. " + table_info_rows = [ + row + for row in conn.execute( + 'PRAGMA table_info("{}")'.format(table) + ).fetchall() + if row[-1] + ] + table_info_rows.sort(key=lambda row: row[-1]) + return [str(r[1]) for r in table_info_rows] + + def get_outbound_foreign_keys(conn, table): infos = conn.execute( 'PRAGMA foreign_key_list([{}])'.format(table) diff --git a/datasette/views/table.py b/datasette/views/table.py index 49b7ac4b..425242dc 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -11,6 +11,7 @@ from datasette.utils import ( InterruptedError, append_querystring, compound_keys_after_sql, + detect_primary_keys, escape_sqlite, filters_should_redirect, is_url, @@ -110,19 +111,16 @@ class RowTableShared(BaseView): ): "Returns columns, rows for specified table - including fancy foreign key treatment" table_metadata = self.ds.table_metadata(database, table) - info = self.ds.inspect()[database] sortable_columns = await self.sortable_columns_for_table(database, table, True) columns = [ {"name": r[0], "sortable": r[0] in sortable_columns} for r in description ] - tables = info["tables"] - table_info = tables.get(table) or {} - pks = table_info.get("primary_keys") or [] + pks = await self.ds.execute_against_connection_in_thread( + database, lambda conn: detect_primary_keys(conn, table) + ) column_to_foreign_key_table = { fk["column"]: fk["other_table"] - for fk in table_info.get( - "foreign_keys", {} - ).get("outgoing", None) or [] + for fk in await self.ds.foreign_keys_for_table(database, table) } cell_rows = [] @@ -600,7 +598,7 @@ class TableView(RowTableShared): if columns_to_expand: expanded_labels = {} - for fk, label_column in expandable_columns: + for fk, _ in expandable_columns: column = fk["column"] if column not in columns_to_expand: continue From f23120a9641e521d669788e9938fb3cf2754d710 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 20:17:47 -0700 Subject: [PATCH 0040/2303] RowView.data() no longer uses inspect refs #420 --- datasette/views/table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 425242dc..6b7c9033 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -803,9 +803,9 @@ class RowView(RowTableShared): async def data(self, request, database, hash, table, pk_path, default_labels=False): pk_values = urlsafe_components(pk_path) - info = self.ds.inspect()[database] - table_info = info["tables"].get(table) or {} - pks = table_info.get("primary_keys") or [] + pks = await self.ds.execute_against_connection_in_thread( + database, lambda conn: detect_primary_keys(conn, table) + ) use_rowid = not pks select = "*" if use_rowid: From db02311ebf3711309b231d007967bcf8b026a026 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 20:23:30 -0700 Subject: [PATCH 0041/2303] foreign_key_tables no longer uses inspect, refs #420 --- datasette/views/table.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 6b7c9033..f2f2eef2 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -14,6 +14,7 @@ from datasette.utils import ( detect_primary_keys, escape_sqlite, filters_should_redirect, + get_all_foreign_keys, is_url, path_from_row_pks, path_with_added_args, @@ -884,11 +885,10 @@ class RowView(RowTableShared): if len(pk_values) != 1: return [] - table_info = self.ds.inspect()[database]["tables"].get(table) - if not table_info: - return [] - - foreign_keys = table_info["foreign_keys"]["incoming"] + all_foreign_keys = await self.ds.execute_against_connection_in_thread( + database, get_all_foreign_keys + ) + foreign_keys = all_foreign_keys[table]["incoming"] if len(foreign_keys) == 0: return [] From 36bd6558b00be027bb4b7dba0c8d200da0ca95df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 20:46:08 -0700 Subject: [PATCH 0042/2303] expandable_columns() no longer uses inspect, refs #420 --- datasette/views/table.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index f2f2eef2..ceef6350 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -44,20 +44,11 @@ class RowTableShared(BaseView): sortable_columns.add("rowid") return sortable_columns - def expandable_columns(self, database, table): + async def expandable_columns(self, database, table): # Returns list of (fk_dict, label_column-or-None) pairs for that table - tables = self.ds.inspect()[database].get("tables", {}) - table_info = tables.get(table) - if not table_info: - return [] expandables = [] - for fk in table_info["foreign_keys"]["outgoing"]: - label_column = ( - self.ds.table_metadata( - database, fk["other_table"] - ).get("label_column") - or tables.get(fk["other_table"], {}).get("label_column") - ) or None + for fk in await self.ds.foreign_keys_for_table(database, table): + label_column = await self.ds.label_column_for_table(database, fk["other_table"]) expandables.append((fk, label_column)) return expandables @@ -582,7 +573,7 @@ class TableView(RowTableShared): # Expand labeled columns if requested expanded_columns = [] - expandable_columns = self.expandable_columns(database, table) + expandable_columns = await self.expandable_columns(database, table) columns_to_expand = None try: all_labels = value_as_boolean(special_args.get("_labels", "")) From 3a208a41d4dce35b97eca8b25f37055c3fda5aed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Apr 2019 20:47:10 -0700 Subject: [PATCH 0043/2303] TableView.data() no longer uses .inspect, refs #420 BUT... it does a count(*) against the whole table which may take unbounded time. Fixing this is part of #422 --- datasette/views/table.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index ceef6350..87feda2d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -11,6 +11,7 @@ from datasette.utils import ( InterruptedError, append_querystring, compound_keys_after_sql, + detect_fts, detect_primary_keys, escape_sqlite, filters_should_redirect, @@ -227,12 +228,12 @@ class TableView(RowTableShared): ) is_view = bool(await self.ds.get_view_definition(database, table)) - info = self.ds.inspect() - table_info = info[database]["tables"].get(table) or {} - if not is_view and not table_info: + table_exists = bool(await self.ds.table_exists(database, table)) + if not is_view and not table_exists: raise NotFound("Table not found: {}".format(table)) - - pks = table_info.get("primary_keys") or [] + pks = await self.ds.execute_against_connection_in_thread( + database, lambda conn: detect_primary_keys(conn, table) + ) use_rowid = not pks and not is_view if use_rowid: select = "rowid, *" @@ -295,8 +296,8 @@ class TableView(RowTableShared): where_clauses, params = filters.build_where_clauses() # _search support: - fts_table = table_metadata.get( - "fts_table", info[database]["tables"].get(table, {}).get("fts_table") + fts_table = await self.ds.execute_against_connection_in_thread( + database, lambda conn: detect_fts(conn, table) ) fts_pk = table_metadata.get("fts_pk", "rowid") search_args = dict( @@ -318,10 +319,9 @@ class TableView(RowTableShared): params["search"] = search else: # More complex: search against specific columns - valid_columns = set(info[database]["tables"][fts_table]["columns"]) for i, (key, search_text) in enumerate(search_args.items()): search_col = key.split("_search_", 1)[1] - if search_col not in valid_columns: + if search_col not in await self.ds.table_columns(database, fts_table): raise DatasetteError("Cannot search by that column", status=400) where_clauses.append( @@ -341,7 +341,9 @@ class TableView(RowTableShared): table_rows_count = None sortable_columns = set() if not is_view: - table_rows_count = table_info["count"] + table_rows_count = (await self.ds.execute( + database, "select count(*) from {}".format(escape_sqlite(table)) + )).rows[0][0] sortable_columns = await self.sortable_columns_for_table(database, table, use_rowid) From 78e45ead4d771007c57b307edf8fc920101f8733 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Apr 2019 08:17:19 -0700 Subject: [PATCH 0044/2303] New ?tags__arraycontains=tag lookup against JSON fields Part one of supporting facet-by-JSON-array, refs #359 --- datasette/utils.py | 25 +++++++++++++++++++++---- datasette/views/table.py | 2 +- tests/fixtures.py | 33 +++++++++++++++++---------------- tests/test_api.py | 21 ++++++++++++++++++--- tests/test_csv.py | 32 ++++++++++++++++---------------- tests/test_utils.py | 2 +- 6 files changed, 74 insertions(+), 41 deletions(-) diff --git a/datasette/utils.py b/datasette/utils.py index c62e0713..bb5c17d6 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -565,6 +565,16 @@ def detect_fts_sql(table): '''.format(table=table) +def detect_json1(conn=None): + if conn is None: + conn = sqlite3.connect(":memory:") + try: + conn.execute("SELECT json('{}')") + return True + except Exception: + return False + + def table_columns(conn, table): return [ r[1] @@ -584,7 +594,7 @@ class Filter: self.numeric = numeric self.no_argument = no_argument - def where_clause(self, column, value, param_counter): + def where_clause(self, table, column, value, param_counter): converted = self.format.format(value) if self.numeric and converted.isdigit(): converted = int(converted) @@ -597,6 +607,7 @@ class Filter: kwargs = { 'c': column, 'p': 'p{}'.format(param_counter), + 't': table, } return self.sql_template.format(**kwargs), converted @@ -613,6 +624,7 @@ class Filter: class Filters: _filters = [ + # key, display, sql_template, human_template, format=, numeric=, no_argument= Filter('exact', '=', '"{c}" = :{p}', lambda c, v: '{c} = {v}' if v.isdigit() else '{c} = "{v}"'), Filter('not', '!=', '"{c}" != :{p}', lambda c, v: '{c} != {v}' if v.isdigit() else '{c} != "{v}"'), Filter('contains', 'contains', '"{c}" like :{p}', '{c} contains "{v}"', format='%{}%'), @@ -624,6 +636,11 @@ class Filters: Filter('lte', '\u2264', '"{c}" <= :{p}', '{c} \u2264 {v}', numeric=True), Filter('glob', 'glob', '"{c}" glob :{p}', '{c} glob "{v}"'), Filter('like', 'like', '"{c}" like :{p}', '{c} like "{v}"'), + ] + ([Filter('arraycontains', 'array contains', """rowid in ( + select {t}.rowid from {t}, json_each({t}.{c}) j + where j.value = :{p} + )""", '{c} contains "{v}"') + ] if detect_json1() else []) + [ Filter('isnull', 'is null', '"{c}" is null', '{c} is null', no_argument=True), Filter('notnull', 'is not null', '"{c}" is not null', '{c} is not null', no_argument=True), Filter('isblank', 'is blank', '("{c}" is null or "{c}" = "")', '{c} is blank', no_argument=True), @@ -677,7 +694,7 @@ class Filters: return bool(self.pairs) def convert_unit(self, column, value): - "If the user has provided a unit in the quey, convert it into the column unit, if present." + "If the user has provided a unit in the query, convert it into the column unit, if present." if column not in self.units: return value @@ -690,13 +707,13 @@ class Filters: column_unit = self.ureg(self.units[column]) return value.to(column_unit).magnitude - def build_where_clauses(self): + def build_where_clauses(self, table): sql_bits = [] params = {} for i, (column, lookup, value) in enumerate(self.selections()): filter = self._filters_by_key.get(lookup, None) if filter: - sql_bit, param = filter.where_clause(column, self.convert_unit(column, value), i) + sql_bit, param = filter.where_clause(table, column, self.convert_unit(column, value), i) sql_bits.append(sql_bit) if param is not None: param_id = 'p{}'.format(i) diff --git a/datasette/views/table.py b/datasette/views/table.py index 87feda2d..2727565b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -293,7 +293,7 @@ class TableView(RowTableShared): table_metadata = self.ds.table_metadata(database, table) units = table_metadata.get("units", {}) filters = Filters(sorted(other_args.items()), units, ureg) - where_clauses, params = filters.build_where_clauses() + where_clauses, params = filters.build_where_clauses(table) # _search support: fts_table = await self.ds.execute_against_connection_in_thread( diff --git a/tests/fixtures.py b/tests/fixtures.py index 8f5f0b68..b3b38c95 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -523,26 +523,27 @@ CREATE TABLE facetable ( state text, city_id integer, neighborhood text, + tags text, FOREIGN KEY ("city_id") REFERENCES [facet_cities](id) ); INSERT INTO facetable - (planet_int, on_earth, state, city_id, neighborhood) + (planet_int, on_earth, state, city_id, neighborhood, tags) VALUES - (1, 1, 'CA', 1, 'Mission'), - (1, 1, 'CA', 1, 'Dogpatch'), - (1, 1, 'CA', 1, 'SOMA'), - (1, 1, 'CA', 1, 'Tenderloin'), - (1, 1, 'CA', 1, 'Bernal Heights'), - (1, 1, 'CA', 1, 'Hayes Valley'), - (1, 1, 'CA', 2, 'Hollywood'), - (1, 1, 'CA', 2, 'Downtown'), - (1, 1, 'CA', 2, 'Los Feliz'), - (1, 1, 'CA', 2, 'Koreatown'), - (1, 1, 'MI', 3, 'Downtown'), - (1, 1, 'MI', 3, 'Greektown'), - (1, 1, 'MI', 3, 'Corktown'), - (1, 1, 'MI', 3, 'Mexicantown'), - (2, 0, 'MC', 4, 'Arcadia Planitia') + (1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]'), + (1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]'), + (1, 1, 'CA', 1, 'SOMA', '[]'), + (1, 1, 'CA', 1, 'Tenderloin', '[]'), + (1, 1, 'CA', 1, 'Bernal Heights', '[]'), + (1, 1, 'CA', 1, 'Hayes Valley', '[]'), + (1, 1, 'CA', 2, 'Hollywood', '[]'), + (1, 1, 'CA', 2, 'Downtown', '[]'), + (1, 1, 'CA', 2, 'Los Feliz', '[]'), + (1, 1, 'CA', 2, 'Koreatown', '[]'), + (1, 1, 'MI', 3, 'Downtown', '[]'), + (1, 1, 'MI', 3, 'Greektown', '[]'), + (1, 1, 'MI', 3, 'Corktown', '[]'), + (1, 1, 'MI', 3, 'Mexicantown', '[]'), + (2, 0, 'MC', 4, 'Arcadia Planitia', '[]') ; INSERT INTO simple_primary_key VALUES (1, 'hello'); diff --git a/tests/test_api.py b/tests/test_api.py index a50148c1..188a60e8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,4 @@ +from datasette.utils import detect_json1 from .fixtures import ( # noqa app_client, app_client_no_files, @@ -115,7 +116,7 @@ def test_database_page(app_client): 'hidden': False, 'primary_keys': ['id'], }, { - 'columns': ['pk', 'planet_int', 'on_earth', 'state', 'city_id', 'neighborhood'], + 'columns': ['pk', 'planet_int', 'on_earth', 'state', 'city_id', 'neighborhood', 'tags'], 'name': 'facetable', 'count': 15, 'foreign_keys': { @@ -882,6 +883,18 @@ def test_table_filter_queries(app_client, path, expected_rows): assert expected_rows == response.json['rows'] +@pytest.mark.skipif( + not detect_json1(), + reason="Requires the SQLite json1 module" +) +def test_table_filter_json_arraycontains(app_client): + response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") + assert [ + [1, 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]'], + [2, 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]'] + ] == response.json['rows'] + + def test_max_returned_rows(app_client): response = app_client.get( '/fixtures.json?sql=select+content+from+no_primary_key' @@ -1244,7 +1257,8 @@ def test_expand_labels(app_client): "value": 1, "label": "San Francisco" }, - "neighborhood": "Dogpatch" + "neighborhood": "Dogpatch", + "tags": '["tag1", "tag3"]' }, "13": { "pk": 13, @@ -1255,7 +1269,8 @@ def test_expand_labels(app_client): "value": 3, "label": "Detroit" }, - "neighborhood": "Corktown" + "neighborhood": "Corktown", + "tags": '[]', } } == response.json diff --git a/tests/test_csv.py b/tests/test_csv.py index 357838f6..aa78620a 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -17,22 +17,22 @@ world '''.replace('\n', '\r\n') EXPECTED_TABLE_WITH_LABELS_CSV = ''' -pk,planet_int,on_earth,state,city_id,city_id_label,neighborhood -1,1,1,CA,1,San Francisco,Mission -2,1,1,CA,1,San Francisco,Dogpatch -3,1,1,CA,1,San Francisco,SOMA -4,1,1,CA,1,San Francisco,Tenderloin -5,1,1,CA,1,San Francisco,Bernal Heights -6,1,1,CA,1,San Francisco,Hayes Valley -7,1,1,CA,2,Los Angeles,Hollywood -8,1,1,CA,2,Los Angeles,Downtown -9,1,1,CA,2,Los Angeles,Los Feliz -10,1,1,CA,2,Los Angeles,Koreatown -11,1,1,MI,3,Detroit,Downtown -12,1,1,MI,3,Detroit,Greektown -13,1,1,MI,3,Detroit,Corktown -14,1,1,MI,3,Detroit,Mexicantown -15,2,0,MC,4,Memnonia,Arcadia Planitia +pk,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags +1,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]" +2,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]" +3,1,1,CA,1,San Francisco,SOMA,[] +4,1,1,CA,1,San Francisco,Tenderloin,[] +5,1,1,CA,1,San Francisco,Bernal Heights,[] +6,1,1,CA,1,San Francisco,Hayes Valley,[] +7,1,1,CA,2,Los Angeles,Hollywood,[] +8,1,1,CA,2,Los Angeles,Downtown,[] +9,1,1,CA,2,Los Angeles,Los Feliz,[] +10,1,1,CA,2,Los Angeles,Koreatown,[] +11,1,1,MI,3,Detroit,Downtown,[] +12,1,1,MI,3,Detroit,Greektown,[] +13,1,1,MI,3,Detroit,Corktown,[] +14,1,1,MI,3,Detroit,Mexicantown,[] +15,2,0,MC,4,Memnonia,Arcadia Planitia,[] '''.lstrip().replace('\n', '\r\n') diff --git a/tests/test_utils.py b/tests/test_utils.py index 9a00b4b4..07074e72 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -187,7 +187,7 @@ def test_custom_json_encoder(obj, expected): ]) def test_build_where(args, expected_where, expected_params): f = utils.Filters(sorted(args.items())) - sql_bits, actual_params = f.build_where_clauses() + sql_bits, actual_params = f.build_where_clauses("table") assert expected_where == sql_bits assert { 'p{}'.format(i): param From 9cd3b44277e6a8ea9273bf659379ff0414e0b8ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Apr 2019 16:13:30 -0700 Subject: [PATCH 0045/2303] Upgrade to Jinja2==2.10.1 (#426) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 524f5243..c4fc93c6 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup( 'click>=6.7', 'click-default-group==1.2', 'Sanic==0.7.0', - 'Jinja2==2.10', + 'Jinja2==2.10.1', 'hupper==1.0', 'pint==0.8.1', 'pluggy>=0.7.1', From db74cf0144df3e2a017c40a3a36ff5e1a78d7e92 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Apr 2019 21:21:17 -0700 Subject: [PATCH 0046/2303] ?_fts_table= and ?_fts_pk= arguments, closes #428 --- datasette/templates/table.html | 3 +++ datasette/views/table.py | 11 +++++++++-- docs/full_text_search.rst | 8 ++++++-- tests/fixtures.py | 10 ++++++++++ tests/test_api.py | 18 ++++++++++++++++++ tests/test_html.py | 14 ++++++++++++++ 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index c9ec4aa1..12887f63 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -84,6 +84,9 @@ {% for facet in sorted_facet_results %} {% endfor %} + {% for key, value in form_hidden_args %} + + {% endfor %} diff --git a/datasette/views/table.py b/datasette/views/table.py index 2727565b..b7c9a4b0 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -296,10 +296,12 @@ class TableView(RowTableShared): where_clauses, params = filters.build_where_clauses(table) # _search support: - fts_table = await self.ds.execute_against_connection_in_thread( + fts_table = special_args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") + fts_table = fts_table or await self.ds.execute_against_connection_in_thread( database, lambda conn: detect_fts(conn, table) ) - fts_pk = table_metadata.get("fts_pk", "rowid") + fts_pk = special_args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) search_args = dict( pair for pair in special_args.items() if pair[0].startswith("_search") ) @@ -731,6 +733,10 @@ class TableView(RowTableShared): table, {} ) self.ds.update_with_inherited_metadata(metadata) + form_hidden_args = [] + for arg in ("_fts_table", "_fts_pk"): + if arg in special_args: + form_hidden_args.append((arg, special_args[arg])) return { "supports_search": bool(fts_table), "search": search or "", @@ -745,6 +751,7 @@ class TableView(RowTableShared): key=lambda f: (len(f["results"]), f["name"]), reverse=True ), + "form_hidden_args": form_hidden_args, "facet_hideable": lambda facet: facet not in metadata_facets, "is_sortable": any(c["sortable"] for c in display_columns), "path_with_replaced_args": path_with_replaced_args, diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index 987e2272..08e85c90 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -78,9 +78,13 @@ Configuring full-text search for a table or view If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown above, Datasette will detect it automatically and add a search interface to the table page for that table. -You can also manually configure which table should be used for full-text search using :ref:`metadata`. You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option. +You can also manually configure which table should be used for full-text search using querystring parameters or :ref:`metadata`. You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option. -The ``fts_table`` property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property. +Use ``?_fts_table=x`` to over-ride the FTS table for a specific page. If the primary key was something other than ``rowid`` you can use ``?_fts_pk=col`` to set that as well. This is particularly useful for views, for example: + +https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk + +The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property. Here is an example which enables full-text search for a ``display_ads`` view which is defined against the ``ads`` table and hence needs to run FTS against the ``ads_fts`` table, using the ``id`` as the primary key:: diff --git a/tests/fixtures.py b/tests/fixtures.py index b3b38c95..cb6f7a39 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -209,6 +209,10 @@ METADATA = { }, 'simple_view': { 'sortable_columns': ['content'], + }, + 'searchable_view_configured_by_metadata': { + 'fts_table': 'searchable_fts', + 'fts_pk': 'pk' } }, 'queries': { @@ -564,6 +568,12 @@ INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); CREATE VIEW simple_view AS SELECT content, upper(content) AS upper_content FROM simple_primary_key; +CREATE VIEW searchable_view AS + SELECT * from searchable; + +CREATE VIEW searchable_view_configured_by_metadata AS + SELECT * from searchable; + ''' + '\n'.join([ 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format(i=i + 1) for i in range(201) diff --git a/tests/test_api.py b/tests/test_api.py index 188a60e8..b822d23f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -847,6 +847,24 @@ def test_searchable(app_client, path, expected_rows): assert expected_rows == response.json['rows'] +@pytest.mark.parametrize('path,expected_rows', [ + ('/fixtures/searchable_view_configured_by_metadata.json?_search=weasel', [ + [2, 'terry dog', 'sara weasel', 'puma'], + ]), + # This should return all results because search is not configured: + ('/fixtures/searchable_view.json?_search=weasel', [ + [1, 'barry cat', 'terry dog', 'panther'], + [2, 'terry dog', 'sara weasel', 'puma'], + ]), + ('/fixtures/searchable_view.json?_search=weasel&_fts_table=searchable_fts&_fts_pk=pk', [ + [2, 'terry dog', 'sara weasel', 'puma'], + ]), +]) +def test_searchable_views(app_client, path, expected_rows): + response = app_client.get(path) + assert expected_rows == response.json['rows'] + + def test_searchable_invalid_column(app_client): response = app_client.get( '/fixtures/searchable.json?_search_invalid=x' diff --git a/tests/test_html.py b/tests/test_html.py index 1babaa60..3e2ea845 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -185,6 +185,20 @@ def test_empty_search_parameter_gets_removed(app_client): ) +def test_searchable_view_persists_fts_table(app_client): + # The search form should persist ?_fts_table as a hidden field + response = app_client.get( + "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk" + ) + inputs = Soup(response.body, "html.parser").find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [ + ('_fts_table', 'searchable_fts'), ('_fts_pk', 'pk') + ] == [ + (hidden['name'], hidden['value']) for hidden in hiddens + ] + + def test_sort_by_desc_redirects(app_client): path_base = '/fixtures/sortable' path = path_base + '?' + urllib.parse.urlencode({ From e11cb4c66442abca2a6b6159521a6cf4da8739c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Apr 2019 22:00:47 -0700 Subject: [PATCH 0047/2303] Persist show/hide state better, closes #425 --- datasette/templates/_codemirror_foot.html | 2 +- datasette/templates/query.html | 5 ++++- tests/test_html.py | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 1e07fc72..4b55bf8d 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -1,5 +1,5 @@ {% endfor %} diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 9fb4d6eb..f168db97 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -9,8 +9,14 @@ {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} +{% block nav %} +

    + home +

    + {{ super() }} +{% endblock %} + {% block content %} -
    home

    {{ metadata.title or database }}

    diff --git a/datasette/templates/index.html b/datasette/templates/index.html index c8ad4148..b394564a 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -21,7 +21,8 @@ {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %} {% endif %}

    -

    {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}

    +

    {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}

    {% endfor %} {% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index bda1e4e2..5703900d 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -15,9 +15,16 @@ {% block body_class %}row db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %} -{% block content %} -
    home / {{ database }} / {{ table }}
    +{% block nav %} +

    + home / + {{ database }} / + {{ table }} +

    + {{ super() }} +{% endblock %} +{% block content %}

    {{ table }}: {{ ', '.join(primary_key_values) }}

    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 2287e901..c7913f60 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -16,8 +16,15 @@ {% block body_class %}table db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %} +{% block nav %} +

    + home / + {{ database }} +

    + {{ super() }} +{% endblock %} + {% block content %} -
    home / {{ database }}

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}

    From fcfcae21e67cc15090942b1d2a47b5f016279337 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jul 2019 17:05:56 -0700 Subject: [PATCH 0199/2303] extra_template_vars plugin hook (#542) * extra_template_vars plugin hook Closes #541 * Workaround for cwd bug Based on https://github.com/pytest-dev/pytest/issues/1235#issuecomment-175295691 --- datasette/hookspecs.py | 5 ++ datasette/views/base.py | 25 ++++++++- datasette/views/index.py | 11 ++-- datasette/views/special.py | 6 +- docs/plugins.rst | 86 +++++++++++++++++++++++++---- tests/conftest.py | 15 +++++ tests/fixtures.py | 23 ++++++++ tests/test_plugins.py | 26 +++++++++ tests/test_templates/show_json.html | 8 +++ 9 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 tests/test_templates/show_json.html diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 42adaae8..3c6726b7 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -35,6 +35,11 @@ def extra_body_script(template, database, table, view_name, datasette): "Extra JavaScript code to be included in diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 4b55bf8d..9bc6d97f 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -1,5 +1,18 @@ diff --git a/datasette/templates/database.html b/datasette/templates/database.html index a934f336..a0d0fcf6 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -26,7 +26,10 @@

    Custom SQL query

    -

    +

    + + +

    {% endif %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 7c6c59f3..34fa78a5 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -37,7 +37,7 @@ {% if editable and config.allow_sql %}

    {% else %} - 
    {% if query %}{{ query.sql }}{% endif %}
    + 
    {% if query %}{{ query.sql }}{% endif %}
    {% endif %} {% else %} @@ -49,7 +49,10 @@

    {% endfor %} {% endif %} -

    +

    + + +

    {% if display_rows %} From 908fc3999e06f3ccd3bb8ad0539490bbc7809748 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 14 Oct 2019 05:52:33 +0200 Subject: [PATCH 0229/2303] Sort databases on homepage by argument order - #591 Closes #585 - thanks, @rixx! --- datasette/app.py | 2 +- datasette/views/index.py | 2 -- tests/test_html.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 41a4eb37..935b1730 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -159,7 +159,7 @@ class Datasette: self.files = [MEMORY] elif memory: self.files = (MEMORY,) + self.files - self.databases = {} + self.databases = collections.OrderedDict() self.inspect_data = inspect_data for file in self.files: path = file diff --git a/datasette/views/index.py b/datasette/views/index.py index fddb04d9..f2e5f774 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -97,8 +97,6 @@ class IndexView(BaseView): } ) - databases.sort(key=lambda database: database["name"]) - if as_format: headers = {} if self.ds.cors: diff --git a/tests/test_html.py b/tests/test_html.py index 0a6df984..ec7765f6 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -26,11 +26,11 @@ def test_homepage(app_client_two_attached_databases): ) # Should be two attached databases assert [ - {"href": "/extra_database", "text": "extra_database"}, {"href": "/fixtures", "text": "fixtures"}, + {"href": "/extra_database", "text": "extra_database"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables - h2 = soup.select("h2")[0] + h2 = soup.select("h2")[1] assert "extra_database" == h2.text.strip() counts_p, links_p = h2.find_all_next("p")[:2] assert ( From 12cec411cae73ba7211429da12cd32c551fe17b1 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 14 Oct 2019 05:53:21 +0200 Subject: [PATCH 0230/2303] Display metadata footer on custom SQL queries (#589) Closes #408 - thanks, @rixx! --- datasette/views/database.py | 10 ++++++---- tests/test_html.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 78af19c5..31d6af59 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,12 +10,17 @@ class DatabaseView(DataView): name = "database" async def data(self, request, database, hash, default_labels=False, _size=None): + metadata = (self.ds.metadata("databases") or {}).get(database, {}) + self.ds.update_with_inherited_metadata(metadata) + if request.args.get("sql"): if not self.ds.config("allow_sql"): raise DatasetteError("sql= is not allowed", status=400) sql = request.raw_args.pop("sql") validate_sql_select(sql) - return await self.custom_sql(request, database, hash, sql, _size=_size) + return await self.custom_sql( + request, database, hash, sql, _size=_size, metadata=metadata + ) db = self.ds.databases[database] @@ -24,9 +29,6 @@ class DatabaseView(DataView): hidden_table_names = set(await db.hidden_table_names()) all_foreign_keys = await db.get_all_foreign_keys() - metadata = (self.ds.metadata("databases") or {}).get(database, {}) - self.ds.update_with_inherited_metadata(metadata) - tables = [] for table in table_counts: table_columns = await db.table_columns(table) diff --git a/tests/test_html.py b/tests/test_html.py index ec7765f6..0bb1c163 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -737,6 +737,18 @@ def test_database_metadata(app_client): assert_footer_links(soup) +def test_database_metadata_with_custom_sql(app_client): + response = app_client.get("/fixtures?sql=select+*+from+simple_primary_key") + assert response.status == 200 + soup = Soup(response.body, "html.parser") + # Page title should be the default + assert "fixtures" == soup.find("h1").text + # Description should be custom + assert "Custom SQL query returning" in soup.find("h3").text + # The source/license should be inherited + assert_footer_links(soup) + + def test_table_metadata(app_client): response = app_client.get("/fixtures/simple_primary_key") assert response.status == 200 From 9366d0bf191daccee6093c54ed51a2855d129cd8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Oct 2019 15:29:16 -0700 Subject: [PATCH 0231/2303] Add Python versions badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 59a6649e..a4db6611 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) +[![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Travis CI](https://travis-ci.org/simonw/datasette.svg?branch=master)](https://travis-ci.org/simonw/datasette) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/master/LICENSE) From 3e864b1625f3142e6ff084f9b41247f2f9f60f80 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Oct 2019 14:51:45 -0700 Subject: [PATCH 0232/2303] Use --platform=managed for publish cloudrun, closes #587 --- datasette/publish/cloudrun.py | 2 +- tests/test_publish_cloudrun.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 32c9cd2a..c2d77746 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -110,7 +110,7 @@ def publish_subcommand(publish): image_id = "gcr.io/{project}/{name}".format(project=project, name=name) check_call("gcloud builds submit --tag {}".format(image_id), shell=True) check_call( - "gcloud beta run deploy --allow-unauthenticated --image {}{}".format( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}{}".format( image_id, " {}".format(service) if service else "" ), shell=True, diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 1e9bb830..481ac04d 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -40,7 +40,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): [ mock.call("gcloud builds submit --tag {}".format(tag), shell=True), mock.call( - "gcloud beta run deploy --allow-unauthenticated --image {}".format( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}".format( tag ), shell=True, From b6ad1fdc7068cb8248787843e7438d1f19fa2e3a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Oct 2019 22:23:01 -0700 Subject: [PATCH 0233/2303] Fixed bug returning non-ascii characters in CSV, closes #584 --- datasette/utils/asgi.py | 2 +- tests/test_csv.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index eaf3428d..bafcfb4a 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -217,7 +217,7 @@ class AsgiWriter: await self.send( { "type": "http.response.body", - "body": chunk.encode("latin-1"), + "body": chunk.encode("utf-8"), "more_body": True, } ) diff --git a/tests/test_csv.py b/tests/test_csv.py index c3cdc241..1d5d2df2 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -80,6 +80,15 @@ def test_table_csv_download(app_client): assert expected_disposition == response.headers["Content-Disposition"] +def test_csv_with_non_ascii_characters(app_client): + response = app_client.get( + "/fixtures.csv?sql=select%0D%0A++%27%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC%27+as+text%2C%0D%0A++1+as+number%0D%0Aunion%0D%0Aselect%0D%0A++%27bob%27+as+text%2C%0D%0A++2+as+number%0D%0Aorder+by%0D%0A++number" + ) + assert response.status == 200 + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" == response.body.decode("utf8") + + def test_max_csv_mb(app_client_csv_max_mb_one): response = app_client_csv_max_mb_one.get( "/fixtures.csv?sql=select+randomblob(10000)+" From b647b5efc29300f715ba656e41b0591f342938e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 15:51:07 -0700 Subject: [PATCH 0234/2303] Fix for /foo v.s. /foo-bar issue, closes #597 Pull request #599 --- datasette/views/base.py | 16 ++++++++-------- tests/fixtures.py | 7 +++++++ tests/test_api.py | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index db1d69d9..219630af 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -193,14 +193,14 @@ class DataView(BaseView): async def resolve_db_name(self, request, db_name, **kwargs): hash = None name = None - if "-" in db_name: - # Might be name-and-hash, or might just be - # a name with a hyphen in it - name, hash = db_name.rsplit("-", 1) - if name not in self.ds.databases: - # Try the whole name - name = db_name - hash = None + if db_name not in self.ds.databases and "-" in db_name: + # No matching DB found, maybe it's a name-hash? + name_bit, hash_bit = db_name.rsplit("-", 1) + if name_bit not in self.ds.databases: + raise NotFound("Database not found: {}".format(name)) + else: + name = name_bit + hash = hash_bit else: name = db_name # Verify the hash diff --git a/tests/fixtures.py b/tests/fixtures.py index dac28dc0..a4c32f36 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -178,6 +178,13 @@ def app_client_two_attached_databases(): ) +@pytest.fixture(scope="session") +def app_client_conflicting_database_names(): + yield from make_app_client( + extra_databases={"foo.db": EXTRA_DATABASE_SQL, "foo-bar.db": EXTRA_DATABASE_SQL} + ) + + @pytest.fixture(scope="session") def app_client_two_attached_databases_one_immutable(): yield from make_app_client( diff --git a/tests/test_api.py b/tests/test_api.py index cc00b780..826c00f3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -7,6 +7,7 @@ from .fixtures import ( # noqa app_client_larger_cache_size, app_client_returned_rows_matches_page_size, app_client_two_attached_databases_one_immutable, + app_client_conflicting_database_names, app_client_with_cors, app_client_with_dot, generate_compound_rows, @@ -1652,3 +1653,20 @@ def test_cors(app_client_with_cors, path, status_code): response = app_client_with_cors.get(path) assert response.status == status_code assert "*" == response.headers["Access-Control-Allow-Origin"] + + +def test_common_prefix_database_names(app_client_conflicting_database_names): + # https://github.com/simonw/datasette/issues/597 + assert ["fixtures", "foo", "foo-bar"] == [ + d["name"] + for d in json.loads( + app_client_conflicting_database_names.get("/-/databases.json").body.decode( + "utf8" + ) + ) + ] + for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): + data = json.loads( + app_client_conflicting_database_names.get(path).body.decode("utf8") + ) + assert db_name == data["database"] From e877b1cb12076946fdbec7ca2fbfbfc75c1c2a28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 16:56:44 -0700 Subject: [PATCH 0235/2303] Don't auto-format SQL on page load (#601) Closes #600 --- datasette/templates/_codemirror_foot.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 9bc6d97f..9aba61ab 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -6,12 +6,6 @@ window.onload = () => { if (sqlFormat && !readOnly) { sqlFormat.hidden = false; } - if (readOnly) { - readOnly.innerHTML = sqlFormatter.format(readOnly.innerHTML); - } - if (sqlInput) { - sqlInput.value = sqlFormatter.format(sqlInput.value); - } var editor = CodeMirror.fromTextArea(sqlInput, { lineNumbers: true, mode: "text/x-sql", From debea4f971c180af64e16b83be98d830e9dee54f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 18:05:47 -0700 Subject: [PATCH 0236/2303] Release 0.30 --- docs/changelog.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 26d0f75c..e8dafa35 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_30: + +0.30 (2019-10-18) +----------------- + +- Added ``/-/threads`` debugging page +- Allow ``EXPLAIN WITH...`` (`#583 `__) +- Button to format SQL - thanks, Tobias Kunze (`#136 `__) +- Sort databases on homepage by argument order - thanks, Tobias Kunze (`#585 `__) +- Display metadata footer on custom SQL queries - thanks, Tobias Kunze (`#589 `__) +- Use ``--platform=managed`` for ``publish cloudrun`` (`#587 `__) +- Fixed bug returning non-ASCII characters in CSV (`#584 `__) +- Fix for ``/foo`` v.s. ``/foo-bar`` bug (`#601 `__) + .. _v0_29_3: 0.29.3 (2019-09-02) From 8050f9e1ece9afd0236ad38c6458c12a4ad917e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 18:08:04 -0700 Subject: [PATCH 0237/2303] Update news in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a4db6611..5894017e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30) * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams. From f4c0830529a9513a83437a9e1550bbe27ebc5c64 Mon Sep 17 00:00:00 2001 From: chris48s Date: Mon, 21 Oct 2019 03:03:08 +0100 Subject: [PATCH 0238/2303] Always pop as_format off args dict (#603) Closes #563. Thanks, @chris48s --- datasette/views/base.py | 2 ++ tests/test_api.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/datasette/views/base.py b/datasette/views/base.py index 219630af..348f0c03 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -362,6 +362,8 @@ class DataView(BaseView): _format = request.args.get("_format", None) if not _format: _format = (args.pop("as_format", None) or "").lstrip(".") + else: + args.pop("as_format", None) if "table_and_format" in args: db = self.ds.databases[database] diff --git a/tests/test_api.py b/tests/test_api.py index 826c00f3..a734b8de 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1107,6 +1107,15 @@ def test_row(app_client): assert [{"id": "1", "content": "hello"}] == response.json["rows"] +def test_row_format_in_querystring(app_client): + # regression test for https://github.com/simonw/datasette/issues/563 + response = app_client.get( + "/fixtures/simple_primary_key/1?_format=json&_shape=objects" + ) + assert response.status == 200 + assert [{"id": "1", "content": "hello"}] == response.json["rows"] + + def test_row_strange_table_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects" From 5dd4d2b2d3abcfd507a6df47e7c2fbad3c552fd8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 11:49:01 -0700 Subject: [PATCH 0239/2303] Update to latest black (#609) --- datasette/views/base.py | 9 ++++++--- datasette/views/table.py | 7 ++++--- setup.py | 2 +- tests/test_api.py | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 348f0c03..1568b084 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -257,9 +257,12 @@ class DataView(BaseView): assert NotImplemented async def get(self, request, db_name, **kwargs): - database, hash, correct_hash_provided, should_redirect = await self.resolve_db_name( - request, db_name, **kwargs - ) + ( + database, + hash, + correct_hash_provided, + should_redirect, + ) = await self.resolve_db_name(request, db_name, **kwargs) if should_redirect: return self.redirect(request, should_redirect, remove_args={"_hash"}) diff --git a/datasette/views/table.py b/datasette/views/table.py index 8ba3abe4..e0362e53 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -586,9 +586,10 @@ class TableView(RowTableShared): ) for facet in facet_instances: - instance_facet_results, instance_facets_timed_out = ( - await facet.facet_results() - ) + ( + instance_facet_results, + instance_facets_timed_out, + ) = await facet.facet_results() facet_results.update(instance_facet_results) facets_timed_out.extend(instance_facets_timed_out) diff --git a/setup.py b/setup.py index cbe545a1..9ae56306 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def get_version(): # Only install black on Python 3.6 or higher maybe_black = [] if sys.version_info > (3, 6): - maybe_black = ["black"] + maybe_black = ["black~=19.10b0"] setup( name="datasette", diff --git a/tests/test_api.py b/tests/test_api.py index a734b8de..4ea95e84 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1245,7 +1245,7 @@ def test_config_json(app_client): def test_page_size_matching_max_returned_rows( - app_client_returned_rows_matches_page_size + app_client_returned_rows_matches_page_size, ): fetched = [] path = "/fixtures/no_primary_key.json" From e2c390500e6782aa476a7edc05c46cf907875a6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 11:49:26 -0700 Subject: [PATCH 0240/2303] Persist _where= in hidden fields, closes #604 --- datasette/views/table.py | 3 +++ tests/test_html.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/datasette/views/table.py b/datasette/views/table.py index e0362e53..652ce994 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -721,6 +721,9 @@ class TableView(RowTableShared): for arg in ("_fts_table", "_fts_pk"): if arg in special_args: form_hidden_args.append((arg, special_args[arg])) + if request.args["_where"]: + for where_text in request.args["_where"]: + form_hidden_args.append(("_where", where_text)) return { "supports_search": bool(fts_table), "search": search or "", diff --git a/tests/test_html.py b/tests/test_html.py index 0bb1c163..aa628dec 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -955,6 +955,12 @@ def test_extra_where_clauses(app_client): "/fixtures/facetable?_where=city_id%3D1", "/fixtures/facetable?_where=neighborhood%3D%27Dogpatch%27", ] == hrefs + # These should also be persisted as hidden fields + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [("_where", "neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [ + (hidden["name"], hidden["value"]) for hidden in hiddens + ] def test_binary_data_display(app_client): From f5f6cbe03cbf05737d848f44779372b5daa79a25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 11:56:04 -0700 Subject: [PATCH 0241/2303] Release 0.30.1 --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e8dafa35..8ac32c45 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _v0_30_1: + +0.30.1 (2019-10-30) +------------------- + +- Fixed bug where ``?_where=`` parameter was not persisted in hidden form fields (`#604 `__) +- Fixed bug with .JSON representation of row pages - thanks, Chris Shaw (`#603 `__) + .. _v0_30: 0.30 (2019-10-18) From 3ca290e0db03bb4747e24203c445873f74512107 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 12:00:21 -0700 Subject: [PATCH 0242/2303] Fixed dumb error --- datasette/views/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 652ce994..44b186cf 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -721,7 +721,7 @@ class TableView(RowTableShared): for arg in ("_fts_table", "_fts_pk"): if arg in special_args: form_hidden_args.append((arg, special_args[arg])) - if request.args["_where"]: + if request.args.get("_where"): for where_text in request.args["_where"]: form_hidden_args.append(("_where", where_text)) return { From 937828f946238c28e77ba50e0b2e649c874560f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Oct 2019 22:39:59 -0700 Subject: [PATCH 0243/2303] Use distinfo.project_name for plugin name if available, closes #606 --- datasette/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 449217b5..3d28a36b 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -633,6 +633,7 @@ def get_plugins(pm): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version + plugin_info["name"] = distinfo.project_name plugins.append(plugin_info) return plugins From 50287e7c6bb0987536e5515f05945721c4515e9a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 12:37:46 -0700 Subject: [PATCH 0244/2303] Only suggest array facet for arrays of strings - closes #562 --- datasette/facets.py | 44 +++++++++++++++++++++++++++++++++----------- tests/fixtures.py | 33 +++++++++++++++++---------------- tests/test_api.py | 20 +++++++++++++++++--- tests/test_csv.py | 32 ++++++++++++++++---------------- tests/test_facets.py | 9 +++++++++ 5 files changed, 92 insertions(+), 46 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 365d9c65..9b5baaa2 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -257,6 +257,16 @@ class ColumnFacet(Facet): class ArrayFacet(Facet): type = "array" + def _is_json_array_of_strings(self, json_string): + try: + array = json.loads(json_string) + except ValueError: + return False + for item in array: + if not isinstance(item, str): + return False + return True + async def suggest(self): columns = await self.get_columns(self.sql, self.params) suggested_facets = [] @@ -282,18 +292,30 @@ class ArrayFacet(Facet): ) types = tuple(r[0] for r in results.rows) if types in (("array",), ("array", None)): - suggested_facets.append( - { - "name": column, - "type": "array", - "toggle_url": self.ds.absolute_url( - self.request, - path_with_added_args( - self.request, {"_facet_array": column} - ), - ), - } + # Now sanity check that first 100 arrays contain only strings + first_100 = await self.ds.execute( + self.database, + "select {column} from ({sql}) where {column} is not null".format( + column=escape_sqlite(column), sql=self.sql + ), + self.params, + truncate=False, + custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), + log_sql_errors=False, ) + if all(self._is_json_array_of_strings(r[0]) for r in first_100): + suggested_facets.append( + { + "name": column, + "type": "array", + "toggle_url": self.ds.absolute_url( + self.request, + path_with_added_args( + self.request, {"_facet_array": column} + ), + ), + } + ) except (QueryInterrupted, sqlite3.OperationalError): continue return suggested_facets diff --git a/tests/fixtures.py b/tests/fixtures.py index a4c32f36..93c3da9f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -661,26 +661,27 @@ CREATE TABLE facetable ( city_id integer, neighborhood text, tags text, + complex_array text, FOREIGN KEY ("city_id") REFERENCES [facet_cities](id) ); INSERT INTO facetable - (created, planet_int, on_earth, state, city_id, neighborhood, tags) + (created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array) VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]'), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]') + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]'), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]'), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]'), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]'), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]'), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]'), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]'), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]') ; CREATE TABLE binary_data ( diff --git a/tests/test_api.py b/tests/test_api.py index 4ea95e84..41557bcf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -195,6 +195,7 @@ def test_database_page(app_client): "city_id", "neighborhood", "tags", + "complex_array", ], "primary_keys": ["pk"], "count": 15, @@ -1029,15 +1030,25 @@ def test_table_filter_queries_multiple_of_same_type(app_client): def test_table_filter_json_arraycontains(app_client): response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") assert [ - [1, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Mission", '["tag1", "tag2"]'], - [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]'], + [ + 1, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Mission", + '["tag1", "tag2"]', + '[{"foo": "bar"}]', + ], + [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]', "[]"], ] == response.json["rows"] def test_table_filter_extra_where(app_client): response = app_client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'") assert [ - [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]'] + [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]', "[]"] ] == response.json["rows"] @@ -1453,6 +1464,7 @@ def test_suggested_facets(app_client): {"name": "city_id", "querystring": "_facet=city_id"}, {"name": "neighborhood", "querystring": "_facet=neighborhood"}, {"name": "tags", "querystring": "_facet=tags"}, + {"name": "complex_array", "querystring": "_facet=complex_array"}, {"name": "created", "querystring": "_facet_date=created"}, ] if detect_json1(): @@ -1488,6 +1500,7 @@ def test_expand_labels(app_client): "city_id": {"value": 1, "label": "San Francisco"}, "neighborhood": "Dogpatch", "tags": '["tag1", "tag3"]', + "complex_array": "[]", }, "13": { "pk": 13, @@ -1498,6 +1511,7 @@ def test_expand_labels(app_client): "city_id": {"value": 3, "label": "Detroit"}, "neighborhood": "Corktown", "tags": "[]", + "complex_array": "[]", }, } == response.json diff --git a/tests/test_csv.py b/tests/test_csv.py index 1d5d2df2..b148b6db 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -21,22 +21,22 @@ world ) EXPECTED_TABLE_WITH_LABELS_CSV = """ -pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags -1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]" -2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]" -3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[] -4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[] -5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[] -6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[] -7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[] -8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[] -9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[] -10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[] -11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[] -12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[] -13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[] -14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[] -15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[] +pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,complex_array +1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]" +2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[] +3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[] +4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[] +5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[] +6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[] +7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[] +8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[] +9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[] +10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[] +11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[] +12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[] +13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[] +14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[] +15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[] """.lstrip().replace( "\n", "\r\n" ) diff --git a/tests/test_facets.py b/tests/test_facets.py index 9169f666..402c155b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -23,6 +23,10 @@ async def test_column_facet_suggest(app_client): {"name": "city_id", "toggle_url": "http://localhost/?_facet=city_id"}, {"name": "neighborhood", "toggle_url": "http://localhost/?_facet=neighborhood"}, {"name": "tags", "toggle_url": "http://localhost/?_facet=tags"}, + { + "name": "complex_array", + "toggle_url": "http://localhost/?_facet=complex_array", + }, ] == suggestions @@ -57,6 +61,10 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client): "name": "tags", "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=tags", }, + { + "name": "complex_array", + "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=complex_array", + }, ] == suggestions @@ -78,6 +86,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client): "state", "neighborhood", "tags", + "complex_array", ] == suggestions From ba5414f16b49781261d0f41a16f2210d5fa3976f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 12:38:15 -0700 Subject: [PATCH 0245/2303] Only inspect first 100 records for #562 --- datasette/facets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/facets.py b/datasette/facets.py index 9b5baaa2..7f350dfe 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -295,7 +295,7 @@ class ArrayFacet(Facet): # Now sanity check that first 100 arrays contain only strings first_100 = await self.ds.execute( self.database, - "select {column} from ({sql}) where {column} is not null".format( + "select {column} from ({sql}) where {column} is not null limit 100".format( column=escape_sqlite(column), sql=self.sql ), self.params, From 7152e76eda9049574643261e7a471958cc16d0b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 14:45:59 -0700 Subject: [PATCH 0246/2303] Don't suggest array facet if column is only [], closes #610 --- datasette/facets.py | 29 ++++++++++++++++++----------- tests/test_facets.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 7f350dfe..0c6459d6 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -293,17 +293,24 @@ class ArrayFacet(Facet): types = tuple(r[0] for r in results.rows) if types in (("array",), ("array", None)): # Now sanity check that first 100 arrays contain only strings - first_100 = await self.ds.execute( - self.database, - "select {column} from ({sql}) where {column} is not null limit 100".format( - column=escape_sqlite(column), sql=self.sql - ), - self.params, - truncate=False, - custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), - log_sql_errors=False, - ) - if all(self._is_json_array_of_strings(r[0]) for r in first_100): + first_100 = [ + v[0] + for v in await self.ds.execute( + self.database, + "select {column} from ({sql}) where {column} is not null and json_array_length({column}) > 0 limit 100".format( + column=escape_sqlite(column), sql=self.sql + ), + self.params, + truncate=False, + custom_time_limit=self.ds.config( + "facet_suggest_time_limit_ms" + ), + log_sql_errors=False, + ) + ] + if first_100 and all( + self._is_json_array_of_strings(r) for r in first_100 + ): suggested_facets.append( { "name": column, diff --git a/tests/test_facets.py b/tests/test_facets.py index 402c155b..e3dc3df3 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -215,6 +215,20 @@ async def test_array_facet_suggest(app_client): ] == suggestions +@pytest.mark.asyncio +@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") +async def test_array_facet_suggest_not_if_all_empty_arrays(app_client): + facet = ArrayFacet( + app_client.ds, + MockRequest("http://localhost/"), + database="fixtures", + sql="select * from facetable where tags = '[]'", + table="facetable", + ) + suggestions = await facet.suggest() + assert [] == suggestions + + @pytest.mark.asyncio @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") async def test_array_facet_results(app_client): From ffae2f0ecde1ca92e78d097665df820d3b7861e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 14:57:49 -0700 Subject: [PATCH 0247/2303] Better documentation of --host, closes #574 --- README.md | 25 +++++++++++++++---------- datasette/cli.py | 11 +++++++++-- docs/datasette-serve-help.txt | 7 +++++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5894017e..9f85f1ba 100644 --- a/README.md +++ b/README.md @@ -89,26 +89,31 @@ Now visiting http://localhost:8001/History/downloads will show you a web interfa ## datasette serve options - $ datasette serve --help - Usage: datasette serve [OPTIONS] [FILES]... Serve up specified SQLite database files with a web UI Options: -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT host for server, defaults to 127.0.0.1 - -p, --port INTEGER port for server, defaults to 8001 + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means + only connections from the local machine will be + allowed. Use 0.0.0.0 to listen to all IPs and + allow access from other machines. + -p, --port INTEGER Port for server, defaults to 8001 --debug Enable debug mode - useful for development - --reload Automatically reload if database or code change detected - - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: * + --reload Automatically reload if database or code change + detected - useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * --load-extension PATH Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette inspect" - -m, --metadata FILENAME Path to JSON file containing license/source metadata + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON file containing license/source + metadata --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --static STATIC MOUNT mountpoint:path-to-directory for serving static + files --memory Make :memory: database available --config CONFIG Set config option using configname:value datasette.readthedocs.io/en/latest/config.html diff --git a/datasette/cli.py b/datasette/cli.py index 181b281c..67c2fe71 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -230,9 +230,16 @@ def package( multiple=True, ) @click.option( - "-h", "--host", default="127.0.0.1", help="host for server, defaults to 127.0.0.1" + "-h", + "--host", + default="127.0.0.1", + help=( + "Host for server. Defaults to 127.0.0.1 which means only connections " + "from the local machine will be allowed. Use 0.0.0.0 to listen to " + "all IPs and allow access from other machines." + ), ) -@click.option("-p", "--port", default=8001, help="port for server, defaults to 8001") +@click.option("-p", "--port", default=8001, help="Port for server, defaults to 8001") @click.option( "--debug", is_flag=True, help="Enable debug mode - useful for development" ) diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 7b7c3b09..1447e84d 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -6,8 +6,11 @@ Usage: datasette serve [OPTIONS] [FILES]... Options: -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT host for server, defaults to 127.0.0.1 - -p, --port INTEGER port for server, defaults to 8001 + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means only + connections from the local machine will be allowed. Use + 0.0.0.0 to listen to all IPs and allow access from other + machines. + -p, --port INTEGER Port for server, defaults to 8001 --debug Enable debug mode - useful for development --reload Automatically reload if database or code change detected - useful for development From ed57e4f99018c1d520858f55f6eee4eb1cc2af3d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 15:15:10 -0700 Subject: [PATCH 0248/2303] Plugin static assets support both hyphens and underscores in names Closes #611 --- datasette/app.py | 13 +++++++++++-- docs/plugins.rst | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 935b1730..203e0991 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -612,8 +612,17 @@ class Datasette: # Mount any plugin static/ directories for plugin in get_plugins(pm): if plugin["static_path"]: - modpath = "/-/static-plugins/{}/(?P.*)$".format(plugin["name"]) - add_route(asgi_static(plugin["static_path"]), modpath) + add_route( + asgi_static(plugin["static_path"]), + "/-/static-plugins/{}/(?P.*)$".format(plugin["name"]), + ) + # Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611 + add_route( + asgi_static(plugin["static_path"]), + "/-/static-plugins/{}/(?P.*)$".format( + plugin["name"].replace("-", "_") + ), + ) add_route( JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata), r"/-/metadata(?P(\.json)?)$", diff --git a/docs/plugins.rst b/docs/plugins.rst index 1d4f1e1a..6df7ff6a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -442,7 +442,7 @@ you have one: @hookimpl def extra_js_urls(): return [ - '/-/static-plugins/your_plugin/app.js' + '/-/static-plugins/your-plugin/app.js' ] .. _plugin_hook_publish_subcommand: From 14da70525b35e1a44cd45c19101385467057f041 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Nov 2019 15:29:40 -0700 Subject: [PATCH 0249/2303] Don't show 'None' as label for nullable foreign key, closes #406 --- datasette/views/table.py | 2 +- tests/fixtures.py | 1 + tests/test_api.py | 18 ++++++++++++++++-- tests/test_html.py | 9 +++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 44b186cf..326c11ae 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -637,7 +637,7 @@ class TableView(RowTableShared): new_row = CustomRow(columns) for column in row.keys(): value = row[column] - if (column, value) in expanded_labels: + if (column, value) in expanded_labels and value is not None: new_row[column] = { "value": value, "label": expanded_labels[(column, value)], diff --git a/tests/fixtures.py b/tests/fixtures.py index 93c3da9f..8aa44687 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -754,6 +754,7 @@ INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); INSERT INTO foreign_key_references VALUES (1, 1, 1); +INSERT INTO foreign_key_references VALUES (2, null, null); INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); INSERT INTO custom_foreign_key_label VALUES (1, 1); diff --git a/tests/test_api.py b/tests/test_api.py index 41557bcf..c6acbab1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -216,7 +216,7 @@ def test_database_page(app_client): "name": "foreign_key_references", "columns": ["pk", "foreign_key_with_label", "foreign_key_with_no_label"], "primary_keys": ["pk"], - "count": 1, + "count": 2, "hidden": False, "fts_table": None, "foreign_keys": { @@ -1519,7 +1519,7 @@ def test_expand_labels(app_client): def test_expand_label(app_client): response = app_client.get( "/fixtures/foreign_key_references.json?_shape=object" - "&_label=foreign_key_with_label" + "&_label=foreign_key_with_label&_size=1" ) assert { "1": { @@ -1693,3 +1693,17 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): app_client_conflicting_database_names.get(path).body.decode("utf8") ) assert db_name == data["database"] + + +def test_null_foreign_keys_are_not_expanded(app_client): + response = app_client.get( + "/fixtures/foreign_key_references.json?_shape=array&_labels=on" + ) + assert [ + { + "pk": "1", + "foreign_key_with_label": {"value": "1", "label": "hello"}, + "foreign_key_with_no_label": {"value": "1", "label": "1"}, + }, + {"pk": "2", "foreign_key_with_label": None, "foreign_key_with_no_label": None,}, + ] == response.json diff --git a/tests/test_html.py b/tests/test_html.py index aa628dec..f63e595b 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -603,7 +603,12 @@ def test_table_html_foreign_key_links(app_client): '1', 'hello\xa01', '1', - ] + ], + [ + '2', + '\xa0', + '\xa0', + ], ] assert expected == [ [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") @@ -611,7 +616,7 @@ def test_table_html_foreign_key_links(app_client): def test_table_html_disable_foreign_key_links_with_labels(app_client): - response = app_client.get("/fixtures/foreign_key_references?_labels=off") + response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") expected = [ From c3181d9a840dff7be8c990b21f5749db393a4ea0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Nov 2019 15:47:20 -0700 Subject: [PATCH 0250/2303] Release notes for 0.30.2 --- docs/changelog.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ac32c45..f4761efe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v0_30_2: + +0.30.2 (2019-11-02) +------------------- + +- ``/-/plugins`` page now uses distribution name e.g. ``datasette-cluster-map`` instead of the name of the underlying Python package (``datasette_cluster_map``) (`#606 `__) +- Array faceting is now only suggested for columns that contain arrays of strings (`#562 `__) +- Better documentation for the ``--host`` argument (`#574 `__) +- Don't show ``None`` with a broken link for the label on a nullable foreign key (`#406 `__) + .. _v0_30_1: 0.30.1 (2019-10-30) @@ -14,6 +24,7 @@ Changelog .. _v0_30: + 0.30 (2019-10-18) ----------------- @@ -82,7 +93,7 @@ Two new plugins take advantage of this hook: New plugin hook: extra_template_vars ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`plugin_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). +The :ref:`plugin_hook_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). Secret plugin configuration options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 2bf7ce5f517d772a16d7855a35a8a75d4456aad7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Nov 2019 16:12:46 -0700 Subject: [PATCH 0251/2303] Fix CSV export for nullable foreign keys, closes #612 --- datasette/views/base.py | 12 ++++++++---- tests/test_csv.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 1568b084..94945304 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -330,10 +330,14 @@ class DataView(BaseView): else: # Look for {"value": "label": } dicts and expand new_row = [] - for cell in row: - if isinstance(cell, dict): - new_row.append(cell["value"]) - new_row.append(cell["label"]) + for heading, cell in zip(data["columns"], row): + if heading in expanded_columns: + if cell is None: + new_row.extend(("", "")) + else: + assert isinstance(cell, dict) + new_row.append(cell["value"]) + new_row.append(cell["label"]) else: new_row.append(cell) await writer.writerow(new_row) diff --git a/tests/test_csv.py b/tests/test_csv.py index b148b6db..13aca489 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -41,6 +41,14 @@ pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,com "\n", "\r\n" ) +EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV = """ +pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_no_label,foreign_key_with_no_label_label +1,1,hello,1,1 +2,,,, +""".lstrip().replace( + "\n", "\r\n" +) + def test_table_csv(app_client): response = app_client.get("/fixtures/simple_primary_key.csv") @@ -63,6 +71,13 @@ def test_table_csv_with_labels(app_client): assert EXPECTED_TABLE_WITH_LABELS_CSV == response.text +def test_table_csv_with_nullable_labels(app_client): + response = app_client.get("/fixtures/foreign_key_references.csv?_labels=1") + assert response.status == 200 + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV == response.text + + def test_custom_sql_csv(app_client): response = app_client.get( "/fixtures.csv?sql=select+content+from+simple_primary_key+limit+2" From ee330222f4c3ee66c2fe41ebc76fed56b9cb9a00 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 4 Nov 2019 03:39:55 +0100 Subject: [PATCH 0252/2303] Offer to format readonly SQL (#602) Following discussion in #601, this PR adds a "Format SQL" button to read-only SQL (if the SQL actually differs from the formatting result). It also removes a console error on readonly SQL queries. Thanks, @rixx! --- datasette/templates/_codemirror_foot.html | 41 ++++++++++++++--------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 9aba61ab..4019d448 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -6,21 +6,32 @@ window.onload = () => { if (sqlFormat && !readOnly) { sqlFormat.hidden = false; } - var editor = CodeMirror.fromTextArea(sqlInput, { - lineNumbers: true, - mode: "text/x-sql", - lineWrapping: true, - }); - editor.setOption("extraKeys", { - "Shift-Enter": function() { - document.getElementsByClassName("sql")[0].submit(); - }, - Tab: false - }); - if (sqlInput && sqlFormat) { - sqlFormat.addEventListener("click", ev => { - editor.setValue(sqlFormatter.format(editor.getValue())); - }) + if (sqlInput) { + var editor = CodeMirror.fromTextArea(sqlInput, { + lineNumbers: true, + mode: "text/x-sql", + lineWrapping: true, + }); + editor.setOption("extraKeys", { + "Shift-Enter": function() { + document.getElementsByClassName("sql")[0].submit(); + }, + Tab: false + }); + if (sqlFormat) { + sqlFormat.addEventListener("click", ev => { + editor.setValue(sqlFormatter.format(editor.getValue())); + }) + } + } + if (sqlFormat && readOnly) { + const formatted = sqlFormatter.format(readOnly.innerHTML); + if (formatted != readOnly.innerHTML) { + sqlFormat.hidden = false; + sqlFormat.addEventListener("click", ev => { + readOnly.innerHTML = formatted; + }) + } } } From 9db22cdf1809fb78a7b183cd2f617cd5e26efc68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Nov 2019 20:11:55 -0800 Subject: [PATCH 0253/2303] pk__notin= filter, closes #614 --- datasette/filters.py | 15 +++++++++++++++ docs/json_api.rst | 3 +++ tests/test_filters.py | 3 +++ 3 files changed, 21 insertions(+) diff --git a/datasette/filters.py b/datasette/filters.py index efe014ae..5897a3ed 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -77,6 +77,20 @@ class InFilter(Filter): return "{} in {}".format(column, json.dumps(self.split_value(value))) +class NotInFilter(InFilter): + key = "notin" + display = "not in" + + def where_clause(self, table, column, value, param_counter): + values = self.split_value(value) + params = [":p{}".format(param_counter + i) for i in range(len(values))] + sql = "{} not in ({})".format(escape_sqlite(column), ", ".join(params)) + return sql, values + + def human_clause(self, column, value): + return "{} not in {}".format(column, json.dumps(self.split_value(value))) + + class Filters: _filters = ( [ @@ -125,6 +139,7 @@ class Filters: TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'), TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'), InFilter(), + NotInFilter(), ] + ( [ diff --git a/docs/json_api.rst b/docs/json_api.rst index 4b365e14..de70362c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -228,6 +228,9 @@ You can filter the data returned by the table based on column values using a que ``?column__in=["value","value,with,commas"]`` +``?column__notin=value1,value2,value3`` + Rows where column does not match any of the provided values. The inverse of ``__in=``. Also supports JSON arrays. + ``?column__arraycontains=value`` Works against columns that contain JSON arrays - matches if any of the values in that array match. diff --git a/tests/test_filters.py b/tests/test_filters.py index fd682cd9..8598087f 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -47,6 +47,9 @@ import pytest ["foo in (:p0, :p1)"], ["dog,cat", "cat[dog]"], ), + # Not in, and JSON array not in + ((("foo__notin", "1,2,3"),), ["foo not in (:p0, :p1, :p2)"], ["1", "2", "3"]), + ((("foo__notin", "[1,2,3]"),), ["foo not in (:p0, :p1, :p2)"], [1, 2, 3]), ], ) def test_build_where(args, expected_where, expected_params): From 52fa79c6075f0830ff635b81d957c64d877a05aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 Nov 2019 15:03:48 -0800 Subject: [PATCH 0254/2303] Use select colnames, not select * for table view - refs #615 --- datasette/views/table.py | 8 ++++++-- tests/test_api.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 326c11ae..139ff80b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -235,13 +235,17 @@ class TableView(RowTableShared): raise NotFound("Table not found: {}".format(table)) pks = await db.primary_keys(table) + table_columns = await db.table_columns(table) + + select_columns = ", ".join(escape_sqlite(t) for t in table_columns) + use_rowid = not pks and not is_view if use_rowid: - select = "rowid, *" + select = "rowid, {}".format(select_columns) order_by = "rowid" order_by_pks = "rowid" else: - select = "*" + select = select_columns order_by_pks = ", ".join([escape_sqlite(pk) for pk in pks]) order_by = order_by_pks diff --git a/tests/test_api.py b/tests/test_api.py index c6acbab1..4a09b238 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -610,7 +610,8 @@ def test_table_json(app_client): assert response.status == 200 data = response.json assert ( - data["query"]["sql"] == "select * from simple_primary_key order by id limit 51" + data["query"]["sql"] + == "select id, content from simple_primary_key order by id limit 51" ) assert data["query"]["params"] == {} assert data["rows"] == [ From 931bfc66613aa3e22f8314df5c0d0758baf31f38 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Tue, 5 Nov 2019 00:16:30 +0100 Subject: [PATCH 0255/2303] Handle spaces in DB names (#590) Closes #503 - thanks, @rixx --- datasette/views/base.py | 3 ++- tests/fixtures.py | 4 ++-- tests/test_api.py | 19 ++++++++++++++++++- tests/test_html.py | 8 ++++---- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 94945304..062c6956 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -203,12 +203,13 @@ class DataView(BaseView): hash = hash_bit else: name = db_name - # Verify the hash + name = urllib.parse.unquote_plus(name) try: db = self.ds.databases[name] except KeyError: raise NotFound("Database not found: {}".format(name)) + # Verify the hash expected = "000" if db.hash is not None: expected = db.hash[:HASH_LENGTH] diff --git a/tests/fixtures.py b/tests/fixtures.py index 8aa44687..dcc414bf 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -174,7 +174,7 @@ def app_client_no_files(): @pytest.fixture(scope="session") def app_client_two_attached_databases(): yield from make_app_client( - extra_databases={"extra_database.db": EXTRA_DATABASE_SQL} + extra_databases={"extra database.db": EXTRA_DATABASE_SQL} ) @@ -188,7 +188,7 @@ def app_client_conflicting_database_names(): @pytest.fixture(scope="session") def app_client_two_attached_databases_one_immutable(): yield from make_app_client( - is_immutable=True, extra_databases={"extra_database.db": EXTRA_DATABASE_SQL} + is_immutable=True, extra_databases={"extra database.db": EXTRA_DATABASE_SQL} ) diff --git a/tests/test_api.py b/tests/test_api.py index 4a09b238..1fa8642f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -6,6 +6,7 @@ from .fixtures import ( # noqa app_client_shorter_time_limit, app_client_larger_cache_size, app_client_returned_rows_matches_page_size, + app_client_two_attached_databases, app_client_two_attached_databases_one_immutable, app_client_conflicting_database_names, app_client_with_cors, @@ -1188,7 +1189,7 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): databases = response.json assert 2 == len(databases) extra_database, fixtures_database = databases - assert "extra_database" == extra_database["name"] + assert "extra database" == extra_database["name"] assert None == extra_database["hash"] assert True == extra_database["is_mutable"] assert False == extra_database["is_memory"] @@ -1679,6 +1680,22 @@ def test_cors(app_client_with_cors, path, status_code): assert "*" == response.headers["Access-Control-Allow-Origin"] +@pytest.mark.parametrize( + "path", + ( + "/", + ".json", + "/searchable", + "/searchable.json", + "/searchable_view", + "/searchable_view.json", + ), +) +def test_database_with_space_in_name(app_client_two_attached_databases, path): + response = app_client_two_attached_databases.get("/extra database" + path) + assert response.status == 200 + + def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 assert ["fixtures", "foo", "foo-bar"] == [ diff --git a/tests/test_html.py b/tests/test_html.py index f63e595b..7f1af86e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -27,11 +27,11 @@ def test_homepage(app_client_two_attached_databases): # Should be two attached databases assert [ {"href": "/fixtures", "text": "fixtures"}, - {"href": "/extra_database", "text": "extra_database"}, + {"href": "/extra database", "text": "extra database"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables h2 = soup.select("h2")[1] - assert "extra_database" == h2.text.strip() + assert "extra database" == h2.text.strip() counts_p, links_p = h2.find_all_next("p")[:2] assert ( "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip() @@ -41,8 +41,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": "/extra_database/searchable", "text": "searchable"}, - {"href": "/extra_database/searchable_view", "text": "searchable_view"}, + {"href": "/extra database/searchable", "text": "searchable"}, + {"href": "/extra database/searchable_view", "text": "searchable_view"}, ] == table_links From c30f07c58e410ee296b28aeabe4dc461dd40b435 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Nov 2019 21:12:55 -0800 Subject: [PATCH 0256/2303] Removed _group_count=col feature, closes #504 --- datasette/views/table.py | 12 ------------ docs/json_api.rst | 9 --------- 2 files changed, 21 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 139ff80b..920693d7 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -499,18 +499,6 @@ class TableView(RowTableShared): if order_by: order_by = "order by {} ".format(order_by) - # _group_count=col1&_group_count=col2 - group_count = special_args_lists.get("_group_count") or [] - if group_count: - sql = 'select {group_cols}, count(*) as "count" from {table_name} {where} group by {group_cols} order by "count" desc limit 100'.format( - group_cols=", ".join( - '"{}"'.format(group_count_col) for group_count_col in group_count - ), - table_name=escape_sqlite(table), - where=where_clause, - ) - return await self.custom_sql(request, database, hash, sql, editable=True) - extra_args = {} # Handle ?_size=500 page_size = _size or request.raw_args.get("_size") diff --git a/docs/json_api.rst b/docs/json_api.rst index de70362c..e369bee7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -321,15 +321,6 @@ Special table arguments Here's `an example `__. - -``?_group_count=COLUMN`` - Executes a SQL query that returns a count of the number of rows matching - each unique value in that column, with the most common ordered first. - -``?_group_count=COLUMN1&_group_count=column2`` - You can pass multiple ``_group_count`` columns to return counts against - unique combinations of those columns. - ``?_next=TOKEN`` Pagination by continuation token - pass the token that was returned in the ``"next"`` property by the previous page. From f9c146b893856a48afa810ebcce1714f30d0d3a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Nov 2019 16:55:44 -0800 Subject: [PATCH 0257/2303] Removed unused special_args_lists variable --- datasette/views/table.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 920693d7..a60a3941 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -261,12 +261,10 @@ class TableView(RowTableShared): # That's so if there is a column that starts with _ # it can still be queried using ?_col__exact=blah special_args = {} - special_args_lists = {} other_args = [] for key, value in args.items(): if key.startswith("_") and "__" not in key: special_args[key] = value[0] - special_args_lists[key] = value else: for v in value: other_args.append((key, v)) From 83fc5165ac724f69cd57d8f15cd3038e7b30f878 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Nov 2019 18:48:39 -0800 Subject: [PATCH 0258/2303] Improved UI for publish cloudrun, closes #608 --- datasette/publish/cloudrun.py | 39 ++++++++++++++++++++++-- tests/test_publish_cloudrun.py | 55 ++++++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index c2d77746..a833a32b 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -60,6 +60,23 @@ def publish_subcommand(publish): "gcloud config get-value project", shell=True, universal_newlines=True ).strip() + if not service: + # Show the user their current services, then prompt for one + click.echo("Please provide a service name for this deployment\n") + click.echo("Using an existing service name will over-write it") + click.echo("") + existing_services = get_existing_services() + if existing_services: + click.echo("Your existing services:\n") + for existing_service in existing_services: + click.echo( + " {name} - created {created} - {url}".format( + **existing_service + ) + ) + click.echo("") + service = click.prompt("Service name", type=str) + extra_metadata = { "title": title, "license": license, @@ -110,8 +127,26 @@ def publish_subcommand(publish): image_id = "gcr.io/{project}/{name}".format(project=project, name=name) check_call("gcloud builds submit --tag {}".format(image_id), shell=True) check_call( - "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}{}".format( - image_id, " {}".format(service) if service else "" + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {} {}".format( + image_id, service, ), shell=True, ) + + +def get_existing_services(): + services = json.loads( + check_output( + "gcloud beta run services list --platform=managed --format json", + shell=True, + universal_newlines=True, + ) + ) + return [ + { + "name": service["metadata"]["name"], + "created": service["metadata"]["creationTimestamp"], + "url": service["status"]["address"]["url"], + } + for service in services + ] diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 481ac04d..a038b60e 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -24,6 +24,53 @@ def test_publish_cloudrun_invalid_database(mock_which): assert 'Path "woop.db" does not exist' in result.output +@mock.patch("shutil.which") +@mock.patch("datasette.publish.cloudrun.check_output") +@mock.patch("datasette.publish.cloudrun.check_call") +@mock.patch("datasette.publish.cloudrun.get_existing_services") +def test_publish_cloudrun_prompts_for_service( + mock_get_existing_services, mock_call, mock_output, mock_which +): + mock_get_existing_services.return_value = [ + {"name": "existing", "created": "2019-01-01", "url": "http://www.example.com/"} + ] + mock_output.return_value = "myproject" + mock_which.return_value = True + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, ["publish", "cloudrun", "test.db"], input="input-service" + ) + assert ( + """ +Please provide a service name for this deployment + +Using an existing service name will over-write it + +Your existing services: + + existing - created 2019-01-01 - http://www.example.com/ + +Service name: input-service +""".strip() + == result.output.strip() + ) + assert 0 == result.exit_code + tag = "gcr.io/myproject/datasette" + mock_call.assert_has_calls( + [ + mock.call("gcloud builds submit --tag {}".format(tag), shell=True), + mock.call( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {} input-service".format( + tag + ), + shell=True, + ), + ] + ) + + @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -33,14 +80,16 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): runner = CliRunner() with runner.isolated_filesystem(): open("test.db", "w").write("data") - result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) + result = runner.invoke( + cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] + ) assert 0 == result.exit_code tag = "gcr.io/{}/datasette".format(mock_output.return_value) mock_call.assert_has_calls( [ mock.call("gcloud builds submit --tag {}".format(tag), shell=True), mock.call( - "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}".format( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {} test".format( tag ), shell=True, @@ -65,6 +114,8 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): "publish", "cloudrun", "test.db", + "--service", + "datasette", "--plugin-secret", "datasette-auth-github", "client_id", From 9f5d19c254d1bfbd99f576dff47a6e32e01c76ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 Nov 2019 18:12:20 -0800 Subject: [PATCH 0259/2303] Improved documentation for "publish cloudrun" --- docs/publish.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index 304be8ef..89d33085 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -43,14 +43,16 @@ You will first need to install and configure the Google Cloud CLI tools by follo You can then publish a database to Google Cloud Run using the following command:: - datasette publish cloudrun mydatabase.db + datasette publish cloudrun mydatabase.db --service=my-database + +A Cloud Run **service** is a single hosted application. The service name you specify will be used as part of the Cloud Run URL. If you deploy to a service name that you have used in the past your new deployment will replace the previous one. + +If you omit the ``--service`` option you will be asked to pick a service name interactively during the deploy. You may need to interact with prompts from the tool. Once it has finished it will output a URL like this one:: - Service [datasette] revision [datasette-00001] has been deployed - and is serving traffic at https://datasette-j7hipcg4aq-uc.a.run.app - -During the deployment the tool will prompt you for the name of your service. You can reuse an existing name to replace your previous deployment with your new version, or pick a new name to deploy to a new URL. + Service [my-service] revision [my-service-00001] has been deployed + and is serving traffic at https://my-service-j7hipcg4aq-uc.a.run.app .. literalinclude:: datasette-publish-cloudrun-help.txt @@ -90,18 +92,18 @@ Custom metadata and plugins You can define your own :ref:`metadata` and deploy that with your instance like so:: - datasette publish nowv1 mydatabase.db -m metadata.json + datasette publish cloudrun --service=my-service mydatabase.db -m metadata.json If you just want to set the title, license or source information you can do that directly using extra options to ``datasette publish``:: - datasette publish nowv1 mydatabase.db \ + datasette publish cloudrun mydatabase.db --service=my-service \ --title="Title of my database" \ --source="Where the data originated" \ --source_url="http://www.example.com/" You can also specify plugins you would like to install. For example, if you want to include the `datasette-vega `_ visualization plugin you can use the following:: - datasette publish nowv1 mydatabase.db --install=datasette-vega + datasette publish cloudrun mydatabase.db --service=my-service --install=datasette-vega If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: From 10b9d85edaaf198879344aa1c498000cfb27dff8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 Nov 2019 18:15:13 -0800 Subject: [PATCH 0260/2303] datasette-csvs on Glitch now uses sqlite-utils It previously used csvs-to-sqlite but that had heavy dependencies. See https://support.glitch.com/t/can-you-upgrade-python-to-latest-version/7980/33 --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index d0c22583..fdf7d23c 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -25,7 +25,7 @@ Glitch allows you to "remix" any project to create your own copy and start editi .. image:: https://cdn.glitch.com/2703baf2-b643-4da7-ab91-7ee2a2d00b5b%2Fremix-button.svg :target: https://glitch.com/edit/#!/remix/datasette-csvs -Find a CSV file and drag it onto the Glitch file explorer panel - ``datasette-csvs`` will automatically convert it to a SQLite database (using `csvs-to-sqlite `__) and allow you to start exploring it using Datasette. +Find a CSV file and drag it onto the Glitch file explorer panel - ``datasette-csvs`` will automatically convert it to a SQLite database (using `sqlite-utils `__) and allow you to start exploring it using Datasette. If your CSV file has a ``latitude`` and ``longitude`` column you can visualize it on a map by uncommenting the ``datasette-cluster-map`` line in the ``requirements.txt`` file using the Glitch file editor. From 28c4a6db5b5e512db630d7ba6127196185de67c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 9 Nov 2019 17:29:36 -0800 Subject: [PATCH 0261/2303] CREATE INDEX statements on table page, closes #618 --- datasette/database.py | 13 ++++++++++++- tests/fixtures.py | 1 + tests/test_html.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index 7e6f7245..3a1cea94 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -232,7 +232,18 @@ class Database: ) if not table_definition_rows: return None - return table_definition_rows[0][0] + bits = [table_definition_rows[0][0] + ";"] + # Add on any indexes + index_rows = list( + await self.ds.execute( + self.name, + "select sql from sqlite_master where tbl_name = :n and type='index' and sql is not null", + {"n": table}, + ) + ) + for index_row in index_rows: + bits.append(index_row[0] + ";") + return "\n".join(bits) async def get_view_definition(self, view): return await self.get_table_definition(view, "view") diff --git a/tests/fixtures.py b/tests/fixtures.py index dcc414bf..87e66f99 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -514,6 +514,7 @@ CREATE TABLE compound_three_primary_keys ( content text, PRIMARY KEY (pk1, pk2, pk3) ); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); CREATE TABLE foreign_key_references ( pk varchar(30) primary key, diff --git a/tests/test_html.py b/tests/test_html.py index 7f1af86e..44627cdc 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -119,6 +119,39 @@ def test_row_strange_table_name_with_url_hash(app_client_with_hash): assert response.status == 200 +@pytest.mark.parametrize( + "path,expected_definition_sql", + [ + ( + "/fixtures/facet_cities", + """ +CREATE TABLE facet_cities ( + id integer primary key, + name text +); + """.strip(), + ), + ( + "/fixtures/compound_three_primary_keys", + """ +CREATE TABLE compound_three_primary_keys ( + pk1 varchar(30), + pk2 varchar(30), + pk3 varchar(30), + content text, + PRIMARY KEY (pk1, pk2, pk3) +); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); + """.strip(), + ), + ], +) +def test_definition_sql(path, expected_definition_sql, app_client): + response = app_client.get(path) + pre = Soup(response.body, "html.parser").select_one("pre.wrapped-sql") + assert expected_definition_sql == pre.string + + def test_table_cell_truncation(): for client in make_app_client(config={"truncate_cells_html": 5}): response = client.get("/fixtures/facetable") From 1c063fae9dba70f70244db010d55a18846640f07 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 Nov 2019 19:45:34 -0800 Subject: [PATCH 0262/2303] Test against Python 3.8 in Travis (#623) * Test against Python 3.8 in Travis * Avoid current_task warnings in Python 3.8 --- .travis.yml | 1 + datasette/tracer.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29388bc1..a6b15b7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ dist: xenial python: - "3.6" - "3.7" + - "3.8" - "3.5" # Executed for 3.5 AND 3.5 as the first "test" stage: diff --git a/datasette/tracer.py b/datasette/tracer.py index e46a6fda..a638b140 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -9,12 +9,19 @@ tracers = {} TRACE_RESERVED_KEYS = {"type", "start", "end", "duration_ms", "traceback"} +# asyncio.current_task was introduced in Python 3.7: +for obj in (asyncio, asyncio.Task): + current_task = getattr(obj, "current_task", None) + if current_task is not None: + break + + def get_task_id(): try: loop = asyncio.get_event_loop() except RuntimeError: return None - return id(asyncio.Task.current_task(loop=loop)) + return id(current_task(loop=loop)) @contextmanager From 42ee3e16a9ba7cc513b8da944cc1609a5407cf42 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 Nov 2019 20:19:01 -0800 Subject: [PATCH 0263/2303] Bump pint to 0.9 (#624) This fixes 2 deprecation warnings in Python 3.8 - refs #623 #622 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ae56306..e8229de1 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( "click-default-group~=1.2.1", "Jinja2~=2.10.1", "hupper~=1.0", - "pint~=0.8.1", + "pint~=0.9", "pluggy~=0.12.0", "uvicorn~=0.8.4", "aiofiles~=0.4.0", From 5bc2570121aea8141ff88790e214765472882b08 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 20:45:12 -0800 Subject: [PATCH 0264/2303] Include uvicorn version in /-/versions, refs #622 --- datasette/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 203e0991..4ba4adfb 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -12,6 +12,7 @@ from pathlib import Path import click from markupsafe import Markup from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader +import uvicorn from .views.base import DatasetteError, ureg, AsgiRouter from .views.database import DatabaseDownload, DatabaseView @@ -433,6 +434,7 @@ class Datasette: }, "datasette": datasette_version, "asgi": "3.0", + "uvicorn": uvicorn.__version__, "sqlite": { "version": sqlite_version, "fts_versions": fts_versions, From cf7776d36fbacefa874cbd6e5fcdc9fff7661203 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:09:11 -0800 Subject: [PATCH 0265/2303] Support Python 3.8, stop supporting Python 3.5 (#627) * Upgrade to uvicorn 0.10.4 * Drop support for Python 3.5 * Bump all dependencies to latest releases * Update docs to reflect we no longer support 3.5 * Removed code that skipped black unit test on 3.5 Closes #622 --- .travis.yml | 1 - README.md | 2 +- docs/contributing.rst | 2 +- docs/installation.rst | 7 +++++-- setup.py | 20 ++++++++++---------- tests/test_black.py | 7 +------ 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6b15b7e..0fc87d93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: - "3.6" - "3.7" - "3.8" - - "3.5" # Executed for 3.5 AND 3.5 as the first "test" stage: script: diff --git a/README.md b/README.md index 9f85f1ba..14c9cfd6 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ sqlite-utils: a Python library and CLI tool for building SQLite databases](https pip3 install datasette -Datasette requires Python 3.5 or higher. We also have [detailed installation instructions](https://datasette.readthedocs.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://datasette.readthedocs.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/docs/contributing.rst b/docs/contributing.rst index 43834edc..078fd841 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -18,7 +18,7 @@ General guidelines Setting up a development environment ------------------------------------ -If you have Python 3.5 or higher installed on your computer (on OS X the easiest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. +If you have Python 3.6 or higher installed on your computer (on OS X the easiest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. If you want to use GitHub to publish your changes, first `create a fork of datasette `__ under your own GitHub account. diff --git a/docs/installation.rst b/docs/installation.rst index e65d8ee3..9ee7eb4e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -69,16 +69,19 @@ You can now run the new custom image like so:: You can confirm that the plugins are installed by visiting http://127.0.0.1:8001/-/plugins - Install using pip ----------------- -To run Datasette without Docker you will need Python 3.5 or higher. +To run Datasette without Docker you will need Python 3.6 or higher. You can install Datasette and its dependencies using ``pip``:: pip install datasette +The last version to support Python 3.5 was 0.30.2 - you can install that version like so:: + + pip install datasette==0.30.2 + If you want to install Datasette in its own virtual environment, use this:: python -mvenv datasette-venv diff --git a/setup.py b/setup.py index e8229de1..7a4cdcb3 100644 --- a/setup.py +++ b/setup.py @@ -42,12 +42,12 @@ setup( include_package_data=True, install_requires=[ "click~=7.0", - "click-default-group~=1.2.1", - "Jinja2~=2.10.1", - "hupper~=1.0", + "click-default-group~=1.2.2", + "Jinja2~=2.10.3", + "hupper~=1.9", "pint~=0.9", - "pluggy~=0.12.0", - "uvicorn~=0.8.4", + "pluggy~=0.13.0", + "uvicorn~=0.10.4", "aiofiles~=0.4.0", ], entry_points=""" @@ -58,11 +58,11 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ - "pytest~=5.0.0", + "pytest~=5.2.2", "pytest-asyncio~=0.10.0", - "aiohttp~=3.5.3", - "beautifulsoup4~=4.6.1", - "asgiref~=3.1.2", + "aiohttp~=3.6.2", + "beautifulsoup4~=4.8.1", + "asgiref~=3.2.3", ] + maybe_black, }, @@ -74,8 +74,8 @@ setup( "Intended Audience :: End Users/Desktop", "Topic :: Database", "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.5", ], ) diff --git a/tests/test_black.py b/tests/test_black.py index 68e2dcc0..b5bfcfd0 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1,3 +1,4 @@ +import black from click.testing import CliRunner from pathlib import Path import pytest @@ -6,13 +7,7 @@ import sys code_root = Path(__file__).parent.parent -@pytest.mark.skipif( - sys.version_info[:2] < (3, 6), reason="Black requires Python 3.6 or later" -) def test_black(): - # Do not import at top of module because Python 3.5 will not have it installed - import black - runner = CliRunner() result = runner.invoke( black.main, [str(code_root / "tests"), str(code_root / "datasette"), "--check"] From 76fc6a9c7317ce4fbf3cc3d327c849f7274d960a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:17:59 -0800 Subject: [PATCH 0266/2303] Release notes for 0.31 --- docs/changelog.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4761efe..6e260be9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,28 @@ Changelog ========= +.. _v0_31: + +0.31 (2019-11-11) +----------------- + +This version adds compatibility with Python 3.8 and breaks compatibility with Python 3.5. + +If you are still running Python 3.5 you should stick with ``0.30.2``, which you can install like this:: + + pip install datasette==0.30.2 + +- Format SQL button now works with read-only SQL queries - thanks, Tobias Kunze (`#602 `__) +- New ``?column__notin=x,y,z`` filter for table views (`#614 `__) +- Table view now uses ``select col1, col2, col3`` instead of ``select *`` +- Database filenames can now contain spaces - thanks, Tobias Kunze (`#590 `__) +- Removed obsolete ``?_group_count=col`` feature (`#504 `__) +- Improved user interface and documentation for ``datasette publish cloudrun`` (`#608 `__) +- Tables with indexes now show the `` CREATE INDEX`` statements on the table page (`#618 `__) +- Current version of `uvicorn `__ is now shown on ``/-/versions`` +- Python 3.8 is now supported! (`#622 `__) +- Python 3.5 is no longer supported. + .. _v0_30_2: 0.30.2 (2019-11-02) From c633c035dc8d4c60f1d13cb074918406bbdb3734 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:26:56 -0800 Subject: [PATCH 0267/2303] Datasette 0.31 in news section --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 14c9cfd6..05995a74 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 11th November 2019: [Datasette 0.31](https://datasette.readthedocs.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30) * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... From 7f89928062b1a1fdb2625a946f7cd5161e597401 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:33:51 -0800 Subject: [PATCH 0268/2303] Removed code that conditionally installs black Since we no longer support Python 3.5 we don't need this any more. --- setup.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 7a4cdcb3..15284779 100644 --- a/setup.py +++ b/setup.py @@ -22,11 +22,6 @@ def get_version(): return g["__version__"] -# Only install black on Python 3.6 or higher -maybe_black = [] -if sys.version_info > (3, 6): - maybe_black = ["black~=19.10b0"] - setup( name="datasette", version=versioneer.get_version(), @@ -63,8 +58,8 @@ setup( "aiohttp~=3.6.2", "beautifulsoup4~=4.8.1", "asgiref~=3.2.3", - ] - + maybe_black, + "black~=19.10b0", + ], }, tests_require=["datasette[test]"], classifiers=[ From 1c518680e9692a9a77022af54f3de3e77fb1aaf4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:57:48 -0800 Subject: [PATCH 0269/2303] Final steps: build stable branch of Read The Docs --- docs/contributing.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 078fd841..48930332 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -150,4 +150,7 @@ Wait long enough for Travis to build and deploy the demo version of that commit git tag 0.25.2 git push --tags -Once the release is out, you can manually update https://github.com/simonw/datasette/releases +Final steps once the release has deployed to https://pypi.org/project/datasette/ + +* Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases +* Manually kick off a build of the `stable` branch on Read The Docs: https://readthedocs.org/projects/datasette/builds/ From f554be39fc14ddc18921ca29d3920d55aad03d46 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 22:00:13 -0800 Subject: [PATCH 0270/2303] ReST fix --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e260be9..763b178e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,7 +21,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you - Database filenames can now contain spaces - thanks, Tobias Kunze (`#590 `__) - Removed obsolete ``?_group_count=col`` feature (`#504 `__) - Improved user interface and documentation for ``datasette publish cloudrun`` (`#608 `__) -- Tables with indexes now show the `` CREATE INDEX`` statements on the table page (`#618 `__) +- Tables with indexes now show the ``CREATE INDEX`` statements on the table page (`#618 `__) - Current version of `uvicorn `__ is now shown on ``/-/versions`` - Python 3.8 is now supported! (`#622 `__) - Python 3.5 is no longer supported. From d977fbadf70a96bf2eea1407d01f99d98e092dec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 22:03:09 -0800 Subject: [PATCH 0271/2303] datasette publish uses python:3.8 base Docker image, closes #629 --- datasette/utils/__init__.py | 2 +- tests/test_publish_cloudrun.py | 2 +- tests/test_publish_now.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 3d28a36b..b8df48cf 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -306,7 +306,7 @@ def make_dockerfile( install = ["datasette"] + list(install) return """ -FROM python:3.6 +FROM python:3.8 COPY . /app WORKDIR /app {spatialite_extras} diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index a038b60e..c5b18cdf 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -128,7 +128,7 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): .split("\n====================\n")[0] .strip() ) - expected = """FROM python:3.6 + expected = """FROM python:3.8 COPY . /app WORKDIR /app diff --git a/tests/test_publish_now.py b/tests/test_publish_now.py index 72aa71db..27fd1245 100644 --- a/tests/test_publish_now.py +++ b/tests/test_publish_now.py @@ -138,7 +138,7 @@ def test_publish_now_plugin_secrets(mock_run, mock_which): .split("\n====================\n")[0] .strip() ) - expected = """FROM python:3.6 + expected = """FROM python:3.8 COPY . /app WORKDIR /app From 16265f6a1a7c547e3925e0fc2d6b88754afb0435 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Nov 2019 18:18:04 -0800 Subject: [PATCH 0272/2303] Release notes for 0.31.1 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 763b178e..746f5b42 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_31_1: + +0.31.1 (2019-11-12) +------------------- + +- Deploymens created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 `__) + .. _v0_31: 0.31 (2019-11-11) From a22c7761b61baa61b8e3da7d30887468d61d6b83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Nov 2019 18:18:39 -0800 Subject: [PATCH 0273/2303] Fixed typo in release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 746f5b42..e527518e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog 0.31.1 (2019-11-12) ------------------- -- Deploymens created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 `__) +- Deployments created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 `__) .. _v0_31: From bbd00e903cdd49067ecdbdb60a4d225833a44b05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Nov 2019 18:38:13 -0800 Subject: [PATCH 0274/2303] Badge linking to datasette on hub.docker.com --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 05995a74..9a22c2b2 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/master/LICENSE) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://black.readthedocs.io/en/stable/) +[![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) *A tool for exploring and publishing data* From 848dec4deb0d3c140a4e0394cac45fbb2593349b Mon Sep 17 00:00:00 2001 From: Stanley Zheng Date: Tue, 12 Nov 2019 23:28:42 -0500 Subject: [PATCH 0275/2303] Fix for datasette publish with just --source_url (#631) Closes #572 --- datasette/templates/_description_source_license.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/_description_source_license.html b/datasette/templates/_description_source_license.html index 3327706e..a2bc18f2 100644 --- a/datasette/templates/_description_source_license.html +++ b/datasette/templates/_description_source_license.html @@ -21,7 +21,7 @@ {% endif %}{{ metadata.source or metadata.source_url }}{% if metadata.source_url %}{% endif %} {% endif %} - {% if metadata.about or metadata.about_url %}{% if metadata.license or metadata.license_url or metadata.source or metadat.source_url %}·{% endif %} + {% if metadata.about or metadata.about_url %}{% if metadata.license or metadata.license_url or metadata.source or metadata.source_url %}·{% endif %} About: {% if metadata.about_url %} {% endif %}{{ metadata.about or metadata.about_url }}{% if metadata.about_url %}{% endif %} From f52451023025579ae9a13de4a7f00d69200184cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Nov 2019 08:42:47 -0800 Subject: [PATCH 0276/2303] Fix "publish heroku" + upgrade to use Python 3.8.0 Closes #633. Closes #632. --- datasette/publish/heroku.py | 7 +++++-- tests/test_publish_heroku.py | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 34d1f773..e75f76df 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -72,7 +72,10 @@ def publish_subcommand(publish): "about_url": about_url, } - environment_variables = {} + environment_variables = { + # Avoid uvicorn error: https://github.com/simonw/datasette/issues/633 + "WEB_CONCURRENCY": "1" + } if plugin_secret: extra_metadata["plugins"] = {} for plugin_name, plugin_setting, setting_value in plugin_secret: @@ -164,7 +167,7 @@ def temporary_heroku_directory( if metadata_content: open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) - open("runtime.txt", "w").write("python-3.6.8") + open("runtime.txt", "w").write("python-3.8.0") if branch: install = [ diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 4cd66219..87386e93 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -57,8 +57,13 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): open("test.db", "w").write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) assert 0 == result.exit_code, result.output - mock_call.assert_called_once_with( - ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] + mock_call.assert_has_calls( + [ + mock.call(["heroku", "config:set", "-a", "f", "WEB_CONCURRENCY=1",]), + mock.call( + ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] + ), + ] ) From b51f258d00bb3c3b401f15d46a1fbd50394dbe1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Nov 2019 08:48:36 -0800 Subject: [PATCH 0277/2303] Release notes for 0.31.2 --- docs/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e527518e..f4958399 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_31_2: + +0.31.2 (2019-11-13) +------------------- + +- Fixed a bug where ``datasette publish heroku`` applications failed to start (`#633 `__) +- Fix for ``datasette publish`` with just ``--source_url`` - thanks, Stanley Zheng (`#572 `__) +- Deployments to Heroku now use Python 3.8.0 (`#632 `__) + .. _v0_31_1: 0.31.1 (2019-11-12) From 8c642f04e0608bf537fdd1f76d64c2367fb04d57 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Nov 2019 15:14:22 -0800 Subject: [PATCH 0278/2303] Render templates using Jinja async mode Closes #628 --- datasette/app.py | 6 ++++-- datasette/views/base.py | 2 +- docs/plugins.rst | 23 ++++++++++++----------- tests/fixtures.py | 8 +++++++- tests/test_plugins.py | 18 ++++++++++++++++++ tests/test_templates/show_json.html | 1 + 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4ba4adfb..02fcf303 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -583,7 +583,9 @@ class Datasette: ), ] ) - self.jinja_env = Environment(loader=template_loader, autoescape=True) + self.jinja_env = Environment( + loader=template_loader, autoescape=True, enable_async=True + ) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters["quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite @@ -730,5 +732,5 @@ class DatasetteRouter(AsgiRouter): else: template = self.ds.jinja_env.select_template(templates) await asgi_send_html( - send, template.render(info), status=status, headers=headers + send, await template.render_async(info), status=status, headers=headers ) diff --git a/datasette/views/base.py b/datasette/views/base.py index 062c6956..5182479c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -139,7 +139,7 @@ class BaseView(AsgiView): extra_template_vars.update(extra_vars) return Response.html( - template.render( + await template.render_async( { **context, **{ diff --git a/docs/plugins.rst b/docs/plugins.rst index 6df7ff6a..e5a3d7dd 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -629,7 +629,9 @@ Function that returns a dictionary If you return a function it will be executed. If it returns a dictionary those values will will be merged into the template context. Function that returns an awaitable function that returns a dictionary - You can also return a function which returns an awaitable function which returns a dictionary. This means you can execute additional SQL queries using ``datasette.execute()``. + You can also return a function which returns an awaitable function which returns a dictionary. + +Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template. Here's an example plugin that returns an authentication object from the ASGI scope: @@ -641,20 +643,19 @@ Here's an example plugin that returns an authentication object from the ASGI sco "auth": request.scope.get("auth") } -And here's an example which returns the current version of SQLite: +And here's an example which adds a ``sql_first(sql_query)`` function which executes a SQL statement and returns the first column of the first row of results: .. code-block:: python @hookimpl - def extra_template_vars(datasette): - async def inner(): - first_db = list(datasette.databases.keys())[0] - return { - "sqlite_version": ( - await datasette.execute(first_db, "select sqlite_version()") - ).rows[0][0] - } - return inner + def extra_template_vars(datasette, database): + async def sql_first(sql, dbname=None): + dbname = dbname or database or next(iter(datasette.databases.keys())) + return (await datasette.execute(dbname, sql)).rows[0][0] + +You can then use the new function in a template like so:: + + SQLite version: {{ sql_first("select sqlite_version()") }} .. _plugin_register_output_renderer: diff --git a/tests/fixtures.py b/tests/fixtures.py index 87e66f99..3e4203f7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -446,13 +446,19 @@ def render_cell(value, database): @hookimpl def extra_template_vars(template, database, table, view_name, request, datasette): + async def query_database(sql): + first_db = list(datasette.databases.keys())[0] + return ( + await datasette.execute(first_db, sql) + ).rows[0][0] async def inner(): return { "extra_template_vars_from_awaitable": json.dumps({ "template": template, "scope_path": request.scope["path"], "awaitable": True, - }, default=lambda b: b.decode("utf8")) + }, default=lambda b: b.decode("utf8")), + "query_database": query_database, } return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index b1c7fd9a..42d063f4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import app_client, make_app_client, TEMP_PLUGIN_SECRET_FILE # noqa +from datasette.utils import sqlite3 import base64 import json import os @@ -214,3 +215,20 @@ def test_plugins_extra_template_vars(restore_working_directory): "awaitable": True, "scope_path": "/-/metadata", } == extra_template_vars_from_awaitable + + +def test_plugins_async_template_function(restore_working_directory): + for client in make_app_client( + template_dir=str(pathlib.Path(__file__).parent / "test_templates") + ): + response = client.get("/-/metadata") + assert response.status == 200 + extra_from_awaitable_function = ( + Soup(response.body, "html.parser") + .select("pre.extra_from_awaitable_function")[0] + .text + ) + expected = ( + sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] + ) + assert expected == extra_from_awaitable_function diff --git a/tests/test_templates/show_json.html b/tests/test_templates/show_json.html index bbf1bc06..cff04fb4 100644 --- a/tests/test_templates/show_json.html +++ b/tests/test_templates/show_json.html @@ -5,4 +5,5 @@ Test data for extra_template_vars: 
    {{ extra_template_vars|safe }}
    {{ extra_template_vars_from_awaitable|safe }}
    +
    {{ query_database("select sqlite_version();") }}
    {% endblock %} From a95bedb9c423fa6d772c93ef47bc40f13a5bea50 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Nov 2019 15:18:53 -0800 Subject: [PATCH 0279/2303] Release notes for 0.32 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4958399..2f909364 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_32: + +0.32 (2019-11-14) +----------------- + +Datasette now renders templates using `Jinja async mode `__. This makes it easy for plugins to provide custom template functions that perform asynchronous actions, for example the new `datasette-template-sql `__ plugin which allows custom templates to directly execute SQL queries and render their results. (`#628 `__) + .. _v0_31_2: 0.31.2 (2019-11-13) From 8fc9a5d877d26dbf2654e125f407ddd2fd767335 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Nov 2019 15:46:37 -0800 Subject: [PATCH 0280/2303] Datasette 0.32 and datasette-template-sql in news --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a22c2b2..030c507f 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 14th November 2019: [Datasette 0.32](https://datasette.readthedocs.io/en/stable/changelog.html#v0-32) now uses asynchronous rendering in Jinja templates, which means template functions can perform asynchronous operations such as executing SQL queries. [datasette-template-sql](https://github.com/simonw/datasette-template-sql) is a new plugin uses this capability to add a new custom `sql(sql_query)` template function. * 11th November 2019: [Datasette 0.31](https://datasette.readthedocs.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30) * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. From a9909c29ccac771c23c2ef22b89d10697b5256b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Nov 2019 14:49:45 -0800 Subject: [PATCH 0281/2303] Move .execute() from Datasette to Database Refs #569 - I split this change out from #579 --- datasette/app.py | 90 ++++++--------------------- datasette/database.py | 137 +++++++++++++++++++++++++++++++----------- 2 files changed, 121 insertions(+), 106 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 02fcf303..119d0e19 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -24,13 +24,11 @@ from .database import Database from .utils import ( QueryInterrupted, - Results, escape_css_string, escape_sqlite, get_plugins, module_from_path, sqlite3, - sqlite_timelimit, to_css_class, ) from .utils.asgi import ( @@ -42,13 +40,12 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) -from .tracer import trace, AsgiTracer +from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS from .version import __version__ app_root = Path(__file__).parent.parent -connections = threading.local() MEMORY = object() ConfigOption = collections.namedtuple("ConfigOption", ("name", "default", "help")) @@ -336,6 +333,25 @@ class Datasette: # pylint: disable=no-member pm.hook.prepare_connection(conn=conn) + async def execute( + self, + db_name, + sql, + params=None, + truncate=False, + custom_time_limit=None, + page_size=None, + log_sql_errors=True, + ): + return await self.databases[db_name].execute( + sql, + params=params, + truncate=truncate, + custom_time_limit=custom_time_limit, + page_size=page_size, + log_sql_errors=log_sql_errors, + ) + async def expand_foreign_keys(self, database, table, column, values): "Returns dict mapping (column, value) -> label" labeled_fks = {} @@ -477,72 +493,6 @@ class Datasette: .get(table, {}) ) - async def execute_against_connection_in_thread(self, db_name, fn): - def in_thread(): - conn = getattr(connections, db_name, None) - if not conn: - conn = self.databases[db_name].connect() - self.prepare_connection(conn) - setattr(connections, db_name, conn) - return fn(conn) - - return await asyncio.get_event_loop().run_in_executor(self.executor, in_thread) - - async def execute( - self, - db_name, - sql, - params=None, - truncate=False, - custom_time_limit=None, - page_size=None, - log_sql_errors=True, - ): - """Executes sql against db_name in a thread""" - page_size = page_size or self.page_size - - def sql_operation_in_thread(conn): - time_limit_ms = self.sql_time_limit_ms - if custom_time_limit and custom_time_limit < time_limit_ms: - time_limit_ms = custom_time_limit - - with sqlite_timelimit(conn, time_limit_ms): - try: - cursor = conn.cursor() - cursor.execute(sql, params or {}) - max_returned_rows = self.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except sqlite3.OperationalError as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - if log_sql_errors: - print( - "ERROR: conn={}, sql = {}, params = {}: {}".format( - conn, repr(sql), params, e - ) - ) - raise - - if truncate: - return Results(rows, truncated, cursor.description) - - else: - return Results(rows, False, cursor.description) - - with trace("sql", database=db_name, sql=sql.strip(), params=params): - results = await self.execute_against_connection_in_thread( - db_name, sql_operation_in_thread - ) - return results - def register_renderers(self): """ Register output renderers which output data in custom formats. """ # Built-in renderers diff --git a/datasette/database.py b/datasette/database.py index 3a1cea94..9a8ae4d4 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,17 +1,25 @@ +import asyncio +import contextlib from pathlib import Path +import threading +from .tracer import trace from .utils import ( QueryInterrupted, + Results, detect_fts, detect_primary_keys, detect_spatialite, get_all_foreign_keys, get_outbound_foreign_keys, + sqlite_timelimit, sqlite3, table_columns, ) from .inspect import inspect_hash +connections = threading.local() + class Database: def __init__(self, ds, path=None, is_mutable=False, is_memory=False): @@ -45,6 +53,73 @@ class Database: "file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False ) + async def execute_against_connection_in_thread(self, fn): + def in_thread(): + conn = getattr(connections, self.name, None) + if not conn: + conn = self.connect() + self.ds.prepare_connection(conn) + setattr(connections, self.name, conn) + return fn(conn) + + return await asyncio.get_event_loop().run_in_executor( + self.ds.executor, in_thread + ) + + async def execute( + self, + sql, + params=None, + truncate=False, + custom_time_limit=None, + page_size=None, + log_sql_errors=True, + ): + """Executes sql against db_name in a thread""" + page_size = page_size or self.ds.page_size + + def sql_operation_in_thread(conn): + time_limit_ms = self.ds.sql_time_limit_ms + if custom_time_limit and custom_time_limit < time_limit_ms: + time_limit_ms = custom_time_limit + + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params or {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except sqlite3.OperationalError as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + print( + "ERROR: conn={}, sql = {}, params = {}: {}".format( + conn, repr(sql), params, e + ) + ) + raise + + if truncate: + return Results(rows, truncated, cursor.description) + + else: + return Results(rows, False, cursor.description) + + with trace("sql", database=self.name, sql=sql.strip(), params=params): + results = await self.execute_against_connection_in_thread( + sql_operation_in_thread + ) + return results + @property def size(self): if self.is_memory: @@ -62,8 +137,7 @@ class Database: for table in await self.table_names(): try: table_count = ( - await self.ds.execute( - self.name, + await self.execute( "select count(*) from [{}]".format(table), custom_time_limit=limit, ) @@ -89,32 +163,30 @@ class Database: return Path(self.path).stem async def table_exists(self, table): - results = await self.ds.execute( - self.name, - "select 1 from sqlite_master where type='table' and name=?", - params=(table,), + results = await self.execute( + "select 1 from sqlite_master where type='table' and name=?", params=(table,) ) return bool(results.rows) async def table_names(self): - results = await self.ds.execute( - self.name, "select name from sqlite_master where type='table'" + results = await self.execute( + "select name from sqlite_master where type='table'" ) return [r[0] for r in results.rows] async def table_columns(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: table_columns(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: table_columns(conn, table) ) async def primary_keys(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: detect_primary_keys(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: detect_primary_keys(conn, table) ) async def fts_table(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: detect_fts(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: detect_fts(conn, table) ) async def label_column_for_table(self, table): @@ -124,8 +196,8 @@ class Database: if explicit_label_column: return explicit_label_column # If a table has two columns, one of which is ID, then label_column is the other one - column_names = await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: table_columns(conn, table) + column_names = await self.execute_against_connection_in_thread( + lambda conn: table_columns(conn, table) ) # Is there a name or title column? name_or_title = [c for c in column_names if c in ("name", "title")] @@ -141,8 +213,8 @@ class Database: return None async def foreign_keys_for_table(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: get_outbound_foreign_keys(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: get_outbound_foreign_keys(conn, table) ) async def hidden_table_names(self): @@ -150,18 +222,17 @@ class Database: hidden_tables = [ r[0] for r in ( - await self.ds.execute( - self.name, + await self.execute( """ select name from sqlite_master where rootpage = 0 and sql like '%VIRTUAL TABLE%USING FTS%' - """, + """ ) ).rows ] - has_spatialite = await self.ds.execute_against_connection_in_thread( - self.name, detect_spatialite + has_spatialite = await self.execute_against_connection_in_thread( + detect_spatialite ) if has_spatialite: # Also hide Spatialite internal tables @@ -178,13 +249,12 @@ class Database: ] + [ r[0] for r in ( - await self.ds.execute( - self.name, + await self.execute( """ select name from sqlite_master where name like "idx_%" and type = "table" - """, + """ ) ).rows ] @@ -207,25 +277,20 @@ class Database: return hidden_tables async def view_names(self): - results = await self.ds.execute( - self.name, "select name from sqlite_master where type='view'" - ) + results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] async def get_all_foreign_keys(self): - return await self.ds.execute_against_connection_in_thread( - self.name, get_all_foreign_keys - ) + return await self.execute_against_connection_in_thread(get_all_foreign_keys) async def get_outbound_foreign_keys(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: get_outbound_foreign_keys(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: get_outbound_foreign_keys(conn, table) ) async def get_table_definition(self, table, type_="table"): table_definition_rows = list( - await self.ds.execute( - self.name, + await self.execute( "select sql from sqlite_master where name = :n and type=:t", {"n": table, "t": type_}, ) From 440a70428c624f6e27b630026acdba2032acc9a7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Nov 2019 15:01:10 -0800 Subject: [PATCH 0282/2303] Include rowid in filter select, closes #636 --- datasette/views/table.py | 6 +----- tests/test_html.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index a60a3941..516b474d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -587,10 +587,6 @@ class TableView(RowTableShared): columns = [r[0] for r in results.description] rows = list(results.rows) - filter_columns = columns[:] - if use_rowid and filter_columns[0] == "rowid": - filter_columns = filter_columns[1:] - # Expand labeled columns if requested expanded_columns = [] expandable_columns = await self.expandable_columns(database, table) @@ -720,7 +716,7 @@ class TableView(RowTableShared): "use_rowid": use_rowid, "filters": filters, "display_columns": display_columns, - "filter_columns": filter_columns, + "filter_columns": columns, "display_rows": display_rows, "facets_timed_out": facets_timed_out, "sorted_facet_results": sorted( diff --git a/tests/test_html.py b/tests/test_html.py index 44627cdc..3b331f38 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -679,6 +679,30 @@ def test_table_html_foreign_key_custom_label_column(app_client): ] +@pytest.mark.parametrize( + "path,expected_column_options", + [ + ("/fixtures/infinity", ["- column -", "rowid", "value"]), + ( + "/fixtures/primary_key_multiple_columns", + ["- column -", "id", "content", "content2"], + ), + ("/fixtures/compound_primary_key", ["- column -", "pk1", "pk2", "content"]), + ], +) +def test_table_html_filter_form_column_options( + path, expected_column_options, app_client +): + response = app_client.get(path) + assert response.status == 200 + form = Soup(response.body, "html.parser").find("form") + column_options = [ + o.attrs.get("value") or o.string + for o in form.select("select[name=_filter_column] option") + ] + assert expected_column_options == column_options + + def test_row_html_compound_primary_key(app_client): response = app_client.get("/fixtures/compound_primary_key/a,b") assert response.status == 200 From c16be14517414a94e1fdbd888e8a3ad0669e3bca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 Nov 2019 10:02:07 -0800 Subject: [PATCH 0283/2303] How to upgrade using Docker --- docs/installation.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 9ee7eb4e..c547f9e4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -33,6 +33,10 @@ Now visit http://127.0.0.1:8001/ to access Datasette. (You can download a copy of ``fixtures.db`` from https://latest.datasette.io/fixtures.db ) +To upgrade to the most recent release of Datasette, run the following:: + + docker pull datasetteproject/datasette + Loading Spatialite ~~~~~~~~~~~~~~~~~~ From fd137da7f83c117b18e189707a1039e319dd5c91 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Nov 2019 16:56:55 -0800 Subject: [PATCH 0284/2303] Suggest column facet only if at least one count > 1 Fixes #638 --- datasette/facets.py | 5 ++++- tests/fixtures.py | 33 +++++++++++++++++---------------- tests/test_api.py | 30 ++++++++++++++++++++++++++++-- tests/test_csv.py | 32 ++++++++++++++++---------------- 4 files changed, 65 insertions(+), 35 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 0c6459d6..a314faaf 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -143,9 +143,10 @@ class ColumnFacet(Facet): if column in already_enabled: continue suggested_facet_sql = """ - select distinct {column} from ( + select {column}, count(*) as n from ( {sql} ) where {column} is not null + group by {column} limit {limit} """.format( column=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 @@ -165,6 +166,8 @@ class ColumnFacet(Facet): and num_distinct_values > 1 and num_distinct_values <= facet_size and num_distinct_values < row_count + # And at least one has n > 1 + and any(r["n"] > 1 for r in distinct_values) ): suggested_facets.append( { diff --git a/tests/fixtures.py b/tests/fixtures.py index 3e4203f7..bb01d171 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -669,26 +669,27 @@ CREATE TABLE facetable ( neighborhood text, tags text, complex_array text, + distinct_some_null, FOREIGN KEY ("city_id") REFERENCES [facet_cities](id) ); INSERT INTO facetable - (created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array) + (created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null) VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]'), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]') + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null) ; CREATE TABLE binary_data ( diff --git a/tests/test_api.py b/tests/test_api.py index 1fa8642f..34eef4ce 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -197,6 +197,7 @@ def test_database_page(app_client): "neighborhood", "tags", "complex_array", + "distinct_some_null", ], "primary_keys": ["pk"], "count": 15, @@ -1042,15 +1043,38 @@ def test_table_filter_json_arraycontains(app_client): "Mission", '["tag1", "tag2"]', '[{"foo": "bar"}]', + "one", + ], + [ + 2, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Dogpatch", + '["tag1", "tag3"]', + "[]", + "two", ], - [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]', "[]"], ] == response.json["rows"] def test_table_filter_extra_where(app_client): response = app_client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'") assert [ - [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]', "[]"] + [ + 2, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Dogpatch", + '["tag1", "tag3"]', + "[]", + "two", + ] ] == response.json["rows"] @@ -1503,6 +1527,7 @@ def test_expand_labels(app_client): "neighborhood": "Dogpatch", "tags": '["tag1", "tag3"]', "complex_array": "[]", + "distinct_some_null": "two", }, "13": { "pk": 13, @@ -1514,6 +1539,7 @@ def test_expand_labels(app_client): "neighborhood": "Corktown", "tags": "[]", "complex_array": "[]", + "distinct_some_null": None, }, } == response.json diff --git a/tests/test_csv.py b/tests/test_csv.py index 13aca489..1030c2bb 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -21,22 +21,22 @@ world ) EXPECTED_TABLE_WITH_LABELS_CSV = """ -pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,complex_array -1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]" -2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[] -3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[] -4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[] -5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[] -6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[] -7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[] -8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[] -9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[] -10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[] -11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[] -12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[] -13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[] -14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[] -15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[] +pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,complex_array,distinct_some_null +1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one +2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two +3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[], +4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[], +5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[], +6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[], +7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[], +8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[], +9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[], +10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[], +11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[], +12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[], +13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[], +14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[], +15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[], """.lstrip().replace( "\n", "\r\n" ) From d3e1c3017ee2f606a731208d59fe48805cdc3259 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Nov 2019 22:07:01 -0800 Subject: [PATCH 0285/2303] Display 0 results, closes #637 --- datasette/static/app.css | 7 +++++ datasette/templates/_table.html | 56 ++++++++++++++++++--------------- datasette/templates/query.html | 2 ++ tests/test_html.py | 14 +++++++++ 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 34eb122c..d7cf6334 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -327,3 +327,10 @@ a.not-underlined { pre.wrapped-sql { white-space: pre-wrap; } + +p.zero-results { + border: 2px solid #ccc; + background-color: #eee; + padding: 0.5em; + font-style: italic; +} diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index c7a72253..42c37c55 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,28 +1,32 @@ - - - - {% for column in display_columns %} - + {% endfor %} + + + + {% for row in display_rows %} + + {% for cell in row %} + + {% endfor %} + + {% endfor %} + +
    - {% if not column.sortable %} - {{ column.name }} - {% else %} - {% if column.name == sort %} - {{ column.name }} ▼ +{% if display_rows %} + + + + {% for column in display_columns %} + - {% endfor %} - - - - {% for row in display_rows %} - - {% for cell in row %} - - {% endfor %} - - {% endfor %} - -
    + {% if not column.sortable %} + {{ column.name }} {% else %} - {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %} + {% if column.name == sort %} + {{ column.name }} ▼ + {% else %} + {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %} + {% endif %} {% endif %} - {% endif %} -
    {{ cell.value }}
    +
    {{ cell.value }}
    +{% else %} +

    0 records

    +{% endif %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 34fa78a5..f10ff000 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -73,6 +73,8 @@ {% endfor %} +{% else %} +

    0 results

    {% endif %} {% include "_codemirror_foot.html" %} diff --git a/tests/test_html.py b/tests/test_html.py index 3b331f38..db73da18 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1059,3 +1059,17 @@ def test_custom_table_include(): '1 - 2 - hello 1' "" ) == str(Soup(response.text, "html.parser").select_one("div.custom-table-row")) + + +@pytest.mark.parametrize( + "path", + [ + "/fixtures?sql=select+*+from+[123_starts_with_digits]", + "/fixtures/123_starts_with_digits", + ], +) +def test_zero_results(app_client, path): + response = app_client.get(path) + soup = Soup(response.text, "html.parser") + assert 0 == len(soup.select("table")) + assert 1 == len(soup.select("p.zero-results")) From aca41618f8761f99c47c8ae8e81b07a6d4af4d7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 Nov 2019 09:04:39 -0800 Subject: [PATCH 0286/2303] index view is also important for plugin hooks --- docs/plugins.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index e5a3d7dd..9bceb961 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -583,7 +583,7 @@ Extra JavaScript to be added to a `` - - - + + + + + + + +
    +

    Pattern Portfolio

    +
    +

    .hd for /database/table/row

    + +

    .bd for /

    +
    +

    Datasette Fixtures

    +
    + An example SQLite database demonstrating Datasette +
    +

    + Data license: + Apache License 2.0 + · + Data source: + + tests/fixtures.py + · + About: + + About Datasette +

    +

    fixtures

    +

    + 1,258 rows in 24 tables, 206 rows in 5 hidden tables, 4 views +

    +

    compound_three_primary_keys, sortable, facetable, roadside_attraction_characteristics, simple_primary_key, ...

    +

    data

    +

    + 6 rows in 2 tables +

    +

    names, foo

    +
    +

    .bd for /database

    +
    +

    fixtures

    +
    + Test tables description +
    +

    + Data license: + Apache License 2.0 + · + Data source: + + tests/fixtures.py + · + About: + + About Datasette +

    +
    +

    Custom SQL query

    +

    +

    + + +

    +
    +
    +

    123_starts_with_digits

    +

    content

    +

    0 rows

    +
    +
    +

    Table With Space In Name

    +

    pk, content

    +

    0 rows

    +
    +
    +

    attraction_characteristic

    +

    pk, name

    +

    2 rows

    +
    +
    +

    .bd for /database/table

    +
    +

    roadside_attraction_characteristics

    +

    + Data license: + Apache License 2.0 + · + Data source: + + tests/fixtures.py + · + About: + + About Datasette +

    +

    3 rows + where characteristic_id = 2 +

    +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    + + +
    +
    +

    ✎ View and edit SQL

    +

    This data as json, CSV (advanced)

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Link + + rowid ▼ + + attraction_id + + characteristic_id +
    11The Mystery Spot 1Paranormal 2
    22Winchester Mystery House 2Paranormal 2
    33Bigfoot Discovery Museum 4Paranormal 2
    +
    +

    Advanced export

    +

    JSON shape: + default, + array, + newline-delimited +

    +
    +

    + CSV options: + + + + + +

    +
    +
    + 
    CREATE TABLE roadside_attraction_characteristics (
+    attraction_id INTEGER REFERENCES roadside_attractions(pk),
+    characteristic_id INTEGER REFERENCES attraction_characteristic(pk)
+);
    +
    +

    .bd for /database/table/row

    +
    +

    roadside_attractions: 2

    +

    This data as json

    + + + + + + + + + + + + + + + + + + + +
    + pk + + name + + address + + latitude + + longitude +
    2Winchester Mystery House525 South Winchester Boulevard, San Jose, CA 9512837.3184-121.9511
    +

    Links from other tables

    +
      +
    • + + 1 row + from attraction_id in roadside_attraction_characteristics +
      • +
    +
    +

    .ft

    +
    Powered by Datasette + · Data license: + Apache License 2.0 + · + Data source: + + tests/fixtures.py + · + About: + + About Datasette +
    + + diff --git a/datasette/views/base.py b/datasette/views/base.py index 2478bd84..e2bce2f9 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -72,7 +72,8 @@ class BaseView(AsgiView): def database_color(self, database): return "ff0000" - async def render(self, templates, request, context): + async def render(self, templates, request, context=None): + context = context or {} template = self.ds.jinja_env.select_template(templates) template_context = { **context, diff --git a/datasette/views/special.py b/datasette/views/special.py index 2b31028d..dfe5ea8c 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -32,3 +32,13 @@ class JsonDataView(BaseView): "data_json": json.dumps(data, indent=4), }, ) + + +class PatternPortfolioView(BaseView): + name = "patterns" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + return await self.render(["patterns.html"], request=request,) diff --git a/tests/test_docs.py b/tests/test_docs.py index d7c5a534..77c2a611 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -65,6 +65,8 @@ def documented_views(): first_word = label.split("_")[0] if first_word.endswith("View"): view_labels.add(first_word) + # We deliberately don't document this one: + view_labels.add("PatternPortfolioView") return view_labels From d996d4122b522eeec3c610f6b2561aa96652ecd2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:46:49 -0700 Subject: [PATCH 0385/2303] Add badges to documentation index --- docs/index.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 070b6f64..2390e263 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,22 @@ Datasette ========= +|PyPI| |Changelog| |Python 3.x| |Travis CI| |License| |docker: +datasette| + +.. |PyPI| image:: https://img.shields.io/pypi/v/datasette.svg + :target: https://pypi.org/project/datasette/ +.. |Changelog| image:: https://img.shields.io/github/v/release/simonw/datasette?include_prereleases&label=changelog + :target: https://datasette.readthedocs.io/en/stable/changelog.html +.. |Python 3.x| image:: https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white + :target: https://pypi.org/project/datasette/ +.. |Travis CI| image:: https://travis-ci.org/simonw/datasette.svg?branch=master + :target: https://travis-ci.org/simonw/datasette +.. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg + :target: https://github.com/simonw/datasette/blob/master/LICENSE +.. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue + :target: https://hub.docker.com/r/datasetteproject/datasette + *A tool for exploring and publishing data* Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API. From 985e59493e44d6fcebf7a30f693f4edecee3e90d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 09:17:48 -0700 Subject: [PATCH 0386/2303] Update aiofiles requirement from ~=0.4.0 to >=0.4,<0.6 (#725) Refs #754 Updates the requirements on [aiofiles](https://github.com/Tinche/aiofiles) to permit the latest version. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.4.0...v0.5.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0de00dc5..0519fa8a 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ setup( "pint~=0.9", "pluggy~=0.13.0", "uvicorn~=0.11", - "aiofiles~=0.4.0", + "aiofiles>=0.4,<0.6", "janus~=0.4.0", "PyYAML~=5.3", "mergedeep~=1.1.1", From e232f77055880b38cc0b738607cd50cde9188eaf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 09:45:49 -0700 Subject: [PATCH 0387/2303] Update mergedeep requirement from ~=1.1.1 to >=1.1.1,<1.4.0 (#728) Updates the requirements on [mergedeep](https://github.com/clarketm/mergedeep) to permit the latest version. - [Release notes](https://github.com/clarketm/mergedeep/releases) - [Commits](https://github.com/clarketm/mergedeep/compare/v1.1.1...v1.3.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0519fa8a..8b54a97b 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ setup( "aiofiles>=0.4,<0.6", "janus~=0.4.0", "PyYAML~=5.3", - "mergedeep~=1.1.1", + "mergedeep>=1.1.1,<1.4.0", ], entry_points=""" [console_scripts] From 109c5a430d53fe38b1300e0daa20f5cef047a08e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 09:48:03 -0700 Subject: [PATCH 0388/2303] Update janus requirement from ~=0.4.0 to >=0.4,<0.6 (#734) Updates the requirements on [janus](https://github.com/aio-libs/janus) to permit the latest version. - [Release notes](https://github.com/aio-libs/janus/releases) - [Changelog](https://github.com/aio-libs/janus/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/janus/compare/v0.4.0...v0.5.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b54a97b..69ac0939 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( "pluggy~=0.13.0", "uvicorn~=0.11", "aiofiles>=0.4,<0.6", - "janus~=0.4.0", + "janus>=0.4,<0.6", "PyYAML~=5.3", "mergedeep>=1.1.1,<1.4.0", ], From aa064de3f400899dbf61f2d33a035fba4017596c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 10:13:15 -0700 Subject: [PATCH 0389/2303] Update jinja2 requirement from ~=2.10.3 to >=2.10.3,<2.12.0 (#722) Updates the requirements on [jinja2](https://github.com/pallets/jinja) to permit the latest version. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.10.3...2.11.1) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 69ac0939..0762ace6 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( install_requires=[ "click~=7.1.1", "click-default-group~=1.2.2", - "Jinja2~=2.10.3", + "Jinja2>=2.10.3,<2.12.0", "hupper~=1.9", "pint~=0.9", "pluggy~=0.13.0", From c91fb9e3d4f0632d4ef25a21165739ab88a9d491 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 10:13:41 -0700 Subject: [PATCH 0390/2303] Update pytest requirement from ~=5.2.2 to >=5.2.2,<5.5.0 (#721) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...5.4.1) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0762ace6..6e4d423a 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ - "pytest~=5.2.2", + "pytest>=5.2.2,<5.5.0", "pytest-asyncio~=0.10.0", "aiohttp~=3.6.2", "beautifulsoup4~=4.8.1", From dbd2d70b3819a7041bb36a527033d77c85683c05 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Mon, 4 May 2020 13:14:25 -0400 Subject: [PATCH 0391/2303] asgi: check raw_path is not None (#719) The ASGI spec (https://asgi.readthedocs.io/en/latest/specs/www.html#http) seems to imply that `None` is a valid value, so we need to check the value itself, not just whether the key is present. In particular, the [mangum](https://github.com/erm/mangum) adapter passes `None` for this key. --- datasette/utils/asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index df358240..73ae562b 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -46,7 +46,7 @@ class Request: @property def path(self): - if "raw_path" in self.scope: + if self.scope.get("raw_path") is not None: return self.scope["raw_path"].decode("latin-1") else: path = self.scope["path"] From 707fe039947b3e48f2b6dcfe8e577d76b617f2a5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 10:14:46 -0700 Subject: [PATCH 0392/2303] Update beautifulsoup4 requirement from ~=4.8.1 to >=4.8.1,<4.10.0 (#720) Updates the requirements on [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) to permit the latest version. Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e4d423a..19f5b4e3 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ setup( "pytest>=5.2.2,<5.5.0", "pytest-asyncio~=0.10.0", "aiohttp~=3.6.2", - "beautifulsoup4~=4.8.1", + "beautifulsoup4>=4.8.1,<4.10.0", "asgiref~=3.2.3", "black~=19.10b0", ], From b314e088c59425122fb2b2abde8741010d9d274a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 10:40:48 -0700 Subject: [PATCH 0393/2303] Update pytest-asyncio requirement from ~=0.10.0 to >=0.10,<0.13 (#753) Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.10.0...v0.12.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 19f5b4e3..c4886a11 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ "pytest>=5.2.2,<5.5.0", - "pytest-asyncio~=0.10.0", + "pytest-asyncio>=0.10,<0.13", "aiohttp~=3.6.2", "beautifulsoup4>=4.8.1,<4.10.0", "asgiref~=3.2.3", From 450d2e2896e07a8ce27f1cf143febe280c97301b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2020 10:40:01 -0700 Subject: [PATCH 0394/2303] Fixed pytest warning about TestClient class --- tests/test_config_dir.py | 6 +++--- tests/test_plugins.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index f262cc59..50e67f80 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -3,7 +3,7 @@ import pytest import sqlite3 from datasette.app import Datasette -from .fixtures import TestClient +from .fixtures import TestClient as _TestClient PLUGIN = """ from datasette import hookimpl @@ -76,7 +76,7 @@ def config_dir_client(tmp_path_factory): ) ds = Datasette([], config_dir=config_dir) - client = TestClient(ds.app()) + client = _TestClient(ds.app()) client.ds = ds yield client @@ -137,7 +137,7 @@ def test_metadata_yaml(tmp_path_factory, filename): config_dir = tmp_path_factory.mktemp("yaml-config-dir") (config_dir / filename).write_text("title: Title from metadata", "utf-8") ds = Datasette([], config_dir=config_dir) - client = TestClient(ds.app()) + client = _TestClient(ds.app()) client.ds = ds response = client.get("/-/metadata.json") assert 200 == response.status diff --git a/tests/test_plugins.py b/tests/test_plugins.py index fce6fd77..8b6a6b41 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -4,7 +4,7 @@ from .fixtures import ( make_app_client, TABLES, TEMP_PLUGIN_SECRET_FILE, - TestClient, + TestClient as _TestClient, ) # noqa from datasette.app import Datasette from datasette.plugins import get_plugins, DEFAULT_PLUGINS @@ -293,7 +293,7 @@ def view_names_client(tmp_path_factory): db_path = str(tmpdir / "fixtures.db") conn = sqlite3.connect(db_path) conn.executescript(TABLES) - return TestClient( + return _TestClient( Datasette( [db_path], template_dir=str(templates), plugins_dir=str(plugins) ).app() From 9424687e9e94401438896116898a071702b09d40 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2020 10:41:58 -0700 Subject: [PATCH 0395/2303] Consistently return charset utf-8, closes #752 --- datasette/app.py | 2 +- datasette/utils/asgi.py | 16 ++++++++++------ docs/custom_templates.rst | 4 ++-- tests/test_html.py | 6 ++++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b541a9a4..8a4b6011 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -828,7 +828,7 @@ class DatasetteRouter(AsgiRouter): view_name="page", ) # Pull content-type out into separate parameter - content_type = "text/html" + content_type = "text/html; charset=utf-8" matches = [k for k in headers if k.lower() == "content-type"] if matches: content_type = headers[matches[0]] diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 73ae562b..20047bb5 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -130,7 +130,7 @@ class AsgiRouter: { "type": "http.response.start", "status": 404, - "headers": [[b"content-type", b"text/html"]], + "headers": [[b"content-type", b"text/html; charset=utf-8"]], } ) await send({"type": "http.response.body", "body": b"

    404

    "}) @@ -140,11 +140,11 @@ class AsgiRouter: { "type": "http.response.start", "status": 404, - "headers": [[b"content-type", b"text/html"]], + "headers": [[b"content-type", b"text/html; charset=utf-8"]], } ) html = "

    500

    ".format(escape(repr(exception))) - await send({"type": "http.response.body", "body": html.encode("latin-1")}) + await send({"type": "http.response.body", "body": html.encode("utf-8")}) class AsgiLifespan: @@ -259,7 +259,11 @@ async def asgi_send_json(send, info, status=200, headers=None): async def asgi_send_html(send, html, status=200, headers=None): headers = headers or {} await asgi_send( - send, html, status=status, headers=headers, content_type="text/html" + send, + html, + status=status, + headers=headers, + content_type="text/html; charset=utf-8", ) @@ -269,13 +273,13 @@ async def asgi_send_redirect(send, location, status=302): "", status=status, headers={"Location": location}, - content_type="text/html", + content_type="text/html; charset=utf-8", ) async def asgi_send(send, content, status, headers=None, content_type="text/plain"): await asgi_start(send, status, headers, content_type) - await send({"type": "http.response.body", "body": content.encode("latin-1")}) + await send({"type": "http.response.body", "body": content.encode("utf-8")}) async def asgi_start(send, status, headers=None, content_type="text/plain"): diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 142ecc97..adbfbc25 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -284,7 +284,7 @@ You can nest directories within pages to create a nested structure. To create a Custom headers and status codes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Custom pages default to being served with a content-type of ``text/html`` and a ``200`` status code. You can change these by calling a custom function from within your template. +Custom pages default to being served with a content-type of ``text/html; charset=utf-8`` and a ``200`` status code. You can change these by calling a custom function from within your template. For example, to serve a custom page with a ``418 I'm a teapot`` HTTP status code, create a file in ``pages/teapot.html`` containing the following:: @@ -314,7 +314,7 @@ You can verify this is working using ``curl`` like this:: date: Sun, 26 Apr 2020 18:38:30 GMT server: uvicorn x-teapot: I am - content-type: text/html + content-type: text/html; charset=utf-8 Custom redirects ~~~~~~~~~~~~~~~~ diff --git a/tests/test_html.py b/tests/test_html.py index b8dc543c..564365ce 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -963,6 +963,12 @@ def test_404_trailing_slash_redirect(app_client, path, expected_redirect): assert expected_redirect == response.headers["Location"] +def test_404_content_type(app_client): + response = app_client.get("/404") + assert 404 == response.status + assert "text/html; charset=utf-8" == response.headers["content-type"] + + def test_canned_query_with_custom_metadata(app_client): response = app_client.get("/fixtures/neighborhood_search?text=town") assert response.status == 200 From cc872b1f50f1d2c0bc2d930c86a6644f154459dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2020 11:42:01 -0700 Subject: [PATCH 0396/2303] Fixed rogue output in tests, closes #755 --- tests/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_database.py b/tests/test_database.py index 35923c0b..a9728019 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -116,7 +116,7 @@ async def test_execute_write_fn_block_false(app_client): def write_fn(conn): with conn: - conn.execute("delete from roadside_attractions where id = 1;") + conn.execute("delete from roadside_attractions where pk = 1;") row = conn.execute("select count(*) from roadside_attractions").fetchone() print("row = ", row) return row[0] From 7e2bb314649baa9e782ad22ff452d90d46aa840b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2020 12:10:31 -0700 Subject: [PATCH 0397/2303] Documented installation using pipx, closes #756 --- docs/installation.rst | 87 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index c547f9e4..990d867b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -97,3 +97,90 @@ You can now run Datasette like so:: datasette fixtures.db If you want to start making contributions to the Datasette project by installing a copy that lets you directly modify the code, take a look at our guide to :ref:`devenvironment`. + +Install using pipx +------------------ + +`pipx `__ is a tool for installing Python software with all of its dependencies in an isolated environment, to ensure that they will not conflict with any other installed Python software. + +If you use `Homebrew `__ on macOS you can install pipx like this:: + + brew install pipx + pipx ensurepath + +Without Homebrew you can install it like so:: + + python3 -m pip install --user pipx + python3 -m pipx ensurepath + +The ``pipx ensurepath`` command configures your shell to ensure it can find commands that have been installed by pipx - generally by making sure ``~/.local/bin`` has been added to your ``PATH``. + +Once pipx is installed you can use it to install Datasette like this:: + + pipx install datasette + +Then run ``datasette --version`` to confirm that it has been successfully installed. + +Installing plugins using pipx +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette plugins need to be installed into the same environment as Datasette itself. You can do this using ``pipx inject datasette name-of-plugin`` - and then confirm that the plugin has been installed using the ``datasette plugins`` command:: + + $ datasette plugins + [] + + $ pipx inject datasette datasette-json-html + injected package datasette-json-html into venv datasette + done! ✨ 🌟 ✨ + + $ datasette plugins + [ + { + "name": "datasette-json-html", + "static": false, + "templates": false, + "version": "0.6" + } + ] + +Upgrading packages using pipx +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can upgrade your pipx installation to the latest release of Datasette using ``pipx upgrade datasette``:: + + $ pipx upgrade datasette + upgraded package datasette from 0.39 to 0.40 (location: /Users/simon/.local/pipx/venvs/datasette) + +To upgrade a plugin within the pipx environment use ``pipx runpip datasette install -U name-of-plugin`` - like this:: + + % datasette plugins + [ + { + "name": "datasette-vega", + "static": true, + "templates": false, + "version": "0.6" + } + ] + + $ pipx runpip datasette install -U datasette-vega + Collecting datasette-vega + Downloading datasette_vega-0.6.2-py3-none-any.whl (1.8 MB) + |████████████████████████████████| 1.8 MB 2.0 MB/s + ... + Installing collected packages: datasette-vega + Attempting uninstall: datasette-vega + Found existing installation: datasette-vega 0.6 + Uninstalling datasette-vega-0.6: + Successfully uninstalled datasette-vega-0.6 + Successfully installed datasette-vega-0.6.2 + + $ datasette plugins + [ + { + "name": "datasette-vega", + "static": true, + "templates": false, + "version": "0.6.2" + } + ] From 0cdf111ae68d46eb2eb51d85e20e1447a42cbdcc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2020 12:31:13 -0700 Subject: [PATCH 0398/2303] Move pip/pipx to top of installation instructions Less intimidating than Docker, hopefully. --- docs/installation.rst | 128 ++++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 61 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 990d867b..c88950c2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -11,67 +11,7 @@ on to your machine, or you can install it using Docker. .. contents:: -Using Docker ------------- - -A Docker image containing the latest release of Datasette is published to Docker -Hub here: https://hub.docker.com/r/datasetteproject/datasette/ - -If you have Docker installed (for example with `Docker for Mac -`_ on OS X) you can download and run this -image like so:: - - docker run -p 8001:8001 -v `pwd`:/mnt \ - datasetteproject/datasette \ - datasette -p 8001 -h 0.0.0.0 /mnt/fixtures.db - -This will start an instance of Datasette running on your machine's port 8001, -serving the ``fixtures.db`` file in your current directory. - -Now visit http://127.0.0.1:8001/ to access Datasette. - -(You can download a copy of ``fixtures.db`` from -https://latest.datasette.io/fixtures.db ) - -To upgrade to the most recent release of Datasette, run the following:: - - docker pull datasetteproject/datasette - -Loading Spatialite -~~~~~~~~~~~~~~~~~~ - -The ``datasetteproject/datasette`` image includes a recent version of the -:ref:`SpatiaLite extension ` for SQLite. To load and enable that -module, use the following command:: - - docker run -p 8001:8001 -v `pwd`:/mnt \ - datasetteproject/datasette \ - datasette -p 8001 -h 0.0.0.0 /mnt/fixtures.db \ - --load-extension=/usr/local/lib/mod_spatialite.so - -You can confirm that SpatiaLite is successfully loaded by visiting -http://127.0.0.1:8001/-/versions - -Installing plugins -~~~~~~~~~~~~~~~~~~ - -If you want to install plugins into your local Datasette Docker image you can do -so using the following recipe. This will install the plugins and then save a -brand new local image called ``datasette-with-plugins``:: - - docker run datasetteproject/datasette \ - pip install datasette-vega - - docker commit $(docker ps -lq) datasette-with-plugins - -You can now run the new custom image like so:: - - docker run -p 8001:8001 -v `pwd`:/mnt \ - datasette-with-plugins \ - datasette -p 8001 -h 0.0.0.0 /mnt/fixtures.db - -You can confirm that the plugins are installed by visiting -http://127.0.0.1:8001/-/plugins +.. _installation_pip: Install using pip ----------------- @@ -98,6 +38,8 @@ You can now run Datasette like so:: If you want to start making contributions to the Datasette project by installing a copy that lets you directly modify the code, take a look at our guide to :ref:`devenvironment`. +.. _installation_pipx: + Install using pipx ------------------ @@ -184,3 +126,67 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst "version": "0.6.2" } ] + +.. _installation_docker: + +Using Docker +------------ + +A Docker image containing the latest release of Datasette is published to Docker +Hub here: https://hub.docker.com/r/datasetteproject/datasette/ + +If you have Docker installed (for example with `Docker for Mac +`_ on OS X) you can download and run this +image like so:: + + docker run -p 8001:8001 -v `pwd`:/mnt \ + datasetteproject/datasette \ + datasette -p 8001 -h 0.0.0.0 /mnt/fixtures.db + +This will start an instance of Datasette running on your machine's port 8001, +serving the ``fixtures.db`` file in your current directory. + +Now visit http://127.0.0.1:8001/ to access Datasette. + +(You can download a copy of ``fixtures.db`` from +https://latest.datasette.io/fixtures.db ) + +To upgrade to the most recent release of Datasette, run the following:: + + docker pull datasetteproject/datasette + +Loading Spatialite +~~~~~~~~~~~~~~~~~~ + +The ``datasetteproject/datasette`` image includes a recent version of the +:ref:`SpatiaLite extension ` for SQLite. To load and enable that +module, use the following command:: + + docker run -p 8001:8001 -v `pwd`:/mnt \ + datasetteproject/datasette \ + datasette -p 8001 -h 0.0.0.0 /mnt/fixtures.db \ + --load-extension=/usr/local/lib/mod_spatialite.so + +You can confirm that SpatiaLite is successfully loaded by visiting +http://127.0.0.1:8001/-/versions + +Installing plugins +~~~~~~~~~~~~~~~~~~ + +If you want to install plugins into your local Datasette Docker image you can do +so using the following recipe. This will install the plugins and then save a +brand new local image called ``datasette-with-plugins``:: + + docker run datasetteproject/datasette \ + pip install datasette-vega + + docker commit $(docker ps -lq) datasette-with-plugins + +You can now run the new custom image like so:: + + docker run -p 8001:8001 -v `pwd`:/mnt \ + datasette-with-plugins \ + datasette -p 8001 -h 0.0.0.0 /mnt/fixtures.db + +You can confirm that the plugins are installed by visiting +http://127.0.0.1:8001/-/plugins From 9212f0c9c3138f005ea8d57acacb8a2a80b252a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2020 12:35:28 -0700 Subject: [PATCH 0399/2303] Removed note about virtual environments Simplifies things now that we also talk about pipx. --- docs/installation.rst | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index c88950c2..cdf1467a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -22,19 +22,13 @@ You can install Datasette and its dependencies using ``pip``:: pip install datasette -The last version to support Python 3.5 was 0.30.2 - you can install that version like so:: +The last version to support Python 3.5 was 0.30.2. If you are running Python 3.5 (check using ``python3 --version``) you can install that version of Datasette like so:: pip install datasette==0.30.2 -If you want to install Datasette in its own virtual environment, use this:: - - python -mvenv datasette-venv - source datasette-venv/bin/activate - pip install datasette - You can now run Datasette like so:: - datasette fixtures.db + datasette If you want to start making contributions to the Datasette project by installing a copy that lets you directly modify the code, take a look at our guide to :ref:`devenvironment`. From 0784f2ef9d3ff6dd9df05f54cb51de29a6d11764 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 May 2020 10:18:31 -0700 Subject: [PATCH 0400/2303] Allow specific pragma functions, closes #761 --- datasette/utils/__init__.py | 21 ++++++++++++++++++++- tests/test_utils.py | 5 ++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 490b71c8..f1c24041 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -171,7 +171,26 @@ allowed_sql_res = [ re.compile(r"^explain with\b"), re.compile(r"^explain query plan with\b"), ] -disallawed_sql_res = [(re.compile("pragma"), "Statement may not contain PRAGMA")] +allowed_pragmas = ( + "database_list", + "foreign_key_list", + "function_list", + "index_info", + "index_list", + "index_xinfo", + "page_count", + "max_page_count", + "page_size", + "schema_version", + "table_info", + "table_xinfo", +) +disallawed_sql_res = [ + ( + re.compile("pragma(?!_({}))".format("|".join(allowed_pragmas))), + "Statement may not contain PRAGMA", + ) +] def validate_sql_select(sql): diff --git a/tests/test_utils.py b/tests/test_utils.py index fe5d9a26..7e4f1a8e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -140,7 +140,8 @@ def test_custom_json_encoder(obj, expected): "update blah;", "-- sql comment to skip\nupdate blah;", "update blah set some_column='# Hello there\n\n* This is a list\n* of items\n--\n[And a link](https://github.com/simonw/datasette-render-markdown).'\nas demo_markdown", - "PRAGMA case_sensitive_like = true" "SELECT * FROM pragma_index_info('idx52')", + "PRAGMA case_sensitive_like = true", + "SELECT * FROM pragma_not_on_allow_list('idx52')", ], ) def test_validate_sql_select_bad(bad_sql): @@ -162,6 +163,8 @@ def test_validate_sql_select_bad(bad_sql): "WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", "explain WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", "explain query plan WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", + "SELECT * FROM pragma_index_info('idx52')", + "select * from pragma_table_xinfo('table')", ], ) def test_validate_sql_select_good(good_sql): From 182e5c8745c94576718315f7596ccc81e5e2417b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 May 2020 11:20:58 -0700 Subject: [PATCH 0401/2303] Release Datasette 0.41 Refs #648 #731 #750 #151 #761 #752 #719 #756 #748 --- README.md | 1 + docs/changelog.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/README.md b/README.md index 12a1ec39..f2a3d81d 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 6th May 2020: [Datasette 0.41](http://datasette.readthedocs.io/en/latest/changelog.html#v0-41) - New mechanism for [creating custom pages](https://datasette.readthedocs.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://datasette.readthedocs.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements. * 21st April 2020: [Datasette 0.40](http://datasette.readthedocs.io/en/latest/changelog.html#v0-40) - Metadata can now be provided as YAML instead of JSON. Publishing to Zeit Now v1 is no longer supported, but Now v2 support is provided by the new [datasette-publish-now](https://github.com/simonw/datasette-publish-now) plugin. Various bug fixes. * 24th March 2020: [Datasette 0.39](http://datasette.readthedocs.io/en/latest/changelog.html#v0-39) - New `base_url` configuration option for running Datasette under a different URL prefix, `"sort"` and `"sort_desc"` metadata options for setting a default sort order for a table. * 8th March 2020: [Datasette 0.38](http://datasette.readthedocs.io/en/latest/changelog.html#v0-38) - New `--memory` option for `datasete publish cloudrun`, [Docker image](https://hub.docker.com/r/datasetteproject/datasette) upgraded to SQLite 3.31.1. diff --git a/docs/changelog.rst b/docs/changelog.rst index 7a58f58b..dc06e4ef 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,35 @@ Changelog ========= +.. _v0_41: + +0.41 (2020-05-06) +----------------- + +You can now create :ref:`custom pages ` within your Datasette instance using a custom template file. For example, adding a template file called ``templates/pages/about.html`` will result in a new page being served at ``/about`` on your instance. See the :ref:`custom pages documentation ` for full details, including how to return custom HTTP headers, redirects and status codes. (`#648 `__) + +:ref:`config_dir` (`#731 `__) allows you to define a custom Datasette instance as a directory. So instead of running the following:: + + $ datasette one.db two.db \ + --metadata.json \ + --template-dir=templates/ \ + --plugins-dir=plugins \ + --static css:css + +You can instead arrange your files in a single directory called ``my-project`` and run this:: + + $ datasette my-project/ + +Also in this release: + +* New ``NOT LIKE`` table filter: ``?colname__notlike=expression``. (`#750 `__) +* Datasette now has a *pattern portfolio* at ``/-/patterns`` - e.g. https://latest.datasette.io/-/patterns. This is a page that shows every Datasette user interface component in one place, to aid core development and people building custom CSS themes. (`#151 `__) +* SQLite `PRAGMA functions `__ such as ``pragma_table_info(tablename)`` are now allowed in Datasette SQL queries. (`#761 `__) +* Datasette pages now consistently return a ``content-type`` of ``text/html; charset=utf-8"``. (`#752 `__) +* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibilty with the `Mangum `__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (`#719 `__) +* Installation documentation now covers how to :ref:`installation_pipx`. (`#756 `__) +* Improved the documentation for :ref:`full_text_search`. (`#748 `__) + .. _v0_40: 0.40 (2020-04-21) From 69e3a855dd7e5a77409d70b18c45ae3c1a145a75 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 07:16:39 -0700 Subject: [PATCH 0402/2303] Rename execute_against_connection_in_thread() to execute_fn(), refs #685 --- datasette/cli.py | 2 +- datasette/database.py | 32 ++++++++++---------------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 919be065..c59fb6e0 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -388,7 +388,7 @@ async def check_databases(ds): # to confirm they are all usable for database in list(ds.databases.values()): try: - await database.execute_against_connection_in_thread(check_connection) + await database.execute_fn(check_connection) except SpatialiteConnectionProblem: raise click.UsageError( "It looks like you're trying to load a SpatiaLite" diff --git a/datasette/database.py b/datasette/database.py index 48c367ef..0f540e01 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -101,7 +101,7 @@ class Database: result = e task.reply_queue.sync_q.put(result) - async def execute_against_connection_in_thread(self, fn): + async def execute_fn(self, fn): def in_thread(): conn = getattr(connections, self.name, None) if not conn: @@ -163,9 +163,7 @@ class Database: return Results(rows, False, cursor.description) with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_against_connection_in_thread( - sql_operation_in_thread - ) + results = await self.execute_fn(sql_operation_in_thread) return results @property @@ -223,19 +221,13 @@ class Database: return [r[0] for r in results.rows] async def table_columns(self, table): - return await self.execute_against_connection_in_thread( - lambda conn: table_columns(conn, table) - ) + return await self.execute_fn(lambda conn: table_columns(conn, table)) async def primary_keys(self, table): - return await self.execute_against_connection_in_thread( - lambda conn: detect_primary_keys(conn, table) - ) + return await self.execute_fn(lambda conn: detect_primary_keys(conn, table)) async def fts_table(self, table): - return await self.execute_against_connection_in_thread( - lambda conn: detect_fts(conn, table) - ) + return await self.execute_fn(lambda conn: detect_fts(conn, table)) async def label_column_for_table(self, table): explicit_label_column = self.ds.table_metadata(self.name, table).get( @@ -244,9 +236,7 @@ class Database: if explicit_label_column: return explicit_label_column # If a table has two columns, one of which is ID, then label_column is the other one - column_names = await self.execute_against_connection_in_thread( - lambda conn: table_columns(conn, table) - ) + column_names = await self.execute_fn(lambda conn: table_columns(conn, table)) # Is there a name or title column? name_or_title = [c for c in column_names if c in ("name", "title")] if name_or_title: @@ -261,7 +251,7 @@ class Database: return None async def foreign_keys_for_table(self, table): - return await self.execute_against_connection_in_thread( + return await self.execute_fn( lambda conn: get_outbound_foreign_keys(conn, table) ) @@ -279,9 +269,7 @@ class Database: ) ).rows ] - has_spatialite = await self.execute_against_connection_in_thread( - detect_spatialite - ) + has_spatialite = await self.execute_fn(detect_spatialite) if has_spatialite: # Also hide Spatialite internal tables hidden_tables += [ @@ -329,10 +317,10 @@ class Database: return [r[0] for r in results.rows] async def get_all_foreign_keys(self): - return await self.execute_against_connection_in_thread(get_all_foreign_keys) + return await self.execute_fn(get_all_foreign_keys) async def get_outbound_foreign_keys(self, table): - return await self.execute_against_connection_in_thread( + return await self.execute_fn( lambda conn: get_outbound_foreign_keys(conn, table) ) From 4433306c1855ad69840cc76cbd41086137572be2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 09:05:46 -0700 Subject: [PATCH 0403/2303] Improvements + docs for db.execute() and Results class * Including new results.first() and results.single_value() methods. Closes #685 --- datasette/app.py | 3 +- datasette/database.py | 39 +++++++++++++++++++++-- datasette/facets.py | 2 +- datasette/utils/__init__.py | 21 ------------- datasette/views/base.py | 2 +- datasette/views/table.py | 2 +- docs/internals.rst | 63 ++++++++++++++++++++++++++++++++++++- tests/test_database.py | 38 ++++++++++++++++++++++ 8 files changed, 141 insertions(+), 29 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8a4b6011..f1fcc5eb 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -26,10 +26,9 @@ from .views.index import IndexView from .views.special import JsonDataView, PatternPortfolioView from .views.table import RowView, TableView from .renderer import json_renderer -from .database import Database +from .database import Database, QueryInterrupted from .utils import ( - QueryInterrupted, escape_css_string, escape_sqlite, format_bytes, diff --git a/datasette/database.py b/datasette/database.py index 0f540e01..e6154caa 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -8,8 +8,6 @@ import uuid from .tracer import trace from .utils import ( - QueryInterrupted, - Results, detect_fts, detect_primary_keys, detect_spatialite, @@ -371,3 +369,40 @@ class WriteTask: self.fn = fn self.task_id = task_id self.reply_queue = reply_queue + + +class QueryInterrupted(Exception): + pass + + +class MultipleValues(Exception): + pass + + +class Results: + def __init__(self, rows, truncated, description): + self.rows = rows + self.truncated = truncated + self.description = description + + @property + def columns(self): + return [d[0] for d in self.description] + + def first(self): + if self.rows: + return self.rows[0] + else: + return None + + def single_value(self): + if self.rows and 1 == len(self.rows) and 1 == len(self.rows[0]): + return self.rows[0][0] + else: + raise MultipleValues + + def __iter__(self): + return iter(self.rows) + + def __len__(self): + return len(self.rows) diff --git a/datasette/facets.py b/datasette/facets.py index 18558754..1712db9b 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -2,12 +2,12 @@ import json import urllib import re from datasette import hookimpl +from datasette.database import QueryInterrupted from datasette.utils import ( escape_sqlite, path_with_added_args, path_with_removed_args, detect_json1, - QueryInterrupted, InvalidSql, sqlite3, ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index f1c24041..26a778d3 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -47,27 +47,6 @@ ENV SQLITE_EXTENSIONS /usr/lib/x86_64-linux-gnu/mod_spatialite.so """ -class QueryInterrupted(Exception): - pass - - -class Results: - def __init__(self, rows, truncated, description): - self.rows = rows - self.truncated = truncated - self.description = description - - @property - def columns(self): - return [d[0] for d in self.description] - - def __iter__(self): - return iter(self.rows) - - def __len__(self): - return len(self.rows) - - def urlsafe_components(token): "Splits token on commas and URL decodes each component" return [urllib.parse.unquote_plus(b) for b in token.split(",")] diff --git a/datasette/views/base.py b/datasette/views/base.py index e2bce2f9..f5eafe63 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -10,8 +10,8 @@ import pint from datasette import __version__ from datasette.plugins import pm +from datasette.database import QueryInterrupted from datasette.utils import ( - QueryInterrupted, InvalidSql, LimitedWriter, is_url, diff --git a/datasette/views/table.py b/datasette/views/table.py index 10e86eeb..c07447d3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -5,9 +5,9 @@ import json import jinja2 from datasette.plugins import pm +from datasette.database import QueryInterrupted from datasette.utils import ( CustomRow, - QueryInterrupted, RequestParameters, append_querystring, compound_keys_after_sql, diff --git a/docs/internals.rst b/docs/internals.rst index d7b6e7cb..0020f96d 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -94,13 +94,74 @@ Database class Instances of the ``Database`` class can be used to execute queries against attached SQLite databases, and to run introspection against their schemas. -SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. +.. _database_execute: + +await db.execute(sql, ...) +-------------------------- + +Executes a SQL query against the database and returns the resulting rows (see :ref:`database_results`). + +``sql`` - string (required) + The SQL query to execute. This can include ``?`` or ``:named`` parameters. + +``params`` - list or dict + A list or dictionary of values to use for the parameters. List for ``?``, dictionary for ``:named``. + +``truncate`` - boolean + Should the rows returned by the query be truncated at the maximum page size? Defaults to ``True``, set this to ``False`` to disable truncation. + +``custom_time_limit`` - integer ms + A custom time limit for this query. This can be set to a lower value than the Datasette configured default. If a query takes longer than this it will be terminated early and raise a ``dataette.database.QueryInterrupted`` exception. + +``page_size`` - integer + Set a custom page size for truncation, over-riding the configured Datasette default. + +``log_sql_errors`` - boolean + Should any SQL errors be logged to the console in addition to being raised as an error? Defaults to ``True``. + +.. _database_results: + +Results +------- + +The ``db.execute()`` method returns a single ``Results`` object. This can be used to access the rows returned by the query. + +Iterating over a ``Results`` object will yield SQLite `Row objects `__. Each of these can be treated as a tuple or can be accessed using ``row["column"]`` syntax: + +.. code-block:: python + + info = [] + results = await db.execute("select name from sqlite_master") + for row in results: + info.append(row["name"]) + +The ``Results`` object also has the following properties and methods: + +``.truncated`` - boolean + Indicates if this query was truncated - if it returned more results than the specified ``page_size``. If this is true then the results object will only provide access to the first ``page_size`` rows in the query result. You can disable truncation by passing ``truncate=False`` to the ``db.query()`` method. + +``.columns`` - list of strings + A list of column names returned by the query. + +``.rows`` - list of sqlite3.Row + This property provides direct access to the list of rows returned by the database. You can access specific rows by index using ``results.rows[0]``. + +``.first()`` - row or None + Returns the first row in the results, or ``None`` if no rows were returned. + +``.single_value()`` + Returns the value of the first column of the first row of results - but only if the query returned a single row with a single column. Raises a ``datasette.database.MultipleValues`` exception otherwise. + +``.__len__()`` + Calling ``len(results)`` returns the (truncated) number of returned results. .. _database_execute_write: await db.execute_write(sql, params=None, block=False) ----------------------------------------------------- +SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. + This method can be used to queue up a non-SELECT SQL query to be executed against a single write connection to the database. You can pass additional SQL parameters as a tuple or dictionary. diff --git a/tests/test_database.py b/tests/test_database.py index a9728019..d4055776 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,9 +1,47 @@ +from datasette.database import Results, MultipleValues +from datasette.utils import sqlite3 from .fixtures import app_client import pytest import time import uuid +@pytest.mark.asyncio +async def test_execute1(app_client): + db = app_client.ds.databases["fixtures"] + results = await db.execute("select * from facetable") + assert isinstance(results, Results) + assert 15 == len(results) + + +@pytest.mark.asyncio +async def test_results_first(app_client): + db = app_client.ds.databases["fixtures"] + assert None is (await db.execute("select * from facetable where pk > 100")).first() + results = await db.execute("select * from facetable") + row = results.first() + assert isinstance(row, sqlite3.Row) + + +@pytest.mark.parametrize( + "query,expected", + [ + ("select 1", 1), + ("select 1, 2", None), + ("select 1 as num union select 2 as num", None), + ], +) +@pytest.mark.asyncio +async def test_results_single_value(app_client, query, expected): + db = app_client.ds.databases["fixtures"] + results = await db.execute(query) + if expected: + assert expected == results.single_value() + else: + with pytest.raises(MultipleValues): + results.single_value() + + @pytest.mark.parametrize( "tables,exists", ( From ec9cdc3ffa7d9a9a214f71fa7864f0cbdf6ccb23 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 09:52:53 -0700 Subject: [PATCH 0404/2303] Documentation for .execute_fn(), refs #685 --- docs/internals.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 0020f96d..526c531c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -155,6 +155,26 @@ The ``Results`` object also has the following properties and methods: ``.__len__()`` Calling ``len(results)`` returns the (truncated) number of returned results. +.. _database_execute_fn: + +await db.execute_fn(fn) +----------------------- + +Executes a given callback function against a read-only database connection running in a thread. The function will be passed a SQLite connection, and the return value from the function will be returned by the ``await``. + +Example usage: + +syntax: + +.. code-block:: python + + def get_version(conn); + return conn.execute( + "select sqlite_version()" + ).fetchall()[0][0] + + version = await db.execute_fn(get_version) + .. _database_execute_write: await db.execute_write(sql, params=None, block=False) From 545c71b6044bbf30caef04976cbd73d519d278a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 09:57:01 -0700 Subject: [PATCH 0405/2303] Small cleanup --- docs/internals.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 526c531c..aa1ff7e7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -163,9 +163,6 @@ await db.execute_fn(fn) Executes a given callback function against a read-only database connection running in a thread. The function will be passed a SQLite connection, and the return value from the function will be returned by the ``await``. Example usage: - -syntax: - .. code-block:: python def get_version(conn); From 5ab848f0b87ad2030088a7259fc1802316b90200 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 10:04:47 -0700 Subject: [PATCH 0406/2303] RST fix --- docs/internals.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/internals.rst b/docs/internals.rst index aa1ff7e7..43944de9 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -163,6 +163,7 @@ await db.execute_fn(fn) Executes a given callback function against a read-only database connection running in a thread. The function will be passed a SQLite connection, and the return value from the function will be returned by the ``await``. Example usage: + .. code-block:: python def get_version(conn); From 2694ddcf14b88955e93a6cfb6c725500bb93e219 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 10:29:17 -0700 Subject: [PATCH 0407/2303] Test for .execute_fn(), refs #685 --- docs/internals.rst | 2 +- tests/test_database.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 43944de9..7b4c1755 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -166,7 +166,7 @@ Example usage: .. code-block:: python - def get_version(conn); + def get_version(conn): return conn.execute( "select sqlite_version()" ).fetchall()[0][0] diff --git a/tests/test_database.py b/tests/test_database.py index d4055776..1f1a3a7e 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -7,7 +7,7 @@ import uuid @pytest.mark.asyncio -async def test_execute1(app_client): +async def test_execute(app_client): db = app_client.ds.databases["fixtures"] results = await db.execute("select * from facetable") assert isinstance(results, Results) @@ -42,6 +42,16 @@ async def test_results_single_value(app_client, query, expected): results.single_value() +@pytest.mark.asyncio +async def test_execute_fn(app_client): + db = app_client.ds.databases["fixtures"] + + def get_1_plus_1(conn): + return conn.execute("select 1 + 1").fetchall()[0][0] + + assert 2 == await db.execute_fn(get_1_plus_1) + + @pytest.mark.parametrize( "tables,exists", ( From af6c6c5d6f929f951c0e63bfd1c82e37a071b50f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 May 2020 10:38:27 -0700 Subject: [PATCH 0408/2303] Release 0.42, refs #685 --- README.md | 1 + docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/README.md b/README.md index f2a3d81d..7351c5c0 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 8th May 2020: [Datasette 0.42](http://datasette.readthedocs.io/en/latest/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database. * 6th May 2020: [Datasette 0.41](http://datasette.readthedocs.io/en/latest/changelog.html#v0-41) - New mechanism for [creating custom pages](https://datasette.readthedocs.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://datasette.readthedocs.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements. * 21st April 2020: [Datasette 0.40](http://datasette.readthedocs.io/en/latest/changelog.html#v0-40) - Metadata can now be provided as YAML instead of JSON. Publishing to Zeit Now v1 is no longer supported, but Now v2 support is provided by the new [datasette-publish-now](https://github.com/simonw/datasette-publish-now) plugin. Various bug fixes. * 24th March 2020: [Datasette 0.39](http://datasette.readthedocs.io/en/latest/changelog.html#v0-39) - New `base_url` configuration option for running Datasette under a different URL prefix, `"sort"` and `"sort_desc"` metadata options for setting a default sort order for a table. diff --git a/docs/changelog.rst b/docs/changelog.rst index dc06e4ef..48d3128b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v0_42: + +0.42 (2020-05-08) +----------------- + +A small release which provides improved internal methods for use in plugins, along with documentation. See `#685 `__. + +* Added documentation for ``db.execute()``, see :ref:`database_execute`. +* Renamed ``db.execute_against_connection_in_thread()`` to ``db.execute_fn()`` and made it a documented method, see :ref:`database_execute_fn`. +* New ``results.first()`` and ``results.single_value()`` methods, plus documentation for the ``Results`` class - see :ref:`database_results`. + .. _v0_41: 0.41 (2020-05-06) From fc24edc153d76bcec917bb23d532981d9862e696 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2020 11:28:53 -0700 Subject: [PATCH 0409/2303] Added project_urls, closes #764 --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index c4886a11..d9c70de5 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,14 @@ setup( author="Simon Willison", license="Apache License, Version 2.0", url="https://github.com/simonw/datasette", + project_urls={ + "Documentation": "https://datasette.readthedocs.io/en/stable/", + "Changelog": "https://datasette.readthedocs.io/en/stable/changelog.html", + "Live demo": "https://latest.datasette.io/", + "Source code": "https://github.com/simonw/datasette", + "Issues": "https://github.com/simonw/datasette/issues", + "CI": "https://travis-ci.org/simonw/datasette", + }, packages=find_packages(exclude="tests"), package_data={"datasette": ["templates/*.html"]}, include_package_data=True, From 504196341c49840270bd75ea1a1871ef386ba7ea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 May 2020 22:51:39 -0700 Subject: [PATCH 0410/2303] Visually distinguish float/int columns, closes #729 --- datasette/static/app.css | 5 ++ datasette/templates/_table.html | 2 +- datasette/views/table.py | 12 ++- tests/test_html.py | 133 ++++++++++++++++---------------- 4 files changed, 85 insertions(+), 67 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index bae091b8..cc33277a 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -345,3 +345,8 @@ p.zero-results { padding: 0.5em; font-style: italic; } + +/* Value types */ +.type-float, .type-int { + color: #666; +} \ No newline at end of file diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index 42c37c55..8fee77b2 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -21,7 +21,7 @@ {% for row in display_rows %} {% for cell in row %} - {{ cell.value }} + {{ cell.value }} {% endfor %} {% endfor %} diff --git a/datasette/views/table.py b/datasette/views/table.py index c07447d3..51b7aa2f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -112,6 +112,7 @@ class RowTableShared(DataView): cells.append( { "column": pks[0] if len(pks) == 1 else "Link", + "value_type": "link", "is_special_link_column": is_special_link_column, "raw": pk_path, "value": jinja2.Markup( @@ -192,7 +193,16 @@ class RowTableShared(DataView): if truncate_cells and len(display_value) > truncate_cells: display_value = display_value[:truncate_cells] + u"\u2026" - cells.append({"column": column, "value": display_value, "raw": value}) + cells.append( + { + "column": column, + "value": display_value, + "raw": value, + "value_type": "none" + if value is None + else str(type(value).__name__), + } + ) cell_rows.append(Row(cells)) if link_column: diff --git a/tests/test_html.py b/tests/test_html.py index 564365ce..a3388c2d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -505,16 +505,16 @@ def test_table_html_simple_primary_key(app_client): assert ["nofollow"] == a["rel"] assert [ [ - '1', - 'hello', + '1', + 'hello', ], [ - '2', - 'world', + '2', + 'world', ], [ - '3', - '\xa0', + '3', + '\xa0', ], ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] @@ -578,9 +578,12 @@ def test_row_html_simple_primary_key(app_client): assert response.status == 200 table = Soup(response.body, "html.parser").find("table") assert ["id", "content"] == [th.string.strip() for th in table.select("thead th")] - assert [['1', 'hello']] == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] + assert [ + [ + '1', + 'hello', + ] + ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] def test_table_not_exists(app_client): @@ -599,14 +602,14 @@ def test_table_html_no_primary_key(app_client): ] expected = [ [ - '{}'.format( + '{}'.format( i, i ), - '{}'.format(i), - '{}'.format(i), - 'a{}'.format(i), - 'b{}'.format(i), - 'c{}'.format(i), + '{}'.format(i), + '{}'.format(i), + 'a{}'.format(i), + 'b{}'.format(i), + 'c{}'.format(i), ] for i in range(1, 51) ] @@ -633,11 +636,11 @@ def test_row_html_no_primary_key(app_client): ] expected = [ [ - '1', - '1', - 'a1', - 'b1', - 'c1', + '1', + '1', + 'a1', + 'b1', + 'c1', ] ] assert expected == [ @@ -658,10 +661,10 @@ def test_table_html_compound_primary_key(app_client): assert a["href"].endswith("/compound_primary_key?_sort={}".format(expected_col)) expected = [ [ - 'a,b', - 'a', - 'b', - 'c', + 'a,b', + 'a', + 'b', + 'c', ] ] assert expected == [ @@ -675,14 +678,14 @@ def test_table_html_foreign_key_links(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1', - 'hello\xa01', - '1', + '1', + 'hello\xa01', + '1', ], [ - '2', - '\xa0', - '\xa0', + '2', + '\xa0', + '\xa0', ], ] assert expected == [ @@ -696,9 +699,9 @@ def test_table_html_disable_foreign_key_links_with_labels(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1', - '1', - '1', + '1', + '1', + '1', ] ] assert expected == [ @@ -712,8 +715,8 @@ def test_table_html_foreign_key_custom_label_column(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1', - 'world2\xa01', + '1', + 'world2\xa01', ] ] assert expected == [ @@ -754,9 +757,9 @@ def test_row_html_compound_primary_key(app_client): ] expected = [ [ - 'a', - 'b', - 'c', + 'a', + 'b', + 'c', ] ] assert expected == [ @@ -771,14 +774,14 @@ def test_compound_primary_key_with_foreign_key_references(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1,feline', - '1\xa01', - 'feline', + '1,feline', + '1\xa01', + 'feline', ], [ - '2,canine', - '2\xa02', - 'canine', + '2,canine', + '2\xa02', + 'canine', ], ] assert expected == [ @@ -799,16 +802,16 @@ def test_view_html(app_client): assert ths[1].string.strip() == "upper_content" expected = [ [ - 'hello', - 'HELLO', + 'hello', + 'HELLO', ], [ - 'world', - 'WORLD', + 'world', + 'WORLD', ], [ - '\xa0', - '\xa0', + '\xa0', + '\xa0', ], ] assert expected == [ @@ -1079,9 +1082,9 @@ def test_binary_data_display(app_client): table = Soup(response.body, "html.parser").find("table") expected_tds = [ [ - '1', - '1', - '<Binary\xa0data:\xa019\xa0bytes>', + '1', + '1', + '<Binary\xa0data:\xa019\xa0bytes>', ] ] assert expected_tds == [ @@ -1154,20 +1157,20 @@ def test_metadata_sort(app_client): rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] expected = [ [ - '3', - 'Detroit', + '3', + 'Detroit', ], [ - '2', - 'Los Angeles', + '2', + 'Los Angeles', ], [ - '4', - 'Memnonia', + '4', + 'Memnonia', ], [ - '1', - 'San Francisco', + '1', + 'San Francisco', ], ] assert expected == rows @@ -1189,12 +1192,12 @@ def test_metadata_sort_desc(app_client): rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] expected = [ [ - '2', - 'Paranormal', + '2', + 'Paranormal', ], [ - '1', - 'Museum', + '1', + 'Museum', ], ] assert expected == rows From 5ea8c6d1cd8ded832718fb1a652b5880c4bf5ebb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 May 2020 22:55:20 -0700 Subject: [PATCH 0411/2303] type-pk instead of type-link CSS class, closes #729 --- datasette/views/table.py | 2 +- tests/test_html.py | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 51b7aa2f..3289e58b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -112,7 +112,7 @@ class RowTableShared(DataView): cells.append( { "column": pks[0] if len(pks) == 1 else "Link", - "value_type": "link", + "value_type": "pk", "is_special_link_column": is_special_link_column, "raw": pk_path, "value": jinja2.Markup( diff --git a/tests/test_html.py b/tests/test_html.py index a3388c2d..445f7b4c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -505,15 +505,15 @@ def test_table_html_simple_primary_key(app_client): assert ["nofollow"] == a["rel"] assert [ [ - '1', + '1', 'hello', ], [ - '2', + '2', 'world', ], [ - '3', + '3', '\xa0', ], ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] @@ -602,7 +602,7 @@ def test_table_html_no_primary_key(app_client): ] expected = [ [ - '{}'.format( + '{}'.format( i, i ), '{}'.format(i), @@ -661,7 +661,7 @@ def test_table_html_compound_primary_key(app_client): assert a["href"].endswith("/compound_primary_key?_sort={}".format(expected_col)) expected = [ [ - 'a,b', + 'a,b', 'a', 'b', 'c', @@ -678,12 +678,12 @@ def test_table_html_foreign_key_links(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1', + '1', 'hello\xa01', '1', ], [ - '2', + '2', '\xa0', '\xa0', ], @@ -699,7 +699,7 @@ def test_table_html_disable_foreign_key_links_with_labels(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1', + '1', '1', '1', ] @@ -715,7 +715,7 @@ def test_table_html_foreign_key_custom_label_column(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1', + '1', 'world2\xa01', ] ] @@ -774,12 +774,12 @@ def test_compound_primary_key_with_foreign_key_references(app_client): table = Soup(response.body, "html.parser").find("table") expected = [ [ - '1,feline', + '1,feline', '1\xa01', 'feline', ], [ - '2,canine', + '2,canine', '2\xa02', 'canine', ], @@ -1082,7 +1082,7 @@ def test_binary_data_display(app_client): table = Soup(response.body, "html.parser").find("table") expected_tds = [ [ - '1', + '1', '1', '<Binary\xa0data:\xa019\xa0bytes>', ] @@ -1157,19 +1157,19 @@ def test_metadata_sort(app_client): rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] expected = [ [ - '3', + '3', 'Detroit', ], [ - '2', + '2', 'Los Angeles', ], [ - '4', + '4', 'Memnonia', ], [ - '1', + '1', 'San Francisco', ], ] @@ -1192,11 +1192,11 @@ def test_metadata_sort_desc(app_client): rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] expected = [ [ - '2', + '2', 'Paranormal', ], [ - '1', + '1', 'Museum', ], ] From faea5093b865031f650da7da6539430f732f511a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 May 2020 11:16:47 -0700 Subject: [PATCH 0412/2303] Column headings now black in mobile view, closes #729 --- datasette/static/app.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index cc33277a..92f268ae 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -79,6 +79,7 @@ table a:visited { .rows-and-columns td:before { display: block; + color: black; margin-left: -10%; font-size: 0.8em; } @@ -349,4 +350,4 @@ p.zero-results { /* Value types */ .type-float, .type-int { color: #666; -} \ No newline at end of file +} From cee671a58f417f827d1735b1abaa40716534ea67 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 May 2020 10:53:51 -0700 Subject: [PATCH 0413/2303] Use dirs_exist_ok=True, refs #744 (#768) --- datasette/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 26a778d3..04bf41af 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -602,9 +602,9 @@ def link_or_copy(src, dst): def link_or_copy_directory(src, dst): try: - shutil.copytree(src, dst, copy_function=os.link) + shutil.copytree(src, dst, copy_function=os.link, dirs_exist_ok=True) except OSError: - shutil.copytree(src, dst) + shutil.copytree(src, dst, dirs_exist_ok=True) def module_from_path(path, name): From 2d099ad9c657d2cab59de91cdb8bfed2da236ef6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 11:17:43 -0700 Subject: [PATCH 0414/2303] Backport of Python 3.8 shutil.copytree, refs #744 (#769) --- datasette/utils/__init__.py | 5 +- datasette/utils/shutil_backport.py | 101 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 datasette/utils/shutil_backport.py diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 04bf41af..cdb1bbc9 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -15,6 +15,7 @@ import shutil import urllib import numbers import yaml +from .shutil_backport import copytree try: import pysqlite3 as sqlite3 @@ -602,9 +603,9 @@ def link_or_copy(src, dst): def link_or_copy_directory(src, dst): try: - shutil.copytree(src, dst, copy_function=os.link, dirs_exist_ok=True) + copytree(src, dst, copy_function=os.link, dirs_exist_ok=True) except OSError: - shutil.copytree(src, dst, dirs_exist_ok=True) + copytree(src, dst, dirs_exist_ok=True) def module_from_path(path, name): diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py new file mode 100644 index 00000000..dbe22404 --- /dev/null +++ b/datasette/utils/shutil_backport.py @@ -0,0 +1,101 @@ +""" +Backported from Python 3.8. + +This code is licensed under the Python License: +https://github.com/python/cpython/blob/v3.8.3/LICENSE +""" +import os +from shutil import copy, copy2, copystat, Error + + +def _copytree( + entries, + src, + dst, + symlinks, + ignore, + copy_function, + ignore_dangling_symlinks, + dirs_exist_ok=False, +): + if ignore is not None: + ignored_names = ignore(src, set(os.listdir(src))) + else: + ignored_names = set() + + os.makedirs(dst, exist_ok=dirs_exist_ok) + errors = [] + use_srcentry = copy_function is copy2 or copy_function is copy + + for srcentry in entries: + if srcentry.name in ignored_names: + continue + srcname = os.path.join(src, srcentry.name) + dstname = os.path.join(dst, srcentry.name) + srcobj = srcentry if use_srcentry else srcname + try: + if srcentry.is_symlink(): + linkto = os.readlink(srcname) + if symlinks: + os.symlink(linkto, dstname) + copystat(srcobj, dstname, follow_symlinks=not symlinks) + else: + if not os.path.exists(linkto) and ignore_dangling_symlinks: + continue + if srcentry.is_dir(): + copytree( + srcobj, + dstname, + symlinks, + ignore, + copy_function, + dirs_exist_ok=dirs_exist_ok, + ) + else: + copy_function(srcobj, dstname) + elif srcentry.is_dir(): + copytree( + srcobj, + dstname, + symlinks, + ignore, + copy_function, + dirs_exist_ok=dirs_exist_ok, + ) + else: + copy_function(srcentry, dstname) + except Error as err: + errors.extend(err.args[0]) + except OSError as why: + errors.append((srcname, dstname, str(why))) + try: + copystat(src, dst) + except OSError as why: + # Copying file access times may fail on Windows + if getattr(why, "winerror", None) is None: + errors.append((src, dst, str(why))) + if errors: + raise Error(errors) + return dst + + +def copytree( + src, + dst, + symlinks=False, + ignore=None, + copy_function=copy2, + ignore_dangling_symlinks=False, + dirs_exist_ok=False, +): + with os.scandir(src) as entries: + return _copytree( + entries=entries, + src=src, + dst=dst, + symlinks=symlinks, + ignore=ignore, + copy_function=copy_function, + ignore_dangling_symlinks=ignore_dangling_symlinks, + dirs_exist_ok=dirs_exist_ok, + ) From 9e6075d21facbfef565ffcdf160a558744fc1c2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 11:35:31 -0700 Subject: [PATCH 0415/2303] rST fixes for register_output_renderer docs --- docs/plugins.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 59d39a62..50271e0a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -743,11 +743,11 @@ Allows the plugin to register a new output renderer, to output data in a custom @hookimpl def register_output_renderer(datasette): return { - 'extension': 'test', - 'callback': render_test + "extension": "test", + "callback": render_test } -This will register `render_test` to be called when paths with the extension `.test` (for example `/database.test`, `/database/table.test`, or `/database/table/row.test`) are requested. When a request is received, the callback function is called with three positional arguments: +This will register ``render_test`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. When a request is received, the callback function is called with three positional arguments: ``args`` - dictionary The GET parameters of the request @@ -756,15 +756,15 @@ This will register `render_test` to be called when paths with the extension `.te The data to be rendered ``view_name`` - string - The name of the view where the renderer is being called. (`index`, `database`, `table`, and `row` are the most important ones.) + The name of the view where the renderer is being called. (``index``, ``database``, ``table``, and ``row`` are the most important ones.) -The callback function can return `None`, if it is unable to render the data, or a dictionary with the following keys: +The callback function can return ``None``, if it is unable to render the data, or a dictionary with the following keys: ``body`` - string or bytes, optional The response body, default empty ``content_type`` - string, optional - The Content-Type header, default `text/plain` + The Content-Type header, default ``text/plain`` ``status_code`` - integer, optional The HTTP status code, default 200 @@ -775,7 +775,7 @@ A simple example of an output renderer callback function: def render_test(args, data, view_name): return { - 'body': 'Hello World' + "body": "Hello World" } Examples: `datasette-atom `_, `datasette-ics `_ From 41a0cd7b6afe0397efbbf27ad822679fc574811a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 12:25:52 -0700 Subject: [PATCH 0416/2303] call_with_supported_arguments() util, refs #581 --- datasette/utils/__init__.py | 11 +++++++++++ tests/test_utils.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index cdb1bbc9..03157072 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -3,6 +3,7 @@ from collections import OrderedDict import base64 import click import hashlib +import inspect import json import mergedeep import os @@ -803,3 +804,13 @@ def parse_metadata(content): return yaml.safe_load(content) except yaml.YAMLError: raise BadMetadataError("Metadata is not valid JSON or YAML") + + +def call_with_supported_arguments(fn, **kwargs): + parameters = inspect.signature(fn).parameters.keys() + call_with = [] + for parameter in parameters: + if parameter not in kwargs: + raise TypeError("{} requires parameters {}".format(fn, tuple(parameters))) + call_with.append(kwargs[parameter]) + return fn(*call_with) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7e4f1a8e..59b80a67 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -446,3 +446,14 @@ async def test_request_post_vars(): request = Request(scope, receive) assert {"foo": "bar", "baz": "1"} == await request.post_vars() + + +def test_call_with_supported_arguments(): + def foo(a, b): + return "{}+{}".format(a, b) + + assert "1+2" == utils.call_with_supported_arguments(foo, a=1, b=2) + assert "1+2" == utils.call_with_supported_arguments(foo, a=1, b=2, c=3) + + with pytest.raises(TypeError): + utils.call_with_supported_arguments(foo, a=1) From da87e963bff24e47878a5bc2025c8bfc63d4bc93 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 13:16:02 -0700 Subject: [PATCH 0417/2303] Test that plugin hooks are unit tested (xfail) This currently fails using xfail. Closes 771. --- tests/test_plugins.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 8b6a6b41..1546de92 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -7,7 +7,7 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from datasette.app import Datasette -from datasette.plugins import get_plugins, DEFAULT_PLUGINS +from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils import sqlite3 import base64 import json @@ -20,6 +20,20 @@ import pytest import urllib +@pytest.mark.xfail +@pytest.mark.parametrize( + "plugin_hook", [name for name in dir(pm.hook) if not name.startswith("_")] +) +def test_plugin_hooks_have_tests(plugin_hook): + "Every plugin hook should be referenced in this test module" + tests_in_this_module = [t for t in globals().keys() if t.startswith("test_")] + ok = False + for test in tests_in_this_module: + if plugin_hook in test: + ok = True + assert ok, "Plugin hook is missing tests: {}".format(plugin_hook) + + def test_plugins_dir_plugin_prepare_connection(app_client): response = app_client.get( "/fixtures.json?sql=select+convert_units(100%2C+'m'%2C+'ft')" From af5702220c06a913746c9657bd33c2108d80c33f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 13:34:12 -0700 Subject: [PATCH 0418/2303] Added datasette-media plugin to the docs --- docs/ecosystem.rst | 5 +++++ docs/plugins.rst | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 8c8785cd..4777cc16 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -97,6 +97,11 @@ datasette-json-html `datasette-json-html `__ renders HTML in Datasette's table view driven by JSON returned from your SQL queries. This provides a way to embed images, links and lists of links directly in Datasette's main interface, defined using custom SQL statements. +datasette-media +--------------- + +`datasette-media `__ adds the ability to serve media files such as images directly, configured through a SQL query that maps a URL parameter to a path to a file on disk. It can also serve resized image thumbnails. + datasette-jellyfish ------------------- diff --git a/docs/plugins.rst b/docs/plugins.rst index 50271e0a..78ad0309 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -887,4 +887,4 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att return add_x_databases_header return wrap_with_databases_header -Examples: `datasette-auth-github `_, `datasette-search-all `_ +Examples: `datasette-auth-github `_, `datasette-search-all `_, `datasette-media `_ From ad88c9b3f3d7886612dae6afed65d43940632b06 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 14:52:03 -0700 Subject: [PATCH 0419/2303] Mechanism for adding a default URL fragment to a canned query Closes #767 --- datasette/templates/database.html | 6 ++--- docs/sql_queries.rst | 40 ++++++++++++++++++++++++++++--- tests/fixtures.py | 1 + tests/test_html.py | 15 ++++++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 7d98f0e5..e47b2418 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -48,7 +48,7 @@ {% endif %} {% if views %} -

    Views

    +

    Views

      {% for view in views %}
    • {{ view }}
      • @@ -57,10 +57,10 @@ {% endif %} {% if queries %} -

      Queries

      +

      Queries

      {% endif %} diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index da10191e..c3efd930 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -72,7 +72,9 @@ Canned queries -------------- As an alternative to adding views to your database, you can define canned -queries inside your ``metadata.json`` file. Here's an example:: +queries inside your ``metadata.json`` file. Here's an example: + +.. code-block:: json { "databases": { @@ -86,7 +88,7 @@ queries inside your ``metadata.json`` file. Here's an example:: } } -Then run datasette like this:: +Then run Datasette like this:: datasette sf-trees.db -m metadata.json @@ -104,6 +106,11 @@ title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). +.. _canned_queries_named_parameters: + +Named parameters +~~~~~~~~~~~~~~~~ + Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create @@ -117,7 +124,9 @@ Here's an example of a canned query with a named parameter: from facetable join facet_cities on facetable.city_id = facet_cities.id where neighborhood like '%' || :text || '%' order by neighborhood; -In the canned query JSON it looks like this:: +In the canned query JSON it looks like this: + +.. code-block:: json { "databases": { @@ -139,6 +148,31 @@ https://latest.datasette.io/fixtures/neighborhood_search?text=town Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. +.. _canned_queries_default_fragment: + +Setting a default fragment +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some plugins, such as `datasette-vega `__, can be configured by including additional data in the fragment hash of the URL - the bit that comes after a ``#`` symbol. + +You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key: + +.. code-block:: json + + { + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;", + "fragment": "fragment-goes-here" + } + } + } + } + +`See here `__ for a demo of this in action. + .. _pagination: Pagination diff --git a/tests/fixtures.py b/tests/fixtures.py index 0284ff9c..a3b75f9f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -327,6 +327,7 @@ METADATA = { ), "title": "Search neighborhoods", "description_html": "Demonstrating simple like search", + "fragment": "fragment-goes-here", }, }, } diff --git a/tests/test_html.py b/tests/test_html.py index 445f7b4c..5a07953e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -85,6 +85,21 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash): assert "fixtures" in response.text +def test_database_page(app_client): + response = app_client.get("/fixtures") + soup = Soup(response.body, "html.parser") + queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") + assert queries_ul is not None + assert [ + ( + "/fixtures/%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC", + "𝐜𝐢𝐭𝐢𝐞𝐬", + ), + ("/fixtures/pragma_cache_size", "pragma_cache_size"), + ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"), + ] == [(a["href"], a.text) for a in queries_ul.find_all("a")] + + def test_invalid_custom_sql(app_client): response = app_client.get("/fixtures?sql=.schema") assert response.status == 400 From 6d7cb02f00010d3cb4b4bac0460d41277652b80e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 15:17:53 -0700 Subject: [PATCH 0420/2303] Documentation for request object, refs #706 --- docs/internals.rst | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 7b4c1755..5535ceb1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -225,3 +225,58 @@ Here's an example of ``block=True`` in action: num_rows_left = await database.execute_write_fn(my_action, block=True) except Exception as e: print("An error occurred:", e) + +.. _internals_request: + +Request object +~~~~~~~~~~~~~~ + +The request object is passed to various plugin hooks. It represents an incoming HTTP request. It has the following properties: + +``.scope`` - dictionary + The ASGI scope that was used to construct this request, described in the `ASGI HTTP connection scope `__ specification. + +``.method`` - string + The HTTP method for this request, usually ``GET`` or ``POST``. + +``.url`` - string + The full URL for this request, e.g. ``https://latest.datasette.io/fixtures``. + +``.scheme`` - string + The request scheme - usually ``https`` or ``http``. + +``.headers`` - dictionary (str -> str) + A dictionary of incoming HTTP request headers. + +``.host`` - string + The host header from the incoming request, e.g. ``latest.datasette.io`` or ``localhost``. + +``.path`` - string + The path of the request, e.g. ``/fixtures``. + +``.query_string`` - string + The querystring component of the request, without the ``?`` - e.g. ``name__contains=sam&age__gt=10``. + +``.args`` - RequestParameters + An object representing the parsed querystring parameters, see below. + +``.raw_args`` - dictionary + A dictionary mapping querystring keys to values. If multiple keys of the same kind are provided, e.g. ``?foo=1&foo=2``, only the first value will be present in this dictionary. + +The object also has one awaitable method: + +``await request.post_vars()`` - dictionary + Returns a dictionary of form variables that were submitted in the request body via ``POST``. + +The RequestParameters class +--------------------------- + +This class, returned by ``request.args``, is a subclass of a Python dictionary that provides methods for working with keys that map to lists of values. + +Conider the querystring ``?foo=1&foo=2``. This will produce a ``request.args`` that looks like this:: + + RequestParameters({"foo": ["1", "2"]}) + +Calling ``request.args.get("foo")`` will return the first value, ``"1"``. If that key is not present it will return ``None`` - or the second argument if you passed one, which will be used as the default. + +Calling ``request.args.getlist("foo")`` will return the full list, ``["1", "2"]``. \ No newline at end of file From 50652f474b94e83d49ee15f219820cdbfc450f11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 15:29:42 -0700 Subject: [PATCH 0421/2303] Stop using .raw_args, deprecate and undocument it - refs #706 --- datasette/utils/asgi.py | 1 + datasette/views/database.py | 4 ++-- datasette/views/table.py | 8 ++++---- docs/internals.rst | 3 --- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 20047bb5..62a2a0c8 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -65,6 +65,7 @@ class Request: @property def raw_args(self): + # Deprecated, undocumented - may be removed in Datasette 1.0 return {key: value[0] for key, value in self.args.items()} async def post_vars(self): diff --git a/datasette/views/database.py b/datasette/views/database.py index 92e24f84..cd27dd5f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -24,7 +24,7 @@ class DatabaseView(DataView): if request.args.get("sql"): if not self.ds.config("allow_sql"): raise DatasetteError("sql= is not allowed", status=400) - sql = request.raw_args.pop("sql") + sql = request.args.get("sql") validate_sql_select(sql) return await QueryView(self.ds).data( request, database, hash, sql, _size=_size, metadata=metadata @@ -107,7 +107,7 @@ class QueryView(DataView): metadata=None, _size=None, ): - params = request.raw_args + params = {key: request.args.get(key) for key in request.args} if "sql" in params: params.pop("sql") if "_shape" in params: diff --git a/datasette/views/table.py b/datasette/views/table.py index 3289e58b..aab4bbe3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -527,7 +527,7 @@ class TableView(RowTableShared): extra_args = {} # Handle ?_size=500 - page_size = _size or request.raw_args.get("_size") + page_size = _size or request.args.get("_size") if page_size: if page_size == "max": page_size = self.ds.max_returned_rows @@ -558,8 +558,8 @@ class TableView(RowTableShared): sql_no_limit=sql_no_limit.rstrip(), limit=page_size + 1, offset=offset ) - if request.raw_args.get("_timelimit"): - extra_args["custom_time_limit"] = int(request.raw_args["_timelimit"]) + if request.args.get("_timelimit"): + extra_args["custom_time_limit"] = int(request.args["_timelimit"]) results = await db.execute(sql, params, truncate=True, **extra_args) @@ -890,7 +890,7 @@ class RowView(RowTableShared): "units": self.ds.table_metadata(database, table).get("units", {}), } - if "foreign_key_tables" in (request.raw_args.get("_extras") or "").split(","): + if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): data["foreign_key_tables"] = await self.foreign_key_tables( database, table, pk_values ) diff --git a/docs/internals.rst b/docs/internals.rst index 5535ceb1..5bcb9da9 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -260,9 +260,6 @@ The request object is passed to various plugin hooks. It represents an incoming ``.args`` - RequestParameters An object representing the parsed querystring parameters, see below. -``.raw_args`` - dictionary - A dictionary mapping querystring keys to values. If multiple keys of the same kind are provided, e.g. ``?foo=1&foo=2``, only the first value will be present in this dictionary. - The object also has one awaitable method: ``await request.post_vars()`` - dictionary From 4b96857f170e329a73186e703cc0d9ca4e8719cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 15:35:25 -0700 Subject: [PATCH 0422/2303] Link to request object documentation, refs #706 --- docs/plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 78ad0309..feb14593 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -669,7 +669,7 @@ Extra template variables that should be made available in the rendered template The name of the view being displayed. (`index`, `database`, `table`, and `row` are the most important ones.) ``request`` - object - The current HTTP request object. ``request.scope`` provides access to the ASGI scope. + The current HTTP :ref:`internals_request`. ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` From 446e5de65d1b9c6c877e38b0ef13bc9285c465a1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 17:57:25 -0700 Subject: [PATCH 0423/2303] Refactored test plugins into tests/plugins, closes #775 --- tests/fixtures.py | 182 +---------------------------------- tests/plugins/my_plugin.py | 89 +++++++++++++++++ tests/plugins/my_plugin_2.py | 94 ++++++++++++++++++ tests/plugins/view_name.py | 9 ++ tests/test_api.py | 1 + tests/test_custom_pages.py | 16 +-- 6 files changed, 197 insertions(+), 194 deletions(-) create mode 100644 tests/plugins/my_plugin.py create mode 100644 tests/plugins/my_plugin_2.py create mode 100644 tests/plugins/view_name.py diff --git a/tests/fixtures.py b/tests/fixtures.py index a3b75f9f..1eaa1dfe 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -19,6 +19,8 @@ from urllib.parse import unquote, quote # This temp file is used by one of the plugin config tests TEMP_PLUGIN_SECRET_FILE = os.path.join(tempfile.gettempdir(), "plugin-secret") +PLUGINS_DIR = str(pathlib.Path(__file__).parent / "plugins") + class TestResponse: def __init__(self, status, headers, body): @@ -109,7 +111,6 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, - extra_plugins=None, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) @@ -130,12 +131,6 @@ def make_app_client( sqlite3.connect(extra_filepath).executescript(extra_sql) files.append(extra_filepath) os.chdir(os.path.dirname(filepath)) - plugins_dir = os.path.join(tmpdir, "plugins") - os.mkdir(plugins_dir) - open(os.path.join(plugins_dir, "my_plugin.py"), "w").write(PLUGIN1) - open(os.path.join(plugins_dir, "my_plugin_2.py"), "w").write(PLUGIN2) - for filename, content in (extra_plugins or {}).items(): - open(os.path.join(plugins_dir, filename), "w").write(content) config = config or {} config.update( { @@ -150,7 +145,7 @@ def make_app_client( memory=memory, cors=cors, metadata=METADATA, - plugins_dir=plugins_dir, + plugins_dir=PLUGINS_DIR, config=config, inspect_data=inspect_data, static_mounts=static_mounts, @@ -334,177 +329,6 @@ METADATA = { }, } -PLUGIN1 = """ -from datasette import hookimpl -import base64 -import pint -import json - -ureg = pint.UnitRegistry() - - -@hookimpl -def prepare_connection(conn, database, datasette): - def convert_units(amount, from_, to_): - "select convert_units(100, 'm', 'ft');" - return (amount * ureg(from_)).to(to_).to_tuple()[0] - conn.create_function('convert_units', 3, convert_units) - def prepare_connection_args(): - return 'database={}, datasette.plugin_config("name-of-plugin")={}'.format( - database, datasette.plugin_config("name-of-plugin") - ) - conn.create_function('prepare_connection_args', 0, prepare_connection_args) - - -@hookimpl -def extra_css_urls(template, database, table, datasette): - return ['https://plugin-example.com/{}/extra-css-urls-demo.css'.format( - base64.b64encode(json.dumps({ - "template": template, - "database": database, - "table": table, - }).encode("utf8")).decode("utf8") - )] - - -@hookimpl -def extra_js_urls(): - return [{ - 'url': 'https://plugin-example.com/jquery.js', - 'sri': 'SRIHASH', - }, 'https://plugin-example.com/plugin1.js'] - - -@hookimpl -def extra_body_script(template, database, table, datasette): - return 'var extra_body_script = {};'.format( - json.dumps({ - "template": template, - "database": database, - "table": table, - "config": datasette.plugin_config( - "name-of-plugin", - database=database, - table=table, - ) - }) - ) - - -@hookimpl -def render_cell(value, column, table, database, datasette): - # Render some debug output in cell with value RENDER_CELL_DEMO - if value != "RENDER_CELL_DEMO": - return None - return json.dumps({ - "column": column, - "table": table, - "database": database, - "config": datasette.plugin_config( - "name-of-plugin", - database=database, - table=table, - ) - }) - - -@hookimpl -def extra_template_vars(template, database, table, view_name, request, datasette): - return { - "extra_template_vars": json.dumps({ - "template": template, - "scope_path": request.scope["path"] if request else None - }, default=lambda b: b.decode("utf8")) - } -""" - -PLUGIN2 = """ -from datasette import hookimpl -from functools import wraps -import jinja2 -import json - - -@hookimpl -def extra_js_urls(): - return [{ - 'url': 'https://plugin-example.com/jquery.js', - 'sri': 'SRIHASH', - }, 'https://plugin-example.com/plugin2.js'] - - -@hookimpl -def render_cell(value, database): - # Render {"href": "...", "label": "..."} as link - if not isinstance(value, str): - return None - stripped = value.strip() - if not stripped.startswith("{") and stripped.endswith("}"): - return None - try: - data = json.loads(value) - except ValueError: - return None - if not isinstance(data, dict): - return None - if set(data.keys()) != {"href", "label"}: - return None - href = data["href"] - if not ( - href.startswith("/") or href.startswith("http://") - or href.startswith("https://") - ): - return None - return jinja2.Markup( - '{label}'.format( - database=database, - href=jinja2.escape(data["href"]), - label=jinja2.escape(data["label"] or "") or " " - ) - ) - - -@hookimpl -def extra_template_vars(template, database, table, view_name, request, datasette): - async def query_database(sql): - first_db = list(datasette.databases.keys())[0] - return ( - await datasette.execute(first_db, sql) - ).rows[0][0] - async def inner(): - return { - "extra_template_vars_from_awaitable": json.dumps({ - "template": template, - "scope_path": request.scope["path"] if request else None, - "awaitable": True, - }, default=lambda b: b.decode("utf8")), - "query_database": query_database, - } - return inner - - -@hookimpl -def asgi_wrapper(datasette): - def wrap_with_databases_header(app): - @wraps(app) - async def add_x_databases_header(scope, recieve, send): - async def wrapped_send(event): - if event["type"] == "http.response.start": - original_headers = event.get("headers") or [] - event = { - "type": event["type"], - "status": event["status"], - "headers": original_headers + [ - [b"x-databases", - ", ".join(datasette.databases.keys()).encode("utf-8")] - ], - } - await send(event) - await app(scope, recieve, wrapped_send) - return add_x_databases_header - return wrap_with_databases_header -""" - TABLES = ( """ CREATE TABLE simple_primary_key ( diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py new file mode 100644 index 00000000..e55a0a32 --- /dev/null +++ b/tests/plugins/my_plugin.py @@ -0,0 +1,89 @@ +from datasette import hookimpl +import base64 +import pint +import json + +ureg = pint.UnitRegistry() + + +@hookimpl +def prepare_connection(conn, database, datasette): + def convert_units(amount, from_, to_): + "select convert_units(100, 'm', 'ft');" + return (amount * ureg(from_)).to(to_).to_tuple()[0] + + conn.create_function("convert_units", 3, convert_units) + + def prepare_connection_args(): + return 'database={}, datasette.plugin_config("name-of-plugin")={}'.format( + database, datasette.plugin_config("name-of-plugin") + ) + + conn.create_function("prepare_connection_args", 0, prepare_connection_args) + + +@hookimpl +def extra_css_urls(template, database, table, datasette): + return [ + "https://plugin-example.com/{}/extra-css-urls-demo.css".format( + base64.b64encode( + json.dumps( + {"template": template, "database": database, "table": table,} + ).encode("utf8") + ).decode("utf8") + ) + ] + + +@hookimpl +def extra_js_urls(): + return [ + {"url": "https://plugin-example.com/jquery.js", "sri": "SRIHASH",}, + "https://plugin-example.com/plugin1.js", + ] + + +@hookimpl +def extra_body_script(template, database, table, datasette): + return "var extra_body_script = {};".format( + json.dumps( + { + "template": template, + "database": database, + "table": table, + "config": datasette.plugin_config( + "name-of-plugin", database=database, table=table, + ), + } + ) + ) + + +@hookimpl +def render_cell(value, column, table, database, datasette): + # Render some debug output in cell with value RENDER_CELL_DEMO + if value != "RENDER_CELL_DEMO": + return None + return json.dumps( + { + "column": column, + "table": table, + "database": database, + "config": datasette.plugin_config( + "name-of-plugin", database=database, table=table, + ), + } + ) + + +@hookimpl +def extra_template_vars(template, database, table, view_name, request, datasette): + return { + "extra_template_vars": json.dumps( + { + "template": template, + "scope_path": request.scope["path"] if request else None, + }, + default=lambda b: b.decode("utf8"), + ) + } diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py new file mode 100644 index 00000000..fdc6956d --- /dev/null +++ b/tests/plugins/my_plugin_2.py @@ -0,0 +1,94 @@ +from datasette import hookimpl +from functools import wraps +import jinja2 +import json + + +@hookimpl +def extra_js_urls(): + return [ + {"url": "https://plugin-example.com/jquery.js", "sri": "SRIHASH",}, + "https://plugin-example.com/plugin2.js", + ] + + +@hookimpl +def render_cell(value, database): + # Render {"href": "...", "label": "..."} as link + if not isinstance(value, str): + return None + stripped = value.strip() + if not stripped.startswith("{") and stripped.endswith("}"): + return None + try: + data = json.loads(value) + except ValueError: + return None + if not isinstance(data, dict): + return None + if set(data.keys()) != {"href", "label"}: + return None + href = data["href"] + if not ( + href.startswith("/") + or href.startswith("http://") + or href.startswith("https://") + ): + return None + return jinja2.Markup( + '{label}'.format( + database=database, + href=jinja2.escape(data["href"]), + label=jinja2.escape(data["label"] or "") or " ", + ) + ) + + +@hookimpl +def extra_template_vars(template, database, table, view_name, request, datasette): + async def query_database(sql): + first_db = list(datasette.databases.keys())[0] + return (await datasette.execute(first_db, sql)).rows[0][0] + + async def inner(): + return { + "extra_template_vars_from_awaitable": json.dumps( + { + "template": template, + "scope_path": request.scope["path"] if request else None, + "awaitable": True, + }, + default=lambda b: b.decode("utf8"), + ), + "query_database": query_database, + } + + return inner + + +@hookimpl +def asgi_wrapper(datasette): + def wrap_with_databases_header(app): + @wraps(app) + async def add_x_databases_header(scope, recieve, send): + async def wrapped_send(event): + if event["type"] == "http.response.start": + original_headers = event.get("headers") or [] + event = { + "type": event["type"], + "status": event["status"], + "headers": original_headers + + [ + [ + b"x-databases", + ", ".join(datasette.databases.keys()).encode("utf-8"), + ] + ], + } + await send(event) + + await app(scope, recieve, wrapped_send) + + return add_x_databases_header + + return wrap_with_databases_header diff --git a/tests/plugins/view_name.py b/tests/plugins/view_name.py new file mode 100644 index 00000000..4d29ab67 --- /dev/null +++ b/tests/plugins/view_name.py @@ -0,0 +1,9 @@ +from datasette import hookimpl + + +@hookimpl +def extra_template_vars(view_name, request): + return { + "view_name": view_name, + "request": request, + } diff --git a/tests/test_api.py b/tests/test_api.py index 7edd7ee6..260d399b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1267,6 +1267,7 @@ def test_plugins_json(app_client): "templates": False, "version": None, }, + {"name": "view_name.py", "static": False, "templates": False, "version": None}, ] == sorted(response.json, key=lambda p: p["name"]) diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 8ac75ec8..c69facb5 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -1,22 +1,10 @@ import pytest from .fixtures import make_app_client -VIEW_NAME_PLUGIN = """ -from datasette import hookimpl - -@hookimpl -def extra_template_vars(view_name, request): - return { - "view_name": view_name, - "request": request, - } -""" - @pytest.fixture(scope="session") def custom_pages_client(tmp_path_factory): template_dir = tmp_path_factory.mktemp("page-templates") - extra_plugins = {"view_name.py": VIEW_NAME_PLUGIN} pages_dir = template_dir / "pages" pages_dir.mkdir() (pages_dir / "about.html").write_text("ABOUT! view_name:{{ view_name }}", "utf-8") @@ -39,9 +27,7 @@ def custom_pages_client(tmp_path_factory): nested_dir = pages_dir / "nested" nested_dir.mkdir() (nested_dir / "nest.html").write_text("Nest!", "utf-8") - for client in make_app_client( - template_dir=str(template_dir), extra_plugins=extra_plugins - ): + for client in make_app_client(template_dir=str(template_dir)): yield client From 52c4387c7d37c867104e3728cc1f4c4d1e100642 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 19:21:41 -0700 Subject: [PATCH 0424/2303] Redesigned register_output_renderer plugin hook, closes #581 --- datasette/app.py | 6 +- datasette/views/base.py | 18 +++++- datasette/views/database.py | 1 + docs/plugins.rst | 53 ++++++++++++--- tests/plugins/register_output_renderer.py | 42 ++++++++++++ tests/test_api.py | 20 +++--- tests/test_html.py | 4 ++ tests/test_plugins.py | 78 +++++++++++++++++++++++ 8 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 tests/plugins/register_output_renderer.py diff --git a/datasette/app.py b/datasette/app.py index f1fcc5eb..941b2895 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -586,7 +586,11 @@ class Datasette: hook_renderers.append(hook) for renderer in hook_renderers: - self.renderers[renderer["extension"]] = renderer["callback"] + self.renderers[renderer["extension"]] = ( + # It used to be called "callback" - remove this in Datasette 1.0 + renderer.get("render") + or renderer["callback"] + ) async def render_template( self, templates, context=None, request=None, view_name=None diff --git a/datasette/views/base.py b/datasette/views/base.py index f5eafe63..5a5fe056 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -14,6 +14,7 @@ from datasette.database import QueryInterrupted from datasette.utils import ( InvalidSql, LimitedWriter, + call_with_supported_arguments, is_url, path_with_added_args, path_with_removed_args, @@ -387,7 +388,21 @@ class DataView(BaseView): if _format in self.ds.renderers.keys(): # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) - result = self.ds.renderers[_format](request.args, data, self.name) + result = call_with_supported_arguments( + self.ds.renderers[_format], + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + # These will be deprecated in Datasette 1.0: + args=request.args, + data=data, + ) if result is None: raise NotFound("No data") @@ -395,6 +410,7 @@ class DataView(BaseView): body=result.get("body"), status=result.get("status_code", 200), content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), ) else: extras = {} diff --git a/datasette/views/database.py b/datasette/views/database.py index cd27dd5f..15545fb8 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -188,6 +188,7 @@ class QueryView(DataView): return ( { "database": database, + "query_name": canned_query, "rows": results.rows, "truncated": results.truncated, "columns": columns, diff --git a/docs/plugins.rst b/docs/plugins.rst index feb14593..27f00476 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -744,19 +744,37 @@ Allows the plugin to register a new output renderer, to output data in a custom def register_output_renderer(datasette): return { "extension": "test", - "callback": render_test + "render": render_test } -This will register ``render_test`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. When a request is received, the callback function is called with three positional arguments: +This will register ``render_test`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. When a request is received, the callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. -``args`` - dictionary - The GET parameters of the request +``datasette`` - :ref:`internals_datasette` + For accessing plugin configuration and executing queries. -``data`` - dictionary - The data to be rendered +``columns`` - list of strings + The names of the columns returned by this query. + +``rows`` - list of ``sqlite3.Row`` objects + The rows returned by the query. + +``sql`` - string + The SQL query that was executed. + +``query_name`` - string or None + If this was the execution of a :ref:`canned query `, the name of that query. + +``database`` - string + The name of the database. + +``table`` - string or None + The table or view, if one is being rendered. + +``request`` - :ref:`internals_request` + The incoming HTTP request. ``view_name`` - string - The name of the view where the renderer is being called. (``index``, ``database``, ``table``, and ``row`` are the most important ones.) + The name of the current view being called. ``index``, ``database``, ``table``, and ``row`` are the most important ones. The callback function can return ``None``, if it is unable to render the data, or a dictionary with the following keys: @@ -769,15 +787,34 @@ The callback function can return ``None``, if it is unable to render the data, o ``status_code`` - integer, optional The HTTP status code, default 200 +``headers`` - dictionary, optional + Extra HTTP headers to be returned in the response. + A simple example of an output renderer callback function: .. code-block:: python - def render_test(args, data, view_name): + def render_test(): return { "body": "Hello World" } +Here is a more complex example: + +.. code-block:: python + + def render_test(columns, rows): + first_row = " | ".join(columns) + lines = [first_row] + lines.append("=" * len(first_row)) + for row in rows: + lines.append(" | ".join(row)) + return { + "body": "Hello World", + "content_type": "text/plain; charset=utf-8", + "headers": {"x-pipes": "yay-pipes"} + } + Examples: `datasette-atom `_, `datasette-ics `_ .. _plugin_register_facet_classes: diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py new file mode 100644 index 00000000..2ea5660e --- /dev/null +++ b/tests/plugins/register_output_renderer.py @@ -0,0 +1,42 @@ +from datasette import hookimpl +import json + + +def render_test_all_parameters( + datasette, columns, rows, sql, query_name, database, table, request, view_name, data +): + headers = {} + for custom_header in request.args.getlist("header") or []: + key, value = custom_header.split(":") + headers[key] = value + return { + "body": json.dumps( + { + "datasette": datasette, + "columns": columns, + "rows": rows, + "sql": sql, + "query_name": query_name, + "database": database, + "table": table, + "request": request, + "view_name": view_name, + }, + default=repr, + ), + "content_type": request.args.get("content_type", "text/plain"), + "status_code": int(request.args.get("status_code", 200)), + "headers": headers, + } + + +def render_test_no_parameters(): + return {"body": "Hello"} + + +@hookimpl +def register_output_renderer(datasette): + return [ + {"extension": "testall", "render": render_test_all_parameters}, + {"extension": "testnone", "callback": render_test_no_parameters}, + ] diff --git a/tests/test_api.py b/tests/test_api.py index 260d399b..f92da45e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1259,16 +1259,16 @@ def test_threads_json(app_client): def test_plugins_json(app_client): response = app_client.get("/-/plugins.json") - assert [ - {"name": "my_plugin.py", "static": False, "templates": False, "version": None}, - { - "name": "my_plugin_2.py", - "static": False, - "templates": False, - "version": None, - }, - {"name": "view_name.py", "static": False, "templates": False, "version": None}, - ] == sorted(response.json, key=lambda p: p["name"]) + expected = [ + {"name": name, "static": False, "templates": False, "version": None} + for name in ( + "my_plugin.py", + "my_plugin_2.py", + "register_output_renderer.py", + "view_name.py", + ) + ] + assert expected == sorted(response.json, key=lambda p: p["name"]) def test_versions_json(app_client): diff --git a/tests/test_html.py b/tests/test_html.py index 5a07953e..e602bf0e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -546,6 +546,8 @@ def test_table_csv_json_export_interface(app_client): actual = [l["href"].split("/")[-1] for l in links] expected = [ "simple_primary_key.json?id__gt=2", + "simple_primary_key.testall?id__gt=2", + "simple_primary_key.testnone?id__gt=2", "simple_primary_key.csv?id__gt=2&_size=max", "#export", ] @@ -582,6 +584,8 @@ def test_csv_json_export_links_include_labels_if_foreign_keys(app_client): actual = [l["href"].split("/")[-1] for l in links] expected = [ "facetable.json?_labels=on", + "facetable.testall?_labels=on", + "facetable.testnone?_labels=on", "facetable.csv?_labels=on&_size=max", "#export", ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1546de92..0e4186d5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -19,6 +19,8 @@ import textwrap import pytest import urllib +at_memory_re = re.compile(r" at 0x\w+") + @pytest.mark.xfail @pytest.mark.parametrize( @@ -329,3 +331,79 @@ def test_view_names(view_names_client, path, view_name): response = view_names_client.get(path) assert response.status == 200 assert "view_name:{}".format(view_name) == response.body.decode("utf8") + + +def test_register_output_renderer_no_parameters(app_client): + response = app_client.get("/fixtures/facetable.testnone") + assert 200 == response.status + assert b"Hello" == response.body + + +def test_register_output_renderer_all_parameters(app_client): + response = app_client.get("/fixtures/facetable.testall") + assert 200 == response.status + # Lots of 'at 0x103a4a690' in here - replace those so we can do + # an easy comparison + body = response.body.decode("utf-8") + body = at_memory_re.sub(" at 0xXXX", body) + assert { + "datasette": "", + "columns": [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "city_id", + "neighborhood", + "tags", + "complex_array", + "distinct_some_null", + ], + "rows": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ], + "sql": "select pk, created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", + "query_name": None, + "database": "fixtures", + "table": "facetable", + "request": "", + "view_name": "table", + } == json.loads(body) + # Test that query_name is set correctly + query_response = app_client.get("/fixtures/pragma_cache_size.testall") + assert "pragma_cache_size" == json.loads(query_response.body)["query_name"] + + +def test_register_output_renderer_custom_status_code(app_client): + response = app_client.get("/fixtures/pragma_cache_size.testall?status_code=202") + assert 202 == response.status + + +def test_register_output_renderer_custom_content_type(app_client): + response = app_client.get( + "/fixtures/pragma_cache_size.testall?content_type=text/blah" + ) + assert "text/blah" == response.headers["content-type"] + + +def test_register_output_renderer_custom_headers(app_client): + response = app_client.get( + "/fixtures/pragma_cache_size.testall?header=x-wow:1&header=x-gosh:2" + ) + assert "1" == response.headers["x-wow"] + assert "2" == response.headers["x-gosh"] From 57f48b8416f5e13df138d63db5bfffd0bb99a9b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 19:43:30 -0700 Subject: [PATCH 0425/2303] Made register_output_renderer callback optionally awaitable, closes #776 --- datasette/views/base.py | 2 ++ docs/plugins.rst | 16 +++++++++++----- tests/plugins/register_output_renderer.py | 4 +++- tests/test_plugins.py | 1 + 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 5a5fe056..d56fd2f6 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -403,6 +403,8 @@ class DataView(BaseView): args=request.args, data=data, ) + if asyncio.iscoroutine(result): + result = await result if result is None: raise NotFound("No data") diff --git a/docs/plugins.rst b/docs/plugins.rst index 27f00476..ebf6adf6 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -736,7 +736,7 @@ register_output_renderer(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` -Allows the plugin to register a new output renderer, to output data in a custom format. The hook function should return a dictionary, or a list of dictionaries, which contain the file extension you want to handle and a callback function: +Registers a new output renderer, to output data in a custom format. The hook function should return a dictionary, or a list of dictionaries, of the following shape: .. code-block:: python @@ -747,7 +747,11 @@ Allows the plugin to register a new output renderer, to output data in a custom "render": render_test } -This will register ``render_test`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. When a request is received, the callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. +This will register ``render_test`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. + +``render_test`` is a Python function. It can be a regular function or an ``async def render_test()`` awaitable function, depending on if it needs to make any asynchronous calls. + +When a request is received, the callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. ``datasette`` - :ref:`internals_datasette` For accessing plugin configuration and executing queries. @@ -803,16 +807,18 @@ Here is a more complex example: .. code-block:: python - def render_test(columns, rows): + async def render_test(datasette, columns, rows): + db = next(iter(datasette.databases.values())) + result = await db.execute("select sqlite_version()") first_row = " | ".join(columns) lines = [first_row] lines.append("=" * len(first_row)) for row in rows: lines.append(" | ".join(row)) return { - "body": "Hello World", + "body": "\n".join(lines), "content_type": "text/plain; charset=utf-8", - "headers": {"x-pipes": "yay-pipes"} + "headers": {"x-sqlite-version": result.first()[0]}, } Examples: `datasette-atom `_, `datasette-ics `_ diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py index 2ea5660e..d4c1228d 100644 --- a/tests/plugins/register_output_renderer.py +++ b/tests/plugins/register_output_renderer.py @@ -2,13 +2,14 @@ from datasette import hookimpl import json -def render_test_all_parameters( +async def render_test_all_parameters( datasette, columns, rows, sql, query_name, database, table, request, view_name, data ): headers = {} for custom_header in request.args.getlist("header") or []: key, value = custom_header.split(":") headers[key] = value + result = await datasette.databases["fixtures"].execute("select 1 + 1") return { "body": json.dumps( { @@ -21,6 +22,7 @@ def render_test_all_parameters( "table": table, "request": request, "view_name": view_name, + "1+1": result.first()[0], }, default=repr, ), diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 0e4186d5..94b69c1f 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -347,6 +347,7 @@ def test_register_output_renderer_all_parameters(app_client): body = response.body.decode("utf-8") body = at_memory_re.sub(" at 0xXXX", body) assert { + "1+1": 2, "datasette": "", "columns": [ "pk", From cbeea23d00b36f72386e68b67d76fdb8a151a486 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 20:13:32 -0700 Subject: [PATCH 0426/2303] Test for prepare_jinja2_environment, refs #773 --- tests/plugins/my_plugin.py | 5 +++++ tests/test_plugins.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index e55a0a32..434a1977 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -87,3 +87,8 @@ def extra_template_vars(template, database, table, view_name, request, datasette default=lambda b: b.decode("utf8"), ) } + + +@hookimpl +def prepare_jinja2_environment(env): + env.filters["format_numeric"] = lambda s: "{:,.0f}".format(float(s)) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 94b69c1f..1bfd9d3f 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,6 +9,7 @@ from .fixtures import ( from datasette.app import Datasette from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils import sqlite3 +from jinja2.environment import Template import base64 import json import os @@ -408,3 +409,12 @@ def test_register_output_renderer_custom_headers(app_client): ) assert "1" == response.headers["x-wow"] assert "2" == response.headers["x-gosh"] + + +@pytest.mark.asyncio +async def test_prepare_jinja2_environment(app_client): + template = app_client.ds.jinja_env.from_string( + "Hello there, {{ a|format_numeric }}", {"a": 3412341} + ) + rendered = await app_client.ds.render_template(template) + assert "Hello there, 3,412,341" == rendered From defead17a4c9d68670ba2d9aeec9c2a70b5b059e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 20:30:32 -0700 Subject: [PATCH 0427/2303] Test for publish_subcommand hook, refs #773 --- tests/test_plugins.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1bfd9d3f..9ebf455a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -7,6 +7,7 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from datasette.app import Datasette +from datasette import cli from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils import sqlite3 from jinja2.environment import Template @@ -418,3 +419,12 @@ async def test_prepare_jinja2_environment(app_client): ) rendered = await app_client.ds.render_template(template) assert "Hello there, 3,412,341" == rendered + + +def test_publish_subcommand(): + # This is hard to test properly, because publish subcommand plugins + # cannot be loaded using the --plugins-dir mechanism - they need + # to be installed using "pip install". So I'm cheating and taking + # advantage of the fact that cloudrun/heroku use the plugin hook + # to register themselves as default plugins. + assert ["cloudrun", "heroku"] == cli.publish.list_commands({}) From 6d95cb4f9146a5c4584a147bdf243c778a0f23f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 21:09:16 -0700 Subject: [PATCH 0428/2303] Unit test for register_facet_classes plugin, closes #773 I was a bit lazy with this one. I didn't hook up a test for the facet_results mechanism. The custom facet hook isn't a great design so I will probably rethink it at some point in the future anyway. --- tests/plugins/my_plugin.py | 34 +++++++++++++++++++++++++++++++ tests/test_plugins.py | 41 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 434a1977..10d7e7e6 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -1,4 +1,6 @@ from datasette import hookimpl +from datasette.facets import Facet +from datasette.utils import path_with_added_args import base64 import pint import json @@ -92,3 +94,35 @@ def extra_template_vars(template, database, table, view_name, request, datasette @hookimpl def prepare_jinja2_environment(env): env.filters["format_numeric"] = lambda s: "{:,.0f}".format(float(s)) + + +@hookimpl +def register_facet_classes(): + return [DummyFacet] + + +class DummyFacet(Facet): + type = "dummy" + + async def suggest(self): + columns = await self.get_columns(self.sql, self.params) + return ( + [ + { + "name": column, + "toggle_url": self.ds.absolute_url( + self.request, + path_with_added_args(self.request, {"_facet_dummy": column}), + ), + "type": "dummy", + } + for column in columns + ] + if self.request.args.get("_dummy_facet") + else [] + ) + + async def facet_results(self): + facet_results = {} + facets_timed_out = [] + return facet_results, facets_timed_out diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9ebf455a..2aadb252 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -428,3 +428,44 @@ def test_publish_subcommand(): # advantage of the fact that cloudrun/heroku use the plugin hook # to register themselves as default plugins. assert ["cloudrun", "heroku"] == cli.publish.list_commands({}) + + +def test_register_facet_classes(app_client): + response = app_client.get( + "/fixtures/compound_three_primary_keys.json?_dummy_facet=1" + ) + data = json.loads(response.body) + assert [ + { + "name": "pk1", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=pk1", + "type": "dummy", + }, + { + "name": "pk2", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=pk2", + "type": "dummy", + }, + { + "name": "pk3", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=pk3", + "type": "dummy", + }, + { + "name": "content", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=content", + "type": "dummy", + }, + { + "name": "pk1", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk1", + }, + { + "name": "pk2", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk2", + }, + { + "name": "pk3", + "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk3", + }, + ] == data["suggested_facets"] From 510c1989d43cd9b7c9f116ad161b7380220ac5d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 21:11:53 -0700 Subject: [PATCH 0429/2303] Removed xfail, refs #773 --- tests/test_plugins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 2aadb252..e9556b31 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -24,7 +24,6 @@ import urllib at_memory_re = re.compile(r" at 0x\w+") -@pytest.mark.xfail @pytest.mark.parametrize( "plugin_hook", [name for name in dir(pm.hook) if not name.startswith("_")] ) From 75cd432e5a96c5fe2577f839c3a059fd6bf41124 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 22:00:04 -0700 Subject: [PATCH 0430/2303] Ability to set custom table/view page size in metadata, closes #751 --- datasette/views/table.py | 2 +- docs/metadata.rst | 23 +++++++++++++++++++++++ tests/fixtures.py | 1 + tests/test_api.py | 4 ++-- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index aab4bbe3..d014db71 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -527,7 +527,7 @@ class TableView(RowTableShared): extra_args = {} # Handle ?_size=500 - page_size = _size or request.args.get("_size") + page_size = _size or request.args.get("_size") or table_metadata.get("size") if page_size: if page_size == "max": page_size = self.ds.max_returned_rows diff --git a/docs/metadata.rst b/docs/metadata.rst index 3cb1f739..88ad5854 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -156,6 +156,29 @@ Or use ``"sort_desc"`` to sort in descending order: } } +.. _metadata_page_size: + +Setting a custom page size +-------------------------- + +Datasette defaults to displaing 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: + +.. code-block:: json + + { + "databases": { + "mydatabase": { + "tables": { + "example_table": { + "size": 10 + } + } + } + } + } + +This size can still be over-ridden by passing e.g. ``?_size=50`` in the querystring. + .. _metadata_sortable_columns: Setting which columns can be used for sorting diff --git a/tests/fixtures.py b/tests/fixtures.py index 1eaa1dfe..9479abf6 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -305,6 +305,7 @@ METADATA = { }, "attraction_characteristic": {"sort_desc": "pk"}, "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, }, "queries": { "𝐜𝐢𝐭𝐢𝐞𝐬": "select id, name from facet_cities order by id limit 1;", diff --git a/tests/test_api.py b/tests/test_api.py index f92da45e..eb80f8e7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -754,9 +754,9 @@ def test_table_with_reserved_word_name(app_client): "path,expected_rows,expected_pages", [ ("/fixtures/no_primary_key.json", 201, 5), - ("/fixtures/paginated_view.json", 201, 5), + ("/fixtures/paginated_view.json", 201, 9), ("/fixtures/no_primary_key.json?_size=25", 201, 9), - ("/fixtures/paginated_view.json?_size=25", 201, 9), + ("/fixtures/paginated_view.json?_size=50", 201, 5), ("/fixtures/paginated_view.json?_size=max", 201, 3), ("/fixtures/123_starts_with_digits.json", 0, 1), # Ensure faceting doesn't break pagination: From 5ab411c733233435d613d04c610a5a41fd0b7735 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2020 22:57:05 -0700 Subject: [PATCH 0431/2303] can_render mechanism for register_output_renderer, closes #770 --- datasette/app.py | 8 ++--- datasette/utils/__init__.py | 6 +++- datasette/views/base.py | 27 ++++++++++++++--- docs/plugins.rst | 22 ++++++++++---- tests/plugins/register_output_renderer.py | 26 +++++++++++++++- tests/test_plugins.py | 37 ++++++++++++++++++++++- 6 files changed, 108 insertions(+), 18 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 941b2895..40d39ac9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -228,7 +228,7 @@ class Datasette: if config_dir and (config_dir / "config.json").exists() and not config: config = json.load((config_dir / "config.json").open()) self._config = dict(DEFAULT_CONFIG, **(config or {})) - self.renderers = {} # File extension -> renderer function + self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( max_workers=self.config("num_sql_threads") @@ -574,7 +574,7 @@ class Datasette: def register_renderers(self): """ Register output renderers which output data in custom formats. """ # Built-in renderers - self.renderers["json"] = json_renderer + self.renderers["json"] = (json_renderer, lambda: True) # Hooks hook_renderers = [] @@ -588,8 +588,8 @@ class Datasette: for renderer in hook_renderers: self.renderers[renderer["extension"]] = ( # It used to be called "callback" - remove this in Datasette 1.0 - renderer.get("render") - or renderer["callback"] + renderer.get("render") or renderer["callback"], + renderer.get("can_render") or (lambda: True), ) async def render_template( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 03157072..2dab8e14 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -811,6 +811,10 @@ def call_with_supported_arguments(fn, **kwargs): call_with = [] for parameter in parameters: if parameter not in kwargs: - raise TypeError("{} requires parameters {}".format(fn, tuple(parameters))) + raise TypeError( + "{} requires parameters {}, missing: {}".format( + fn, tuple(parameters), set(parameters) - set(kwargs.keys()) + ) + ) call_with.append(kwargs[parameter]) return fn(*call_with) diff --git a/datasette/views/base.py b/datasette/views/base.py index d56fd2f6..06b78d5f 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -389,7 +389,7 @@ class DataView(BaseView): # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( - self.ds.renderers[_format], + self.ds.renderers[_format][0], datasette=self.ds, columns=data.get("columns") or [], rows=data.get("rows") or [], @@ -426,10 +426,27 @@ class DataView(BaseView): if data.get("expandable_columns"): url_labels_extra = {"_labels": "on"} - renderers = { - key: path_with_format(request, key, {**url_labels_extra}) - for key in self.ds.renderers.keys() - } + renderers = {} + for key, (_, can_render) in self.ds.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + ) + if asyncio.iscoroutine(it_can_render): + it_can_render = await it_can_render + if it_can_render: + renderers[key] = path_with_format( + request, key, {**url_labels_extra} + ) + url_csv_args = {"_size": "max", **url_labels_extra} url_csv = path_with_format(request, "csv", url_csv_args) url_csv_path = url_csv.split("?")[0] diff --git a/docs/plugins.rst b/docs/plugins.rst index ebf6adf6..b27daf3f 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -744,14 +744,17 @@ Registers a new output renderer, to output data in a custom format. The hook fun def register_output_renderer(datasette): return { "extension": "test", - "render": render_test + "render": render_demo, + "can_render": can_render_demo, # Optional } -This will register ``render_test`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. +This will register ``render_demo`` to be called when paths with the extension ``.test`` (for example ``/database.test``, ``/database/table.test``, or ``/database/table/row.test``) are requested. -``render_test`` is a Python function. It can be a regular function or an ``async def render_test()`` awaitable function, depending on if it needs to make any asynchronous calls. +``render_demo`` is a Python function. It can be a regular function or an ``async def render_demo()`` awaitable function, depending on if it needs to make any asynchronous calls. -When a request is received, the callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. +``can_render_demo`` is a Python function (or ``async def`` function) which acepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. + +When a request is received, the ``"render"`` callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. ``datasette`` - :ref:`internals_datasette` For accessing plugin configuration and executing queries. @@ -798,7 +801,7 @@ A simple example of an output renderer callback function: .. code-block:: python - def render_test(): + def render_demo(): return { "body": "Hello World" } @@ -807,7 +810,7 @@ Here is a more complex example: .. code-block:: python - async def render_test(datasette, columns, rows): + async def render_demo(datasette, columns, rows): db = next(iter(datasette.databases.values())) result = await db.execute("select sqlite_version()") first_row = " | ".join(columns) @@ -821,6 +824,13 @@ Here is a more complex example: "headers": {"x-sqlite-version": result.first()[0]}, } +And here is an example ``can_render`` function which returns ``True`` only if the query results contain the columns ``atom_id``, ``atom_title`` and ``atom_updated``: + +.. code-block:: python + + def can_render_demo(columns): + return {"atom_id", "atom_title", "atom_updated"}.issubset(columns) + Examples: `datasette-atom `_, `datasette-ics `_ .. _plugin_register_facet_classes: diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py index d4c1228d..a9f0f157 100644 --- a/tests/plugins/register_output_renderer.py +++ b/tests/plugins/register_output_renderer.py @@ -2,6 +2,26 @@ from datasette import hookimpl import json +async def can_render( + datasette, columns, rows, sql, query_name, database, table, request, view_name +): + # We stash this on datasette so the calling unit test can see it + datasette._can_render_saw = { + "datasette": datasette, + "columns": columns, + "rows": rows, + "sql": sql, + "query_name": query_name, + "database": database, + "table": table, + "request": request, + "view_name": view_name, + } + if request.args.get("_no_can_render"): + return False + return True + + async def render_test_all_parameters( datasette, columns, rows, sql, query_name, database, table, request, view_name, data ): @@ -39,6 +59,10 @@ def render_test_no_parameters(): @hookimpl def register_output_renderer(datasette): return [ - {"extension": "testall", "render": render_test_all_parameters}, + { + "extension": "testall", + "render": render_test_all_parameters, + "can_render": can_render, + }, {"extension": "testnone", "callback": render_test_no_parameters}, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index e9556b31..a34328a9 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,7 +9,7 @@ from .fixtures import ( from datasette.app import Datasette from datasette import cli from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm -from datasette.utils import sqlite3 +from datasette.utils import sqlite3, CustomRow from jinja2.environment import Template import base64 import json @@ -411,6 +411,41 @@ def test_register_output_renderer_custom_headers(app_client): assert "2" == response.headers["x-gosh"] +def test_register_output_renderer_can_render(app_client): + response = app_client.get("/fixtures/facetable?_no_can_render=1") + assert response.status == 200 + links = ( + Soup(response.body, "html.parser") + .find("p", {"class": "export-links"}) + .findAll("a") + ) + actual = [l["href"].split("/")[-1] for l in links] + # Should not be present because we sent ?_no_can_render=1 + assert "facetable.testall?_labels=on" not in actual + # Check that it was passed the values we expected + assert hasattr(app_client.ds, "_can_render_saw") + assert { + "datasette": app_client.ds, + "columns": [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "city_id", + "neighborhood", + "tags", + "complex_array", + "distinct_some_null", + ], + "sql": "select pk, created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", + "query_name": None, + "database": "fixtures", + "table": "facetable", + "view_name": "table", + }.items() <= app_client.ds._can_render_saw.items() + + @pytest.mark.asyncio async def test_prepare_jinja2_environment(app_client): template = app_client.ds.jinja_env.from_string( From d56f402822df102f9cf1a9a056449d01a15e3aae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2020 07:10:21 -0700 Subject: [PATCH 0432/2303] Release notes for 0.43 Refs #581, #770, #729, #706, #751, #706, #744, #771, #773 --- README.md | 1 + docs/changelog.rst | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 7351c5c0..90df75de 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 28th May 2020: [Datasette 0.43](http://datasette.readthedocs.io/en/latest/changelog.html#v0-43) - Redesigned [register_output_renderer](https://datasette.readthedocs.io/en/latest/plugins.html#plugin-register-output-renderer) plugin hook and various small improvements and fixes. * 8th May 2020: [Datasette 0.42](http://datasette.readthedocs.io/en/latest/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database. * 6th May 2020: [Datasette 0.41](http://datasette.readthedocs.io/en/latest/changelog.html#v0-41) - New mechanism for [creating custom pages](https://datasette.readthedocs.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://datasette.readthedocs.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements. * 21st April 2020: [Datasette 0.40](http://datasette.readthedocs.io/en/latest/changelog.html#v0-40) - Metadata can now be provided as YAML instead of JSON. Publishing to Zeit Now v1 is no longer supported, but Now v2 support is provided by the new [datasette-publish-now](https://github.com/simonw/datasette-publish-now) plugin. Various bug fixes. diff --git a/docs/changelog.rst b/docs/changelog.rst index 48d3128b..8f375dd1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,21 @@ Changelog ========= +.. _v0_43: + +0.43 (2020-05-28) +----------------- + +The main focus of this release is a major upgrade to the :ref:`plugin_register_output_renderer` plugin hook, which allows plugins to provide new output formats for Datasette such as `datasette-atom `__ and `datasette-ics `__. + +* Redesign of :ref:`plugin_register_output_renderer` to provide more context to the render callback and support an optional ``"can_render"`` callback that controls if a suggested link to the output format is provided. (`#581 `__, `#770 `__) +* Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (`#729 `__) +* The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (`#706 `__) +* New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`metadata_page_size`. (`#751 `__) +* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_default_fragment`. (`#706 `__) +* Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (`#744 `__) +* Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (`#771 `__, `#773 `__) + .. _v0_42: 0.42 (2020-05-08) From 40885ef24e32d91502b6b8bbad1c7376f50f2830 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2020 07:41:22 -0700 Subject: [PATCH 0433/2303] Noted tool for converting release notes to Markdown --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 48930332..567c4f47 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -145,12 +145,12 @@ To release a new version, first create a commit that updates :ref:`the changelog For non-bugfix releases you may want to update the news section of ``README.md`` as part of the same commit. -Wait long enough for Travis to build and deploy the demo version of that commit (otherwise the tag deployment may fail to alias to it properly). Then run the following:: +To tag and push the releaes, run the following:: git tag 0.25.2 git push --tags Final steps once the release has deployed to https://pypi.org/project/datasette/ -* Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases +* Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases - you can convert the release notes to Markdown by copying and pasting the rendered HTML into this tool: https://euangoddard.github.io/clipboard2markdown/ * Manually kick off a build of the `stable` branch on Read The Docs: https://readthedocs.org/projects/datasette/builds/ From 7bb30c1f11f7246baf7bb6a229f6b93572c4cbe3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2020 10:09:32 -0700 Subject: [PATCH 0434/2303] request.url now respects force_https_urls, closes #781 --- datasette/app.py | 7 +++++++ tests/plugins/my_plugin_2.py | 3 +++ tests/test_api.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 40d39ac9..07190c16 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -786,6 +786,13 @@ class DatasetteRouter(AsgiRouter): base_url = self.ds.config("base_url") if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] + # Apply force_https_urls, if set + if ( + self.ds.config("force_https_urls") + and scope["type"] == "http" + and scope.get("scheme") != "https" + ): + scope = dict(scope, scheme="https") return await super().route_path(scope, receive, send, path) async def handle_404(self, scope, receive, send, exception=None): diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index fdc6956d..c9e7c78f 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -46,6 +46,9 @@ def render_cell(value, database): @hookimpl def extra_template_vars(template, database, table, view_name, request, datasette): + # This helps unit tests that want to run assertions against the request object: + datasette._last_request = request + async def query_database(sql): first_db = list(datasette.databases.keys())[0] return (await datasette.execute(first_db, sql)).rows[0][0] diff --git a/tests/test_api.py b/tests/test_api.py index eb80f8e7..d7e7c03f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1676,6 +1676,10 @@ def test_config_force_https_urls(): "toggle_url" ].startswith("https://") assert response.json["suggested_facets"][0]["toggle_url"].startswith("https://") + # Also confirm that request.url and request.scheme are set correctly + response = client.get("/") + assert client.ds._last_request.url.startswith("https://") + assert client.ds._last_request.scheme == "https" def test_infinity_returned_as_null(app_client): From 21a8ffc82dcf5e8e5f484ce39ee9713f959e0ad5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2020 10:49:58 -0700 Subject: [PATCH 0435/2303] Tip about referencing issues in release notes commit --- docs/contributing.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 567c4f47..da4dc35a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -137,12 +137,16 @@ We increment ``minor`` for new features. We increment ``patch`` for bugfix releass. -To release a new version, first create a commit that updates :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: +To release a new version, first create a commit that updates :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: # Update changelog - git commit -m "Release 0.25.2" -a + git commit -m "Release notes for 0.43 + + Refs #581, #770, #729, #706, #751, #706, #744, #771, #773" -a git push +Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. + For non-bugfix releases you may want to update the news section of ``README.md`` as part of the same commit. To tag and push the releaes, run the following:: From 3c1a60589e14849344acd8aa6da0a60b40fbfc60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2020 11:27:24 -0700 Subject: [PATCH 0436/2303] Consistent capitalization of SpatiaLite in the docs --- docs/changelog.rst | 2 +- docs/installation.rst | 2 +- docs/metadata.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8f375dd1..8b6272cb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -947,7 +947,7 @@ request all rows where that column is less than 50 meters or more than 20 feet f 404s for missing tables/databases closes `#184 `_ - long_description in markdown for the new PyPI -- Hide Spatialite system tables. [Russ Garrett] +- Hide SpatiaLite system tables. [Russ Garrett] - Allow ``explain select`` / ``explain query plan select`` `#201 `_ - Datasette inspect now finds primary_keys `#195 `_ - Ability to sort using form fields (for mobile portrait mode) `#199 `_ diff --git a/docs/installation.rst b/docs/installation.rst index cdf1467a..aacfed1d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -149,7 +149,7 @@ To upgrade to the most recent release of Datasette, run the following:: docker pull datasetteproject/datasette -Loading Spatialite +Loading SpatiaLite ~~~~~~~~~~~~~~~~~~ The ``datasetteproject/datasette`` image includes a recent version of the diff --git a/docs/metadata.rst b/docs/metadata.rst index 88ad5854..18766bac 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -260,7 +260,7 @@ Hiding tables ------------- You can hide tables from the database listing view (in the same way that FTS and -Spatialite tables are automatically hidden) using ``"hidden": true``: +SpatiaLite tables are automatically hidden) using ``"hidden": true``: .. code-block:: json From 3e8932bf6443bd5168f22d559597aed619205995 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 29 May 2020 15:12:10 -0700 Subject: [PATCH 0437/2303] Upgrade to actions/cache@v2 --- .github/workflows/deploy-latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 33490972..fd53f754 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: 3.8 - - uses: actions/cache@v1 + - uses: actions/cache@v2 name: Configure pip caching with: path: ~/.cache/pip From 7ccd55a1638d7d2762f2789f192e5bb81fb0d0c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2020 11:54:57 -0700 Subject: [PATCH 0438/2303] Views do support sorting now, refs #508 --- docs/metadata.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metadata.rst b/docs/metadata.rst index 18766bac..024af01e 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -210,7 +210,7 @@ This will restrict sorting of ``example_table`` to just the ``height`` and You can also disable sorting entirely by setting ``"sortable_columns": []`` -By default, database views in Datasette do not support sorting. You can use ``sortable_columns`` to enable specific sort orders for a view called ``name_of_view`` in the database ``my_database`` like so: +You can use ``sortable_columns`` to enable specific sort orders for a view called ``name_of_view`` in the database ``my_database`` like so: .. code-block:: json From 84616a2364df56f966f579eecc0716b9877f0d70 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 29 May 2020 15:51:30 -0700 Subject: [PATCH 0439/2303] request.args.getlist() returns [] if missing, refs #774 Also added some unit tests for request.args --- datasette/utils/__init__.py | 4 ++-- docs/internals.rst | 2 +- tests/plugins/register_output_renderer.py | 2 +- tests/test_utils.py | 10 ++++++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2dab8e14..9b4f21ba 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -761,9 +761,9 @@ class RequestParameters(dict): except (KeyError, TypeError): return default - def getlist(self, name, default=None): + def getlist(self, name): "Return full list" - return super().get(name, default) + return super().get(name) or [] class ConnectionProblem(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index 5bcb9da9..bbf10cae 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -276,4 +276,4 @@ Conider the querystring ``?foo=1&foo=2``. This will produce a ``request.args`` t Calling ``request.args.get("foo")`` will return the first value, ``"1"``. If that key is not present it will return ``None`` - or the second argument if you passed one, which will be used as the default. -Calling ``request.args.getlist("foo")`` will return the full list, ``["1", "2"]``. \ No newline at end of file +Calling ``request.args.getlist("foo")`` will return the full list, ``["1", "2"]``. If you call it on a missing key it will return ``[]``. diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py index a9f0f157..82b60d01 100644 --- a/tests/plugins/register_output_renderer.py +++ b/tests/plugins/register_output_renderer.py @@ -26,7 +26,7 @@ async def render_test_all_parameters( datasette, columns, rows, sql, query_name, database, table, request, view_name, data ): headers = {} - for custom_header in request.args.getlist("header") or []: + for custom_header in request.args.getlist("header"): key, value = custom_header.split(":") headers[key] = value result = await datasette.databases["fixtures"].execute("select 1 + 1") diff --git a/tests/test_utils.py b/tests/test_utils.py index 59b80a67..ffb66ca5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -448,6 +448,16 @@ async def test_request_post_vars(): assert {"foo": "bar", "baz": "1"} == await request.post_vars() +def test_request_args(): + request = Request.fake("/foo?multi=1&multi=2&single=3") + assert "1" == request.args.get("multi") + assert "3" == request.args.get("single") + assert ["1", "2"] == request.args.getlist("multi") + assert [] == request.args.getlist("missing") + with pytest.raises(KeyError): + request.args["missing"] + + def test_call_with_supported_arguments(): def foo(a, b): return "{}+{}".format(a, b) From f272cbc65fbf56368413320e21c87dc842e0a083 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 29 May 2020 15:57:46 -0700 Subject: [PATCH 0440/2303] Use request.args.getlist instead of request.args[...], refs #774 --- datasette/views/table.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index d014db71..d1d92bb1 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -319,19 +319,19 @@ class TableView(RowTableShared): if not self.ds.config("allow_sql"): raise DatasetteError("_where= is not allowed", status=400) else: - where_clauses.extend(request.args["_where"]) + where_clauses.extend(request.args.getlist("_where")) extra_wheres_for_ui = [ { "text": text, "remove_url": path_with_removed_args(request, {"_where": text}), } - for text in request.args["_where"] + for text in request.args.getlist("_where") ] # Support for ?_through={table, column, value} extra_human_descriptions = [] if "_through" in request.args: - for through in request.args["_through"]: + for through in request.args.getlist("_through"): through_data = json.loads(through) through_table = through_data["table"] other_column = through_data["column"] @@ -559,7 +559,7 @@ class TableView(RowTableShared): ) if request.args.get("_timelimit"): - extra_args["custom_time_limit"] = int(request.args["_timelimit"]) + extra_args["custom_time_limit"] = int(request.args.get("_timelimit")) results = await db.execute(sql, params, truncate=True, **extra_args) @@ -633,7 +633,7 @@ class TableView(RowTableShared): all_labels = default_labels # Check for explicit _label= if "_label" in request.args: - columns_to_expand = request.args["_label"] + columns_to_expand = request.args.getlist("_label") if columns_to_expand is None and all_labels: # expand all columns with foreign keys columns_to_expand = [fk["column"] for fk, _ in expandable_columns] @@ -746,7 +746,7 @@ class TableView(RowTableShared): if arg in special_args: form_hidden_args.append((arg, special_args[arg])) if request.args.get("_where"): - for where_text in request.args["_where"]: + for where_text in request.args.getlist("_where"): form_hidden_args.append(("_where", where_text)) # if no sort specified AND table has a single primary key, From 81be31322a968d23cf57cee62b58df55433385e3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 29 May 2020 16:18:01 -0700 Subject: [PATCH 0441/2303] New implementation for RequestParams - no longer subclasses dict - request.args[key] now returns first item, not all items - removed request.raw_args entirely Closes #774 --- datasette/renderer.py | 2 +- datasette/utils/__init__.py | 30 +++++++++++++++++++++++++++--- datasette/utils/asgi.py | 5 ----- datasette/views/table.py | 6 +++--- docs/internals.rst | 12 ++++++++---- tests/test_utils.py | 10 ++++++++++ 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 349c2922..3f921fe7 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -32,7 +32,7 @@ def json_renderer(args, data, view_name): # Handle the _json= parameter which may modify data["rows"] json_cols = [] if "_json" in args: - json_cols = args["_json"] + json_cols = args.getlist("_json") if json_cols and "rows" in data and "columns" in data: data["rows"] = convert_specific_columns_to_json( data["rows"], data["columns"], json_cols diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 9b4f21ba..bf965413 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -753,17 +753,41 @@ def escape_fts(query): ) -class RequestParameters(dict): +class RequestParameters: + def __init__(self, data): + # data is a dictionary of key => [list, of, values] + assert isinstance(data, dict), "data should be a dictionary of key => [list]" + for key in data: + assert isinstance( + data[key], list + ), "data should be a dictionary of key => [list]" + self._data = data + + def __contains__(self, key): + return key in self._data + + def __getitem__(self, key): + return self._data[key][0] + + def keys(self): + return self._data.keys() + + def __iter__(self): + yield from self._data.keys() + + def __len__(self): + return len(self._data) + def get(self, name, default=None): "Return first value in the list, if available" try: - return super().get(name)[0] + return self._data.get(name)[0] except (KeyError, TypeError): return default def getlist(self, name): "Return full list" - return super().get(name) or [] + return self._data.get(name) or [] class ConnectionProblem(Exception): diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 62a2a0c8..24398b77 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -63,11 +63,6 @@ class Request: def args(self): return RequestParameters(parse_qs(qs=self.query_string)) - @property - def raw_args(self): - # Deprecated, undocumented - may be removed in Datasette 1.0 - return {key: value[0] for key, value in self.args.items()} - async def post_vars(self): body = [] body = b"" diff --git a/datasette/views/table.py b/datasette/views/table.py index d1d92bb1..a629346f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -277,11 +277,11 @@ class TableView(RowTableShared): # it can still be queried using ?_col__exact=blah special_args = {} other_args = [] - for key, value in args.items(): + for key in args: if key.startswith("_") and "__" not in key: - special_args[key] = value[0] + special_args[key] = args[key] else: - for v in value: + for v in args.getlist(key): other_args.append((key, v)) # Handle ?_filter_column and redirect, if present diff --git a/docs/internals.rst b/docs/internals.rst index bbf10cae..ea015dbc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -268,12 +268,16 @@ The object also has one awaitable method: The RequestParameters class --------------------------- -This class, returned by ``request.args``, is a subclass of a Python dictionary that provides methods for working with keys that map to lists of values. +This class, returned by ``request.args``, is a dictionary-like object. -Conider the querystring ``?foo=1&foo=2``. This will produce a ``request.args`` that looks like this:: +Consider the querystring ``?foo=1&foo=2``. This will produce a ``request.args`` that looks like this:: RequestParameters({"foo": ["1", "2"]}) -Calling ``request.args.get("foo")`` will return the first value, ``"1"``. If that key is not present it will return ``None`` - or the second argument if you passed one, which will be used as the default. +``request.args["foo"]`` returns the first value, ``"1"`` - or raises ``KeyError`` if that key is missing. -Calling ``request.args.getlist("foo")`` will return the full list, ``["1", "2"]``. If you call it on a missing key it will return ``[]``. +``request.args.get("foo")`` returns ``"1"`` - or ``None`` if the key is missing. A second argument can be used to specify a different default value. + +``request.args.getlist("foo")`` returns the full list, ``["1", "2"]``. If you call it on a missing key it will return ``[]``. + +You can use ``if key in request.args`` to check if a key is present. ``for key in request.args`` will iterate through the keys, or you can use ``request.args.keys()`` to get all of the keys. diff --git a/tests/test_utils.py b/tests/test_utils.py index ffb66ca5..9d6f45b0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -452,8 +452,18 @@ def test_request_args(): request = Request.fake("/foo?multi=1&multi=2&single=3") assert "1" == request.args.get("multi") assert "3" == request.args.get("single") + assert "1" == request.args["multi"] + assert "3" == request.args["single"] assert ["1", "2"] == request.args.getlist("multi") assert [] == request.args.getlist("missing") + assert "multi" in request.args + assert "single" in request.args + assert "missing" not in request.args + expected = ["multi", "single"] + assert expected == list(request.args.keys()) + for i, key in enumerate(request.args): + assert expected[i] == key + assert 2 == len(request.args) with pytest.raises(KeyError): request.args["missing"] From 31fb006a9b05067a8eb2f774ad3a3b15b4565924 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 07:28:29 -0700 Subject: [PATCH 0442/2303] Added datasette.get_database() method Refs #576 --- datasette/app.py | 5 +++++ docs/internals.rst | 10 ++++++++++ docs/plugins.rst | 2 +- tests/test_database.py | 3 +++ tests/test_internals_datasette.py | 23 +++++++++++++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/test_internals_datasette.py diff --git a/datasette/app.py b/datasette/app.py index 07190c16..30eb3dba 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -281,6 +281,11 @@ class Datasette: self.register_renderers() + def get_database(self, name=None): + if name is None: + return next(iter(self.databases.values())) + return self.databases[name] + def add_database(self, name, db): self.databases[name] = db diff --git a/docs/internals.rst b/docs/internals.rst index ea015dbc..886cb7e7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -44,6 +44,16 @@ This method lets you read plugin configuration values that were set in ``metadat Renders a `Jinja template `__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins. +.. _datasette_get_database: + +.get_database(name) +------------------- + +``name`` - string, optional + The name of the database - optional. + +Returns the specified database object. Raises a ``KeyError`` if the database does not exist. Call this method without an argument to return the first connected database. + .. _datasette_add_database: .add_database(name, db) diff --git a/docs/plugins.rst b/docs/plugins.rst index b27daf3f..f08f1217 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -811,7 +811,7 @@ Here is a more complex example: .. code-block:: python async def render_demo(datasette, columns, rows): - db = next(iter(datasette.databases.values())) + db = datasette.get_database() result = await db.execute("select sqlite_version()") first_row = " | ".join(columns) lines = [first_row] diff --git a/tests/test_database.py b/tests/test_database.py index 1f1a3a7e..bd7e7666 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,3 +1,6 @@ +""" +Tests for the datasette.database.Database class +""" from datasette.database import Results, MultipleValues from datasette.utils import sqlite3 from .fixtures import app_client diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py new file mode 100644 index 00000000..4993250d --- /dev/null +++ b/tests/test_internals_datasette.py @@ -0,0 +1,23 @@ +""" +Tests for the datasette.app.Datasette class +""" +from .fixtures import app_client +import pytest + + +@pytest.fixture +def datasette(app_client): + return app_client.ds + + +def test_get_database(datasette): + db = datasette.get_database("fixtures") + assert "fixtures" == db.name + with pytest.raises(KeyError): + datasette.get_database("missing") + + +def test_get_database_no_argument(datasette): + # Returns the first available database: + db = datasette.get_database() + assert "fixtures" == db.name From ca56c226a9f1b02e871d7d7b392619a805b7f1ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 07:33:02 -0700 Subject: [PATCH 0443/2303] Renamed test_database.py to test_internals_database.py Also added a db fixture to remove some boilerplate. --- ...database.py => test_internals_database.py} | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) rename tests/{test_database.py => test_internals_database.py} (80%) diff --git a/tests/test_database.py b/tests/test_internals_database.py similarity index 80% rename from tests/test_database.py rename to tests/test_internals_database.py index bd7e7666..fde7ad2c 100644 --- a/tests/test_database.py +++ b/tests/test_internals_database.py @@ -9,17 +9,20 @@ import time import uuid +@pytest.fixture +def db(app_client): + return app_client.ds.get_database("fixtures") + + @pytest.mark.asyncio -async def test_execute(app_client): - db = app_client.ds.databases["fixtures"] +async def test_execute(db): results = await db.execute("select * from facetable") assert isinstance(results, Results) assert 15 == len(results) @pytest.mark.asyncio -async def test_results_first(app_client): - db = app_client.ds.databases["fixtures"] +async def test_results_first(db): assert None is (await db.execute("select * from facetable where pk > 100")).first() results = await db.execute("select * from facetable") row = results.first() @@ -35,8 +38,7 @@ async def test_results_first(app_client): ], ) @pytest.mark.asyncio -async def test_results_single_value(app_client, query, expected): - db = app_client.ds.databases["fixtures"] +async def test_results_single_value(db, query, expected): results = await db.execute(query) if expected: assert expected == results.single_value() @@ -46,9 +48,7 @@ async def test_results_single_value(app_client, query, expected): @pytest.mark.asyncio -async def test_execute_fn(app_client): - db = app_client.ds.databases["fixtures"] - +async def test_execute_fn(db): def get_1_plus_1(conn): return conn.execute("select 1 + 1").fetchall()[0][0] @@ -63,16 +63,14 @@ async def test_execute_fn(app_client): ), ) @pytest.mark.asyncio -async def test_table_exists(app_client, tables, exists): - db = app_client.ds.databases["fixtures"] +async def test_table_exists(db, tables, exists): for table in tables: actual = await db.table_exists(table) assert exists == actual @pytest.mark.asyncio -async def test_get_all_foreign_keys(app_client): - db = app_client.ds.databases["fixtures"] +async def test_get_all_foreign_keys(db): all_foreign_keys = await db.get_all_foreign_keys() assert { "incoming": [], @@ -102,8 +100,7 @@ async def test_get_all_foreign_keys(app_client): @pytest.mark.asyncio -async def test_table_names(app_client): - db = app_client.ds.databases["fixtures"] +async def test_table_names(db): table_names = await db.table_names() assert [ "simple_primary_key", @@ -139,8 +136,7 @@ async def test_table_names(app_client): @pytest.mark.asyncio -async def test_execute_write_block_true(app_client): - db = app_client.ds.databases["fixtures"] +async def test_execute_write_block_true(db): await db.execute_write( "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], @@ -151,8 +147,7 @@ async def test_execute_write_block_true(app_client): @pytest.mark.asyncio -async def test_execute_write_block_false(app_client): - db = app_client.ds.databases["fixtures"] +async def test_execute_write_block_false(db): await db.execute_write( "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], ) @@ -162,9 +157,7 @@ async def test_execute_write_block_false(app_client): @pytest.mark.asyncio -async def test_execute_write_fn_block_false(app_client): - db = app_client.ds.databases["fixtures"] - +async def test_execute_write_fn_block_false(db): def write_fn(conn): with conn: conn.execute("delete from roadside_attractions where pk = 1;") @@ -177,9 +170,7 @@ async def test_execute_write_fn_block_false(app_client): @pytest.mark.asyncio -async def test_execute_write_fn_block_true(app_client): - db = app_client.ds.databases["fixtures"] - +async def test_execute_write_fn_block_true(db): def write_fn(conn): with conn: conn.execute("delete from roadside_attractions where pk = 1;") @@ -191,9 +182,7 @@ async def test_execute_write_fn_block_true(app_client): @pytest.mark.asyncio -async def test_execute_write_fn_exception(app_client): - db = app_client.ds.databases["fixtures"] - +async def test_execute_write_fn_exception(db): def write_fn(conn): assert False From 012c76901af65442e90eac4b36db43455e3c922f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 07:38:46 -0700 Subject: [PATCH 0444/2303] _ prefix for many private methods of Datasette, refs #576 --- datasette/app.py | 28 ++++++++++++++-------------- datasette/database.py | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 30eb3dba..4b9807b0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -279,7 +279,7 @@ class Datasette: # pylint: disable=no-member pm.hook.prepare_jinja2_environment(env=self.jinja_env) - self.register_renderers() + self._register_renderers() def get_database(self, name=None): if name is None: @@ -392,7 +392,7 @@ class Datasette: } ) - def prepare_connection(self, conn, database): + def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") for name, num_args, func in self.sqlite_functions: @@ -468,12 +468,12 @@ class Datasette: url = "https://" + url[len("http://") :] return url - def register_custom_units(self): + def _register_custom_units(self): "Register any custom units defined in the metadata.json with Pint" for unit in self.metadata("custom_units") or []: ureg.define(unit) - def connected_databases(self): + def _connected_databases(self): return [ { "name": d.name, @@ -486,9 +486,9 @@ class Datasette: for d in sorted(self.databases.values(), key=lambda d: d.name) ] - def versions(self): + def _versions(self): conn = sqlite3.connect(":memory:") - self.prepare_connection(conn, ":memory:") + self._prepare_connection(conn, ":memory:") sqlite_version = conn.execute("select sqlite_version()").fetchone()[0] sqlite_extensions = {} for extension, testsql, hasversion in ( @@ -534,7 +534,7 @@ class Datasette: }, } - def plugins(self, show_all=False): + def _plugins(self, show_all=False): ps = list(get_plugins()) if not show_all: ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] @@ -548,7 +548,7 @@ class Datasette: for p in ps ] - def threads(self): + def _threads(self): threads = list(threading.enumerate()) d = { "num_threads": len(threads), @@ -576,7 +576,7 @@ class Datasette: .get(table, {}) ) - def register_renderers(self): + def _register_renderers(self): """ Register output renderers which output data in custom formats. """ # Built-in renderers self.renderers["json"] = (json_renderer, lambda: True) @@ -724,11 +724,11 @@ class Datasette: r"/-/metadata(?P(\.json)?)$", ) add_route( - JsonDataView.as_asgi(self, "versions.json", self.versions), + JsonDataView.as_asgi(self, "versions.json", self._versions), r"/-/versions(?P(\.json)?)$", ) add_route( - JsonDataView.as_asgi(self, "plugins.json", self.plugins), + JsonDataView.as_asgi(self, "plugins.json", self._plugins), r"/-/plugins(?P(\.json)?)$", ) add_route( @@ -736,11 +736,11 @@ class Datasette: r"/-/config(?P(\.json)?)$", ) add_route( - JsonDataView.as_asgi(self, "threads.json", self.threads), + JsonDataView.as_asgi(self, "threads.json", self._threads), r"/-/threads(?P(\.json)?)$", ) add_route( - JsonDataView.as_asgi(self, "databases.json", self.connected_databases), + JsonDataView.as_asgi(self, "databases.json", self._connected_databases), r"/-/databases(?P(\.json)?)$", ) add_route( @@ -765,7 +765,7 @@ class Datasette: + renderer_regex + r")?$", ) - self.register_custom_units() + self._register_custom_units() async def setup_db(): # First time server starts up, calculate table counts for immutable databases diff --git a/datasette/database.py b/datasette/database.py index e6154caa..89bf47f4 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -104,7 +104,7 @@ class Database: conn = getattr(connections, self.name, None) if not conn: conn = self.connect() - self.ds.prepare_connection(conn, self.name) + self.ds._prepare_connection(conn, self.name) setattr(connections, self.name, conn) return fn(conn) From de1cde65a67cf9acb227b4df67230b47fdfc9a0e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 10:45:11 -0700 Subject: [PATCH 0445/2303] Moved request tests to test_internals_request.py --- tests/test_internals_request.py | 42 +++++++++++++++++++++++++++++++++ tests/test_utils.py | 40 ------------------------------- 2 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 tests/test_internals_request.py diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py new file mode 100644 index 00000000..5c9b254b --- /dev/null +++ b/tests/test_internals_request.py @@ -0,0 +1,42 @@ +from datasette.utils.asgi import Request +import pytest + + +@pytest.mark.asyncio +async def test_request_post_vars(): + scope = { + "http_version": "1.1", + "method": "POST", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "scheme": "http", + "type": "http", + "headers": [[b"content-type", b"application/x-www-form-urlencoded"]], + } + + async def receive(): + return {"type": "http.request", "body": b"foo=bar&baz=1", "more_body": False} + + request = Request(scope, receive) + assert {"foo": "bar", "baz": "1"} == await request.post_vars() + + +def test_request_args(): + request = Request.fake("/foo?multi=1&multi=2&single=3") + assert "1" == request.args.get("multi") + assert "3" == request.args.get("single") + assert "1" == request.args["multi"] + assert "3" == request.args["single"] + assert ["1", "2"] == request.args.getlist("multi") + assert [] == request.args.getlist("missing") + assert "multi" in request.args + assert "single" in request.args + assert "missing" not in request.args + expected = ["multi", "single"] + assert expected == list(request.args.keys()) + for i, key in enumerate(request.args): + assert expected[i] == key + assert 2 == len(request.args) + with pytest.raises(KeyError): + request.args["missing"] diff --git a/tests/test_utils.py b/tests/test_utils.py index 9d6f45b0..01a10468 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -428,46 +428,6 @@ def test_check_connection_passes(): utils.check_connection(conn) -@pytest.mark.asyncio -async def test_request_post_vars(): - scope = { - "http_version": "1.1", - "method": "POST", - "path": "/", - "raw_path": b"/", - "query_string": b"", - "scheme": "http", - "type": "http", - "headers": [[b"content-type", b"application/x-www-form-urlencoded"]], - } - - async def receive(): - return {"type": "http.request", "body": b"foo=bar&baz=1", "more_body": False} - - request = Request(scope, receive) - assert {"foo": "bar", "baz": "1"} == await request.post_vars() - - -def test_request_args(): - request = Request.fake("/foo?multi=1&multi=2&single=3") - assert "1" == request.args.get("multi") - assert "3" == request.args.get("single") - assert "1" == request.args["multi"] - assert "3" == request.args["single"] - assert ["1", "2"] == request.args.getlist("multi") - assert [] == request.args.getlist("missing") - assert "multi" in request.args - assert "single" in request.args - assert "missing" not in request.args - expected = ["multi", "single"] - assert expected == list(request.args.keys()) - for i, key in enumerate(request.args): - assert expected[i] == key - assert 2 == len(request.args) - with pytest.raises(KeyError): - request.args["missing"] - - def test_call_with_supported_arguments(): def foo(a, b): return "{}+{}".format(a, b) From 5ae14c9f20e0dc59c588f0e93eedfefe0f0f3e8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 10:54:22 -0700 Subject: [PATCH 0446/2303] Improved documentation for RequestParameters class --- docs/internals.rst | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 886cb7e7..ca725cc4 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -278,16 +278,27 @@ The object also has one awaitable method: The RequestParameters class --------------------------- -This class, returned by ``request.args``, is a dictionary-like object. +``request.args`` is a ``RequestParameters`` object - a dictionary-like object which provides access to querystring parameters that may have multiple values. -Consider the querystring ``?foo=1&foo=2``. This will produce a ``request.args`` that looks like this:: +Consider the querystring ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` and one value for ``bar``. - RequestParameters({"foo": ["1", "2"]}) +``request.args[key]`` - string + Returns the first value for that key, or raises a ``KeyError`` if the key is missing. For the above example ``request.args["foo"]`` would return ``"1"``. -``request.args["foo"]`` returns the first value, ``"1"`` - or raises ``KeyError`` if that key is missing. +``request.args.get(key)`` - string or None + Returns the first value for that key, or ``None`` if the key is missing. Pass a second argument to specify a different default, e.g. ``q = request.args.get("q", "")``. -``request.args.get("foo")`` returns ``"1"`` - or ``None`` if the key is missing. A second argument can be used to specify a different default value. +``request.args.getlist(key)`` - list of strings + Returns the list of strings for that key. ``request.args.getlist("foo")`` would return ``["1", "2"]`` in the above example. ``request.args.getlist("bar")`` would return ``["3"]``. If the key is missing an empty list will be returned. -``request.args.getlist("foo")`` returns the full list, ``["1", "2"]``. If you call it on a missing key it will return ``[]``. +``request.args.keys()`` - list of strings + Returns the list of available keys - for the example this would be ``["foo", "bar"]``. -You can use ``if key in request.args`` to check if a key is present. ``for key in request.args`` will iterate through the keys, or you can use ``request.args.keys()`` to get all of the keys. +``key in request.args`` - True or False + You can use ``if key in request.args`` to check if a key is present. + +``for key in request.args`` - iterator + This lets you loop through every available key. + +``len(request.args)`` - integer + Returns the number of keys. From 3c5afaeb231c94a55309f1c0187ff6dedd5b5fb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 11:06:13 -0700 Subject: [PATCH 0447/2303] Re-arranged internals documentation Request is more useful to most people than Database. --- docs/internals.rst | 136 ++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index ca725cc4..4db710c0 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -3,7 +3,74 @@ Internals for plugins ===================== -Many :ref:`plugin_hooks` are passed objects that provide access to internal Datasette functionality. The interface to these objects should not be considered stable (at least until Datasette 1.0) with the exception of methods that are documented on this page. +Many :ref:`plugin_hooks` are passed objects that provide access to internal Datasette functionality. The interface to these objects should not be considered stable with the exception of methods that are documented here. + +.. _internals_request: + +Request object +~~~~~~~~~~~~~~ + +The request object is passed to various plugin hooks. It represents an incoming HTTP request. It has the following properties: + +``.scope`` - dictionary + The ASGI scope that was used to construct this request, described in the `ASGI HTTP connection scope `__ specification. + +``.method`` - string + The HTTP method for this request, usually ``GET`` or ``POST``. + +``.url`` - string + The full URL for this request, e.g. ``https://latest.datasette.io/fixtures``. + +``.scheme`` - string + The request scheme - usually ``https`` or ``http``. + +``.headers`` - dictionary (str -> str) + A dictionary of incoming HTTP request headers. + +``.host`` - string + The host header from the incoming request, e.g. ``latest.datasette.io`` or ``localhost``. + +``.path`` - string + The path of the request, e.g. ``/fixtures``. + +``.query_string`` - string + The querystring component of the request, without the ``?`` - e.g. ``name__contains=sam&age__gt=10``. + +``.args`` - RequestParameters + An object representing the parsed querystring parameters, see below. + +The object also has one awaitable method: + +``await request.post_vars()`` - dictionary + Returns a dictionary of form variables that were submitted in the request body via ``POST``. + +The RequestParameters class +--------------------------- + +``request.args`` is a ``RequestParameters`` object - a dictionary-like object which provides access to querystring parameters that may have multiple values. + +Consider the querystring ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` and one value for ``bar``. + +``request.args[key]`` - string + Returns the first value for that key, or raises a ``KeyError`` if the key is missing. For the above example ``request.args["foo"]`` would return ``"1"``. + +``request.args.get(key)`` - string or None + Returns the first value for that key, or ``None`` if the key is missing. Pass a second argument to specify a different default, e.g. ``q = request.args.get("q", "")``. + +``request.args.getlist(key)`` - list of strings + Returns the list of strings for that key. ``request.args.getlist("foo")`` would return ``["1", "2"]`` in the above example. ``request.args.getlist("bar")`` would return ``["3"]``. If the key is missing an empty list will be returned. + +``request.args.keys()`` - list of strings + Returns the list of available keys - for the example this would be ``["foo", "bar"]``. + +``key in request.args`` - True or False + You can use ``if key in request.args`` to check if a key is present. + +``for key in request.args`` - iterator + This lets you loop through every available key. + +``len(request.args)`` - integer + Returns the number of keys. .. _internals_datasette: @@ -235,70 +302,3 @@ Here's an example of ``block=True`` in action: num_rows_left = await database.execute_write_fn(my_action, block=True) except Exception as e: print("An error occurred:", e) - -.. _internals_request: - -Request object -~~~~~~~~~~~~~~ - -The request object is passed to various plugin hooks. It represents an incoming HTTP request. It has the following properties: - -``.scope`` - dictionary - The ASGI scope that was used to construct this request, described in the `ASGI HTTP connection scope `__ specification. - -``.method`` - string - The HTTP method for this request, usually ``GET`` or ``POST``. - -``.url`` - string - The full URL for this request, e.g. ``https://latest.datasette.io/fixtures``. - -``.scheme`` - string - The request scheme - usually ``https`` or ``http``. - -``.headers`` - dictionary (str -> str) - A dictionary of incoming HTTP request headers. - -``.host`` - string - The host header from the incoming request, e.g. ``latest.datasette.io`` or ``localhost``. - -``.path`` - string - The path of the request, e.g. ``/fixtures``. - -``.query_string`` - string - The querystring component of the request, without the ``?`` - e.g. ``name__contains=sam&age__gt=10``. - -``.args`` - RequestParameters - An object representing the parsed querystring parameters, see below. - -The object also has one awaitable method: - -``await request.post_vars()`` - dictionary - Returns a dictionary of form variables that were submitted in the request body via ``POST``. - -The RequestParameters class ---------------------------- - -``request.args`` is a ``RequestParameters`` object - a dictionary-like object which provides access to querystring parameters that may have multiple values. - -Consider the querystring ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` and one value for ``bar``. - -``request.args[key]`` - string - Returns the first value for that key, or raises a ``KeyError`` if the key is missing. For the above example ``request.args["foo"]`` would return ``"1"``. - -``request.args.get(key)`` - string or None - Returns the first value for that key, or ``None`` if the key is missing. Pass a second argument to specify a different default, e.g. ``q = request.args.get("q", "")``. - -``request.args.getlist(key)`` - list of strings - Returns the list of strings for that key. ``request.args.getlist("foo")`` would return ``["1", "2"]`` in the above example. ``request.args.getlist("bar")`` would return ``["3"]``. If the key is missing an empty list will be returned. - -``request.args.keys()`` - list of strings - Returns the list of available keys - for the example this would be ``["foo", "bar"]``. - -``key in request.args`` - True or False - You can use ``if key in request.args`` to check if a key is present. - -``for key in request.args`` - iterator - This lets you loop through every available key. - -``len(request.args)`` - integer - Returns the number of keys. From 4d798ca0e3df246bd47f0600cc7b5118ba33ac16 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 11:17:20 -0700 Subject: [PATCH 0448/2303] Added test for db.mtime_ns --- datasette/database.py | 4 +++- tests/test_internals_database.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 89bf47f4..ed119542 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -33,7 +33,7 @@ class Database: self.cached_table_counts = None self._write_thread = None self._write_queue = None - if not self.is_mutable: + if not self.is_mutable and not self.is_memory: p = Path(path) self.hash = inspect_hash(p) self.cached_size = p.stat().st_size @@ -197,6 +197,8 @@ class Database: @property def mtime_ns(self): + if self.is_memory: + return None return Path(self.path).stat().st_mtime_ns @property diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index fde7ad2c..5d5520dd 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,7 +1,7 @@ """ Tests for the datasette.database.Database class """ -from datasette.database import Results, MultipleValues +from datasette.database import Database, Results, MultipleValues from datasette.utils import sqlite3 from .fixtures import app_client import pytest @@ -188,3 +188,13 @@ async def test_execute_write_fn_exception(db): with pytest.raises(AssertionError): await db.execute_write_fn(write_fn, block=True) + + +@pytest.mark.asyncio +async def test_mtime_ns(db): + assert isinstance(db.mtime_ns, int) + + +def test_mtime_ns_is_none_for_memory(app_client): + memory_db = Database(app_client.ds, is_memory=True) + assert None is memory_db.mtime_ns From 124acf34a678f0af438dc31a2dceebf28612f249 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 11:39:46 -0700 Subject: [PATCH 0449/2303] Removed db.get_outbound_foreign_keys method It duplicated the functionality of db.foreign_keys_for_table. --- datasette/database.py | 5 ----- datasette/utils/__init__.py | 2 +- datasette/views/table.py | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index ed119542..ab3c82c9 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -319,11 +319,6 @@ class Database: async def get_all_foreign_keys(self): return await self.execute_fn(get_all_foreign_keys) - async def get_outbound_foreign_keys(self, table): - return await self.execute_fn( - lambda conn: get_outbound_foreign_keys(conn, table) - ) - async def get_table_definition(self, table, type_="table"): table_definition_rows = list( await self.execute( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bf965413..2eb31502 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -428,7 +428,7 @@ def get_outbound_foreign_keys(conn, table): if info is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = info fks.append( - {"other_table": table_name, "column": from_, "other_column": to_} + {"column": from_, "other_table": table_name, "other_column": to_} ) return fks diff --git a/datasette/views/table.py b/datasette/views/table.py index a629346f..2e9515c3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -336,9 +336,7 @@ class TableView(RowTableShared): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] - outgoing_foreign_keys = await db.get_outbound_foreign_keys( - through_table - ) + outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) try: fk_to_us = [ fk for fk in outgoing_foreign_keys if fk["other_table"] == table From c4fbe50676929b512940aab90de590a78ac5d7fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 11:40:30 -0700 Subject: [PATCH 0450/2303] Documentation for Database introspection methods, closes #684 Refs #576 --- docs/internals.rst | 68 ++++++++++++++++++++++++++++++++++++++++++++++ docs/metadata.rst | 2 ++ 2 files changed, 70 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 4db710c0..e9ba9567 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -302,3 +302,71 @@ Here's an example of ``block=True`` in action: num_rows_left = await database.execute_write_fn(my_action, block=True) except Exception as e: print("An error occurred:", e) + +Database introspection +---------------------- + +The ``Database`` class also provides properties and methods for introspecting the database. + +``db.name`` - string + The name of the database - usually the filename without the ``.db`` prefix. + +``db.size`` - integer + The size of the database file in bytes. 0 for ``:memory:`` databases. + +``db.mtime_ns`` - integer or None + The last modification time of the database file in nanoseconds since the epoch. ``None`` for ``:memory:`` databases. + +``await db.table_exists(table)`` - boolean + Check if a table called ``table`` exists. + +``await db.table_names()`` - list of strings + List of names of tables in the database. + +``await db.view_names()`` - list of strings + List of names of views in tha database. + +``await db.table_columns(table)`` - list of strings + Names of columns in a specific table. + +``await db.primary_keys(table)`` - list of strings + Names of the columns that are part of the primary key for this table. + +``await db.fts_table(table)`` - string or None + The name of the FTS table associated with this table, if one exists. + +``await db.label_column_for_table(table)`` - string or None + The label column that is associated with this table - either automatically detected or using the ``"label_column"`` key from :ref:`metadata`, see :ref:`label_columns`. + +``await db.foreign_keys_for_table(table)`` - list of dictionaries + Details of columns in this table which are foreign keys to other tables. A list of dictionaries where each dictionary is shaped like this: ``{"column": string, "other_table": string, "other_column": string}``. + +``await db.hidden_table_names()`` - list of strings + List of tables which Datasette "hides" by default - usually these are tables associated with SQLite's full-text search feature, the SpatiaLite extension or tables hidden using the :ref:`metadata_hiding_tables` feature. + +``await db.get_table_definition(table)`` - string + Returns the SQL definition for the table - the ``CREATE TABLE`` statement and any associated ``CREATE INDEX`` statements. + +``await db.get_view_definition(view)`` - string + Returns the SQL definition of the named view. + +``await db.get_all_foreign_keys()`` - dictionary + Dictionary representing both incoming and outgoing foreign keys for this table. It has two keys, ``"incoming"`` and ``"outgoing"``, each of which is a list of dictionaries with keys ``"column"``, ``"other_table"`` and ``"other_column"``. For example: + + .. code-block:: json + + { + "incoming": [], + "outgoing": [ + { + "other_table": "attraction_characteristic", + "column": "characteristic_id", + "other_column": "pk", + }, + { + "other_table": "roadside_attractions", + "column": "attraction_id", + "other_column": "pk", + } + ] + } diff --git a/docs/metadata.rst b/docs/metadata.rst index 024af01e..471a52e3 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -256,6 +256,8 @@ used for the link label with the ``label_column`` property: } } +.. _metadata_hiding_tables: + Hiding tables ------------- From 060a56735c1d3bde0a4c7674e82b5f45bef34dee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 13:24:00 -0700 Subject: [PATCH 0451/2303] actor_from_request and permission_allowed hookspecs, refs #699 --- datasette/hookspecs.py | 10 ++++++++++ docs/plugins.rst | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index c2fc0126..65c1c859 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -58,3 +58,13 @@ def register_output_renderer(datasette): @hookspec def register_facet_classes(): "Register Facet subclasses" + + +@hookspec +def actor_from_request(datasette, request): + "Return an actor dictionary based on the incoming request" + + +@hookspec +def permission_allowed(actor, action, resource_type, resource_identifier): + "Check if actor is allowed to perfom this action - return True, False or None" diff --git a/docs/plugins.rst b/docs/plugins.rst index f08f1217..09e8f5e3 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -941,3 +941,40 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att return wrap_with_databases_header Examples: `datasette-auth-github `_, `datasette-search-all `_, `datasette-media `_ + +.. _plugin_actor_from_request: + +actor_from_request(datasette, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``request`` - object + The current HTTP :ref:`internals_request`. + +This is part of Datasette's authentication and permissions system. The function should attempt to authenticate an actor (either a user or an API actor of some sort) based on information in the request. + +If it cannot authenticate an actor, it should return ``None``. Otherwise it should return a dictionary representing that actor. + +.. _plugin_permission_allowed: + +permission_allowed(datasette, actor, action, resource_type, resource_identifier) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary + The current actor, as decided by :ref:`plugin_actor_from_request`. + +``action`` - string + The action to be performed, e.g. ``"edit-table"``. + +``resource_type`` - string + The type of resource being acted on, e.g. ``"table"``. + +``resource`` - string + An identifier for the individual resource, e.g. the name of the table. + +Called to check that an actor has permission to perform an action on a resource. Can return ``True`` if the action is allowed, ``False`` if the action is not allowed or ``None`` if the plugin does not have an opinion one way or the other. From 461c82838d65dd9f61c5be725343a82c61b5c3f3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 15:06:33 -0700 Subject: [PATCH 0452/2303] Implemented actor_from_request with tests, refs #699 Also added datasette argument to permission_allowed hook --- datasette/app.py | 13 ++++++++++++- datasette/hookspecs.py | 2 +- docs/plugins.rst | 23 +++++++++++++++++++++++ tests/plugins/my_plugin.py | 8 ++++++++ tests/plugins/my_plugin_2.py | 12 ++++++++++++ tests/test_plugins.py | 24 ++++++++++++++++++++++++ 6 files changed, 80 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4b9807b0..3f2876ec 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -798,7 +798,18 @@ class DatasetteRouter(AsgiRouter): and scope.get("scheme") != "https" ): scope = dict(scope, scheme="https") - return await super().route_path(scope, receive, send, path) + # Handle authentication + actor = None + for actor in pm.hook.actor_from_request( + datasette=self.ds, request=Request(scope, receive) + ): + if callable(actor): + actor = actor() + if asyncio.iscoroutine(actor): + actor = await actor + if actor: + break + return await super().route_path(dict(scope, actor=actor), receive, send, path) async def handle_404(self, scope, receive, send, exception=None): # If URL has a trailing slash, redirect to URL without it diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 65c1c859..71d06661 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -66,5 +66,5 @@ def actor_from_request(datasette, request): @hookspec -def permission_allowed(actor, action, resource_type, resource_identifier): +def permission_allowed(datasette, actor, action, resource_type, resource_identifier): "Check if actor is allowed to perfom this action - return True, False or None" diff --git a/docs/plugins.rst b/docs/plugins.rst index 09e8f5e3..fb2843f4 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -957,6 +957,29 @@ This is part of Datasette's authentication and permissions system. The function If it cannot authenticate an actor, it should return ``None``. Otherwise it should return a dictionary representing that actor. +Instead of returning a dictionary, this function can return an awaitable function which itself returns either ``None`` or a dictionary. This is useful for authentication functions that need to make a database query - for example: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def actor_from_request(datasette, request): + async def inner(): + token = request.args.get("_token") + if not token: + return None + # Look up ?_token=xxx in sessions table + result = await datasette.get_database().execute( + "select count(*) from sessions where token = ?", [token] + ) + if result.first()[0]: + return {"token": token} + else: + return None + + return inner + .. _plugin_permission_allowed: permission_allowed(datasette, actor, action, resource_type, resource_identifier) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 10d7e7e6..305cb3b7 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -126,3 +126,11 @@ class DummyFacet(Facet): facet_results = {} facets_timed_out = [] return facet_results, facets_timed_out + + +@hookimpl +def actor_from_request(datasette, request): + if request.args.get("_bot"): + return {"id": "bot"} + else: + return None diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index c9e7c78f..0a5cbba5 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -95,3 +95,15 @@ def asgi_wrapper(datasette): return add_x_databases_header return wrap_with_databases_header + + +@hookimpl +def actor_from_request(datasette, request): + async def inner(): + if request.args.get("_bot2"): + result = await datasette.get_database().execute("select 1 + 1") + return {"id": "bot2", "1+1": result.first()[0]} + else: + return None + + return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a34328a9..3ad26986 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -503,3 +503,27 @@ def test_register_facet_classes(app_client): "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk3", }, ] == data["suggested_facets"] + + +def test_actor_from_request(app_client): + app_client.get("/") + # Should have no actor + assert None == app_client.ds._last_request.scope["actor"] + app_client.get("/?_bot=1") + # Should have bot actor + assert {"id": "bot"} == app_client.ds._last_request.scope["actor"] + + +def test_actor_from_request_async(app_client): + app_client.get("/") + # Should have no actor + assert None == app_client.ds._last_request.scope["actor"] + app_client.get("/?_bot2=1") + # Should have bot2 actor + assert {"id": "bot2", "1+1": 2} == app_client.ds._last_request.scope["actor"] + + +@pytest.mark.xfail +def test_permission_allowed(app_client): + # TODO + assert False From 9315bacf6f63e20781d21d170e55a55b2c54fcdd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 15:24:43 -0700 Subject: [PATCH 0453/2303] Implemented datasette.permission_allowed(), refs #699 --- datasette/app.py | 19 +++++++++++++++++++ docs/internals.rst | 19 +++++++++++++++++++ tests/plugins/my_plugin.py | 8 ++++++++ tests/plugins/my_plugin_2.py | 13 +++++++++++++ tests/test_plugins.py | 20 ++++++++++++++++---- 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3f2876ec..773dee31 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -406,6 +406,25 @@ class Datasette: # pylint: disable=no-member pm.hook.prepare_connection(conn=conn, database=database, datasette=self) + async def permission_allowed( + self, actor, action, resource_type=None, resource_identifier=None, default=False + ): + "Check permissions using the permissions_allowed plugin hook" + for check in pm.hook.permission_allowed( + datasette=self, + actor=actor, + action=action, + resource_type=resource_type, + resource_identifier=resource_identifier, + ): + if callable(check): + check = check() + if asyncio.iscoroutine(check): + check = await check + if check is not None: + return check + return default + async def execute( self, db_name, diff --git a/docs/internals.rst b/docs/internals.rst index e9ba9567..2ba70722 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -111,6 +111,25 @@ This method lets you read plugin configuration values that were set in ``metadat Renders a `Jinja template `__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins. +await .permission_allowed(actor, action, resource_type=None, resource_identifier=None, default=False) +----------------------------------------------------------------------------------------------------- + +``actor`` - dictionary + The authenticated actor. This is usually ``request.scope.get("actor")``. + +``action`` - string + The name of the action that is being permission checked. + +``resource_type`` - string, optional + The type of resource being checked, e.g. ``"table"``. + +``resource_identifier`` - string, optional + The resource identifier, e.g. the name of the table. + +Check if the given actor has permission to perform the given action on the given resource. This uses plugins that implement the :ref:`plugin_permission_allowed` plugin hook to decide if the action is allowed or not. + +If none of the plugins express an opinion, the return value will be the ``default`` argument. This is deny, but you can pass ``default=True`` to default allow instead. + .. _datasette_get_database: .get_database(name) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 305cb3b7..46893710 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -134,3 +134,11 @@ def actor_from_request(datasette, request): return {"id": "bot"} else: return None + + +@hookimpl +def permission_allowed(actor, action): + if action == "this_is_allowed": + return True + elif action == "this_is_denied": + return False diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 0a5cbba5..039112f4 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -107,3 +107,16 @@ def actor_from_request(datasette, request): return None return inner + + +@hookimpl +def permission_allowed(datasette, actor, action): + # Testing asyncio version of permission_allowed + async def inner(): + assert 2 == (await datasette.get_database().execute("select 1 + 1")).first()[0] + if action == "this_is_allowed_async": + return True + elif action == "this_is_denied_async": + return False + + return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 3ad26986..e123b7a0 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -523,7 +523,19 @@ def test_actor_from_request_async(app_client): assert {"id": "bot2", "1+1": 2} == app_client.ds._last_request.scope["actor"] -@pytest.mark.xfail -def test_permission_allowed(app_client): - # TODO - assert False +@pytest.mark.asyncio +@pytest.mark.parametrize( + "action,expected", + [ + ("this_is_allowed", True), + ("this_is_denied", False), + ("this_is_allowed_async", True), + ("this_is_denied_async", False), + ("no_match", None), + ], +) +async def test_permission_allowed(app_client, action, expected): + actual = await app_client.ds.permission_allowed( + {"id": "actor"}, action, default=None + ) + assert expected == actual From 1fc6ceefb9eddd29844e7bfe3e06a83df6ce3dc4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 30 May 2020 18:51:00 -0700 Subject: [PATCH 0454/2303] Added /-/actor.json - refs #699 Also added JSON highlighting to introspection documentation. --- datasette/app.py | 7 ++++++ datasette/views/special.py | 8 +++++-- docs/introspection.rst | 44 ++++++++++++++++++++++++++++++++------ tests/test_plugins.py | 7 ++++++ 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 773dee31..37b4ed3d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -586,6 +586,9 @@ class Datasette: ) return d + def _actor(self, request): + return {"actor": request.scope.get("actor", None)} + def table_metadata(self, database, table): "Fetch table-specific metadata." return ( @@ -762,6 +765,10 @@ class Datasette: JsonDataView.as_asgi(self, "databases.json", self._connected_databases), r"/-/databases(?P(\.json)?)$", ) + add_route( + JsonDataView.as_asgi(self, "actor.json", self._actor, needs_request=True), + r"/-/actor(?P(\.json)?)$", + ) add_route( PatternPortfolioView.as_asgi(self), r"/-/patterns$", ) diff --git a/datasette/views/special.py b/datasette/views/special.py index dfe5ea8c..840473a7 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -6,13 +6,17 @@ from .base import BaseView class JsonDataView(BaseView): name = "json_data" - def __init__(self, datasette, filename, data_callback): + def __init__(self, datasette, filename, data_callback, needs_request=False): self.ds = datasette self.filename = filename self.data_callback = data_callback + self.needs_request = needs_request async def get(self, request, as_format): - data = self.data_callback() + if self.needs_request: + data = self.data_callback(request) + else: + data = self.data_callback() if as_format: headers = {} if self.ds.cors: diff --git a/docs/introspection.rst b/docs/introspection.rst index 3cd4a40f..e5d08dbc 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -10,7 +10,9 @@ Each of these pages can be viewed in your browser. Add ``.json`` to the URL to g /-/metadata ----------- -Shows the contents of the ``metadata.json`` file that was passed to ``datasette serve``, if any. `Metadata example `_:: +Shows the contents of the ``metadata.json`` file that was passed to ``datasette serve``, if any. `Metadata example `_: + +.. code-block:: json { "license": "CC Attribution 4.0 License", @@ -18,7 +20,9 @@ Shows the contents of the ``metadata.json`` file that was passed to ``datasette "source": "fivethirtyeight/data on GitHub", "source_url": "https://github.com/fivethirtyeight/data", "title": "Five Thirty Eight", - "databases": {...} + "databases": { + + } } .. _JsonDataView_versions: @@ -26,7 +30,9 @@ Shows the contents of the ``metadata.json`` file that was passed to ``datasette /-/versions ----------- -Shows the version of Datasette, Python and SQLite. `Versions example `_:: +Shows the version of Datasette, Python and SQLite. `Versions example `_: + +.. code-block:: json { "datasette": { @@ -63,7 +69,9 @@ Shows the version of Datasette, Python and SQLite. `Versions example `_:: +Shows a list of currently installed plugins and their versions. `Plugins example `_: + +.. code-block:: json [ { @@ -79,7 +87,9 @@ Shows a list of currently installed plugins and their versions. `Plugins example /-/config --------- -Shows the :ref:`config` options for this instance of Datasette. `Config example `_:: +Shows the :ref:`config` options for this instance of Datasette. `Config example `_: + +.. code-block:: json { "default_facet_size": 30, @@ -95,7 +105,9 @@ Shows the :ref:`config` options for this instance of Datasette. `Config example /-/databases ------------ -Shows currently attached databases. `Databases example `_:: +Shows currently attached databases. `Databases example `_: + +.. code-block:: json [ { @@ -113,7 +125,9 @@ Shows currently attached databases. `Databases example `_:: +Shows details of threads and ``asyncio`` tasks. `Threads example `_: + +.. code-block:: json { "num_threads": 2, @@ -136,3 +150,19 @@ Shows details of threads and ``asyncio`` tasks. `Threads example wait_for=()]>>" ] } + +.. _JsonDataView_actor: + +/-/actor +-------- + +Shows the currently authenticated actor. Useful for debugging Datasette authentication plugins. + +.. code-block:: json + + { + "actor": { + "id": 1, + "username": "some-user" + } + } diff --git a/tests/test_plugins.py b/tests/test_plugins.py index e123b7a0..7a3fb49a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -539,3 +539,10 @@ async def test_permission_allowed(app_client, action, expected): {"id": "actor"}, action, default=None ) assert expected == actual + + +def test_actor_json(app_client): + assert {"actor": None} == app_client.get("/-/actor.json").json + assert {"actor": {"id": "bot2", "1+1": 2}} == app_client.get( + "/-/actor.json/?_bot2=1" + ).json From fa27e44fe09f57dcb87157be97f15b6add7f14ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2020 15:42:08 -0700 Subject: [PATCH 0455/2303] datasette.sign() and datasette.unsign() methods, refs #785 --- datasette/app.py | 9 +++++++++ datasette/cli.py | 7 +++++++ docs/datasette-serve-help.txt | 3 +++ docs/internals.rst | 28 ++++++++++++++++++++++++++++ setup.py | 1 + tests/test_cli.py | 1 + tests/test_internals_datasette.py | 12 ++++++++++++ 7 files changed, 61 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 37b4ed3d..5e3d3af5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -14,6 +14,7 @@ from pathlib import Path import click from markupsafe import Markup +from itsdangerous import URLSafeSerializer import jinja2 from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape from jinja2.environment import Template @@ -163,12 +164,14 @@ class Datasette: static_mounts=None, memory=False, config=None, + secret=None, version_note=None, config_dir=None, ): assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" + self._secret = secret or os.urandom(32).hex() self.files = tuple(files) + tuple(immutables or []) if config_dir: self.files += tuple([str(p) for p in config_dir.glob("*.db")]) @@ -281,6 +284,12 @@ class Datasette: self._register_renderers() + def sign(self, value, namespace="default"): + return URLSafeSerializer(self._secret, namespace).dumps(value) + + def unsign(self, signed, namespace="default"): + return URLSafeSerializer(self._secret, namespace).loads(signed) + def get_database(self, name=None): if name is None: return next(iter(self.databases.values())) diff --git a/datasette/cli.py b/datasette/cli.py index c59fb6e0..dba3a612 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -299,6 +299,11 @@ def package( help="Set config option using configname:value datasette.readthedocs.io/en/latest/config.html", multiple=True, ) +@click.option( + "--secret", + help="Secret used for signing secure values, such as signed cookies", + envvar="DATASETTE_SECRET", +) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-config", is_flag=True, help="Show available config options") def serve( @@ -317,6 +322,7 @@ def serve( static, memory, config, + secret, version_note, help_config, return_instance=False, @@ -362,6 +368,7 @@ def serve( static_mounts=static, config=dict(config), memory=memory, + secret=secret, version_note=version_note, ) diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 5265c294..ab27714a 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -29,6 +29,9 @@ Options: --config CONFIG Set config option using configname:value datasette.readthedocs.io/en/latest/config.html + --secret TEXT Secret used for signing secure values, such as signed + cookies + --version-note TEXT Additional note to show on /-/versions --help-config Show available config options --help Show this message and exit. diff --git a/docs/internals.rst b/docs/internals.rst index 2ba70722..68a35312 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -183,6 +183,34 @@ Use ``is_memory`` if the connection is to an in-memory SQLite database. This removes a database that has been previously added. ``name=`` is the unique name of that database, also used in the URL for it. +.. _datasette_sign: + +.sign(value, namespace="default") +--------------------------------- + +``value`` - any serializable type + The value to be signed. + +``namespace`` - string, optional + An alternative namespace, see the `itsdangerous salt documentation `__. + +Utility method for signing values, such that you can safely pass data to and from an untrusted environment. This is a wrapper around the `itsdangerous `__ library. + +This method returns a signed string, which can be decoded and verified using :ref:`datasette_unsign`. + +.. _datasette_unsign: + +.unsign(value, namespace="default") +----------------------------------- + +``signed`` - any serializable type + The signed string that was created using :ref:`datasette_sign`. + +``namespace`` - string, optional + The alternative namespace, if one was used. + +Returns the original, decoded object that was passed to :ref:`datasette_sign`. If the signature is not valid this raises a ``itsdangerous.BadSignature`` exception. + .. _internals_database: Database class diff --git a/setup.py b/setup.py index d9c70de5..93628266 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ setup( "janus>=0.4,<0.6", "PyYAML~=5.3", "mergedeep>=1.1.1,<1.4.0", + "itsdangerous~=1.1", ], entry_points=""" [console_scripts] diff --git a/tests/test_cli.py b/tests/test_cli.py index ac5746c6..f52f17b4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -75,6 +75,7 @@ def test_metadata_yaml(): static=[], memory=False, config=[], + secret=None, version_note=None, help_config=False, return_instance=True, diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 4993250d..0be0b932 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,6 +1,7 @@ """ Tests for the datasette.app.Datasette class """ +from itsdangerous import BadSignature from .fixtures import app_client import pytest @@ -21,3 +22,14 @@ def test_get_database_no_argument(datasette): # Returns the first available database: db = datasette.get_database() assert "fixtures" == db.name + + +@pytest.mark.parametrize("value", ["hello", 123, {"key": "value"}]) +@pytest.mark.parametrize("namespace", [None, "two"]) +def test_sign_unsign(datasette, value, namespace): + extra_args = [namespace] if namespace else [] + signed = datasette.sign(value, *extra_args) + assert value != signed + assert value == datasette.unsign(signed, *extra_args) + with pytest.raises(BadSignature): + datasette.unsign(signed[:-1] + ("!" if signed[-1] != "!" else ":")) From 7690d5ba40fda37ba4ba38ad56fe06c3aed071de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2020 17:18:06 -0700 Subject: [PATCH 0456/2303] Docs for --secret/DATASETTE_SECRET - closes #785 --- docs/config.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/config.rst b/docs/config.rst index d8c2f550..da93e40a 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -288,3 +288,30 @@ For example, if you are sending traffic from ``https://www.example.com/tools/dat You can do that like so:: datasette mydatabase.db --config base_url:/tools/datasette/ + +.. _config_secret: + +Configuring the secret +---------------------- + +Datasette uses a secret string to sign secure values such as cookies. + +If you do not provide a secret, Datasette will create one when it starts up. This secret will reset every time the Datasette server restarts though, so things like authentication cookies will not stay valid between restarts. + +You can pass a secret to Datasette in two ways: with the ``--secret`` command-line option or by setting a ``DATASETTE_SECRET`` environment variable. + +:: + + $ datasette mydb.db --secret=SECRET_VALUE_HERE + +Or:: + + $ export DATASETTE_SECRET=SECRET_VALUE_HERE + $ datasette mydb.db + +One way to generate a secure random secret is to use Python like this:: + + $ python3 -c 'import os; print(os.urandom(32).hex())' + cdb19e94283a20f9d42cca50c5a4871c0aa07392db308755d60a1a5b9bb0fa52 + +Plugin authors make use of this signing mechanism in their plugins using :ref:`datasette_sign` and :ref:`datasette_unsign`. From 9f3d4aba31baf1e2de1910a40bc9663ef53b94e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2020 18:03:17 -0700 Subject: [PATCH 0457/2303] --root option and /-/auth-token view, refs #784 --- datasette/app.py | 6 +++++- datasette/cli.py | 8 ++++++++ datasette/plugins.py | 1 + datasette/views/special.py | 32 +++++++++++++++++++++++++++++++- docs/datasette-serve-help.txt | 1 + tests/fixtures.py | 19 +++++++++++++++---- tests/test_auth.py | 25 +++++++++++++++++++++++++ tests/test_cli.py | 1 + tests/test_docs.py | 4 ++-- 9 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 tests/test_auth.py diff --git a/datasette/app.py b/datasette/app.py index 5e3d3af5..6b39ce12 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -24,7 +24,7 @@ import uvicorn from .views.base import DatasetteError, ureg, AsgiRouter from .views.database import DatabaseDownload, DatabaseView from .views.index import IndexView -from .views.special import JsonDataView, PatternPortfolioView +from .views.special import JsonDataView, PatternPortfolioView, AuthTokenView from .views.table import RowView, TableView from .renderer import json_renderer from .database import Database, QueryInterrupted @@ -283,6 +283,7 @@ class Datasette: pm.hook.prepare_jinja2_environment(env=self.jinja_env) self._register_renderers() + self._root_token = os.urandom(32).hex() def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -778,6 +779,9 @@ class Datasette: JsonDataView.as_asgi(self, "actor.json", self._actor, needs_request=True), r"/-/actor(?P(\.json)?)$", ) + add_route( + AuthTokenView.as_asgi(self), r"/-/auth-token$", + ) add_route( PatternPortfolioView.as_asgi(self), r"/-/patterns$", ) diff --git a/datasette/cli.py b/datasette/cli.py index dba3a612..23f9e36b 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -304,6 +304,11 @@ def package( help="Secret used for signing secure values, such as signed cookies", envvar="DATASETTE_SECRET", ) +@click.option( + "--root", + help="Output URL that sets a cookie authenticating the root user", + is_flag=True, +) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-config", is_flag=True, help="Show available config options") def serve( @@ -323,6 +328,7 @@ def serve( memory, config, secret, + root, version_note, help_config, return_instance=False, @@ -387,6 +393,8 @@ def serve( asyncio.get_event_loop().run_until_complete(check_databases(ds)) # Start the server + if root: + print("http://{}:{}/-/auth-token?token={}".format(host, port, ds._root_token)) uvicorn.run(ds.app(), host=host, port=port, log_level="info") diff --git a/datasette/plugins.py b/datasette/plugins.py index 6c9677d0..487fce4d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -9,6 +9,7 @@ DEFAULT_PLUGINS = ( "datasette.publish.cloudrun", "datasette.facets", "datasette.sql_functions", + "datasette.actor_auth_cookie", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/views/special.py b/datasette/views/special.py index 840473a7..910193e8 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,6 +1,8 @@ import json from datasette.utils.asgi import Response from .base import BaseView +from http.cookies import SimpleCookie +import secrets class JsonDataView(BaseView): @@ -45,4 +47,32 @@ class PatternPortfolioView(BaseView): self.ds = datasette async def get(self, request): - return await self.render(["patterns.html"], request=request,) + return await self.render(["patterns.html"], request=request) + + +class AuthTokenView(BaseView): + name = "auth_token" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + token = request.args.get("token") or "" + if not self.ds._root_token: + return Response("Root token has already been used", status=403) + if secrets.compare_digest(token, self.ds._root_token): + self.ds._root_token = None + cookie = SimpleCookie() + cookie["ds_actor"] = self.ds.sign({"id": "root"}, "actor") + cookie["ds_actor"]["path"] = "/" + response = Response( + body="", + status=302, + headers={ + "Location": "/", + "set-cookie": cookie.output(header="").lstrip(), + }, + ) + return response + else: + return Response("Invalid token", status=403) diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index ab27714a..183ecc14 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -32,6 +32,7 @@ Options: --secret TEXT Secret used for signing secure values, such as signed cookies + --root Output URL that sets a cookie authenticating the root user --version-note TEXT Additional note to show on /-/versions --help-config Show available config options --help Show this message and exit. diff --git a/tests/fixtures.py b/tests/fixtures.py index 9479abf6..b2cfd3d6 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -2,6 +2,7 @@ from datasette.app import Datasette from datasette.utils import sqlite3 from asgiref.testing import ApplicationCommunicator from asgiref.sync import async_to_sync +from http.cookies import SimpleCookie import itertools import json import os @@ -44,10 +45,14 @@ class TestClient: self.asgi_app = asgi_app @async_to_sync - async def get(self, path, allow_redirects=True, redirect_count=0, method="GET"): - return await self._get(path, allow_redirects, redirect_count, method) + async def get( + self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None + ): + return await self._get(path, allow_redirects, redirect_count, method, cookies) - async def _get(self, path, allow_redirects=True, redirect_count=0, method="GET"): + async def _get( + self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None + ): query_string = b"" if "?" in path: path, _, query_string = path.partition("?") @@ -56,6 +61,12 @@ class TestClient: raw_path = path.encode("latin-1") else: raw_path = quote(path, safe="/:,").encode("latin-1") + headers = [[b"host", b"localhost"]] + if cookies: + sc = SimpleCookie() + for key, value in cookies.items(): + sc[key] = value + headers.append([b"cookie", sc.output(header="").encode("utf-8")]) scope = { "type": "http", "http_version": "1.0", @@ -63,7 +74,7 @@ class TestClient: "path": unquote(path), "raw_path": raw_path, "query_string": query_string, - "headers": [[b"host", b"localhost"]], + "headers": headers, } instance = ApplicationCommunicator(self.asgi_app, scope) await instance.send_input({"type": "http.request"}) diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 00000000..6b69ab93 --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,25 @@ +from .fixtures import app_client + + +def test_auth_token(app_client): + "The /-/auth-token endpoint sets the correct cookie" + assert app_client.ds._root_token is not None + path = "/-/auth-token?token={}".format(app_client.ds._root_token) + response = app_client.get(path, allow_redirects=False,) + assert 302 == response.status + assert "/" == response.headers["Location"] + set_cookie = response.headers["set-cookie"] + assert set_cookie.endswith("; Path=/") + assert set_cookie.startswith("ds_actor=") + cookie_value = set_cookie.split("ds_actor=")[1].split("; Path=/")[0] + assert {"id": "root"} == app_client.ds.unsign(cookie_value, "actor") + # Check that a second with same token fails + assert app_client.ds._root_token is None + assert 403 == app_client.get(path, allow_redirects=False,).status + + +def test_actor_cookie(app_client): + "A valid actor cookie sets request.scope['actor']" + cookie = app_client.ds.sign({"id": "test"}, "actor") + response = app_client.get("/", cookies={"ds_actor": cookie}) + assert {"id": "test"} == app_client.ds._last_request.scope["actor"] diff --git a/tests/test_cli.py b/tests/test_cli.py index f52f17b4..529661ce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -76,6 +76,7 @@ def test_metadata_yaml(): memory=False, config=[], secret=None, + root=False, version_note=None, help_config=False, return_instance=True, diff --git a/tests/test_docs.py b/tests/test_docs.py index 77c2a611..09c00ddf 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -65,8 +65,8 @@ def documented_views(): first_word = label.split("_")[0] if first_word.endswith("View"): view_labels.add(first_word) - # We deliberately don't document this one: - view_labels.add("PatternPortfolioView") + # We deliberately don't document these: + view_labels.update(("PatternPortfolioView", "AuthTokenView")) return view_labels From 57cf5139c552cb7feab9947daa949ca434cc0a66 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2020 18:06:16 -0700 Subject: [PATCH 0458/2303] Default actor_from_request hook supporting ds_actor signed cookie Refs #784, refs #699 --- datasette/actor_auth_cookie.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 datasette/actor_auth_cookie.py diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py new file mode 100644 index 00000000..41f33fe9 --- /dev/null +++ b/datasette/actor_auth_cookie.py @@ -0,0 +1,18 @@ +from datasette import hookimpl +from itsdangerous import BadSignature +from http.cookies import SimpleCookie + + +@hookimpl +def actor_from_request(datasette, request): + cookies = SimpleCookie() + cookies.load( + dict(request.scope.get("headers") or []).get(b"cookie", b"").decode("utf-8") + ) + if "ds_actor" not in cookies: + return None + ds_actor = cookies["ds_actor"].value + try: + return datasette.unsign(ds_actor, "actor") + except BadSignature: + return None From dfdbdf378aba9afb66666f66b78df2f2069d2595 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2020 22:00:36 -0700 Subject: [PATCH 0459/2303] Added /-/permissions debug tool, closes #788 Also started the authentication.rst docs page, refs #786. Part of authentication work, refs #699. --- datasette/app.py | 32 +++++++++++-- datasette/default_permissions.py | 7 +++ datasette/plugins.py | 1 + datasette/templates/permissions_debug.html | 55 ++++++++++++++++++++++ datasette/views/special.py | 18 +++++++ docs/authentication.rst | 18 +++++++ docs/index.rst | 1 + tests/test_auth.py | 23 +++++++++ 8 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 datasette/default_permissions.py create mode 100644 datasette/templates/permissions_debug.html create mode 100644 docs/authentication.rst diff --git a/datasette/app.py b/datasette/app.py index 6b39ce12..b8a5e23d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,5 +1,6 @@ import asyncio import collections +import datetime import hashlib import itertools import json @@ -24,7 +25,12 @@ import uvicorn from .views.base import DatasetteError, ureg, AsgiRouter from .views.database import DatabaseDownload, DatabaseView from .views.index import IndexView -from .views.special import JsonDataView, PatternPortfolioView, AuthTokenView +from .views.special import ( + JsonDataView, + PatternPortfolioView, + AuthTokenView, + PermissionsDebugView, +) from .views.table import RowView, TableView from .renderer import json_renderer from .database import Database, QueryInterrupted @@ -283,6 +289,7 @@ class Datasette: pm.hook.prepare_jinja2_environment(env=self.jinja_env) self._register_renderers() + self.permission_checks = collections.deque(maxlen=30) self._root_token = os.urandom(32).hex() def sign(self, value, namespace="default"): @@ -420,6 +427,7 @@ class Datasette: self, actor, action, resource_type=None, resource_identifier=None, default=False ): "Check permissions using the permissions_allowed plugin hook" + result = None for check in pm.hook.permission_allowed( datasette=self, actor=actor, @@ -432,8 +440,23 @@ class Datasette: if asyncio.iscoroutine(check): check = await check if check is not None: - return check - return default + result = check + used_default = False + if result is None: + result = default + used_default = True + self.permission_checks.append( + { + "when": datetime.datetime.utcnow().isoformat(), + "actor": actor, + "action": action, + "resource_type": resource_type, + "resource_identifier": resource_identifier, + "used_default": used_default, + "result": result, + } + ) + return result async def execute( self, @@ -782,6 +805,9 @@ class Datasette: add_route( AuthTokenView.as_asgi(self), r"/-/auth-token$", ) + add_route( + PermissionsDebugView.as_asgi(self), r"/-/permissions$", + ) add_route( PatternPortfolioView.as_asgi(self), r"/-/patterns$", ) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py new file mode 100644 index 00000000..0b0d17f9 --- /dev/null +++ b/datasette/default_permissions.py @@ -0,0 +1,7 @@ +from datasette import hookimpl + + +@hookimpl +def permission_allowed(actor, action, resource_type, resource_identifier): + if actor and actor.get("id") == "root" and action == "permissions-debug": + return True diff --git a/datasette/plugins.py b/datasette/plugins.py index 487fce4d..26d4fd63 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -10,6 +10,7 @@ DEFAULT_PLUGINS = ( "datasette.facets", "datasette.sql_functions", "datasette.actor_auth_cookie", + "datasette.default_permissions", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html new file mode 100644 index 00000000..fb098c5c --- /dev/null +++ b/datasette/templates/permissions_debug.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} + +{% block title %}Debug permissions{% endblock %} + +{% block extra_head %} + +{% endblock %} + +{% block nav %} +

      + home +

      + {{ super() }} +{% endblock %} + +{% block content %} + +

      Recent permissions checks

      + +{% for check in permission_checks %} +
      +

      + {{ check.action }} + checked at + {{ check.when }} + {% if check.result %} + ✓ + {% else %} + ✗ + {% endif %} + {% if check.used_default %} + (used default) + {% endif %} +

      +

      Actor: {{ check.actor|tojson }}

      + {% if check.resource_type %} +

      Resource: {{ check.resource_type }}: {{ check.resource_identifier }}

      + {% endif %} +
      +{% endfor %} + +{% endblock %} diff --git a/datasette/views/special.py b/datasette/views/special.py index 910193e8..b75355fb 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -76,3 +76,21 @@ class AuthTokenView(BaseView): return response else: return Response("Invalid token", status=403) + + +class PermissionsDebugView(BaseView): + name = "permissions_debug" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + if not await self.ds.permission_allowed( + request.scope.get("actor"), "permissions-debug" + ): + return Response("Permission denied", status=403) + return await self.render( + ["permissions_debug.html"], + request, + {"permission_checks": reversed(self.ds.permission_checks)}, + ) diff --git a/docs/authentication.rst b/docs/authentication.rst new file mode 100644 index 00000000..0a9a4c0d --- /dev/null +++ b/docs/authentication.rst @@ -0,0 +1,18 @@ +.. _authentication: + +================================ + Authentication and permissions +================================ + +Datasette's authentication system is currently under construction. Follow `issue 699 `__ to track the development of this feature. + +.. _PermissionsDebugView: + +Permissions Debug +================= + +The debug tool at ``/-/permissions`` is only available to the root user. + +It shows the thirty most recent permission checks that have been carried out by the Datasette instance. + +This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system. diff --git a/docs/index.rst b/docs/index.rst index 2390e263..03988c8e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,7 @@ Contents publish json_api sql_queries + authentication performance csv_export facets diff --git a/tests/test_auth.py b/tests/test_auth.py index 6b69ab93..ddf328af 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,4 +1,5 @@ from .fixtures import app_client +from bs4 import BeautifulSoup as Soup def test_auth_token(app_client): @@ -23,3 +24,25 @@ def test_actor_cookie(app_client): cookie = app_client.ds.sign({"id": "test"}, "actor") response = app_client.get("/", cookies={"ds_actor": cookie}) assert {"id": "test"} == app_client.ds._last_request.scope["actor"] + + +def test_permissions_debug(app_client): + assert 403 == app_client.get("/-/permissions").status + # With the cookie it should work + cookie = app_client.ds.sign({"id": "root"}, "actor") + response = app_client.get("/-/permissions", cookies={"ds_actor": cookie}) + # Should show one failure and one success + soup = Soup(response.body, "html.parser") + check_divs = soup.findAll("div", {"class": "check"}) + checks = [ + { + "action": div.select_one(".check-action").text, + "result": bool(div.select(".check-result-true")), + "used_default": bool(div.select(".check-used-default")), + } + for div in check_divs + ] + assert [ + {"action": "permissions-debug", "result": True, "used_default": False}, + {"action": "permissions-debug", "result": False, "used_default": True}, + ] == checks From b4cd8797b8592a8bf060a76eb7227f3f1ba61d32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 10:43:50 -0700 Subject: [PATCH 0460/2303] permission_checks is now _permission_checks --- datasette/app.py | 4 ++-- datasette/views/special.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b8a5e23d..e3ad5fc7 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -289,7 +289,7 @@ class Datasette: pm.hook.prepare_jinja2_environment(env=self.jinja_env) self._register_renderers() - self.permission_checks = collections.deque(maxlen=30) + self._permission_checks = collections.deque(maxlen=30) self._root_token = os.urandom(32).hex() def sign(self, value, namespace="default"): @@ -445,7 +445,7 @@ class Datasette: if result is None: result = default used_default = True - self.permission_checks.append( + self._permission_checks.append( { "when": datetime.datetime.utcnow().isoformat(), "actor": actor, diff --git a/datasette/views/special.py b/datasette/views/special.py index b75355fb..811ed4cb 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -92,5 +92,5 @@ class PermissionsDebugView(BaseView): return await self.render( ["permissions_debug.html"], request, - {"permission_checks": reversed(self.ds.permission_checks)}, + {"permission_checks": reversed(self.ds._permission_checks)}, ) From 1d0bea157ac7074f23229af247565a78fa71c03f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 14:06:53 -0700 Subject: [PATCH 0461/2303] New request.cookies property --- datasette/actor_auth_cookie.py | 9 ++------- datasette/utils/asgi.py | 7 +++++++ docs/internals.rst | 3 +++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 41f33fe9..f3a0f306 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -5,14 +5,9 @@ from http.cookies import SimpleCookie @hookimpl def actor_from_request(datasette, request): - cookies = SimpleCookie() - cookies.load( - dict(request.scope.get("headers") or []).get(b"cookie", b"").decode("utf-8") - ) - if "ds_actor" not in cookies: + if "ds_actor" not in request.cookies: return None - ds_actor = cookies["ds_actor"].value try: - return datasette.unsign(ds_actor, "actor") + return datasette.unsign(request.cookies["ds_actor"], "actor") except BadSignature: return None diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 24398b77..960532ca 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -4,6 +4,7 @@ from mimetypes import guess_type from urllib.parse import parse_qs, urlunparse, parse_qsl from pathlib import Path from html import escape +from http.cookies import SimpleCookie import re import aiofiles @@ -44,6 +45,12 @@ class Request: def host(self): return self.headers.get("host") or "localhost" + @property + def cookies(self): + cookies = SimpleCookie() + cookies.load(self.headers.get("cookie", "")) + return {key: value.value for key, value in cookies.items()} + @property def path(self): if self.scope.get("raw_path") is not None: diff --git a/docs/internals.rst b/docs/internals.rst index 68a35312..b3ad623f 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -27,6 +27,9 @@ The request object is passed to various plugin hooks. It represents an incoming ``.headers`` - dictionary (str -> str) A dictionary of incoming HTTP request headers. +``.cookies`` - dictionary (str -> str) + A dictionary of incoming cookies + ``.host`` - string The host header from the incoming request, e.g. ``latest.datasette.io`` or ``localhost``. From 4fa7cf68536628344356d3ef8c92c25c249067a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 14:08:12 -0700 Subject: [PATCH 0462/2303] Flash messages mechanism, closes #790 --- datasette/app.py | 42 +++++++++++++++++++++++ datasette/static/app.css | 16 +++++++++ datasette/templates/base.html | 8 +++++ datasette/templates/messages_debug.html | 26 ++++++++++++++ datasette/utils/asgi.py | 4 +-- datasette/views/base.py | 16 +++++++++ datasette/views/special.py | 24 +++++++++++++ docs/internals.rst | 18 ++++++++++ docs/introspection.rst | 8 +++++ tests/fixtures.py | 6 ++++ tests/plugins/messages_output_renderer.py | 21 ++++++++++++ tests/test_api.py | 1 + tests/test_auth.py | 6 +--- tests/test_messages.py | 28 +++++++++++++++ 14 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 datasette/templates/messages_debug.html create mode 100644 tests/plugins/messages_output_renderer.py create mode 100644 tests/test_messages.py diff --git a/datasette/app.py b/datasette/app.py index e3ad5fc7..41c73900 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,6 +2,7 @@ import asyncio import collections import datetime import hashlib +from http.cookies import SimpleCookie import itertools import json import os @@ -30,6 +31,7 @@ from .views.special import ( PatternPortfolioView, AuthTokenView, PermissionsDebugView, + MessagesDebugView, ) from .views.table import RowView, TableView from .renderer import json_renderer @@ -156,6 +158,11 @@ async def favicon(scope, receive, send): class Datasette: + # Message constants: + INFO = 1 + WARNING = 2 + ERROR = 3 + def __init__( self, files, @@ -423,6 +430,38 @@ class Datasette: # pylint: disable=no-member pm.hook.prepare_connection(conn=conn, database=database, datasette=self) + def add_message(self, request, message, type=INFO): + if not hasattr(request, "_messages"): + request._messages = [] + request._messages_should_clear = False + request._messages.append((message, type)) + + def _write_messages_to_response(self, request, response): + if getattr(request, "_messages", None): + # Set those messages + cookie = SimpleCookie() + cookie["ds_messages"] = self.sign(request._messages, "messages") + cookie["ds_messages"]["path"] = "/" + # TODO: Co-exist with existing set-cookie headers + assert "set-cookie" not in response.headers + response.headers["set-cookie"] = cookie.output(header="").lstrip() + elif getattr(request, "_messages_should_clear", False): + cookie = SimpleCookie() + cookie["ds_messages"] = "" + cookie["ds_messages"]["path"] = "/" + # TODO: Co-exist with existing set-cookie headers + assert "set-cookie" not in response.headers + response.headers["set-cookie"] = cookie.output(header="").lstrip() + + def _show_messages(self, request): + if getattr(request, "_messages", None): + request._messages_should_clear = True + messages = request._messages + request._messages = [] + return messages + else: + return [] + async def permission_allowed( self, actor, action, resource_type=None, resource_identifier=None, default=False ): @@ -808,6 +847,9 @@ class Datasette: add_route( PermissionsDebugView.as_asgi(self), r"/-/permissions$", ) + add_route( + MessagesDebugView.as_asgi(self), r"/-/messages$", + ) add_route( PatternPortfolioView.as_asgi(self), r"/-/patterns$", ) diff --git a/datasette/static/app.css b/datasette/static/app.css index 92f268ae..774a2235 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -351,3 +351,19 @@ p.zero-results { .type-float, .type-int { color: #666; } + +.message-info { + padding: 1em; + border: 1px solid green; + background-color: #c7fbc7; +} +.message-warning { + padding: 1em; + border: 1px solid #ae7100; + background-color: #fbdda5; +} +.message-error { + padding: 1em; + border: 1px solid red; + background-color: pink; +} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index d9fd945b..9b871d03 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -17,6 +17,14 @@
      +{% block messages %} +{% if show_messages %} + {% for message, message_type in show_messages() %} +

      {{ message }}

      + {% endfor %} +{% endif %} +{% endblock %} + {% block content %} {% endblock %}
      diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html new file mode 100644 index 00000000..b2e1bc7c --- /dev/null +++ b/datasette/templates/messages_debug.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block title %}Debug messages{% endblock %} + +{% block content %} + +

      Debug messages

      + +

      Set a message:

      + +
      +
      + +
      + +
      + +
      +
      + +{% endblock %} diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 960532ca..5682da48 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -180,9 +180,9 @@ class AsgiLifespan: class AsgiView: - def dispatch_request(self, request, *args, **kwargs): + async def dispatch_request(self, request, *args, **kwargs): handler = getattr(self, request.method.lower(), None) - return handler(request, *args, **kwargs) + return await handler(request, *args, **kwargs) @classmethod def as_asgi(cls, *class_args, **class_kwargs): diff --git a/datasette/views/base.py b/datasette/views/base.py index 06b78d5f..2402406a 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,6 +1,7 @@ import asyncio import csv import itertools +from itsdangerous import BadSignature import json import re import time @@ -73,6 +74,20 @@ class BaseView(AsgiView): def database_color(self, database): return "ff0000" + async def dispatch_request(self, request, *args, **kwargs): + # Populate request_messages if ds_messages cookie is present + if self.ds: + try: + request._messages = self.ds.unsign( + request.cookies.get("ds_messages", ""), "messages" + ) + except BadSignature: + pass + response = await super().dispatch_request(request, *args, **kwargs) + if self.ds: + self.ds._write_messages_to_response(request, response) + return response + async def render(self, templates, request, context=None): context = context or {} template = self.ds.jinja_env.select_template(templates) @@ -81,6 +96,7 @@ class BaseView(AsgiView): **{ "database_url": self.database_url, "database_color": self.database_color, + "show_messages": lambda: self.ds._show_messages(request), "select_templates": [ "{}{}".format( "*" if template_name == template.name else "", template_name diff --git a/datasette/views/special.py b/datasette/views/special.py index 811ed4cb..37c04697 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -94,3 +94,27 @@ class PermissionsDebugView(BaseView): request, {"permission_checks": reversed(self.ds._permission_checks)}, ) + + +class MessagesDebugView(BaseView): + name = "messages_debug" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + return await self.render(["messages_debug.html"], request) + + async def post(self, request): + post = await request.post_vars() + message = post.get("message", "") + message_type = post.get("message_type") or "INFO" + assert message_type in ("INFO", "WARNING", "ERROR", "all") + datasette = self.ds + if message_type == "all": + datasette.add_message(request, message, datasette.INFO) + datasette.add_message(request, message, datasette.WARNING) + datasette.add_message(request, message, datasette.ERROR) + else: + datasette.add_message(request, message, getattr(datasette, message_type)) + return Response.redirect("/") diff --git a/docs/internals.rst b/docs/internals.rst index b3ad623f..4d51d614 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -214,6 +214,24 @@ This method returns a signed string, which can be decoded and verified using :re Returns the original, decoded object that was passed to :ref:`datasette_sign`. If the signature is not valid this raises a ``itsdangerous.BadSignature`` exception. +.. _datasette_add_message: + +.add_message(request, message, message_type=datasette.INFO) +----------------------------------------------------------- + +``request`` - Request + The current Request object + +``message`` - string + The message string + +``message_type`` - constant, optional + The message type - ``datasette.INFO``, ``datasette.WARNING`` or ``datasette.ERROR`` + +Datasette's flash messaging mechanism allows you to add a message that will be displayed to the user on the next page that they visit. Messages are persisted in a ``ds_messages`` cookie. This method adds a message to that cookie. + +You can try out these messages (including the different visual styling of the three message types) using the ``/-/messages`` debugging tool. + .. _internals_database: Database class diff --git a/docs/introspection.rst b/docs/introspection.rst index e5d08dbc..084ee144 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -166,3 +166,11 @@ Shows the currently authenticated actor. Useful for debugging Datasette authenti "username": "some-user" } } + + +.. _MessagesDebugView: + +/-/messages +----------- + +The debug tool at ``/-/messages`` can be used to set flash messages to try out that feature. See :ref:`datasette_add_message` for details of this feature. diff --git a/tests/fixtures.py b/tests/fixtures.py index b2cfd3d6..daff0168 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -29,6 +29,12 @@ class TestResponse: self.headers = headers self.body = body + @property + def cookies(self): + cookie = SimpleCookie() + cookie.load(self.headers.get("set-cookie") or "") + return {key: value.value for key, value in cookie.items()} + @property def json(self): return json.loads(self.text) diff --git a/tests/plugins/messages_output_renderer.py b/tests/plugins/messages_output_renderer.py new file mode 100644 index 00000000..6b52f801 --- /dev/null +++ b/tests/plugins/messages_output_renderer.py @@ -0,0 +1,21 @@ +from datasette import hookimpl + + +def render_message_debug(datasette, request): + if request.args.get("add_msg"): + msg_type = request.args.get("type", "INFO") + datasette.add_message( + request, request.args["add_msg"], getattr(datasette, msg_type) + ) + return {"body": "Hello from message debug"} + + +@hookimpl +def register_output_renderer(datasette): + return [ + { + "extension": "message", + "render": render_message_debug, + "can_render": lambda: False, + } + ] diff --git a/tests/test_api.py b/tests/test_api.py index d7e7c03f..a5c6f6a2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1262,6 +1262,7 @@ def test_plugins_json(app_client): expected = [ {"name": name, "static": False, "templates": False, "version": None} for name in ( + "messages_output_renderer.py", "my_plugin.py", "my_plugin_2.py", "register_output_renderer.py", diff --git a/tests/test_auth.py b/tests/test_auth.py index ddf328af..ac8d7abe 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -9,11 +9,7 @@ def test_auth_token(app_client): response = app_client.get(path, allow_redirects=False,) assert 302 == response.status assert "/" == response.headers["Location"] - set_cookie = response.headers["set-cookie"] - assert set_cookie.endswith("; Path=/") - assert set_cookie.startswith("ds_actor=") - cookie_value = set_cookie.split("ds_actor=")[1].split("; Path=/")[0] - assert {"id": "root"} == app_client.ds.unsign(cookie_value, "actor") + assert {"id": "root"} == app_client.ds.unsign(response.cookies["ds_actor"], "actor") # Check that a second with same token fails assert app_client.ds._root_token is None assert 403 == app_client.get(path, allow_redirects=False,).status diff --git a/tests/test_messages.py b/tests/test_messages.py new file mode 100644 index 00000000..d17e015c --- /dev/null +++ b/tests/test_messages.py @@ -0,0 +1,28 @@ +from .fixtures import app_client +import pytest + + +@pytest.mark.parametrize( + "qs,expected", + [ + ("add_msg=added-message", [["added-message", 1]]), + ("add_msg=added-warning&type=WARNING", [["added-warning", 2]]), + ("add_msg=added-error&type=ERROR", [["added-error", 3]]), + ], +) +def test_add_message_sets_cookie(app_client, qs, expected): + response = app_client.get("/fixtures.message?{}".format(qs)) + signed = response.cookies["ds_messages"] + decoded = app_client.ds.unsign(signed, "messages") + assert expected == decoded + + +def test_messages_are_displayed_and_cleared(app_client): + # First set the message cookie + set_msg_response = app_client.get("/fixtures.message?add_msg=xmessagex") + # Now access a page that displays messages + response = app_client.get("/", cookies=set_msg_response.cookies) + # Messages should be in that HTML + assert "xmessagex" in response.text + # Cookie should have been set that clears messages + assert "" == response.cookies["ds_messages"] From 5278c04682929f0b155102827f9150c7b2112215 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 14:29:12 -0700 Subject: [PATCH 0463/2303] More consistent use of response.text/response.json in tests, closes #792 --- tests/test_api.py | 10 ++-------- tests/test_config_dir.py | 13 ++++++------- tests/test_csv.py | 2 +- tests/test_html.py | 4 +--- tests/test_plugins.py | 10 ++++------ 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index a5c6f6a2..7ed4cced 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1762,16 +1762,10 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 assert ["fixtures", "foo", "foo-bar"] == [ d["name"] - for d in json.loads( - app_client_conflicting_database_names.get("/-/databases.json").body.decode( - "utf8" - ) - ) + for d in app_client_conflicting_database_names.get("/-/databases.json").json ] for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): - data = json.loads( - app_client_conflicting_database_names.get(path).body.decode("utf8") - ) + data = app_client_conflicting_database_names.get(path).json assert db_name == data["database"] diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 50e67f80..490b1f1d 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -84,21 +84,20 @@ def config_dir_client(tmp_path_factory): def test_metadata(config_dir_client): response = config_dir_client.get("/-/metadata.json") assert 200 == response.status - assert METADATA == json.loads(response.text) + assert METADATA == response.json def test_config(config_dir_client): response = config_dir_client.get("/-/config.json") assert 200 == response.status - config = json.loads(response.text) - assert 60 == config["default_cache_ttl"] - assert not config["allow_sql"] + assert 60 == response.json["default_cache_ttl"] + assert not response.json["allow_sql"] def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status - assert "hooray.py" in {p["name"] for p in json.loads(response.text)} + assert "hooray.py" in {p["name"] for p in response.json} def test_templates_and_plugin(config_dir_client): @@ -123,7 +122,7 @@ def test_static_directory_browsing_not_allowed(config_dir_client): def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status - databases = json.loads(response.text) + databases = response.json assert 2 == len(databases) databases.sort(key=lambda d: d["name"]) assert "demo" == databases[0]["name"] @@ -141,4 +140,4 @@ def test_metadata_yaml(tmp_path_factory, filename): client.ds = ds response = client.get("/-/metadata.json") assert 200 == response.status - assert {"title": "Title from metadata"} == json.loads(response.text) + assert {"title": "Title from metadata"} == response.json diff --git a/tests/test_csv.py b/tests/test_csv.py index 1030c2bb..42022726 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -101,7 +101,7 @@ def test_csv_with_non_ascii_characters(app_client): ) assert response.status == 200 assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" == response.body.decode("utf8") + assert "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" == response.text def test_max_csv_mb(app_client_csv_max_mb_one): diff --git a/tests/test_html.py b/tests/test_html.py index e602bf0e..2d2a141a 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -606,9 +606,7 @@ def test_row_html_simple_primary_key(app_client): def test_table_not_exists(app_client): - assert "Table not found: blah" in app_client.get("/fixtures/blah").body.decode( - "utf8" - ) + assert "Table not found: blah" in app_client.get("/fixtures/blah").text def test_table_html_no_primary_key(app_client): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 7a3fb49a..f69e7fa7 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -218,7 +218,7 @@ def test_plugin_config_file(app_client): ) def test_plugins_extra_body_script(app_client, path, expected_extra_body_script): r = re.compile(r"") - json_data = r.search(app_client.get(path).body.decode("utf8")).group(1) + json_data = r.search(app_client.get(path).text).group(1) actual_data = json.loads(json_data) assert expected_extra_body_script == actual_data @@ -331,7 +331,7 @@ def view_names_client(tmp_path_factory): def test_view_names(view_names_client, path, view_name): response = view_names_client.get(path) assert response.status == 200 - assert "view_name:{}".format(view_name) == response.body.decode("utf8") + assert "view_name:{}".format(view_name) == response.text def test_register_output_renderer_no_parameters(app_client): @@ -345,8 +345,7 @@ def test_register_output_renderer_all_parameters(app_client): assert 200 == response.status # Lots of 'at 0x103a4a690' in here - replace those so we can do # an easy comparison - body = response.body.decode("utf-8") - body = at_memory_re.sub(" at 0xXXX", body) + body = at_memory_re.sub(" at 0xXXX", response.text) assert { "1+1": 2, "datasette": "", @@ -468,7 +467,6 @@ def test_register_facet_classes(app_client): response = app_client.get( "/fixtures/compound_three_primary_keys.json?_dummy_facet=1" ) - data = json.loads(response.body) assert [ { "name": "pk1", @@ -502,7 +500,7 @@ def test_register_facet_classes(app_client): "name": "pk3", "toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk3", }, - ] == data["suggested_facets"] + ] == response.json["suggested_facets"] def test_actor_from_request(app_client): From a7137dfe069e5fceca56f78631baebd4a6a19967 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 14:49:28 -0700 Subject: [PATCH 0464/2303] /-/plugins now shows details of hooks, closes #794 Also added /-/plugins?all=1 parameter to see default plugins. --- datasette/app.py | 9 ++++--- datasette/plugins.py | 1 + docs/introspection.rst | 5 +++- tests/test_api.py | 61 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 41c73900..22fb04c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -625,9 +625,9 @@ class Datasette: }, } - def _plugins(self, show_all=False): + def _plugins(self, request): ps = list(get_plugins()) - if not show_all: + if not request.args.get("all"): ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] return [ { @@ -635,6 +635,7 @@ class Datasette: "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), + "hooks": p["hooks"], } for p in ps ] @@ -822,7 +823,9 @@ class Datasette: r"/-/versions(?P(\.json)?)$", ) add_route( - JsonDataView.as_asgi(self, "plugins.json", self._plugins), + JsonDataView.as_asgi( + self, "plugins.json", self._plugins, needs_request=True + ), r"/-/plugins(?P(\.json)?)$", ) add_route( diff --git a/datasette/plugins.py b/datasette/plugins.py index 26d4fd63..b35b750f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -49,6 +49,7 @@ def get_plugins(): "name": plugin.__name__, "static_path": static_path, "templates_path": templates_path, + "hooks": [h.name for h in pm.get_hookcallers(plugin)], } distinfo = plugin_to_distinfo.get(plugin) if distinfo: diff --git a/docs/introspection.rst b/docs/introspection.rst index 084ee144..08006529 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -78,10 +78,13 @@ Shows a list of currently installed plugins and their versions. `Plugins example "name": "datasette_cluster_map", "static": true, "templates": false, - "version": "0.4" + "version": "0.10", + "hooks": ["extra_css_urls", "extra_js_urls", "extra_body_script"] } ] +Add ``?all=1`` to include details of the default plugins baked into Datasette. + .. _JsonDataView_config: /-/config diff --git a/tests/test_api.py b/tests/test_api.py index 7ed4cced..4b752f31 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1260,14 +1260,59 @@ def test_threads_json(app_client): def test_plugins_json(app_client): response = app_client.get("/-/plugins.json") expected = [ - {"name": name, "static": False, "templates": False, "version": None} - for name in ( - "messages_output_renderer.py", - "my_plugin.py", - "my_plugin_2.py", - "register_output_renderer.py", - "view_name.py", - ) + { + "name": "messages_output_renderer.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["register_output_renderer"], + }, + { + "name": "my_plugin.py", + "static": False, + "templates": False, + "version": None, + "hooks": [ + "actor_from_request", + "extra_body_script", + "extra_css_urls", + "extra_js_urls", + "extra_template_vars", + "permission_allowed", + "prepare_connection", + "prepare_jinja2_environment", + "register_facet_classes", + "render_cell", + ], + }, + { + "name": "my_plugin_2.py", + "static": False, + "templates": False, + "version": None, + "hooks": [ + "actor_from_request", + "asgi_wrapper", + "extra_js_urls", + "extra_template_vars", + "permission_allowed", + "render_cell", + ], + }, + { + "name": "register_output_renderer.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["register_output_renderer"], + }, + { + "name": "view_name.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["extra_template_vars"], + }, ] assert expected == sorted(response.json, key=lambda p: p["name"]) From 3c5e4f266dfa07bd0bbb530d17019207f787d806 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 15:34:50 -0700 Subject: [PATCH 0465/2303] Added messages to pattern portfolio, refs #790 --- datasette/templates/patterns.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 9ea4ae42..73443ac2 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -20,6 +20,12 @@ attraction_characteristic

      +

      Messages

      +
      +

      Example message

      +

      Example message

      +

      Example message

      +

      .bd for /

      Datasette Fixtures

      From 9690ce606823bbfceb0c50d59e03adf7bb1a8475 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Jun 2020 17:05:33 -0700 Subject: [PATCH 0466/2303] More efficient modifiation of scope --- datasette/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 22fb04c6..f9bf91a8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -901,13 +901,14 @@ class DatasetteRouter(AsgiRouter): base_url = self.ds.config("base_url") if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] + scope_modifications = {} # Apply force_https_urls, if set if ( self.ds.config("force_https_urls") and scope["type"] == "http" and scope.get("scheme") != "https" ): - scope = dict(scope, scheme="https") + scope_modifications["scheme"] = "https" # Handle authentication actor = None for actor in pm.hook.actor_from_request( @@ -919,7 +920,10 @@ class DatasetteRouter(AsgiRouter): actor = await actor if actor: break - return await super().route_path(dict(scope, actor=actor), receive, send, path) + scope_modifications["actor"] = actor + return await super().route_path( + dict(scope, **scope_modifications), receive, send, path + ) async def handle_404(self, scope, receive, send, exception=None): # If URL has a trailing slash, redirect to URL without it From 0934844c0b6d124163d0185fb6a41ba5a71433da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 3 Jun 2020 06:48:39 -0700 Subject: [PATCH 0467/2303] request.post_vars() no longer discards empty values --- datasette/utils/asgi.py | 2 +- tests/test_internals_request.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 5682da48..c7810a50 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -80,7 +80,7 @@ class Request: body += message.get("body", b"") more_body = message.get("more_body", False) - return dict(parse_qsl(body.decode("utf-8"))) + return dict(parse_qsl(body.decode("utf-8"), keep_blank_values=True)) @classmethod def fake(cls, path_with_query_string, method="GET", scheme="http"): diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index 5c9b254b..433b23d5 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -16,10 +16,14 @@ async def test_request_post_vars(): } async def receive(): - return {"type": "http.request", "body": b"foo=bar&baz=1", "more_body": False} + return { + "type": "http.request", + "body": b"foo=bar&baz=1&empty=", + "more_body": False, + } request = Request(scope, receive) - assert {"foo": "bar", "baz": "1"} == await request.post_vars() + assert {"foo": "bar", "baz": "1", "empty": ""} == await request.post_vars() def test_request_args(): From aa82d0370463580f2cb10d9617f1bcbe45cc994a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 3 Jun 2020 08:16:50 -0700 Subject: [PATCH 0468/2303] Basic writable canned queries Refs #698. First working version of this feature. * request.post_vars() no longer discards empty values --- datasette/templates/query.html | 7 ++- datasette/views/database.py | 60 +++++++++++++++++++++-- datasette/views/table.py | 18 +++++++ docs/sql_queries.rst | 65 +++++++++++++++++++++++-- tests/fixtures.py | 37 +++++++++++--- tests/test_canned_write.py | 88 ++++++++++++++++++++++++++++++++++ 6 files changed, 256 insertions(+), 19 deletions(-) create mode 100644 tests/test_canned_write.py diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 2c8c05a0..52896e96 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -27,11 +27,12 @@ {% endblock %} {% block content %} +

      {{ metadata.title or database }}

      {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -
      +

      Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %} {% if hide_sql %}(show){% else %}(hide){% endif %}

      {% if not hide_sql %} {% if editable and config.allow_sql %} @@ -74,7 +75,9 @@ {% else %} -

      0 results

      + {% if not canned_write %} +

      0 results

      + {% endif %} {% endif %} {% include "_codemirror_foot.html" %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 15545fb8..558dd0f0 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -106,6 +106,8 @@ class QueryView(DataView): canned_query=None, metadata=None, _size=None, + named_parameters=None, + write=False, ): params = {key: request.args.get(key) for key in request.args} if "sql" in params: @@ -113,7 +115,7 @@ class QueryView(DataView): if "_shape" in params: params.pop("_shape") # Extract any :named parameters - named_parameters = self.re_named_parameter.findall(sql) + named_parameters = named_parameters or self.re_named_parameter.findall(sql) named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters @@ -129,12 +131,60 @@ class QueryView(DataView): extra_args["custom_time_limit"] = int(params["_timelimit"]) if _size: extra_args["page_size"] = _size - results = await self.ds.execute( - database, sql, params, truncate=True, **extra_args - ) - columns = [r[0] for r in results.description] templates = ["query-{}.html".format(to_css_class(database)), "query.html"] + + # Execute query - as write or as read + if write: + if request.method == "POST": + params = await request.post_vars() + try: + cursor = await self.ds.databases[database].execute_write( + sql, params, block=True + ) + message = metadata.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + message_type = self.ds.INFO + redirect_url = metadata.get("on_success_redirect") + except Exception as e: + message = metadata.get("on_error_message") or str(e) + message_type = self.ds.ERROR + redirect_url = metadata.get("on_error_redirect") + self.ds.add_message(request, message, message_type) + return self.redirect(request, redirect_url or request.path) + else: + + async def extra_template(): + return { + "request": request, + "path_with_added_args": path_with_added_args, + "path_with_removed_args": path_with_removed_args, + "named_parameter_values": named_parameter_values, + "canned_query": canned_query, + "success_message": request.args.get("_success") or "", + "canned_write": True, + } + + return ( + { + "database": database, + "rows": [], + "truncated": False, + "columns": [], + "query": {"sql": sql, "params": params}, + }, + extra_template, + templates, + ) + else: # Not a write + results = await self.ds.execute( + database, sql, params, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + if canned_query: templates.insert( 0, diff --git a/datasette/views/table.py b/datasette/views/table.py index 2e9515c3..79bf8b08 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -221,6 +221,22 @@ class RowTableShared(DataView): class TableView(RowTableShared): name = "table" + async def post(self, request, db_name, table_and_format): + # Handle POST to a canned query + canned_query = self.ds.get_canned_query(db_name, table_and_format) + assert canned_query, "You may only POST to a canned query" + return await QueryView(self.ds).data( + request, + db_name, + None, + canned_query["sql"], + metadata=canned_query, + editable=False, + canned_query=table_and_format, + named_parameters=canned_query.get("params"), + write=bool(canned_query.get("write")), + ) + async def data( self, request, @@ -241,6 +257,8 @@ class TableView(RowTableShared): metadata=canned_query, editable=False, canned_query=table, + named_parameters=canned_query.get("params"), + write=bool(canned_query.get("write")), ) db = self.ds.databases[database] diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index c3efd930..dc239a84 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -161,11 +161,12 @@ You can set a default fragment hash that will be included in the link to the can { "databases": { - "fixtures": { - "queries": { - "neighborhood_search": { - "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;", - "fragment": "fragment-goes-here" + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;", + "fragment": "fragment-goes-here" + } } } } @@ -173,6 +174,60 @@ You can set a default fragment hash that will be included in the link to the can `See here `__ for a demo of this in action. +.. _canned_queries_writable: + +Writable canned queries +~~~~~~~~~~~~~~~~~~~~~~~ + +Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. + +.. code-block:: json + + { + "databases": { + "mydatabase": { + "queries": { + "add_name": { + "sql": "INSERT INTO names (name) VALUES (:name)", + "write": true + } + } + } + } + } + +This configuration will create a page at ``/mydatabase/add_name`` displaying a form with a ``name`` field. Submitting that form will execute the configured ``INSERT`` query. + +You can customize how Datasette represents success and errors using the following optional properties: + +- ``on_success_message`` - the message shown when a query is successful +- ``on_success_redirect`` - the path or URL the user is redirected to on success +- ``on_error_message`` - the message shown when a query throws an error +- ``on_error_redirect`` - the path or URL the user is redirected to on error + +For example: + +.. code-block:: json + + { + "databases": { + "mydatabase": { + "queries": { + "add_name": { + "sql": "INSERT INTO names (name) VALUES (:name)", + "write": true, + "on_success_message": "Name inserted", + "on_success_redirect": "/mydatabase/names", + "on_error_message": "Name insert failed", + "on_error_redirect": "/mydatabase" + } + } + } + } + } + +You may wish to use this feature in conjunction with :ref:`authentication`. + .. _pagination: Pagination diff --git a/tests/fixtures.py b/tests/fixtures.py index daff0168..78a54c68 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -14,7 +14,7 @@ import string import tempfile import textwrap import time -from urllib.parse import unquote, quote +from urllib.parse import unquote, quote, urlencode # This temp file is used by one of the plugin config tests @@ -54,10 +54,26 @@ class TestClient: async def get( self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None ): - return await self._get(path, allow_redirects, redirect_count, method, cookies) + return await self._request( + path, allow_redirects, redirect_count, method, cookies + ) - async def _get( - self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None + @async_to_sync + async def post( + self, path, post_data=None, allow_redirects=True, redirect_count=0, cookies=None + ): + return await self._request( + path, allow_redirects, redirect_count, "POST", cookies, post_data + ) + + async def _request( + self, + path, + allow_redirects=True, + redirect_count=0, + method="GET", + cookies=None, + post_data=None, ): query_string = b"" if "?" in path: @@ -83,7 +99,13 @@ class TestClient: "headers": headers, } instance = ApplicationCommunicator(self.asgi_app, scope) - await instance.send_input({"type": "http.request"}) + + if post_data: + body = urlencode(post_data, doseq=True).encode("utf-8") + await instance.send_input({"type": "http.request", "body": body}) + else: + await instance.send_input({"type": "http.request"}) + # First message back should be response.start with headers and status messages = [] start = await instance.receive_output(2) @@ -110,7 +132,7 @@ class TestClient: redirect_count, self.max_redirects ) location = response.headers["Location"] - return await self._get( + return await self._request( location, allow_redirects=True, redirect_count=redirect_count + 1 ) return response @@ -128,6 +150,7 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, + metadata=None, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) @@ -161,7 +184,7 @@ def make_app_client( immutables=immutables, memory=memory, cors=cors, - metadata=METADATA, + metadata=metadata or METADATA, plugins_dir=PLUGINS_DIR, config=config, inspect_data=inspect_data, diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py new file mode 100644 index 00000000..52c8aec2 --- /dev/null +++ b/tests/test_canned_write.py @@ -0,0 +1,88 @@ +import pytest +from .fixtures import make_app_client + + +@pytest.fixture +def canned_write_client(): + for client in make_app_client( + extra_databases={"data.db": "create table names (name text)"}, + metadata={ + "databases": { + "data": { + "queries": { + "add_name": { + "sql": "insert into names (name) values (:name)", + "write": True, + "on_success_redirect": "/data/add_name?success", + }, + "add_name_specify_id": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "write": True, + "on_error_redirect": "/data/add_name_specify_id?error", + }, + "delete_name": { + "sql": "delete from names where rowid = :rowid", + "write": True, + "on_success_message": "Name deleted", + }, + "update_name": { + "sql": "update names set name = :name where rowid = :rowid", + "params": ["rowid", "name"], + "write": True, + }, + } + } + } + }, + ): + yield client + + +def test_insert(canned_write_client): + response = canned_write_client.post( + "/data/add_name", {"name": "Hello"}, allow_redirects=False + ) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Query executed, 1 row affected", 1]] == messages + + +def test_custom_success_message(canned_write_client): + response = canned_write_client.post( + "/data/delete_name", {"rowid": 1}, allow_redirects=False + ) + assert 302 == response.status + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Name deleted", 1]] == messages + + +def test_insert_error(canned_write_client): + canned_write_client.post("/data/add_name", {"name": "Hello"}) + response = canned_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + allow_redirects=False, + ) + assert 302 == response.status + assert "/data/add_name_specify_id?error" == response.headers["Location"] + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["UNIQUE constraint failed: names.rowid", 3]] == messages + # How about with a custom error message? + canned_write_client.ds._metadata["databases"]["data"]["queries"][ + "add_name_specify_id" + ]["on_error_message"] = "ERROR" + response = canned_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + allow_redirects=False, + ) + assert [["ERROR", 3]] == canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) From 9cb44be42f012a68c8c3904a37008200cc7bb1f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 3 Jun 2020 14:04:40 -0700 Subject: [PATCH 0469/2303] Docs and tests for "params", closes #797 --- docs/sql_queries.rst | 70 ++++++++++++++++++++++++++++++-------- tests/test_canned_write.py | 7 +++- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index dc239a84..aa1edc98 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -121,32 +121,68 @@ Here's an example of a canned query with a named parameter: .. code-block:: sql select neighborhood, facet_cities.name, state - from facetable join facet_cities on facetable.city_id = facet_cities.id - where neighborhood like '%' || :text || '%' order by neighborhood; + from facetable + join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' + order by neighborhood; -In the canned query JSON it looks like this: +In the canned query metadata (here :ref:`metadata_yaml` as ``metadata.yaml``) it looks like this: + +.. code-block:: yaml + + databases: + fixtures: + queries: + neighborhood_search: + sql: |- + select neighborhood, facet_cities.name, state + from facetable + join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' + order by neighborhood + title: Search neighborhoods + +Here's the equivalent using JSON (as ``metadata.json``): .. code-block:: json { "databases": { - "fixtures": { - "queries": { - "neighborhood_search": { - "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;", - "title": "Search neighborhoods", - "description_html": "Demonstrating simple like search" - } - } - } + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": "select neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%'\norder by neighborhood", + "title": "Search neighborhoods" + } + } + } } } +Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. + You can try this canned query out here: https://latest.datasette.io/fixtures/neighborhood_search?text=town -Note that we are using SQLite string concatenation here - the ``||`` operator - -to add wildcard ``%`` characters to the string provided by the user. +In this example the ``:text`` named parameter is automatically extracted from the query using a regular expression. + +You can alternatively provide an explicit list of named parameters using the ``"params"`` key, like this: + +.. code-block:: yaml + + databases: + fixtures: + queries: + neighborhood_search: + params: + - text + sql: |- + select neighborhood, facet_cities.name, state + from facetable + join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' + order by neighborhood + title: Search neighborhoods .. _canned_queries_default_fragment: @@ -181,6 +217,8 @@ Writable canned queries Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. +You may wish to use this feature in conjunction with :ref:`authentication`. + .. code-block:: json { @@ -226,7 +264,9 @@ For example: } } -You may wish to use this feature in conjunction with :ref:`authentication`. +You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. + +You can pre-populate form fields when the page first loads using a querystring, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query. .. _pagination: diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py index 52c8aec2..692d726e 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_write.py @@ -27,7 +27,7 @@ def canned_write_client(): }, "update_name": { "sql": "update names set name = :name where rowid = :rowid", - "params": ["rowid", "name"], + "params": ["rowid", "name", "extra"], "write": True, }, } @@ -86,3 +86,8 @@ def test_insert_error(canned_write_client): assert [["ERROR", 3]] == canned_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) + + +def test_custom_params(canned_write_client): + response = canned_write_client.get("/data/update_name?extra=foo") + assert '' in response.text From 8524866fdf0b43a68e1ee24c419c80b5cddaaeca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 4 Jun 2020 16:58:19 -0700 Subject: [PATCH 0470/2303] Link to authentication docs --- docs/plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index fb2843f4..3777bba7 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -953,7 +953,7 @@ actor_from_request(datasette, request) ``request`` - object The current HTTP :ref:`internals_request`. -This is part of Datasette's authentication and permissions system. The function should attempt to authenticate an actor (either a user or an API actor of some sort) based on information in the request. +This is part of Datasette's :ref:`authentication and permissions system `. The function should attempt to authenticate an actor (either a user or an API actor of some sort) based on information in the request. If it cannot authenticate an actor, it should return ``None``. Otherwise it should return a dictionary representing that actor. From 2074efa5a49f72cf1c47c28894de6c0b1f0fb3b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 4 Jun 2020 18:38:32 -0700 Subject: [PATCH 0471/2303] Another actor_from_request example --- docs/plugins.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index 3777bba7..8004e118 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -957,6 +957,27 @@ This is part of Datasette's :ref:`authentication and permissions system Date: Thu, 4 Jun 2020 20:10:40 -0700 Subject: [PATCH 0472/2303] More things you can do with plugins --- docs/plugins.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index 8004e118..ecc7cbf1 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -18,6 +18,8 @@ Things you can do with plugins include: * Make new custom SQL functions available for use within Datasette, for example `datasette-haversine `__ and `datasette-jellyfish `__. +* Define custom output formats with custom extensions, for example `datasette-atom `__ and + `datasette-ics `__. * Add template functions that can be called within your Jinja custom templates, for example `datasette-render-markdown `__. * Customize how database values are rendered in the Datasette interface, for example From 0da7f49b24e429e81317e370cb01de941f1b873e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 10:52:50 -0700 Subject: [PATCH 0473/2303] Rename RequestParameters to MultiParams, refs #799 --- datasette/utils/__init__.py | 25 +++++++++++++++++-------- datasette/utils/asgi.py | 4 ++-- datasette/views/table.py | 4 ++-- docs/internals.rst | 10 ++++++---- tests/test_utils.py | 9 +++++++++ 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2eb31502..083fba0c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -753,15 +753,24 @@ def escape_fts(query): ) -class RequestParameters: +class MultiParams: def __init__(self, data): - # data is a dictionary of key => [list, of, values] - assert isinstance(data, dict), "data should be a dictionary of key => [list]" - for key in data: - assert isinstance( - data[key], list - ), "data should be a dictionary of key => [list]" - self._data = data + # data is a dictionary of key => [list, of, values] or a list of [["key", "value"]] pairs + if isinstance(data, dict): + for key in data: + assert isinstance( + data[key], list + ), "dictionary data should be a dictionary of key => [list]" + self._data = data + elif isinstance(data, list): + new_data = {} + for item in data: + assert ( + isinstance(item, list) and len(item) == 2 + ), "list data should be a list of [key, value] pairs" + key, value = item + new_data.setdefault(key, []).append(value) + self._data = new_data def __contains__(self, key): return key in self._data diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index c7810a50..ba131dc8 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,5 +1,5 @@ import json -from datasette.utils import RequestParameters +from datasette.utils import MultiParams from mimetypes import guess_type from urllib.parse import parse_qs, urlunparse, parse_qsl from pathlib import Path @@ -68,7 +68,7 @@ class Request: @property def args(self): - return RequestParameters(parse_qs(qs=self.query_string)) + return MultiParams(parse_qs(qs=self.query_string)) async def post_vars(self): body = [] diff --git a/datasette/views/table.py b/datasette/views/table.py index 79bf8b08..ec1b6c7c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,7 +8,7 @@ from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( CustomRow, - RequestParameters, + MultiParams, append_querystring, compound_keys_after_sql, escape_sqlite, @@ -286,7 +286,7 @@ class TableView(RowTableShared): order_by = "" # Ensure we don't drop anything with an empty value e.g. ?name__exact= - args = RequestParameters( + args = MultiParams( urllib.parse.parse_qs(request.query_string, keep_blank_values=True) ) diff --git a/docs/internals.rst b/docs/internals.rst index 4d51d614..4b4adc5e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -39,7 +39,7 @@ The request object is passed to various plugin hooks. It represents an incoming ``.query_string`` - string The querystring component of the request, without the ``?`` - e.g. ``name__contains=sam&age__gt=10``. -``.args`` - RequestParameters +``.args`` - MultiParams An object representing the parsed querystring parameters, see below. The object also has one awaitable method: @@ -47,10 +47,12 @@ The object also has one awaitable method: ``await request.post_vars()`` - dictionary Returns a dictionary of form variables that were submitted in the request body via ``POST``. -The RequestParameters class ---------------------------- +.. _internals_multiparams: -``request.args`` is a ``RequestParameters`` object - a dictionary-like object which provides access to querystring parameters that may have multiple values. +The MultiParams class +--------------------- + +``request.args`` is a ``MultiParams`` object - a dictionary-like object which provides access to querystring parameters that may have multiple values. Consider the querystring ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` and one value for ``bar``. diff --git a/tests/test_utils.py b/tests/test_utils.py index 01a10468..ffe14587 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -437,3 +437,12 @@ def test_call_with_supported_arguments(): with pytest.raises(TypeError): utils.call_with_supported_arguments(foo, a=1) + + +def test_multi_params_list(): + p1 = utils.MultiParams([["foo", "bar"], ["foo", "baz"]]) + assert "bar" == p1["foo"] + assert ["bar", "baz"] == p1.getlist("foo") + # Should raise an error if list isn't pairs + with pytest.raises(AssertionError): + utils.MultiParams([["foo", "bar"], ["foo", "baz", "bar"]]) From d96ac1d52cacf34bae09705eb8f9a0e3f81c426b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 11:01:06 -0700 Subject: [PATCH 0474/2303] Allow tuples as well as lists in MultiParams, refs #799 --- datasette/utils/__init__.py | 6 +++--- tests/test_utils.py | 22 ++++++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 083fba0c..69e288e6 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -759,14 +759,14 @@ class MultiParams: if isinstance(data, dict): for key in data: assert isinstance( - data[key], list + data[key], (list, tuple) ), "dictionary data should be a dictionary of key => [list]" self._data = data - elif isinstance(data, list): + elif isinstance(data, list) or isinstance(data, tuple): new_data = {} for item in data: assert ( - isinstance(item, list) and len(item) == 2 + isinstance(item, (list, tuple)) and len(item) == 2 ), "list data should be a list of [key, value] pairs" key, value = item new_data.setdefault(key, []).append(value) diff --git a/tests/test_utils.py b/tests/test_utils.py index ffe14587..a7968e54 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -439,10 +439,20 @@ def test_call_with_supported_arguments(): utils.call_with_supported_arguments(foo, a=1) -def test_multi_params_list(): - p1 = utils.MultiParams([["foo", "bar"], ["foo", "baz"]]) +@pytest.mark.parametrize("data,should_raise", [ + ([["foo", "bar"], ["foo", "baz"]], False), + ([("foo", "bar"), ("foo", "baz")], False), + ((["foo", "bar"], ["foo", "baz"]), False), + ([["foo", "bar"], ["foo", "baz", "bax"]], True), + ({"foo": ["bar", "baz"]}, False), + ({"foo": ("bar", "baz")}, False), + ({"foo": "bar"}, True), +]) +def test_multi_params(data, should_raise): + if should_raise: + with pytest.raises(AssertionError): + utils.MultiParams(data) + return + p1 = utils.MultiParams(data) assert "bar" == p1["foo"] - assert ["bar", "baz"] == p1.getlist("foo") - # Should raise an error if list isn't pairs - with pytest.raises(AssertionError): - utils.MultiParams([["foo", "bar"], ["foo", "baz", "bar"]]) + assert ["bar", "baz"] == list(p1.getlist("foo")) From 84a9c4ff75460f91c049bd30bba3cee1fd89d9e2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 12:05:57 -0700 Subject: [PATCH 0475/2303] CSRF protection (#798) Closes #793. * Rename RequestParameters to MultiParams, refs #799 * Allow tuples as well as lists in MultiParams, refs #799 * Use csrftokens when running tests, refs #799 * Use new csrftoken() function, refs https://github.com/simonw/asgi-csrf/issues/7 * Check for Vary: Cookie hedaer, refs https://github.com/simonw/asgi-csrf/issues/8 --- datasette/app.py | 10 ++++++- datasette/templates/messages_debug.html | 3 +- datasette/templates/query.html | 1 + datasette/utils/__init__.py | 3 ++ datasette/views/base.py | 1 + setup.py | 1 + tests/fixtures.py | 38 +++++++++++++++++++++---- tests/test_canned_write.py | 8 ++++-- tests/test_utils.py | 21 ++++++++------ 9 files changed, 67 insertions(+), 19 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f9bf91a8..54cf02f8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,4 +1,5 @@ import asyncio +import asgi_csrf import collections import datetime import hashlib @@ -884,7 +885,14 @@ class Datasette: await database.table_counts(limit=60 * 60 * 1000) asgi = AsgiLifespan( - AsgiTracer(DatasetteRouter(self, routes)), on_startup=setup_db + AsgiTracer( + asgi_csrf.asgi_csrf( + DatasetteRouter(self, routes), + signing_secret=self._secret, + cookie_name="ds_csrftoken", + ) + ), + on_startup=setup_db, ) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html index b2e1bc7c..e83d2a2f 100644 --- a/datasette/templates/messages_debug.html +++ b/datasette/templates/messages_debug.html @@ -8,7 +8,7 @@

      Set a message:

      - +
      @@ -19,6 +19,7 @@
      +
      diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 52896e96..a7cb6647 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -52,6 +52,7 @@ {% endif %}

      + {% if canned_query %}{% endif %}

      diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 69e288e6..059db184 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -772,6 +772,9 @@ class MultiParams: new_data.setdefault(key, []).append(value) self._data = new_data + def __repr__(self): + return "".format(self._data) + def __contains__(self, key): return key in self._data diff --git a/datasette/views/base.py b/datasette/views/base.py index 2402406a..315c96fe 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -95,6 +95,7 @@ class BaseView(AsgiView): **context, **{ "database_url": self.database_url, + "csrftoken": request.scope["csrftoken"], "database_color": self.database_color, "show_messages": lambda: self.ds._show_messages(request), "select_templates": [ diff --git a/setup.py b/setup.py index 93628266..c0316deb 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ setup( "uvicorn~=0.11", "aiofiles>=0.4,<0.6", "janus>=0.4,<0.6", + "asgi-csrf>=0.4", "PyYAML~=5.3", "mergedeep>=1.1.1,<1.4.0", "itsdangerous~=1.1", diff --git a/tests/fixtures.py b/tests/fixtures.py index 78a54c68..a64a8295 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from datasette.app import Datasette -from datasette.utils import sqlite3 +from datasette.utils import sqlite3, MultiParams from asgiref.testing import ApplicationCommunicator from asgiref.sync import async_to_sync from http.cookies import SimpleCookie @@ -60,10 +60,35 @@ class TestClient: @async_to_sync async def post( - self, path, post_data=None, allow_redirects=True, redirect_count=0, cookies=None + self, + path, + post_data=None, + allow_redirects=True, + redirect_count=0, + content_type="application/x-www-form-urlencoded", + cookies=None, + csrftoken_from=None, ): + cookies = cookies or {} + post_data = post_data or {} + # Maybe fetch a csrftoken first + if csrftoken_from is not None: + if csrftoken_from is True: + csrftoken_from = path + token_response = await self._request(csrftoken_from) + # Check this had a Vary: Cookie header + assert "Cookie" == token_response.headers["vary"] + csrftoken = token_response.cookies["ds_csrftoken"] + cookies["ds_csrftoken"] = csrftoken + post_data["csrftoken"] = csrftoken return await self._request( - path, allow_redirects, redirect_count, "POST", cookies, post_data + path, + allow_redirects, + redirect_count, + "POST", + cookies, + post_data, + content_type, ) async def _request( @@ -74,6 +99,7 @@ class TestClient: method="GET", cookies=None, post_data=None, + content_type=None, ): query_string = b"" if "?" in path: @@ -84,6 +110,8 @@ class TestClient: else: raw_path = quote(path, safe="/:,").encode("latin-1") headers = [[b"host", b"localhost"]] + if content_type: + headers.append((b"content-type", content_type.encode("utf-8"))) if cookies: sc = SimpleCookie() for key, value in cookies.items(): @@ -111,7 +139,7 @@ class TestClient: start = await instance.receive_output(2) messages.append(start) assert start["type"] == "http.response.start" - headers = dict( + response_headers = MultiParams( [(k.decode("utf8"), v.decode("utf8")) for k, v in start["headers"]] ) status = start["status"] @@ -124,7 +152,7 @@ class TestClient: body += message["body"] if not message.get("more_body"): break - response = TestResponse(status, headers, body) + response = TestResponse(status, response_headers, body) if allow_redirects and response.status in (301, 302): assert ( redirect_count < self.max_redirects diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py index 692d726e..be838063 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_write.py @@ -40,7 +40,7 @@ def canned_write_client(): def test_insert(canned_write_client): response = canned_write_client.post( - "/data/add_name", {"name": "Hello"}, allow_redirects=False + "/data/add_name", {"name": "Hello"}, allow_redirects=False, csrftoken_from=True, ) assert 302 == response.status assert "/data/add_name?success" == response.headers["Location"] @@ -52,7 +52,7 @@ def test_insert(canned_write_client): def test_custom_success_message(canned_write_client): response = canned_write_client.post( - "/data/delete_name", {"rowid": 1}, allow_redirects=False + "/data/delete_name", {"rowid": 1}, allow_redirects=False, csrftoken_from=True ) assert 302 == response.status messages = canned_write_client.ds.unsign( @@ -62,11 +62,12 @@ def test_custom_success_message(canned_write_client): def test_insert_error(canned_write_client): - canned_write_client.post("/data/add_name", {"name": "Hello"}) + canned_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) response = canned_write_client.post( "/data/add_name_specify_id", {"rowid": 1, "name": "Should fail"}, allow_redirects=False, + csrftoken_from=True, ) assert 302 == response.status assert "/data/add_name_specify_id?error" == response.headers["Location"] @@ -82,6 +83,7 @@ def test_insert_error(canned_write_client): "/data/add_name_specify_id", {"rowid": 1, "name": "Should fail"}, allow_redirects=False, + csrftoken_from=True, ) assert [["ERROR", 3]] == canned_write_client.ds.unsign( response.cookies["ds_messages"], "messages" diff --git a/tests/test_utils.py b/tests/test_utils.py index a7968e54..cf714215 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -439,15 +439,18 @@ def test_call_with_supported_arguments(): utils.call_with_supported_arguments(foo, a=1) -@pytest.mark.parametrize("data,should_raise", [ - ([["foo", "bar"], ["foo", "baz"]], False), - ([("foo", "bar"), ("foo", "baz")], False), - ((["foo", "bar"], ["foo", "baz"]), False), - ([["foo", "bar"], ["foo", "baz", "bax"]], True), - ({"foo": ["bar", "baz"]}, False), - ({"foo": ("bar", "baz")}, False), - ({"foo": "bar"}, True), -]) +@pytest.mark.parametrize( + "data,should_raise", + [ + ([["foo", "bar"], ["foo", "baz"]], False), + ([("foo", "bar"), ("foo", "baz")], False), + ((["foo", "bar"], ["foo", "baz"]), False), + ([["foo", "bar"], ["foo", "baz", "bax"]], True), + ({"foo": ["bar", "baz"]}, False), + ({"foo": ("bar", "baz")}, False), + ({"foo": "bar"}, True), + ] +) def test_multi_params(data, should_raise): if should_raise: with pytest.raises(AssertionError): From 033a1bb22c70a955d9fd1d3b4675a0e2e5c8b8cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 12:06:43 -0700 Subject: [PATCH 0476/2303] Removed rogue print() from test --- tests/test_internals_database.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 5d5520dd..2d288cc8 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -162,7 +162,6 @@ async def test_execute_write_fn_block_false(db): with conn: conn.execute("delete from roadside_attractions where pk = 1;") row = conn.execute("select count(*) from roadside_attractions").fetchone() - print("row = ", row) return row[0] task_id = await db.execute_write_fn(write_fn) From f786033a5f0098371cb1df1ce83959b27c588115 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 16:46:37 -0700 Subject: [PATCH 0477/2303] Fixed 'datasette plugins' command, with tests - closes #802 --- datasette/app.py | 4 ++-- datasette/cli.py | 2 +- tests/fixtures.py | 56 +++++++++++++++++++++++++++++++++++++++++++ tests/test_api.py | 58 ++------------------------------------------- tests/test_cli.py | 31 +++++++++++++++++++++++- tests/test_utils.py | 2 +- 6 files changed, 92 insertions(+), 61 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 54cf02f8..444a065a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -626,9 +626,9 @@ class Datasette: }, } - def _plugins(self, request): + def _plugins(self, request=None, all=False): ps = list(get_plugins()) - if not request.args.get("all"): + if all is False or (request is not None and request.args.get("all")): ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] return [ { diff --git a/datasette/cli.py b/datasette/cli.py index 23f9e36b..2e3c8e36 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -126,7 +126,7 @@ pm.hook.publish_subcommand(publish=publish) def plugins(all, plugins_dir): "List currently available plugins" app = Datasette([], plugins_dir=plugins_dir) - click.echo(json.dumps(app.plugins(all), indent=4)) + click.echo(json.dumps(app._plugins(all=all), indent=4)) @cli.command() diff --git a/tests/fixtures.py b/tests/fixtures.py index a64a8295..4ca7b10f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -22,6 +22,62 @@ TEMP_PLUGIN_SECRET_FILE = os.path.join(tempfile.gettempdir(), "plugin-secret") PLUGINS_DIR = str(pathlib.Path(__file__).parent / "plugins") +EXPECTED_PLUGINS = [ + { + "name": "messages_output_renderer.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["register_output_renderer"], + }, + { + "name": "my_plugin.py", + "static": False, + "templates": False, + "version": None, + "hooks": [ + "actor_from_request", + "extra_body_script", + "extra_css_urls", + "extra_js_urls", + "extra_template_vars", + "permission_allowed", + "prepare_connection", + "prepare_jinja2_environment", + "register_facet_classes", + "render_cell", + ], + }, + { + "name": "my_plugin_2.py", + "static": False, + "templates": False, + "version": None, + "hooks": [ + "actor_from_request", + "asgi_wrapper", + "extra_js_urls", + "extra_template_vars", + "permission_allowed", + "render_cell", + ], + }, + { + "name": "register_output_renderer.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["register_output_renderer"], + }, + { + "name": "view_name.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["extra_template_vars"], + }, +] + class TestResponse: def __init__(self, status, headers, body): diff --git a/tests/test_api.py b/tests/test_api.py index 4b752f31..0aa62a95 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -15,6 +15,7 @@ from .fixtures import ( # noqa generate_compound_rows, generate_sortable_rows, make_app_client, + EXPECTED_PLUGINS, METADATA, ) import json @@ -1259,62 +1260,7 @@ def test_threads_json(app_client): def test_plugins_json(app_client): response = app_client.get("/-/plugins.json") - expected = [ - { - "name": "messages_output_renderer.py", - "static": False, - "templates": False, - "version": None, - "hooks": ["register_output_renderer"], - }, - { - "name": "my_plugin.py", - "static": False, - "templates": False, - "version": None, - "hooks": [ - "actor_from_request", - "extra_body_script", - "extra_css_urls", - "extra_js_urls", - "extra_template_vars", - "permission_allowed", - "prepare_connection", - "prepare_jinja2_environment", - "register_facet_classes", - "render_cell", - ], - }, - { - "name": "my_plugin_2.py", - "static": False, - "templates": False, - "version": None, - "hooks": [ - "actor_from_request", - "asgi_wrapper", - "extra_js_urls", - "extra_template_vars", - "permission_allowed", - "render_cell", - ], - }, - { - "name": "register_output_renderer.py", - "static": False, - "templates": False, - "version": None, - "hooks": ["register_output_renderer"], - }, - { - "name": "view_name.py", - "static": False, - "templates": False, - "version": None, - "hooks": ["extra_template_vars"], - }, - ] - assert expected == sorted(response.json, key=lambda p: p["name"]) + assert EXPECTED_PLUGINS == sorted(response.json, key=lambda p: p["name"]) def test_versions_json(app_client): diff --git a/tests/test_cli.py b/tests/test_cli.py index 529661ce..c53e9a3e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,9 @@ -from .fixtures import app_client, make_app_client, TestClient as _TestClient +from .fixtures import ( + app_client, + make_app_client, + TestClient as _TestClient, + EXPECTED_PLUGINS, +) from datasette.cli import cli, serve from click.testing import CliRunner import io @@ -50,6 +55,30 @@ def test_spatialite_error_if_attempt_to_open_spatialite(): assert "trying to load a SpatiaLite database" in result.output +def test_plugins_cli(app_client): + runner = CliRunner() + result1 = runner.invoke(cli, ["plugins"]) + assert sorted(EXPECTED_PLUGINS, key=lambda p: p["name"]) == sorted( + json.loads(result1.output), key=lambda p: p["name"] + ) + # Try with --all + result2 = runner.invoke(cli, ["plugins", "--all"]) + names = [p["name"] for p in json.loads(result2.output)] + # Should have all the EXPECTED_PLUGINS + assert set(names).issuperset(set(p["name"] for p in EXPECTED_PLUGINS)) + # And the following too: + assert set(names).issuperset( + [ + "datasette.sql_functions", + "datasette.actor_auth_cookie", + "datasette.facets", + "datasette.publish.cloudrun", + "datasette.default_permissions", + "datasette.publish.heroku", + ] + ) + + def test_metadata_yaml(): yaml_file = io.StringIO( textwrap.dedent( diff --git a/tests/test_utils.py b/tests/test_utils.py index cf714215..4931ef3b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -449,7 +449,7 @@ def test_call_with_supported_arguments(): ({"foo": ["bar", "baz"]}, False), ({"foo": ("bar", "baz")}, False), ({"foo": "bar"}, True), - ] + ], ) def test_multi_params(data, should_raise): if should_raise: From 75c143a84cee2fad878c6318755582522b9afff3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 16:55:08 -0700 Subject: [PATCH 0478/2303] Fixed /-/plugins?all=1, refs #802 --- datasette/app.py | 7 ++++++- tests/test_api.py | 6 ++++++ tests/test_cli.py | 12 ++---------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 444a065a..1624f6ea 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -628,7 +628,12 @@ class Datasette: def _plugins(self, request=None, all=False): ps = list(get_plugins()) - if all is False or (request is not None and request.args.get("all")): + should_show_all = False + if request is not None: + should_show_all = request.args.get("all") + else: + should_show_all = all + if not should_show_all: ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] return [ { diff --git a/tests/test_api.py b/tests/test_api.py index 0aa62a95..b35c0a2d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,4 @@ +from datasette.plugins import DEFAULT_PLUGINS from datasette.utils import detect_json1 from .fixtures import ( # noqa app_client, @@ -1261,6 +1262,11 @@ def test_threads_json(app_client): def test_plugins_json(app_client): response = app_client.get("/-/plugins.json") assert EXPECTED_PLUGINS == sorted(response.json, key=lambda p: p["name"]) + # Try with ?all=1 + response = app_client.get("/-/plugins.json?all=1") + names = {p["name"] for p in response.json} + assert names.issuperset(p["name"] for p in EXPECTED_PLUGINS) + assert names.issuperset(DEFAULT_PLUGINS) def test_versions_json(app_client): diff --git a/tests/test_cli.py b/tests/test_cli.py index c53e9a3e..2616f1d1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ from .fixtures import ( TestClient as _TestClient, EXPECTED_PLUGINS, ) +from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from click.testing import CliRunner import io @@ -67,16 +68,7 @@ def test_plugins_cli(app_client): # Should have all the EXPECTED_PLUGINS assert set(names).issuperset(set(p["name"] for p in EXPECTED_PLUGINS)) # And the following too: - assert set(names).issuperset( - [ - "datasette.sql_functions", - "datasette.actor_auth_cookie", - "datasette.facets", - "datasette.publish.cloudrun", - "datasette.default_permissions", - "datasette.publish.heroku", - ] - ) + assert set(names).issuperset(DEFAULT_PLUGINS) def test_metadata_yaml(): From 9c563d6aed072f14d3d25f58e84659f9caa1a243 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jun 2020 17:15:52 -0700 Subject: [PATCH 0479/2303] Bump asgi-csrf to 0.5.1 for a bug fix Refs https://github.com/simonw/asgi-csrf/issues/10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c0316deb..678a022f 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ setup( "uvicorn~=0.11", "aiofiles>=0.4,<0.6", "janus>=0.4,<0.6", - "asgi-csrf>=0.4", + "asgi-csrf>=0.5.1", "PyYAML~=5.3", "mergedeep>=1.1.1,<1.4.0", "itsdangerous~=1.1", From 30a8132d58a89fed0e034e058b62fab5180fae0f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 11:18:46 -0700 Subject: [PATCH 0480/2303] Docs for authentication + canned query permissions, refs #800 Closes #786 --- docs/authentication.rst | 108 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 0a9a4c0d..2c07f75a 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -4,14 +4,118 @@ Authentication and permissions ================================ -Datasette's authentication system is currently under construction. Follow `issue 699 `__ to track the development of this feature. +Datasette does not require authentication by default. Any visitor to a Datasette instance can explore the full data and execute SQL queries. + +Datasette's plugin system can be used to add many different styles of authentication, such as user accounts, single sign-on or API keys. + +.. _authentication_actor: + +Actors +====== + +Through plugins, Datasette can support both authenticated users (with cookies) and authenticated API agents (via authentication tokens). The word "actor" is used to cover both of these cases. + +Every request to Datasette has an associated actor value. This can be ``None`` for unauthenticated requests, or a JSON compatible Python dictionary for authenticated users or API agents. + +The only required field in an actor is ``"id"``, which must be a string. Plugins may decide to add any other fields to the actor dictionary. + +Plugins can use the :ref:`plugin_actor_from_request` hook to implement custom logic for authenticating an actor based on the incoming HTTP request. + +.. _authentication_root: + +Using the "root" actor +====================== + +Datasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github `__ for example. + +The one exception is the "root" account, which you can sign into while using Datasette on your local machine. This provides access to a small number of debugging features. + +To sign in as root, start Datasette using the ``--root`` command-line option, like this:: + + $ datasette --root + http://127.0.0.1:8001/-/auth-token?token=786fc524e0199d70dc9a581d851f466244e114ca92f33aa3b42a139e9388daa7 + INFO: Started server process [25801] + INFO: Waiting for application startup. + INFO: Application startup complete. + INFO: Uvicorn running on http://127.0.0.1:8001 (Press CTRL+C to quit) + +The URL on the first line includes a one-use token which can be used to sign in as the "root" actor in your browser. Click on that link and then visit ``http://127.0.0.1:8001/-/actor`` to confirm that you are authenticated as an actor that looks like this: + +.. code-block:: json + + { + "id": "root" + } + + +.. _authentication_permissions_canned_queries: + +Setting permissions for canned queries +====================================== + +Datasette's :ref:`canned_queries` default to allowing any user to execute them. + +You can limit who is allowed to execute a specific query with the ``"allow"`` key in the :ref:`metadata` configuration for that query. + +Here's how to restrict access to a write query to just the "root" user: + +.. code-block:: json + + { + "databases": { + "mydatabase": { + "queries": { + "add_name": { + "sql": "INSERT INTO names (name) VALUES (:name)", + "write": true, + "allow": { + "id": ["root"] + } + } + } + } + } + } + +To allow any of the actors with an ``id`` matching a specific list of values, use this: + +.. code-block:: json + + { + "allow": { + "id": ["simon", "cleopaws"] + } + } + +This works for other keys as well. Imagine an actor that looks like this: + +.. code-block:: json + + { + "id": "simon", + "roles": ["staff", "developer"] + } + +You can provide access to any user that has "developer" as one of their roles like so: + +.. code-block:: json + + { + "allow": { + "roles": ["developer"] + } + } + +Note that "roles" is not a concept that is baked into Datasette - it's more of a convention that plugins can choose to implement and act on. + +These keys act as an "or" mechanism. A actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. .. _PermissionsDebugView: Permissions Debug ================= -The debug tool at ``/-/permissions`` is only available to the root user. +The debug tool at ``/-/permissions`` is only available to the :ref:`authenticated root user ` (or any actor granted the ``permissions-debug`` action according to a plugin). It shows the thirty most recent permission checks that have been carried out by the Datasette instance. From d4c7b85f556230923d37ff327a068ed08aa9b62b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 11:23:54 -0700 Subject: [PATCH 0481/2303] Documentation for "id": "*", refs #800 --- docs/authentication.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/authentication.rst b/docs/authentication.rst index 2c07f75a..a90dcc41 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -108,6 +108,16 @@ You can provide access to any user that has "developer" as one of their roles li Note that "roles" is not a concept that is baked into Datasette - it's more of a convention that plugins can choose to implement and act on. +If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to spceify that a query can be accessed by any logged-in user use this: + +.. code-block:: json + + { + "allow": { + "id": "*" + } + } + These keys act as an "or" mechanism. A actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. .. _PermissionsDebugView: From 14f6b4d200f24940a795ddc0825319ab2891bde2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 11:39:11 -0700 Subject: [PATCH 0482/2303] actor_matches_allow utility function, refs #800 --- datasette/utils/__init__.py | 19 +++++++++++++++++++ docs/authentication.rst | 18 ++++++++++++++++-- tests/test_utils.py | 27 +++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 059db184..eb118f38 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -854,3 +854,22 @@ def call_with_supported_arguments(fn, **kwargs): ) call_with.append(kwargs[parameter]) return fn(*call_with) + + +def actor_matches_allow(actor, allow): + if allow is None: + return True + for key, values in allow.items(): + if values == "*" and key in actor: + return True + if isinstance(values, str): + values = [values] + actor_values = actor.get(key) + if actor_values is None: + return False + if isinstance(actor_values, str): + actor_values = [actor_values] + actor_values = set(actor_values) + if actor_values.intersection(values): + return True + return False diff --git a/docs/authentication.rst b/docs/authentication.rst index a90dcc41..85bbbbbd 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -50,8 +50,8 @@ The URL on the first line includes a one-use token which can be used to sign in .. _authentication_permissions_canned_queries: -Setting permissions for canned queries -====================================== +Permissions for canned queries +============================== Datasette's :ref:`canned_queries` default to allowing any user to execute them. @@ -120,6 +120,20 @@ If you want to provide access to any actor with a value for a specific key, use These keys act as an "or" mechanism. A actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. +.. _authentication_actor_matches_allow: + +actor_matches_allow() +===================== + +Plugins that wish to implement the same permissions scheme as canned queries can take advantage of the ``datasette.utils.actor_matches_allow(actor, allow)`` function: + +.. code-block:: python + + from datasette.utils import actor_matches_allow + + actor_matches_allow({"id": "root"}, {"id": "*"}) + # returns True + .. _PermissionsDebugView: Permissions Debug diff --git a/tests/test_utils.py b/tests/test_utils.py index 4931ef3b..7c24648a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -459,3 +459,30 @@ def test_multi_params(data, should_raise): p1 = utils.MultiParams(data) assert "bar" == p1["foo"] assert ["bar", "baz"] == list(p1.getlist("foo")) + + +@pytest.mark.parametrize( + "actor,allow,expected", + [ + ({"id": "root"}, None, True), + ({"id": "root"}, {}, False), + # Special "*" value for any key: + ({"id": "root"}, {"id": "*"}, True), + ({}, {"id": "*"}, False), + ({"name": "root"}, {"id": "*"}, False), + # Supports single strings or list of values: + ({"id": "root"}, {"id": "bob"}, False), + ({"id": "root"}, {"id": ["bob"]}, False), + ({"id": "root"}, {"id": "root"}, True), + ({"id": "root"}, {"id": ["root"]}, True), + # Any matching role will work: + ({"id": "garry", "roles": ["staff", "dev"]}, {"roles": ["staff"]}, True), + ({"id": "garry", "roles": ["staff", "dev"]}, {"roles": ["dev"]}, True), + ({"id": "garry", "roles": ["staff", "dev"]}, {"roles": ["otter"]}, False), + ({"id": "garry", "roles": ["staff", "dev"]}, {"roles": ["dev", "otter"]}, True), + ({"id": "garry", "roles": []}, {"roles": ["staff"]}, False), + ({"id": "garry"}, {"roles": ["staff"]}, False), + ], +) +def test_actor_matches_allow(actor, allow, expected): + assert expected == utils.actor_matches_allow(actor, allow) From 3f83d4632a643266f46ccd955d951be7aacbab99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 12:05:22 -0700 Subject: [PATCH 0483/2303] Respect query permissions on database page, refs #800 --- datasette/templates/database.html | 2 +- datasette/utils/__init__.py | 1 + datasette/views/database.py | 13 ++++++++++++- tests/test_canned_write.py | 31 ++++++++++++++++++++++++++++++- tests/test_utils.py | 3 +++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index e47b2418..fc88003c 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -60,7 +60,7 @@

      Queries

      {% endif %} diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index eb118f38..077728f4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -857,6 +857,7 @@ def call_with_supported_arguments(fn, **kwargs): def actor_matches_allow(actor, allow): + actor = actor or {} if allow is None: return True for key, values in allow.items(): diff --git a/datasette/views/database.py b/datasette/views/database.py index 558dd0f0..abc7d3bb 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -2,6 +2,7 @@ import os import jinja2 from datasette.utils import ( + actor_matches_allow, to_css_class, validate_sql_select, is_url, @@ -53,6 +54,16 @@ class DatabaseView(DataView): ) tables.sort(key=lambda t: (t["hidden"], t["name"])) + canned_queries = [ + dict( + query, + requires_auth=not actor_matches_allow(None, query.get("allow", None)), + ) + for query in self.ds.get_canned_queries(database) + if actor_matches_allow( + request.scope.get("actor", None), query.get("allow", None) + ) + ] return ( { "database": database, @@ -60,7 +71,7 @@ class DatabaseView(DataView): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": views, - "queries": self.ds.get_canned_queries(database), + "queries": canned_queries, }, { "show_hidden": request.args.get("_show_hidden"), diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py index be838063..5b5756b0 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_write.py @@ -24,6 +24,7 @@ def canned_write_client(): "sql": "delete from names where rowid = :rowid", "write": True, "on_success_message": "Name deleted", + "allow": {"id": "root"}, }, "update_name": { "sql": "update names set name = :name where rowid = :rowid", @@ -52,7 +53,11 @@ def test_insert(canned_write_client): def test_custom_success_message(canned_write_client): response = canned_write_client.post( - "/data/delete_name", {"rowid": 1}, allow_redirects=False, csrftoken_from=True + "/data/delete_name", + {"rowid": 1}, + cookies={"ds_actor": canned_write_client.ds.sign({"id": "root"}, "actor")}, + allow_redirects=False, + csrftoken_from=True, ) assert 302 == response.status messages = canned_write_client.ds.unsign( @@ -93,3 +98,27 @@ def test_insert_error(canned_write_client): def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") assert '' in response.text + + +def test_canned_query_permissions_on_database_page(canned_write_client): + # Without auth only shows three queries + query_names = [ + q["name"] for q in canned_write_client.get("/data.json").json["queries"] + ] + assert ["add_name", "add_name_specify_id", "update_name"] == query_names + + # With auth shows four + response = canned_write_client.get( + "/data.json", + cookies={"ds_actor": canned_write_client.ds.sign({"id": "root"}, "actor")}, + ) + assert 200 == response.status + assert [ + {"name": "add_name", "requires_auth": False}, + {"name": "add_name_specify_id", "requires_auth": False}, + {"name": "delete_name", "requires_auth": True}, + {"name": "update_name", "requires_auth": False}, + ] == [ + {"name": q["name"], "requires_auth": q["requires_auth"]} + for q in response.json["queries"] + ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 7c24648a..975ed0fd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -466,6 +466,9 @@ def test_multi_params(data, should_raise): [ ({"id": "root"}, None, True), ({"id": "root"}, {}, False), + (None, None, True), + (None, {}, False), + (None, {"id": "root"}, False), # Special "*" value for any key: ({"id": "root"}, {"id": "*"}, True), ({}, {"id": "*"}, False), From 070838bfa19b177f59ef3bd8f0139266adecda90 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 12:26:19 -0700 Subject: [PATCH 0484/2303] Better test for Vary header --- tests/fixtures.py | 2 -- tests/test_canned_write.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 4ca7b10f..2268ef4d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -132,8 +132,6 @@ class TestClient: if csrftoken_from is True: csrftoken_from = path token_response = await self._request(csrftoken_from) - # Check this had a Vary: Cookie header - assert "Cookie" == token_response.headers["vary"] csrftoken = token_response.cookies["ds_csrftoken"] cookies["ds_csrftoken"] = csrftoken post_data["csrftoken"] = csrftoken diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py index 5b5756b0..aacc586f 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_write.py @@ -100,6 +100,12 @@ def test_custom_params(canned_write_client): assert '' in response.text +def test_vary_header(canned_write_client): + # These forms embed a csrftoken so they should be served with Vary: Cookie + assert "vary" not in canned_write_client.get("/data").headers + assert "Cookie" == canned_write_client.get("/data/update_name").headers["vary"] + + def test_canned_query_permissions_on_database_page(canned_write_client): # Without auth only shows three queries query_names = [ From 966eec7f75d2e1b809b001bb7e82f35d477f77ea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 12:27:00 -0700 Subject: [PATCH 0485/2303] Check permissions on canned query page, refs #800 --- datasette/views/database.py | 10 +++++++++- tests/test_canned_write.py | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index abc7d3bb..4e9a6da7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -9,7 +9,7 @@ from datasette.utils import ( path_with_added_args, path_with_removed_args, ) -from datasette.utils.asgi import AsgiFileDownload +from datasette.utils.asgi import AsgiFileDownload, Response from datasette.plugins import pm from .base import DatasetteError, DataView @@ -125,6 +125,14 @@ class QueryView(DataView): params.pop("sql") if "_shape" in params: params.pop("_shape") + + # Respect canned query permissions + if canned_query: + if not actor_matches_allow( + request.scope.get("actor", None), metadata.get("allow") + ): + return Response("Permission denied", status=403) + # Extract any :named parameters named_parameters = named_parameters or self.re_named_parameter.findall(sql) named_parameter_values = { diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py index aacc586f..73b01e51 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_write.py @@ -128,3 +128,11 @@ def test_canned_query_permissions_on_database_page(canned_write_client): {"name": q["name"], "requires_auth": q["requires_auth"]} for q in response.json["queries"] ] + + +def test_canned_query_permissions(canned_write_client): + assert 403 == canned_write_client.get("/data/delete_name").status + assert 200 == canned_write_client.get("/data/update_name").status + cookies = {"ds_actor": canned_write_client.ds.sign({"id": "root"}, "actor")} + assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status + assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status From 3359d54a4eb9c9725c27a85437661b5180c4099a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 12:33:08 -0700 Subject: [PATCH 0486/2303] Use cookies when accessing csrftoken_from --- tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 2268ef4d..75bd6b94 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -131,7 +131,7 @@ class TestClient: if csrftoken_from is not None: if csrftoken_from is True: csrftoken_from = path - token_response = await self._request(csrftoken_from) + token_response = await self._request(csrftoken_from, cookies=cookies) csrftoken = token_response.cookies["ds_csrftoken"] cookies["ds_csrftoken"] = csrftoken post_data["csrftoken"] = csrftoken From f1daf64e722f9aedc61bea1636a9df715c4c4a8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 12:46:40 -0700 Subject: [PATCH 0487/2303] Link to canned query permissions documentation --- docs/authentication.rst | 2 +- docs/sql_queries.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 85bbbbbd..8b24a44a 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -53,7 +53,7 @@ The URL on the first line includes a one-use token which can be used to sign in Permissions for canned queries ============================== -Datasette's :ref:`canned_queries` default to allowing any user to execute them. +Datasette's :ref:`canned queries ` default to allowing any user to execute them. You can limit who is allowed to execute a specific query with the ``"allow"`` key in the :ref:`metadata` configuration for that query. diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index aa1edc98..5df8bdb0 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -217,7 +217,7 @@ Writable canned queries Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. -You may wish to use this feature in conjunction with :ref:`authentication`. +See :ref:`authentication_permissions_canned_queries` for details on how to add permission checks to canned queries, using the ``"allow"`` key. .. code-block:: json From 7dc23cd71aeb5a0e194f25fd1b8e569e3bb2149b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 13:05:09 -0700 Subject: [PATCH 0488/2303] Whitespace --- docs/authentication.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8b24a44a..730a86c8 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -47,7 +47,6 @@ The URL on the first line includes a one-use token which can be used to sign in "id": "root" } - .. _authentication_permissions_canned_queries: Permissions for canned queries From bd4de0647d660709de122303a1aece3a8ef88394 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 19:09:59 -0700 Subject: [PATCH 0489/2303] Improved permissions documentation --- docs/authentication.rst | 7 +++++++ docs/internals.rst | 2 ++ 2 files changed, 9 insertions(+) diff --git a/docs/authentication.rst b/docs/authentication.rst index 730a86c8..fd70000e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -47,6 +47,13 @@ The URL on the first line includes a one-use token which can be used to sign in "id": "root" } +.. _authentication_permissions: + +Permissions +=========== + +Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method. This method is also used by Datasette core code itself, which allows plugins to help make decisions on which actions are allowed by implementing the :ref:`permission_allowed(...) ` plugin hook. + .. _authentication_permissions_canned_queries: Permissions for canned queries diff --git a/docs/internals.rst b/docs/internals.rst index 4b4adc5e..25b2d875 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -116,6 +116,8 @@ This method lets you read plugin configuration values that were set in ``metadat Renders a `Jinja template `__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins. +.. _datasette_permission_allowed: + await .permission_allowed(actor, action, resource_type=None, resource_identifier=None, default=False) ----------------------------------------------------------------------------------------------------- From 86dec9e8fffd6c4efec928ae9b5713748dec7e74 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jun 2020 22:30:36 -0700 Subject: [PATCH 0490/2303] Added permission check to every view, closes #808 --- datasette/app.py | 5 ++ datasette/templates/permissions_debug.html | 2 +- datasette/utils/asgi.py | 4 + datasette/views/base.py | 14 ++++ datasette/views/database.py | 8 ++ datasette/views/index.py | 1 + datasette/views/table.py | 5 ++ docs/authentication.rst | 88 ++++++++++++++++++++++ tests/conftest.py | 38 ++++++++++ tests/fixtures.py | 16 ++++ tests/test_api.py | 2 +- tests/test_auth.py | 1 + tests/test_html.py | 38 ++++++++++ 13 files changed, 220 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1624f6ea..f433a10a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -49,6 +49,7 @@ from .utils import ( ) from .utils.asgi import ( AsgiLifespan, + Forbidden, NotFound, Request, Response, @@ -1003,6 +1004,10 @@ class DatasetteRouter(AsgiRouter): status = 404 info = {} message = exception.args[0] + elif isinstance(exception, Forbidden): + status = 403 + info = {} + message = exception.args[0] elif isinstance(exception, DatasetteError): status = exception.status info = exception.error_dict diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index fb098c5c..dda57dfa 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -47,7 +47,7 @@

      Actor: {{ check.actor|tojson }}

      {% if check.resource_type %} -

      Resource: {{ check.resource_type }}: {{ check.resource_identifier }}

      +

      Resource: {{ check.resource_type }} = {{ check.resource_identifier }}

      {% endif %}
      {% endfor %} diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index ba131dc8..fa78c8df 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -13,6 +13,10 @@ class NotFound(Exception): pass +class Forbidden(Exception): + pass + + class Request: def __init__(self, scope, receive): self.scope = scope diff --git a/datasette/views/base.py b/datasette/views/base.py index 315c96fe..9c2cbbcc 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -29,6 +29,7 @@ from datasette.utils.asgi import ( AsgiWriter, AsgiRouter, AsgiView, + Forbidden, NotFound, Response, ) @@ -63,6 +64,19 @@ class BaseView(AsgiView): response.body = b"" return response + async def check_permission( + self, request, action, resource_type=None, resource_identifier=None + ): + ok = await self.ds.permission_allowed( + request.scope.get("actor"), + action, + resource_type=resource_type, + resource_identifier=resource_identifier, + default=True, + ) + if not ok: + raise Forbidden(action) + def database_url(self, database): db = self.ds.databases[database] base_url = self.ds.config("base_url") diff --git a/datasette/views/database.py b/datasette/views/database.py index 4e9a6da7..eb7c29ca 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -19,6 +19,7 @@ class DatabaseView(DataView): name = "database" async def data(self, request, database, hash, default_labels=False, _size=None): + await self.check_permission(request, "view-database", "database", database) metadata = (self.ds.metadata("databases") or {}).get(database, {}) self.ds.update_with_inherited_metadata(metadata) @@ -89,6 +90,9 @@ class DatabaseDownload(DataView): name = "database_download" async def view_get(self, request, database, hash, correct_hash_present, **kwargs): + await self.check_permission( + request, "view-database-download", "database", database + ) if database not in self.ds.databases: raise DatasetteError("Invalid database", status=404) db = self.ds.databases[database] @@ -128,6 +132,10 @@ class QueryView(DataView): # Respect canned query permissions if canned_query: + await self.check_permission( + request, "view-query", "query", (database, canned_query) + ) + # TODO: fix this to use that permission check if not actor_matches_allow( request.scope.get("actor", None), metadata.get("allow") ): diff --git a/datasette/views/index.py b/datasette/views/index.py index fe88a38c..40c41002 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -22,6 +22,7 @@ class IndexView(BaseView): self.ds = datasette async def get(self, request, as_format): + await self.check_permission(request, "view-index") databases = [] for name, db in self.ds.databases.items(): table_names = await db.table_names() diff --git a/datasette/views/table.py b/datasette/views/table.py index ec1b6c7c..32c7f839 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -267,6 +267,8 @@ class TableView(RowTableShared): if not is_view and not table_exists: raise NotFound("Table not found: {}".format(table)) + await self.check_permission(request, "view-table", "table", (database, table)) + pks = await db.primary_keys(table) table_columns = await db.table_columns(table) @@ -844,6 +846,9 @@ class RowView(RowTableShared): async def data(self, request, database, hash, table, pk_path, default_labels=False): pk_values = urlsafe_components(pk_path) + await self.check_permission( + request, "view-row", "row", tuple([database, table] + list(pk_values)) + ) db = self.ds.databases[database] pks = await db.primary_keys(table) use_rowid = not pks diff --git a/docs/authentication.rst b/docs/authentication.rst index fd70000e..b0473ee8 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -150,3 +150,91 @@ The debug tool at ``/-/permissions`` is only available to the :ref:`authenticate It shows the thirty most recent permission checks that have been carried out by the Datasette instance. This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system. + + +.. _permissions: + +Permissions +=========== + +This section lists all of the permission checks that are carried out by Datasette core, along with their ``resource_type`` and ``resource_identifier`` if those are passed. + +.. _permissions_view_index: + +view-index +---------- + +Actor is allowed to view the index page, e.g. https://latest.datasette.io/ + + +.. _permissions_view_database: + +view-database +------------- + +Actor is allowed to view a database page, e.g. https://latest.datasette.io/fixtures + +``resource_type`` - string + "database" + +``resource_identifier`` - string + The name of the database + +.. _permissions_view_database_download: + +view-database-download +----------------------- + +Actor is allowed to download a database, e.g. https://latest.datasette.io/fixtures.db + +``resource_type`` - string + "database" + +``resource_identifier`` - string + The name of the database + +.. _permissions_view_table: + +view-table +---------- + +Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys + +``resource_type`` - string + "table" - even if this is actually a SQL view + +``resource_identifier`` - tuple: (string, string) + The name of the database, then the name of the table + +.. _permissions_view_row: + +view-row +-------- + +Actor is allowed to view a row page, e.g. https://latest.datasette.io/fixtures/compound_primary_key/a,b + +``resource_type`` - string + "row" + +``resource_identifier`` - tuple: (string, string, strings...) + The name of the database, then the name of the table, then the primary key of the row. The primary key may be a single value or multiple values, so the ``resource_identifier`` tuple may be three or more items long. + +.. _permissions_view_query: + +view-query +---------- + +Actor is allowed to view a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size + +``resource_type`` - string + "query" + +``resource_identifier`` - string + The name of the canned query + +.. _permissions_permissions_debug: + +permissions-debug +----------------- + +Actor is allowed to view the ``/-/permissions`` debug page. diff --git a/tests/conftest.py b/tests/conftest.py index a19ad18d..1921ae3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,15 @@ import os +import pathlib import pytest +import re + +UNDOCUMENTED_PERMISSIONS = { + "this_is_allowed", + "this_is_denied", + "this_is_allowed_async", + "this_is_denied_async", + "no_match", +} def pytest_configure(config): @@ -39,3 +49,31 @@ def restore_working_directory(tmpdir, request): os.chdir(previous_cwd) request.addfinalizer(return_to_previous) + + +@pytest.fixture(scope="session", autouse=True) +def check_permission_actions_are_documented(): + from datasette.plugins import pm + + content = ( + (pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst") + .open() + .read() + ) + permissions_re = re.compile(r"\.\. _permissions_([^\s:]+):") + documented_permission_actions = set(permissions_re.findall(content)).union( + UNDOCUMENTED_PERMISSIONS + ) + + def before(hook_name, hook_impls, kwargs): + if hook_name == "permission_allowed": + action = kwargs.get("action").replace("-", "_") + assert ( + action in documented_permission_actions + ), "Undocumented permission action: {}, resource_type: {}, resource_identifier: {}".format( + action, kwargs["resource_type"], kwargs["resource_identifier"] + ) + + pm.add_hookcall_monitoring( + before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None + ) diff --git a/tests/fixtures.py b/tests/fixtures.py index 75bd6b94..d175dfd5 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -840,3 +840,19 @@ if __name__ == "__main__": sys.argv[0] ) ) + + +def assert_permission_checked( + datasette, action, resource_type=None, resource_identifier=None +): + assert [ + pc + for pc in datasette._permission_checks + if pc["action"] == action + and pc["resource_type"] == resource_type + and pc["resource_identifier"] == resource_identifier + ], """Missing expected permission check: action={}, resource_type={}, resource_identifier={} + Permission checks seen: {} + """.format( + action, resource_type, resource_identifier, datasette._permission_checks + ) diff --git a/tests/test_api.py b/tests/test_api.py index b35c0a2d..555e394a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1721,7 +1721,7 @@ def test_trace(app_client): assert isinstance(trace["traceback"], list) assert isinstance(trace["database"], str) assert isinstance(trace["sql"], str) - assert isinstance(trace["params"], (list, dict)) + assert isinstance(trace["params"], (list, dict, None.__class__)) @pytest.mark.parametrize( diff --git a/tests/test_auth.py b/tests/test_auth.py index ac8d7abe..40dc2587 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -23,6 +23,7 @@ def test_actor_cookie(app_client): def test_permissions_debug(app_client): + app_client.ds._permission_checks.clear() assert 403 == app_client.get("/-/permissions").status # With the cookie it should work cookie = app_client.ds.sign({"id": "root"}, "actor") diff --git a/tests/test_html.py b/tests/test_html.py index 2d2a141a..3569b92c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -4,6 +4,7 @@ from .fixtures import ( # noqa app_client_shorter_time_limit, app_client_two_attached_databases, app_client_with_hash, + assert_permission_checked, make_app_client, METADATA, ) @@ -17,6 +18,7 @@ import urllib.parse def test_homepage(app_client_two_attached_databases): response = app_client_two_attached_databases.get("/") + assert_permission_checked(app_client_two_attached_databases.ds, "view-index") assert response.status == 200 assert "text/html; charset=utf-8" == response.headers["content-type"] soup = Soup(response.body, "html.parser") @@ -75,6 +77,12 @@ def test_static_mounts(): def test_memory_database_page(): for client in make_app_client(memory=True): response = client.get("/:memory:") + assert_permission_checked( + client.ds, + "view-database", + resource_type="database", + resource_identifier=":memory:", + ) assert response.status == 200 @@ -87,6 +95,12 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash): def test_database_page(app_client): response = app_client.get("/fixtures") + assert_permission_checked( + app_client.ds, + "view-database", + resource_type="database", + resource_identifier="fixtures", + ) soup = Soup(response.body, "html.parser") queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") assert queries_ul is not None @@ -197,6 +211,12 @@ def test_row_page_does_not_truncate(): for client in make_app_client(config={"truncate_cells_html": 5}): response = client.get("/fixtures/facetable/1") assert response.status == 200 + assert_permission_checked( + client.ds, + "view-row", + resource_type="row", + resource_identifier=("fixtures", "facetable", "1"), + ) table = Soup(response.body, "html.parser").find("table") assert table["class"] == ["rows-and-columns"] assert ["Mission"] == [ @@ -506,6 +526,12 @@ def test_templates_considered(app_client, path, expected_considered): def test_table_html_simple_primary_key(app_client): response = app_client.get("/fixtures/simple_primary_key?_size=3") + assert_permission_checked( + app_client.ds, + "view-table", + resource_type="table", + resource_identifier=("fixtures", "simple_primary_key"), + ) assert response.status == 200 table = Soup(response.body, "html.parser").find("table") assert table["class"] == ["rows-and-columns"] @@ -896,6 +922,12 @@ def test_database_download_allowed_for_immutable(): assert len(soup.findAll("a", {"href": re.compile(r"\.db$")})) # Check we can actually download it assert 200 == client.get("/fixtures.db").status + assert_permission_checked( + client.ds, + "view-database-download", + resource_type="database", + resource_identifier="fixtures", + ) def test_database_download_disallowed_for_mutable(app_client): @@ -991,6 +1023,12 @@ def test_404_content_type(app_client): def test_canned_query_with_custom_metadata(app_client): response = app_client.get("/fixtures/neighborhood_search?text=town") + assert_permission_checked( + app_client.ds, + "view-query", + resource_type="query", + resource_identifier=("fixtures", "neighborhood_search"), + ) assert response.status == 200 soup = Soup(response.body, "html.parser") assert "Search neighborhoods" == soup.find("h1").text From 4340845754e90fe778a7da8668b4fd9bf6ccc2c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jun 2020 13:03:08 -0700 Subject: [PATCH 0491/2303] Nested permission checks for all views, refs #811 --- datasette/views/database.py | 10 +++++- datasette/views/index.py | 2 +- datasette/views/table.py | 5 +++ docs/authentication.rst | 21 ++++++++--- tests/fixtures.py | 36 +++++++++++-------- tests/test_html.py | 71 ++++++++++++++++++++++--------------- 6 files changed, 97 insertions(+), 48 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index eb7c29ca..4eae9e33 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -19,6 +19,7 @@ class DatabaseView(DataView): name = "database" async def data(self, request, database, hash, default_labels=False, _size=None): + await self.check_permission(request, "view-instance") await self.check_permission(request, "view-database", "database", database) metadata = (self.ds.metadata("databases") or {}).get(database, {}) self.ds.update_with_inherited_metadata(metadata) @@ -90,6 +91,8 @@ class DatabaseDownload(DataView): name = "database_download" async def view_get(self, request, database, hash, correct_hash_present, **kwargs): + await self.check_permission(request, "view-instance") + await self.check_permission(request, "view-database", "database", database) await self.check_permission( request, "view-database-download", "database", database ) @@ -132,6 +135,8 @@ class QueryView(DataView): # Respect canned query permissions if canned_query: + await self.check_permission(request, "view-instance") + await self.check_permission(request, "view-database", "database", database) await self.check_permission( request, "view-query", "query", (database, canned_query) ) @@ -140,7 +145,10 @@ class QueryView(DataView): request.scope.get("actor", None), metadata.get("allow") ): return Response("Permission denied", status=403) - + else: + await self.check_permission(request, "view-instance") + await self.check_permission(request, "view-database", "database", database) + await self.check_permission(request, "execute-query", "database", database) # Extract any :named parameters named_parameters = named_parameters or self.re_named_parameter.findall(sql) named_parameter_values = { diff --git a/datasette/views/index.py b/datasette/views/index.py index 40c41002..5f903474 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -22,7 +22,7 @@ class IndexView(BaseView): self.ds = datasette async def get(self, request, as_format): - await self.check_permission(request, "view-index") + await self.check_permission(request, "view-instance") databases = [] for name, db in self.ds.databases.items(): table_names = await db.table_names() diff --git a/datasette/views/table.py b/datasette/views/table.py index 32c7f839..10d6725a 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -267,6 +267,8 @@ class TableView(RowTableShared): if not is_view and not table_exists: raise NotFound("Table not found: {}".format(table)) + await self.check_permission(request, "view-instance") + await self.check_permission(request, "view-database", "database", database) await self.check_permission(request, "view-table", "table", (database, table)) pks = await db.primary_keys(table) @@ -846,6 +848,9 @@ class RowView(RowTableShared): async def data(self, request, database, hash, table, pk_path, default_labels=False): pk_values = urlsafe_components(pk_path) + await self.check_permission(request, "view-instance") + await self.check_permission(request, "view-database", "database", database) + await self.check_permission(request, "view-table", "table", (database, table)) await self.check_permission( request, "view-row", "row", tuple([database, table] + list(pk_values)) ) diff --git a/docs/authentication.rst b/docs/authentication.rst index b0473ee8..7fa96b35 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -159,12 +159,12 @@ Permissions This section lists all of the permission checks that are carried out by Datasette core, along with their ``resource_type`` and ``resource_identifier`` if those are passed. -.. _permissions_view_index: +.. _permissions_view_instance: -view-index ----------- +view-instance +------------- -Actor is allowed to view the index page, e.g. https://latest.datasette.io/ +Top level permission - Actor is allowed to view any pages within this instance, starting at https://latest.datasette.io/ .. _permissions_view_database: @@ -232,6 +232,19 @@ Actor is allowed to view a :ref:`canned query ` page, e.g. https ``resource_identifier`` - string The name of the canned query +.. _permissions_execute_query: + +execute-query +------------- + +Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures?sql=select+100 + +``resource_type`` - string + "database" + +``resource_identifier`` - string + The name of the database + .. _permissions_permissions_debug: permissions-debug diff --git a/tests/fixtures.py b/tests/fixtures.py index d175dfd5..f767dc84 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -842,17 +842,25 @@ if __name__ == "__main__": ) -def assert_permission_checked( - datasette, action, resource_type=None, resource_identifier=None -): - assert [ - pc - for pc in datasette._permission_checks - if pc["action"] == action - and pc["resource_type"] == resource_type - and pc["resource_identifier"] == resource_identifier - ], """Missing expected permission check: action={}, resource_type={}, resource_identifier={} - Permission checks seen: {} - """.format( - action, resource_type, resource_identifier, datasette._permission_checks - ) +def assert_permissions_checked(datasette, actions): + # actions is a list of "action" or (action, resource_type, resource_identifier) tuples + for action in actions: + if isinstance(action, str): + resource_type = None + resource_identifier = None + else: + action, resource_type, resource_identifier = action + assert [ + pc + for pc in datasette._permission_checks + if pc["action"] == action + and pc["resource_type"] == resource_type + and pc["resource_identifier"] == resource_identifier + ], """Missing expected permission check: action={}, resource_type={}, resource_identifier={} + Permission checks seen: {} + """.format( + action, + resource_type, + resource_identifier, + json.dumps(list(datasette._permission_checks), indent=4), + ) diff --git a/tests/test_html.py b/tests/test_html.py index 3569b92c..b41c1943 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -4,7 +4,7 @@ from .fixtures import ( # noqa app_client_shorter_time_limit, app_client_two_attached_databases, app_client_with_hash, - assert_permission_checked, + assert_permissions_checked, make_app_client, METADATA, ) @@ -18,7 +18,7 @@ import urllib.parse def test_homepage(app_client_two_attached_databases): response = app_client_two_attached_databases.get("/") - assert_permission_checked(app_client_two_attached_databases.ds, "view-index") + assert_permissions_checked(app_client_two_attached_databases.ds, ["view-instance"]) assert response.status == 200 assert "text/html; charset=utf-8" == response.headers["content-type"] soup = Soup(response.body, "html.parser") @@ -77,11 +77,8 @@ def test_static_mounts(): def test_memory_database_page(): for client in make_app_client(memory=True): response = client.get("/:memory:") - assert_permission_checked( - client.ds, - "view-database", - resource_type="database", - resource_identifier=":memory:", + assert_permissions_checked( + client.ds, ["view-instance", ("view-database", "database", ":memory:")] ) assert response.status == 200 @@ -95,11 +92,8 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash): def test_database_page(app_client): response = app_client.get("/fixtures") - assert_permission_checked( - app_client.ds, - "view-database", - resource_type="database", - resource_identifier="fixtures", + assert_permissions_checked( + app_client.ds, ["view-instance", ("view-database", "database", "fixtures")] ) soup = Soup(response.body, "html.parser") queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") @@ -211,11 +205,13 @@ def test_row_page_does_not_truncate(): for client in make_app_client(config={"truncate_cells_html": 5}): response = client.get("/fixtures/facetable/1") assert response.status == 200 - assert_permission_checked( + assert_permissions_checked( client.ds, - "view-row", - resource_type="row", - resource_identifier=("fixtures", "facetable", "1"), + [ + "view-instance", + ("view-table", "table", ("fixtures", "facetable")), + ("view-row", "row", ("fixtures", "facetable", "1")), + ], ) table = Soup(response.body, "html.parser").find("table") assert table["class"] == ["rows-and-columns"] @@ -526,11 +522,13 @@ def test_templates_considered(app_client, path, expected_considered): def test_table_html_simple_primary_key(app_client): response = app_client.get("/fixtures/simple_primary_key?_size=3") - assert_permission_checked( + assert_permissions_checked( app_client.ds, - "view-table", - resource_type="table", - resource_identifier=("fixtures", "simple_primary_key"), + [ + "view-instance", + ("view-database", "database", "fixtures"), + ("view-table", "table", ("fixtures", "simple_primary_key")), + ], ) assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -887,6 +885,19 @@ def test_database_metadata(app_client): assert_footer_links(soup) +def test_database_query_permission_checks(app_client): + response = app_client.get("/fixtures?sql=select+1") + assert response.status == 200 + assert_permissions_checked( + app_client.ds, + [ + "view-instance", + ("view-database", "database", "fixtures"), + ("execute-query", "database", "fixtures"), + ], + ) + + def test_database_metadata_with_custom_sql(app_client): response = app_client.get("/fixtures?sql=select+*+from+simple_primary_key") assert response.status == 200 @@ -922,11 +933,13 @@ def test_database_download_allowed_for_immutable(): assert len(soup.findAll("a", {"href": re.compile(r"\.db$")})) # Check we can actually download it assert 200 == client.get("/fixtures.db").status - assert_permission_checked( + assert_permissions_checked( client.ds, - "view-database-download", - resource_type="database", - resource_identifier="fixtures", + [ + "view-instance", + ("view-database", "database", "fixtures"), + ("view-database-download", "database", "fixtures"), + ], ) @@ -1023,11 +1036,13 @@ def test_404_content_type(app_client): def test_canned_query_with_custom_metadata(app_client): response = app_client.get("/fixtures/neighborhood_search?text=town") - assert_permission_checked( + assert_permissions_checked( app_client.ds, - "view-query", - resource_type="query", - resource_identifier=("fixtures", "neighborhood_search"), + [ + "view-instance", + ("view-database", "database", "fixtures"), + ("view-query", "query", ("fixtures", "neighborhood_search")), + ], ) assert response.status == 200 soup = Soup(response.body, "html.parser") From a1e801453aaeb540d2aea8cccb90b425af737c44 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jun 2020 13:20:59 -0700 Subject: [PATCH 0492/2303] Renamed execute-query permission to execute-sql, refs #811 --- datasette/views/database.py | 13 +++---------- docs/authentication.rst | 4 ++-- tests/test_html.py | 2 +- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 4eae9e33..961ab61e 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -134,21 +134,14 @@ class QueryView(DataView): params.pop("_shape") # Respect canned query permissions + await self.check_permission(request, "view-instance") + await self.check_permission(request, "view-database", "database", database) if canned_query: - await self.check_permission(request, "view-instance") - await self.check_permission(request, "view-database", "database", database) await self.check_permission( request, "view-query", "query", (database, canned_query) ) - # TODO: fix this to use that permission check - if not actor_matches_allow( - request.scope.get("actor", None), metadata.get("allow") - ): - return Response("Permission denied", status=403) else: - await self.check_permission(request, "view-instance") - await self.check_permission(request, "view-database", "database", database) - await self.check_permission(request, "execute-query", "database", database) + await self.check_permission(request, "execute-sql", "database", database) # Extract any :named parameters named_parameters = named_parameters or self.re_named_parameter.findall(sql) named_parameter_values = { diff --git a/docs/authentication.rst b/docs/authentication.rst index 7fa96b35..ee8e7125 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -234,8 +234,8 @@ Actor is allowed to view a :ref:`canned query ` page, e.g. https .. _permissions_execute_query: -execute-query -------------- +execute-sql +----------- Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures?sql=select+100 diff --git a/tests/test_html.py b/tests/test_html.py index b41c1943..ac7432d7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -893,7 +893,7 @@ def test_database_query_permission_checks(app_client): [ "view-instance", ("view-database", "database", "fixtures"), - ("execute-query", "database", "fixtures"), + ("execute-sql", "database", "fixtures"), ], ) From 5ed2853cf3432a0f5a3511df8d2ffe9c6c79a584 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jun 2020 14:01:22 -0700 Subject: [PATCH 0493/2303] Fix permissions documenation test --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index ee8e7125..1bf2a1a5 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -232,7 +232,7 @@ Actor is allowed to view a :ref:`canned query ` page, e.g. https ``resource_identifier`` - string The name of the canned query -.. _permissions_execute_query: +.. _permissions_execute_sql: execute-sql ----------- From abc733912447f284b38ddc389d18ba0a8cef8bcf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jun 2020 14:14:10 -0700 Subject: [PATCH 0494/2303] Nicer pattern for make_app_client() in tests, closes #395 --- tests/fixtures.py | 44 +++++++++++++++++++++++++------------- tests/test_api.py | 10 ++++----- tests/test_canned_write.py | 4 ++-- tests/test_cli.py | 2 +- tests/test_custom_pages.py | 2 +- tests/test_html.py | 28 ++++++++++++------------ tests/test_plugins.py | 8 +++---- 7 files changed, 56 insertions(+), 42 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index f767dc84..2ac73fb1 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -2,6 +2,7 @@ from datasette.app import Datasette from datasette.utils import sqlite3, MultiParams from asgiref.testing import ApplicationCommunicator from asgiref.sync import async_to_sync +import contextlib from http.cookies import SimpleCookie import itertools import json @@ -220,6 +221,7 @@ class TestClient: return response +@contextlib.contextmanager def make_app_client( sql_time_limit_ms=None, max_returned_rows=None, @@ -281,7 +283,8 @@ def make_app_client( @pytest.fixture(scope="session") def app_client(): - yield from make_app_client() + with make_app_client() as client: + yield client @pytest.fixture(scope="session") @@ -294,64 +297,75 @@ def app_client_no_files(): @pytest.fixture(scope="session") def app_client_two_attached_databases(): - yield from make_app_client( + with make_app_client( extra_databases={"extra database.db": EXTRA_DATABASE_SQL} - ) + ) as client: + yield client @pytest.fixture(scope="session") def app_client_conflicting_database_names(): - yield from make_app_client( + with make_app_client( extra_databases={"foo.db": EXTRA_DATABASE_SQL, "foo-bar.db": EXTRA_DATABASE_SQL} - ) + ) as client: + yield client @pytest.fixture(scope="session") def app_client_two_attached_databases_one_immutable(): - yield from make_app_client( + with make_app_client( is_immutable=True, extra_databases={"extra database.db": EXTRA_DATABASE_SQL} - ) + ) as client: + yield client @pytest.fixture(scope="session") def app_client_with_hash(): - yield from make_app_client(config={"hash_urls": True}, is_immutable=True) + with make_app_client(config={"hash_urls": True}, is_immutable=True) as client: + yield client @pytest.fixture(scope="session") def app_client_shorter_time_limit(): - yield from make_app_client(20) + with make_app_client(20) as client: + yield client @pytest.fixture(scope="session") def app_client_returned_rows_matches_page_size(): - yield from make_app_client(max_returned_rows=50) + with make_app_client(max_returned_rows=50) as client: + yield client @pytest.fixture(scope="session") def app_client_larger_cache_size(): - yield from make_app_client(config={"cache_size_kb": 2500}) + with make_app_client(config={"cache_size_kb": 2500}) as client: + yield client @pytest.fixture(scope="session") def app_client_csv_max_mb_one(): - yield from make_app_client(config={"max_csv_mb": 1}) + with make_app_client(config={"max_csv_mb": 1}) as client: + yield client @pytest.fixture(scope="session") def app_client_with_dot(): - yield from make_app_client(filename="fixtures.dot.db") + with make_app_client(filename="fixtures.dot.db") as client: + yield client @pytest.fixture(scope="session") def app_client_with_cors(): - yield from make_app_client(cors=True) + with make_app_client(cors=True) as client: + yield client @pytest.fixture(scope="session") def app_client_immutable_and_inspect_file(): inspect_data = {"fixtures": {"tables": {"sortable": {"count": 100}}}} - yield from make_app_client(is_immutable=True, inspect_data=inspect_data) + with make_app_client(is_immutable=True, inspect_data=inspect_data) as client: + yield client def generate_compound_rows(num): diff --git a/tests/test_api.py b/tests/test_api.py index 555e394a..22378946 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -605,7 +605,7 @@ def test_invalid_custom_sql(app_client): def test_allow_sql_off(): - for client in make_app_client(config={"allow_sql": False}): + with make_app_client(config={"allow_sql": False}) as client: response = client.get("/fixtures.json?sql=select+sleep(0.01)") assert 400 == response.status assert "sql= is not allowed" == response.json["error"] @@ -1107,7 +1107,7 @@ def test_table_filter_extra_where_invalid(app_client): def test_table_filter_extra_where_disabled_if_no_sql_allowed(): - for client in make_app_client(config={"allow_sql": False}): + with make_app_client(config={"allow_sql": False}) as client: response = client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'") assert 400 == response.status assert "_where= is not allowed" == response.json["error"] @@ -1528,14 +1528,14 @@ def test_suggested_facets(app_client): def test_allow_facet_off(): - for client in make_app_client(config={"allow_facet": False}): + with make_app_client(config={"allow_facet": False}) as client: assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status # Should not suggest any facets either: assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] def test_suggest_facets_off(): - for client in make_app_client(config={"suggest_facets": False}): + with make_app_client(config={"suggest_facets": False}) as client: # Now suggested_facets should be [] assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] @@ -1667,7 +1667,7 @@ def test_config_cache_size(app_client_larger_cache_size): def test_config_force_https_urls(): - for client in make_app_client(config={"force_https_urls": True}): + with make_app_client(config={"force_https_urls": True}) as client: response = client.get("/fixtures/facetable.json?_size=3&_facet=state") assert response.json["next_url"].startswith("https://") assert response.json["facet_results"]["state"]["results"][0][ diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py index 73b01e51..c217be8f 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_write.py @@ -4,7 +4,7 @@ from .fixtures import make_app_client @pytest.fixture def canned_write_client(): - for client in make_app_client( + with make_app_client( extra_databases={"data.db": "create table names (name text)"}, metadata={ "databases": { @@ -35,7 +35,7 @@ def canned_write_client(): } } }, - ): + ) as client: yield client diff --git a/tests/test_cli.py b/tests/test_cli.py index 2616f1d1..6939fe57 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,7 +41,7 @@ def test_inspect_cli_writes_to_file(app_client): def test_serve_with_inspect_file_prepopulates_table_counts_cache(): inspect_data = {"fixtures": {"tables": {"hithere": {"count": 44}}}} - for client in make_app_client(inspect_data=inspect_data, is_immutable=True): + with make_app_client(inspect_data=inspect_data, is_immutable=True) as client: assert inspect_data == client.ds.inspect_data db = client.ds.databases["fixtures"] assert {"hithere": 44} == db.cached_table_counts diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index c69facb5..4e4b2a67 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -27,7 +27,7 @@ def custom_pages_client(tmp_path_factory): nested_dir = pages_dir / "nested" nested_dir.mkdir() (nested_dir / "nest.html").write_text("Nest!", "utf-8") - for client in make_app_client(template_dir=str(template_dir)): + with make_app_client(template_dir=str(template_dir)) as client: yield client diff --git a/tests/test_html.py b/tests/test_html.py index ac7432d7..4e913bcf 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -63,9 +63,9 @@ def test_static(app_client): def test_static_mounts(): - for client in make_app_client( + with make_app_client( static_mounts=[("custom-static", str(pathlib.Path(__file__).parent))] - ): + ) as client: response = client.get("/custom-static/test_html.py") assert response.status == 200 response = client.get("/custom-static/not_exists.py") @@ -75,7 +75,7 @@ def test_static_mounts(): def test_memory_database_page(): - for client in make_app_client(memory=True): + with make_app_client(memory=True) as client: response = client.get("/:memory:") assert_permissions_checked( client.ds, ["view-instance", ("view-database", "database", ":memory:")] @@ -177,7 +177,7 @@ def test_definition_sql(path, expected_definition_sql, app_client): def test_table_cell_truncation(): - for client in make_app_client(config={"truncate_cells_html": 5}): + with make_app_client(config={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -202,7 +202,7 @@ def test_table_cell_truncation(): def test_row_page_does_not_truncate(): - for client in make_app_client(config={"truncate_cells_html": 5}): + with make_app_client(config={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable/1") assert response.status == 200 assert_permissions_checked( @@ -925,7 +925,7 @@ def test_table_metadata(app_client): def test_database_download_allowed_for_immutable(): - for client in make_app_client(is_immutable=True): + with make_app_client(is_immutable=True) as client: assert not client.ds.databases["fixtures"].is_mutable # Regular page should have a download link response = client.get("/fixtures") @@ -951,7 +951,7 @@ def test_database_download_disallowed_for_mutable(app_client): def test_database_download_disallowed_for_memory(): - for client in make_app_client(memory=True): + with make_app_client(memory=True) as client: # Memory page should NOT have a download link response = client.get("/:memory:") soup = Soup(response.body, "html.parser") @@ -960,7 +960,7 @@ def test_database_download_disallowed_for_memory(): def test_allow_download_off(): - for client in make_app_client(is_immutable=True, config={"allow_download": False}): + with make_app_client(is_immutable=True, config={"allow_download": False}) as client: response = client.get("/fixtures") soup = Soup(response.body, "html.parser") assert not len(soup.findAll("a", {"href": re.compile(r"\.db$")})) @@ -978,7 +978,7 @@ def test_allow_sql_on(app_client): def test_allow_sql_off(): - for client in make_app_client(config={"allow_sql": False}): + with make_app_client(config={"allow_sql": False}) as client: response = client.get("/fixtures") soup = Soup(response.body, "html.parser") assert not len(soup.findAll("textarea", {"name": "sql"})) @@ -1170,9 +1170,9 @@ def test_metadata_json_html(app_client): def test_custom_table_include(): - for client in make_app_client( + with make_app_client( template_dir=str(pathlib.Path(__file__).parent / "test_templates") - ): + ) as client: response = client.get("/fixtures/complex_foreign_keys") assert response.status == 200 assert ( @@ -1197,7 +1197,7 @@ def test_zero_results(app_client, path): def test_config_template_debug_on(): - for client in make_app_client(config={"template_debug": True}): + with make_app_client(config={"template_debug": True}) as client: response = client.get("/fixtures/facetable?_context=1") assert response.status == 200 assert response.text.startswith("
      {")
@@ -1211,7 +1211,7 @@ def test_config_template_debug_off(app_client):
 
 def test_debug_context_includes_extra_template_vars():
     # https://github.com/simonw/datasette/issues/693
-    for client in make_app_client(config={"template_debug": True}):
+    with make_app_client(config={"template_debug": True}) as client:
         response = client.get("/fixtures/facetable?_context=1")
         # scope_path is added by PLUGIN1
         assert "scope_path" in response.text
@@ -1292,7 +1292,7 @@ def test_metadata_sort_desc(app_client):
     ],
 )
 def test_base_url_config(base_url, path):
-    for client in make_app_client(config={"base_url": base_url}):
+    with make_app_client(config={"base_url": base_url}) as client:
         response = client.get(base_url + path.lstrip("/"))
         soup = Soup(response.body, "html.parser")
         for el in soup.findAll(["a", "link", "script"]):
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index f69e7fa7..c782b87b 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -229,9 +229,9 @@ def test_plugins_asgi_wrapper(app_client):
 
 
 def test_plugins_extra_template_vars(restore_working_directory):
-    for client in make_app_client(
+    with make_app_client(
         template_dir=str(pathlib.Path(__file__).parent / "test_templates")
-    ):
+    ) as client:
         response = client.get("/-/metadata")
         assert response.status == 200
         extra_template_vars = json.loads(
@@ -254,9 +254,9 @@ def test_plugins_extra_template_vars(restore_working_directory):
 
 
 def test_plugins_async_template_function(restore_working_directory):
-    for client in make_app_client(
+    with make_app_client(
         template_dir=str(pathlib.Path(__file__).parent / "test_templates")
-    ):
+    ) as client:
         response = client.get("/-/metadata")
         assert response.status == 200
         extra_from_awaitable_function = (

From ece0ba6f4bc152af6f605fc5f536ffa46af95274 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 14:23:16 -0700
Subject: [PATCH 0495/2303] Test + default impl for view-query permission, refs
 #811

---
 datasette/default_permissions.py | 21 ++++++++++++++++++---
 tests/test_permissions.py        | 22 ++++++++++++++++++++++
 2 files changed, 40 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_permissions.py

diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index 0b0d17f9..40ae54ab 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -1,7 +1,22 @@
 from datasette import hookimpl
+from datasette.utils import actor_matches_allow
 
 
 @hookimpl
-def permission_allowed(actor, action, resource_type, resource_identifier):
-    if actor and actor.get("id") == "root" and action == "permissions-debug":
-        return True
+def permission_allowed(datasette, actor, action, resource_type, resource_identifier):
+    if action == "permissions-debug":
+        if actor and actor.get("id") == "root":
+            return True
+    elif action == "view-query":
+        # Check if this query has a "allow" block in metadata
+        assert resource_type == "query"
+        database, query_name = resource_identifier
+        queries_metadata = datasette.metadata("queries", database=database)
+        assert query_name in queries_metadata
+        if isinstance(queries_metadata[query_name], str):
+            return True
+        allow = queries_metadata[query_name].get("allow")
+        print("checking allow - actor = {}, allow = {}".format(actor, allow))
+        if allow is None:
+            return True
+        return actor_matches_allow(actor, allow)
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
new file mode 100644
index 00000000..c90fdf7a
--- /dev/null
+++ b/tests/test_permissions.py
@@ -0,0 +1,22 @@
+from .fixtures import make_app_client
+import pytest
+
+
+@pytest.mark.parametrize(
+    "allow,expected_anon,expected_auth",
+    [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
+)
+def test_execute_sql(allow, expected_anon, expected_auth):
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {"queries": {"q": {"sql": "select 1 + 1", "allow": allow}}}
+            }
+        }
+    ) as client:
+        anon_response = client.get("/fixtures/q")
+        assert expected_anon == anon_response.status
+        auth_response = client.get(
+            "/fixtures/q", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+        )
+        assert expected_auth == auth_response.status

From 8571ce388a23dd98adbdc1b7eff6c6eef5a9d1af Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 14:30:39 -0700
Subject: [PATCH 0496/2303] Implemented view-instance permission, refs #811

---
 datasette/default_permissions.py |  4 ++++
 tests/test_permissions.py        | 20 ++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index 40ae54ab..ee182c85 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -7,6 +7,10 @@ def permission_allowed(datasette, actor, action, resource_type, resource_identif
     if action == "permissions-debug":
         if actor and actor.get("id") == "root":
             return True
+    elif action == "view-instance":
+        allow = datasette.metadata("allow")
+        if allow is not None:
+            return actor_matches_allow(actor, allow)
     elif action == "view-query":
         # Check if this query has a "allow" block in metadata
         assert resource_type == "query"
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index c90fdf7a..b5c2e00c 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -20,3 +20,23 @@ def test_execute_sql(allow, expected_anon, expected_auth):
             "/fixtures/q", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
         )
         assert expected_auth == auth_response.status
+
+
+@pytest.mark.parametrize(
+    "allow,expected_anon,expected_auth",
+    [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
+)
+def test_view_instance(allow, expected_anon, expected_auth):
+    with make_app_client(metadata={"allow": allow}) as client:
+        for path in (
+            "/",
+            "/fixtures",
+            "/fixtures/compound_three_primary_keys",
+            "/fixtures/compound_three_primary_keys/a,a,a",
+        ):
+            anon_response = client.get(path)
+            assert expected_anon == anon_response.status
+            auth_response = client.get(
+                path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+            )
+            assert expected_auth == auth_response.status

From cd92e4fe2a47039a8c780e4e7183a0d2e7446884 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 14:33:52 -0700
Subject: [PATCH 0497/2303] Fixed test name, this executes view-query, not
 execute-sql - refs #811

---
 tests/test_permissions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index b5c2e00c..bf66bc9c 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -6,7 +6,7 @@ import pytest
     "allow,expected_anon,expected_auth",
     [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
 )
-def test_execute_sql(allow, expected_anon, expected_auth):
+def test_view_query(allow, expected_anon, expected_auth):
     with make_app_client(
         metadata={
             "databases": {

From 613fa551a1be31645deb0ece4b46638c181827e0 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 20:14:27 -0700
Subject: [PATCH 0498/2303] Removed view-row permission, for the moment - refs
 #811

https://github.com/simonw/datasette/issues/811#issuecomment-640338347
---
 datasette/views/table.py |  3 ---
 docs/authentication.rst  | 13 -------------
 tests/test_html.py       |  1 -
 3 files changed, 17 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 10d6725a..935fed3d 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -851,9 +851,6 @@ class RowView(RowTableShared):
         await self.check_permission(request, "view-instance")
         await self.check_permission(request, "view-database", "database", database)
         await self.check_permission(request, "view-table", "table", (database, table))
-        await self.check_permission(
-            request, "view-row", "row", tuple([database, table] + list(pk_values))
-        )
         db = self.ds.databases[database]
         pks = await db.primary_keys(table)
         use_rowid = not pks
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 1bf2a1a5..2caca66f 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -206,19 +206,6 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i
 ``resource_identifier`` - tuple: (string, string)
     The name of the database, then the name of the table
 
-.. _permissions_view_row:
-
-view-row
---------
-
-Actor is allowed to view a row page, e.g. https://latest.datasette.io/fixtures/compound_primary_key/a,b
-
-``resource_type`` - string
-    "row"
-
-``resource_identifier`` - tuple: (string, string, strings...)
-    The name of the database, then the name of the table, then the primary key of the row. The primary key may be a single value or multiple values, so the ``resource_identifier`` tuple may be three or more items long.
-
 .. _permissions_view_query:
 
 view-query
diff --git a/tests/test_html.py b/tests/test_html.py
index 4e913bcf..e05640d7 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -210,7 +210,6 @@ def test_row_page_does_not_truncate():
             [
                 "view-instance",
                 ("view-table", "table", ("fixtures", "facetable")),
-                ("view-row", "row", ("fixtures", "facetable", "1")),
             ],
         )
         table = Soup(response.body, "html.parser").find("table")

From 9b42e1a4f5902fb7d6ad0111189900e2656ffda3 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 20:50:37 -0700
Subject: [PATCH 0499/2303] view-database permission
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Also now using 🔒 to indicate private resources - resources that
would not be available to the anonymous user. Refs #811
---
 datasette/default_permissions.py  |  7 +++++-
 datasette/templates/database.html |  2 +-
 datasette/templates/index.html    |  2 +-
 datasette/views/database.py       |  3 +--
 datasette/views/index.py          | 19 +++++++++++++++-
 tests/test_canned_write.py        | 11 +++++-----
 tests/test_html.py                |  5 +----
 tests/test_permissions.py         | 36 +++++++++++++++++++++++++++++++
 8 files changed, 69 insertions(+), 16 deletions(-)

diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index ee182c85..40be8d34 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -11,6 +11,12 @@ def permission_allowed(datasette, actor, action, resource_type, resource_identif
         allow = datasette.metadata("allow")
         if allow is not None:
             return actor_matches_allow(actor, allow)
+    elif action == "view-database":
+        assert resource_type == "database"
+        database_allow = datasette.metadata("allow", database=resource_identifier)
+        if database_allow is None:
+            return True
+        return actor_matches_allow(actor, database_allow)
     elif action == "view-query":
         # Check if this query has a "allow" block in metadata
         assert resource_type == "query"
@@ -20,7 +26,6 @@ def permission_allowed(datasette, actor, action, resource_type, resource_identif
         if isinstance(queries_metadata[query_name], str):
             return True
         allow = queries_metadata[query_name].get("allow")
-        print("checking allow - actor = {}, allow = {}".format(actor, allow))
         if allow is None:
             return True
         return actor_matches_allow(actor, allow)
diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index fc88003c..eaebfdf7 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -60,7 +60,7 @@
     
      Queries
      
     
 {% endif %}
diff --git a/datasette/templates/index.html b/datasette/templates/index.html
index b394564a..3b8568b3 100644
--- a/datasette/templates/index.html
+++ b/datasette/templates/index.html
@@ -10,7 +10,7 @@
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
 {% for database in databases %}
-    
      {{ database.name }}
      
+    
      {{ database.name }}{% if database.private %} 🔒{% endif %}
      
     
      
         {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%}
         {% if database.hidden_tables_count -%}
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 961ab61e..4804b2a9 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -58,8 +58,7 @@ class DatabaseView(DataView):
         tables.sort(key=lambda t: (t["hidden"], t["name"]))
         canned_queries = [
             dict(
-                query,
-                requires_auth=not actor_matches_allow(None, query.get("allow", None)),
+                query, private=not actor_matches_allow(None, query.get("allow", None)),
             )
             for query in self.ds.get_canned_queries(database)
             if actor_matches_allow(
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 5f903474..7b88028b 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -2,7 +2,7 @@ import hashlib
 import json
 
 from datasette.utils import CustomJSONEncoder
-from datasette.utils.asgi import Response
+from datasette.utils.asgi import Response, Forbidden
 from datasette.version import __version__
 
 from .base import BaseView
@@ -25,6 +25,22 @@ class IndexView(BaseView):
         await self.check_permission(request, "view-instance")
         databases = []
         for name, db in self.ds.databases.items():
+            # Check permission
+            allowed = await self.ds.permission_allowed(
+                request.scope.get("actor"),
+                "view-database",
+                resource_type="database",
+                resource_identifier=name,
+                default=True,
+            )
+            if not allowed:
+                continue
+            private = not await self.ds.permission_allowed(
+                None,
+                "view-database",
+                resource_type="database",
+                resource_identifier=name,
+            )
             table_names = await db.table_names()
             hidden_table_names = set(await db.hidden_table_names())
             views = await db.view_names()
@@ -95,6 +111,7 @@ class IndexView(BaseView):
                     ),
                     "hidden_tables_count": len(hidden_tables),
                     "views_count": len(views),
+                    "private": private,
                 }
             )
 
diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py
index c217be8f..dc3fba3f 100644
--- a/tests/test_canned_write.py
+++ b/tests/test_canned_write.py
@@ -120,13 +120,12 @@ def test_canned_query_permissions_on_database_page(canned_write_client):
     )
     assert 200 == response.status
     assert [
-        {"name": "add_name", "requires_auth": False},
-        {"name": "add_name_specify_id", "requires_auth": False},
-        {"name": "delete_name", "requires_auth": True},
-        {"name": "update_name", "requires_auth": False},
+        {"name": "add_name", "private": False},
+        {"name": "add_name_specify_id", "private": False},
+        {"name": "delete_name", "private": True},
+        {"name": "update_name", "private": False},
     ] == [
-        {"name": q["name"], "requires_auth": q["requires_auth"]}
-        for q in response.json["queries"]
+        {"name": q["name"], "private": q["private"]} for q in response.json["queries"]
     ]
 
 
diff --git a/tests/test_html.py b/tests/test_html.py
index e05640d7..3f6dc4df 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -207,10 +207,7 @@ def test_row_page_does_not_truncate():
         assert response.status == 200
         assert_permissions_checked(
             client.ds,
-            [
-                "view-instance",
-                ("view-table", "table", ("fixtures", "facetable")),
-            ],
+            ["view-instance", ("view-table", "table", ("fixtures", "facetable")),],
         )
         table = Soup(response.body, "html.parser").find("table")
         assert table["class"] == ["rows-and-columns"]
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index bf66bc9c..21014a25 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -40,3 +40,39 @@ def test_view_instance(allow, expected_anon, expected_auth):
                 path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
             )
             assert expected_auth == auth_response.status
+
+
+@pytest.mark.parametrize(
+    "allow,expected_anon,expected_auth",
+    [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
+)
+def test_view_database(allow, expected_anon, expected_auth):
+    with make_app_client(
+        metadata={"databases": {"fixtures": {"allow": allow}}}
+    ) as client:
+        for path in (
+            "/fixtures",
+            "/fixtures/compound_three_primary_keys",
+            "/fixtures/compound_three_primary_keys/a,a,a",
+        ):
+            anon_response = client.get(path)
+            assert expected_anon == anon_response.status
+            auth_response = client.get(
+                path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+            )
+            assert expected_auth == auth_response.status
+
+
+def test_database_list_respects_view_database():
+    with make_app_client(
+        metadata={"databases": {"fixtures": {"allow": {"id": "root"}}}},
+        extra_databases={"data.db": "create table names (name text)"},
+    ) as client:
+        anon_response = client.get("/")
+        assert 'data' in anon_response.text
+        assert 'fixtures' not in anon_response.text
+        auth_response = client.get(
+            "/", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+        )
+        assert 'data' in auth_response.text
+        assert 'fixtures 🔒' in auth_response.text

From b26292a4582ea7fe16c59d0ac99f3bd8c3d4b1d0 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 20:56:49 -0700
Subject: [PATCH 0500/2303] Test that view-query is respected by query list,
 refs #811

---
 datasette/templates/database.html |  2 +-
 tests/test_permissions.py         | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index eaebfdf7..dfafc049 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -60,7 +60,7 @@
     
      Queries
      
     
 {% endif %}
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 21014a25..e66b9291 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -22,6 +22,26 @@ def test_view_query(allow, expected_anon, expected_auth):
         assert expected_auth == auth_response.status
 
 
+def test_query_list_respects_view_query():
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {
+                    "queries": {"q": {"sql": "select 1 + 1", "allow": {"id": "root"}}}
+                }
+            }
+        }
+    ) as client:
+        html_fragment = '
    • q 🔒
      • '
+        anon_response = client.get("/fixtures")
+        assert html_fragment not in anon_response.text
+        assert '"/fixtures/q"' not in anon_response.text
+        auth_response = client.get(
+            "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+        )
+        assert html_fragment in auth_response.text
+
+
 @pytest.mark.parametrize(
     "allow,expected_anon,expected_auth",
     [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],

From 9397d718345c4b35d2a5c55bfcbd1468876b5ab9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 7 Jun 2020 21:47:22 -0700
Subject: [PATCH 0501/2303] Implemented view-table, refs #811

---
 datasette/default_permissions.py  |   8 ++
 datasette/templates/database.html |   2 +-
 datasette/views/database.py       |  16 ++++
 tests/test_permissions.py         | 123 ++++++++++++++++++++----------
 4 files changed, 108 insertions(+), 41 deletions(-)

diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index 40be8d34..dd1770a3 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -17,6 +17,14 @@ def permission_allowed(datasette, actor, action, resource_type, resource_identif
         if database_allow is None:
             return True
         return actor_matches_allow(actor, database_allow)
+    elif action == "view-table":
+        assert resource_type == "table"
+        database, table = resource_identifier
+        tables = datasette.metadata("tables", database=database) or {}
+        table_allow = (tables.get(table) or {}).get("allow")
+        if table_allow is None:
+            return True
+        return actor_matches_allow(actor, table_allow)
     elif action == "view-query":
         # Check if this query has a "allow" block in metadata
         assert resource_type == "query"
diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index dfafc049..1187267d 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -36,7 +36,7 @@
 {% for table in tables %}
 {% if show_hidden or not table.hidden %}
 
      
-    
      {{ table.name }}{% if table.hidden %} (hidden){% endif %}
      
+    
      {{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}
      
     
      {% for column in table.columns[:9] %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}{% if table.columns|length > 9 %}...{% endif %}
      
     
      {% if table.count is none %}Many rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}
      
 
      
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 4804b2a9..ba3d22d9 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -42,6 +42,21 @@ class DatabaseView(DataView):
 
         tables = []
         for table in table_counts:
+            allowed = await self.ds.permission_allowed(
+                request.scope.get("actor"),
+                "view-table",
+                resource_type="table",
+                resource_identifier=(database, table),
+                default=True,
+            )
+            if not allowed:
+                continue
+            private = not await self.ds.permission_allowed(
+                None,
+                "view-table",
+                resource_type="table",
+                resource_identifier=(database, table),
+            )
             table_columns = await db.table_columns(table)
             tables.append(
                 {
@@ -52,6 +67,7 @@ class DatabaseView(DataView):
                     "hidden": table in hidden_table_names,
                     "fts_table": await db.fts_table(table),
                     "foreign_keys": all_foreign_keys[table],
+                    "private": private,
                 }
             )
 
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index e66b9291..7c5b02c0 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -2,46 +2,6 @@ from .fixtures import make_app_client
 import pytest
 
 
-@pytest.mark.parametrize(
-    "allow,expected_anon,expected_auth",
-    [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
-)
-def test_view_query(allow, expected_anon, expected_auth):
-    with make_app_client(
-        metadata={
-            "databases": {
-                "fixtures": {"queries": {"q": {"sql": "select 1 + 1", "allow": allow}}}
-            }
-        }
-    ) as client:
-        anon_response = client.get("/fixtures/q")
-        assert expected_anon == anon_response.status
-        auth_response = client.get(
-            "/fixtures/q", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
-        )
-        assert expected_auth == auth_response.status
-
-
-def test_query_list_respects_view_query():
-    with make_app_client(
-        metadata={
-            "databases": {
-                "fixtures": {
-                    "queries": {"q": {"sql": "select 1 + 1", "allow": {"id": "root"}}}
-                }
-            }
-        }
-    ) as client:
-        html_fragment = '
    • q 🔒
      • '
-        anon_response = client.get("/fixtures")
-        assert html_fragment not in anon_response.text
-        assert '"/fixtures/q"' not in anon_response.text
-        auth_response = client.get(
-            "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
-        )
-        assert html_fragment in auth_response.text
-
-
 @pytest.mark.parametrize(
     "allow,expected_anon,expected_auth",
     [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
@@ -96,3 +56,86 @@ def test_database_list_respects_view_database():
         )
         assert 'data' in auth_response.text
         assert 'fixtures 🔒' in auth_response.text
+
+
+@pytest.mark.parametrize(
+    "allow,expected_anon,expected_auth",
+    [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
+)
+def test_view_table(allow, expected_anon, expected_auth):
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {
+                    "tables": {"compound_three_primary_keys": {"allow": allow}}
+                }
+            }
+        }
+    ) as client:
+        anon_response = client.get("/fixtures/compound_three_primary_keys")
+        assert expected_anon == anon_response.status
+        auth_response = client.get(
+            "/fixtures/compound_three_primary_keys",
+            cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+        )
+        assert expected_auth == auth_response.status
+
+
+def test_table_list_respects_view_table():
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {
+                    "tables": {"compound_three_primary_keys": {"allow": {"id": "root"}}}
+                }
+            }
+        }
+    ) as client:
+        html_fragment = 'compound_three_primary_keys 🔒'
+        anon_response = client.get("/fixtures")
+        assert html_fragment not in anon_response.text
+        assert '"/fixtures/compound_three_primary_keys"' not in anon_response.text
+        auth_response = client.get(
+            "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+        )
+        assert html_fragment in auth_response.text
+
+
+@pytest.mark.parametrize(
+    "allow,expected_anon,expected_auth",
+    [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],
+)
+def test_view_query(allow, expected_anon, expected_auth):
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {"queries": {"q": {"sql": "select 1 + 1", "allow": allow}}}
+            }
+        }
+    ) as client:
+        anon_response = client.get("/fixtures/q")
+        assert expected_anon == anon_response.status
+        auth_response = client.get(
+            "/fixtures/q", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+        )
+        assert expected_auth == auth_response.status
+
+
+def test_query_list_respects_view_query():
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {
+                    "queries": {"q": {"sql": "select 1 + 1", "allow": {"id": "root"}}}
+                }
+            }
+        }
+    ) as client:
+        html_fragment = '
    • q 🔒
      • '
+        anon_response = client.get("/fixtures")
+        assert html_fragment not in anon_response.text
+        assert '"/fixtures/q"' not in anon_response.text
+        auth_response = client.get(
+            "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+        )
+        assert html_fragment in auth_response.text

From e18f8c3f871fe1e9e00554b5c6c75409cc1a5e6d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 06:49:55 -0700
Subject: [PATCH 0502/2303] New check_visibility() utility function, refs #811

---
 datasette/utils/__init__.py | 23 +++++++++++++++++++++++
 datasette/views/database.py | 35 ++++++++++++++++-------------------
 datasette/views/index.py    | 19 ++++---------------
 3 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 077728f4..3d964049 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -874,3 +874,26 @@ def actor_matches_allow(actor, allow):
         if actor_values.intersection(values):
             return True
     return False
+
+
+async def check_visibility(
+    datasette, actor, action, resource_type, resource_identifier, default=True
+):
+    "Returns (visible, private) - visible = can you see it, private = can others see it too"
+    visible = await datasette.permission_allowed(
+        actor,
+        action,
+        resource_type=resource_type,
+        resource_identifier=resource_identifier,
+        default=default,
+    )
+    if not visible:
+        return (False, False)
+    private = not await datasette.permission_allowed(
+        None,
+        action,
+        resource_type=resource_type,
+        resource_identifier=resource_identifier,
+        default=default,
+    )
+    return visible, private
diff --git a/datasette/views/database.py b/datasette/views/database.py
index ba3d22d9..afbb6b05 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -3,6 +3,7 @@ import jinja2
 
 from datasette.utils import (
     actor_matches_allow,
+    check_visibility,
     to_css_class,
     validate_sql_select,
     is_url,
@@ -42,21 +43,15 @@ class DatabaseView(DataView):
 
         tables = []
         for table in table_counts:
-            allowed = await self.ds.permission_allowed(
+            visible, private = await check_visibility(
+                self.ds,
                 request.scope.get("actor"),
                 "view-table",
-                resource_type="table",
-                resource_identifier=(database, table),
-                default=True,
+                "table",
+                (database, table),
             )
-            if not allowed:
+            if not visible:
                 continue
-            private = not await self.ds.permission_allowed(
-                None,
-                "view-table",
-                resource_type="table",
-                resource_identifier=(database, table),
-            )
             table_columns = await db.table_columns(table)
             tables.append(
                 {
@@ -72,15 +67,17 @@ class DatabaseView(DataView):
             )
 
         tables.sort(key=lambda t: (t["hidden"], t["name"]))
-        canned_queries = [
-            dict(
-                query, private=not actor_matches_allow(None, query.get("allow", None)),
+        canned_queries = []
+        for query in self.ds.get_canned_queries(database):
+            visible, private = await check_visibility(
+                self.ds,
+                request.scope.get("actor"),
+                "view-query",
+                "query",
+                (database, query["name"]),
             )
-            for query in self.ds.get_canned_queries(database)
-            if actor_matches_allow(
-                request.scope.get("actor", None), query.get("allow", None)
-            )
-        ]
+            if visible:
+                canned_queries.append(dict(query, private=private))
         return (
             {
                 "database": database,
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 7b88028b..0f7fb613 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -1,7 +1,7 @@
 import hashlib
 import json
 
-from datasette.utils import CustomJSONEncoder
+from datasette.utils import check_visibility, CustomJSONEncoder
 from datasette.utils.asgi import Response, Forbidden
 from datasette.version import __version__
 
@@ -25,22 +25,11 @@ class IndexView(BaseView):
         await self.check_permission(request, "view-instance")
         databases = []
         for name, db in self.ds.databases.items():
-            # Check permission
-            allowed = await self.ds.permission_allowed(
-                request.scope.get("actor"),
-                "view-database",
-                resource_type="database",
-                resource_identifier=name,
-                default=True,
+            visible, private = await check_visibility(
+                self.ds, request.scope.get("actor"), "view-database", "database", name,
             )
-            if not allowed:
+            if not visible:
                 continue
-            private = not await self.ds.permission_allowed(
-                None,
-                "view-database",
-                resource_type="database",
-                resource_identifier=name,
-            )
             table_names = await db.table_names()
             hidden_table_names = set(await db.hidden_table_names())
             views = await db.view_names()

From cc218fa9be55842656d030545c308392e3736053 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 07:02:31 -0700
Subject: [PATCH 0503/2303] Move assert_permissions_checked() calls from
 test_html.py to test_permissions.py, refs #811

---
 datasette/app.py          |  2 +-
 tests/test_html.py        | 49 ------------------------------------
 tests/test_permissions.py | 52 ++++++++++++++++++++++++++++++++++++++-
 3 files changed, 52 insertions(+), 51 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index f433a10a..23c293c9 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -298,7 +298,7 @@ class Datasette:
         pm.hook.prepare_jinja2_environment(env=self.jinja_env)
 
         self._register_renderers()
-        self._permission_checks = collections.deque(maxlen=30)
+        self._permission_checks = collections.deque(maxlen=200)
         self._root_token = os.urandom(32).hex()
 
     def sign(self, value, namespace="default"):
diff --git a/tests/test_html.py b/tests/test_html.py
index 3f6dc4df..cb0e0c90 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -4,7 +4,6 @@ from .fixtures import (  # noqa
     app_client_shorter_time_limit,
     app_client_two_attached_databases,
     app_client_with_hash,
-    assert_permissions_checked,
     make_app_client,
     METADATA,
 )
@@ -18,7 +17,6 @@ import urllib.parse
 
 def test_homepage(app_client_two_attached_databases):
     response = app_client_two_attached_databases.get("/")
-    assert_permissions_checked(app_client_two_attached_databases.ds, ["view-instance"])
     assert response.status == 200
     assert "text/html; charset=utf-8" == response.headers["content-type"]
     soup = Soup(response.body, "html.parser")
@@ -77,9 +75,6 @@ def test_static_mounts():
 def test_memory_database_page():
     with make_app_client(memory=True) as client:
         response = client.get("/:memory:")
-        assert_permissions_checked(
-            client.ds, ["view-instance", ("view-database", "database", ":memory:")]
-        )
         assert response.status == 200
 
 
@@ -92,9 +87,6 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash):
 
 def test_database_page(app_client):
     response = app_client.get("/fixtures")
-    assert_permissions_checked(
-        app_client.ds, ["view-instance", ("view-database", "database", "fixtures")]
-    )
     soup = Soup(response.body, "html.parser")
     queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul")
     assert queries_ul is not None
@@ -205,10 +197,6 @@ def test_row_page_does_not_truncate():
     with make_app_client(config={"truncate_cells_html": 5}) as client:
         response = client.get("/fixtures/facetable/1")
         assert response.status == 200
-        assert_permissions_checked(
-            client.ds,
-            ["view-instance", ("view-table", "table", ("fixtures", "facetable")),],
-        )
         table = Soup(response.body, "html.parser").find("table")
         assert table["class"] == ["rows-and-columns"]
         assert ["Mission"] == [
@@ -518,14 +506,6 @@ def test_templates_considered(app_client, path, expected_considered):
 
 def test_table_html_simple_primary_key(app_client):
     response = app_client.get("/fixtures/simple_primary_key?_size=3")
-    assert_permissions_checked(
-        app_client.ds,
-        [
-            "view-instance",
-            ("view-database", "database", "fixtures"),
-            ("view-table", "table", ("fixtures", "simple_primary_key")),
-        ],
-    )
     assert response.status == 200
     table = Soup(response.body, "html.parser").find("table")
     assert table["class"] == ["rows-and-columns"]
@@ -881,19 +861,6 @@ def test_database_metadata(app_client):
     assert_footer_links(soup)
 
 
-def test_database_query_permission_checks(app_client):
-    response = app_client.get("/fixtures?sql=select+1")
-    assert response.status == 200
-    assert_permissions_checked(
-        app_client.ds,
-        [
-            "view-instance",
-            ("view-database", "database", "fixtures"),
-            ("execute-sql", "database", "fixtures"),
-        ],
-    )
-
-
 def test_database_metadata_with_custom_sql(app_client):
     response = app_client.get("/fixtures?sql=select+*+from+simple_primary_key")
     assert response.status == 200
@@ -929,14 +896,6 @@ def test_database_download_allowed_for_immutable():
         assert len(soup.findAll("a", {"href": re.compile(r"\.db$")}))
         # Check we can actually download it
         assert 200 == client.get("/fixtures.db").status
-        assert_permissions_checked(
-            client.ds,
-            [
-                "view-instance",
-                ("view-database", "database", "fixtures"),
-                ("view-database-download", "database", "fixtures"),
-            ],
-        )
 
 
 def test_database_download_disallowed_for_mutable(app_client):
@@ -1032,14 +991,6 @@ def test_404_content_type(app_client):
 
 def test_canned_query_with_custom_metadata(app_client):
     response = app_client.get("/fixtures/neighborhood_search?text=town")
-    assert_permissions_checked(
-        app_client.ds,
-        [
-            "view-instance",
-            ("view-database", "database", "fixtures"),
-            ("view-query", "query", ("fixtures", "neighborhood_search")),
-        ],
-    )
     assert response.status == 200
     soup = Soup(response.body, "html.parser")
     assert "Search neighborhoods" == soup.find("h1").text
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 7c5b02c0..df905aa1 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -1,4 +1,4 @@
-from .fixtures import make_app_client
+from .fixtures import app_client, assert_permissions_checked, make_app_client
 import pytest
 
 
@@ -139,3 +139,53 @@ def test_query_list_respects_view_query():
             "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
         )
         assert html_fragment in auth_response.text
+
+
+@pytest.mark.parametrize(
+    "path,permissions",
+    [
+        ("/", ["view-instance"]),
+        ("/fixtures", ["view-instance", ("view-database", "database", "fixtures")]),
+        (
+            "/fixtures/facetable/1",
+            ["view-instance", ("view-table", "table", ("fixtures", "facetable"))],
+        ),
+        (
+            "/fixtures/simple_primary_key",
+            [
+                "view-instance",
+                ("view-database", "database", "fixtures"),
+                ("view-table", "table", ("fixtures", "simple_primary_key")),
+            ],
+        ),
+        (
+            "/fixtures?sql=select+1",
+            [
+                "view-instance",
+                ("view-database", "database", "fixtures"),
+                ("execute-sql", "database", "fixtures"),
+            ],
+        ),
+        (
+            "/fixtures.db",
+            [
+                "view-instance",
+                ("view-database", "database", "fixtures"),
+                ("view-database-download", "database", "fixtures"),
+            ],
+        ),
+        (
+            "/fixtures/neighborhood_search",
+            [
+                "view-instance",
+                ("view-database", "database", "fixtures"),
+                ("view-query", "query", ("fixtures", "neighborhood_search")),
+            ],
+        ),
+    ],
+)
+def test_permissions_checked(app_client, path, permissions):
+    app_client.ds._permission_checks.clear()
+    response = app_client.get(path)
+    assert response.status in (200, 403)
+    assert_permissions_checked(app_client.ds, permissions)

From 1cf86e5eccf3f92b483bacbad860879cf39b0ad6 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 07:18:37 -0700
Subject: [PATCH 0504/2303] Show padlock on private index page, refs #811

---
 datasette/templates/index.html | 2 +-
 datasette/views/index.py       | 3 +++
 tests/test_permissions.py      | 6 ++++++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/index.html b/datasette/templates/index.html
index 3b8568b3..5a8dccae 100644
--- a/datasette/templates/index.html
+++ b/datasette/templates/index.html
@@ -5,7 +5,7 @@
 {% block body_class %}index{% endblock %}
 
 {% block content %}
-
      {{ metadata.title or "Datasette" }}
      
+
      {{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}
      
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 0f7fb613..8cbe28f0 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -121,5 +121,8 @@ class IndexView(BaseView):
                     "databases": databases,
                     "metadata": self.ds.metadata(),
                     "datasette_version": __version__,
+                    "private": not await self.ds.permission_allowed(
+                        None, "view-instance"
+                    ),
                 },
             )
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index df905aa1..5dcf46ad 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -16,10 +16,16 @@ def test_view_instance(allow, expected_anon, expected_auth):
         ):
             anon_response = client.get(path)
             assert expected_anon == anon_response.status
+            if allow and path == "/" and anon_response.status == 200:
+                # Should be no padlock
+                assert "
      Datasette 🔒
      " not in anon_response.text
             auth_response = client.get(
                 path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
             )
             assert expected_auth == auth_response.status
+            # Check for the padlock
+            if allow and path == "/" and expected_anon == 403 and expected_auth == 200:
+                assert "
      Datasette 🔒
      " in auth_response.text
 
 
 @pytest.mark.parametrize(

From 3ce7f2e7dae010de97b67618c111ea5853164a69 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 07:23:10 -0700
Subject: [PATCH 0505/2303] Show padlock on private database page, refs #811

---
 datasette/templates/database.html |  2 +-
 datasette/views/database.py       |  3 +++
 tests/test_permissions.py         | 10 ++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index 1187267d..089142e2 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -18,7 +18,7 @@
 
 {% block content %}
 
-
      {{ metadata.title or database }}
      
+
      {{ metadata.title or database }}{% if private %} 🔒{% endif %}
      
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index afbb6b05..2d7e6b31 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -86,6 +86,9 @@ class DatabaseView(DataView):
                 "hidden_count": len([t for t in tables if t["hidden"]]),
                 "views": views,
                 "queries": canned_queries,
+                "private": not await self.ds.permission_allowed(
+                    None, "view-database", "database", database
+                ),
             },
             {
                 "show_hidden": request.args.get("_show_hidden"),
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 5dcf46ad..d76d1e15 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -43,10 +43,20 @@ def test_view_database(allow, expected_anon, expected_auth):
         ):
             anon_response = client.get(path)
             assert expected_anon == anon_response.status
+            if allow and path == "/fixtures" and anon_response.status == 200:
+                # Should be no padlock
+                assert ">fixtures 🔒" not in anon_response.text
             auth_response = client.get(
                 path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
             )
             assert expected_auth == auth_response.status
+            if (
+                allow
+                and path == "/fixtures"
+                and expected_anon == 403
+                and expected_auth == 200
+            ):
+                assert ">fixtures 🔒" in auth_response.text
 
 
 def test_database_list_respects_view_database():

From 2a8b39800f194925658bd9e1b5e4cc12619d5e9c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 07:50:06 -0700
Subject: [PATCH 0506/2303] Updated tests, refs #811

---
 tests/test_api.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/tests/test_api.py b/tests/test_api.py
index 22378946..13a98b6a 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -70,6 +70,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "Table With Space In Name",
@@ -79,6 +80,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "attraction_characteristic",
@@ -97,6 +99,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "binary_data",
@@ -106,6 +109,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "complex_foreign_keys",
@@ -134,6 +138,7 @@ def test_database_page(app_client):
                     },
                 ],
             },
+            "private": False,
         },
         {
             "name": "compound_primary_key",
@@ -143,6 +148,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "compound_three_primary_keys",
@@ -152,6 +158,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "custom_foreign_key_label",
@@ -170,6 +177,7 @@ def test_database_page(app_client):
                     }
                 ],
             },
+            "private": False,
         },
         {
             "name": "facet_cities",
@@ -188,6 +196,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "facetable",
@@ -217,6 +226,7 @@ def test_database_page(app_client):
                     }
                 ],
             },
+            "private": False,
         },
         {
             "name": "foreign_key_references",
@@ -240,6 +250,7 @@ def test_database_page(app_client):
                     },
                 ],
             },
+            "private": False,
         },
         {
             "name": "infinity",
@@ -249,6 +260,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "primary_key_multiple_columns",
@@ -267,6 +279,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "primary_key_multiple_columns_explicit_label",
@@ -285,6 +298,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "roadside_attraction_characteristics",
@@ -308,6 +322,7 @@ def test_database_page(app_client):
                     },
                 ],
             },
+            "private": False,
         },
         {
             "name": "roadside_attractions",
@@ -326,6 +341,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "searchable",
@@ -344,6 +360,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "searchable_tags",
@@ -363,6 +380,7 @@ def test_database_page(app_client):
                     },
                 ],
             },
+            "private": False,
         },
         {
             "name": "select",
@@ -372,6 +390,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "simple_primary_key",
@@ -405,6 +424,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "sortable",
@@ -422,6 +442,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "table/with/slashes.csv",
@@ -431,6 +452,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "tags",
@@ -449,6 +471,7 @@ def test_database_page(app_client):
                 ],
                 "outgoing": [],
             },
+            "private": False,
         },
         {
             "name": "units",
@@ -458,6 +481,7 @@ def test_database_page(app_client):
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "no_primary_key",
@@ -467,6 +491,7 @@ def test_database_page(app_client):
             "hidden": True,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "searchable_fts",
@@ -476,6 +501,7 @@ def test_database_page(app_client):
             "hidden": True,
             "fts_table": "searchable_fts",
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "searchable_fts_content",
@@ -491,6 +517,7 @@ def test_database_page(app_client):
             "hidden": True,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "searchable_fts_segdir",
@@ -507,6 +534,7 @@ def test_database_page(app_client):
             "hidden": True,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
         {
             "name": "searchable_fts_segments",
@@ -516,6 +544,7 @@ def test_database_page(app_client):
             "hidden": True,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
         },
     ] == data["tables"]
 
@@ -537,6 +566,7 @@ def test_no_files_uses_memory_database(app_client_no_files):
             "tables_and_views_more": False,
             "tables_and_views_truncated": [],
             "views_count": 0,
+            "private": False,
         }
     } == response.json
     # Try that SQL query

From 177059284dc953e6c76f86213aa470db2ff3eaca Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 10:05:32 -0700
Subject: [PATCH 0507/2303] New request.actor property, refs #811

---
 datasette/app.py            | 2 +-
 datasette/utils/asgi.py     | 4 ++++
 datasette/views/base.py     | 2 +-
 datasette/views/database.py | 4 ++--
 datasette/views/index.py    | 2 +-
 datasette/views/special.py  | 2 +-
 docs/authentication.rst     | 2 ++
 docs/internals.rst          | 5 ++++-
 8 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 23c293c9..87e542c1 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -667,7 +667,7 @@ class Datasette:
         return d
 
     def _actor(self, request):
-        return {"actor": request.scope.get("actor", None)}
+        return {"actor": request.actor}
 
     def table_metadata(self, database, table):
         "Fetch table-specific metadata."
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index fa78c8df..bca9c9ab 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -74,6 +74,10 @@ class Request:
     def args(self):
         return MultiParams(parse_qs(qs=self.query_string))
 
+    @property
+    def actor(self):
+        return self.scope.get("actor", None)
+
     async def post_vars(self):
         body = []
         body = b""
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 9c2cbbcc..000d354b 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -68,7 +68,7 @@ class BaseView(AsgiView):
         self, request, action, resource_type=None, resource_identifier=None
     ):
         ok = await self.ds.permission_allowed(
-            request.scope.get("actor"),
+            request.actor,
             action,
             resource_type=resource_type,
             resource_identifier=resource_identifier,
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 2d7e6b31..dee6c9c8 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -45,7 +45,7 @@ class DatabaseView(DataView):
         for table in table_counts:
             visible, private = await check_visibility(
                 self.ds,
-                request.scope.get("actor"),
+                request.actor,
                 "view-table",
                 "table",
                 (database, table),
@@ -71,7 +71,7 @@ class DatabaseView(DataView):
         for query in self.ds.get_canned_queries(database):
             visible, private = await check_visibility(
                 self.ds,
-                request.scope.get("actor"),
+                request.actor,
                 "view-query",
                 "query",
                 (database, query["name"]),
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 8cbe28f0..609bfa6a 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -26,7 +26,7 @@ class IndexView(BaseView):
         databases = []
         for name, db in self.ds.databases.items():
             visible, private = await check_visibility(
-                self.ds, request.scope.get("actor"), "view-database", "database", name,
+                self.ds, request.actor, "view-database", "database", name,
             )
             if not visible:
                 continue
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 37c04697..b8bd57c6 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -86,7 +86,7 @@ class PermissionsDebugView(BaseView):
 
     async def get(self, request):
         if not await self.ds.permission_allowed(
-            request.scope.get("actor"), "permissions-debug"
+            request.actor, "permissions-debug"
         ):
             return Response("Permission denied", status=403)
         return await self.render(
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 2caca66f..bda6a0b7 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -140,6 +140,8 @@ Plugins that wish to implement the same permissions scheme as canned queries can
     actor_matches_allow({"id": "root"}, {"id": "*"})
     # returns True
 
+The currently authenticated actor is made available to plugins as ``request.actor``.
+
 .. _PermissionsDebugView:
 
 Permissions Debug
diff --git a/docs/internals.rst b/docs/internals.rst
index 25b2d875..7498f017 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -42,6 +42,9 @@ The request object is passed to various plugin hooks. It represents an incoming
 ``.args`` - MultiParams
     An object representing the parsed querystring parameters, see below.
 
+``.actor`` - dictionary (str -> Any) or None
+    The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated.
+
 The object also has one awaitable method:
 
 ``await request.post_vars()`` - dictionary
@@ -122,7 +125,7 @@ await .permission_allowed(actor, action, resource_type=None, resource_identifier
 -----------------------------------------------------------------------------------------------------
 
 ``actor`` - dictionary
-    The authenticated actor. This is usually ``request.scope.get("actor")``.
+    The authenticated actor. This is usually ``request.actor``.
 
 ``action`` - string
     The name of the action that is being permission checked.

From ab14b20b248dafbe7f9f9487985614939c83b517 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 10:16:24 -0700
Subject: [PATCH 0508/2303] Get tests working again

---
 datasette/views/database.py | 6 +-----
 datasette/views/index.py    | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index dee6c9c8..6f6404a7 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -44,11 +44,7 @@ class DatabaseView(DataView):
         tables = []
         for table in table_counts:
             visible, private = await check_visibility(
-                self.ds,
-                request.actor,
-                "view-table",
-                "table",
-                (database, table),
+                self.ds, request.actor, "view-table", "table", (database, table),
             )
             if not visible:
                 continue
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 609bfa6a..59d3e042 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -122,7 +122,7 @@ class IndexView(BaseView):
                     "metadata": self.ds.metadata(),
                     "datasette_version": __version__,
                     "private": not await self.ds.permission_allowed(
-                        None, "view-instance"
+                        None, "view-instance", default=True
                     ),
                 },
             )

From dfff34e1987976e72f58ee7b274952840b1f4b71 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:03:33 -0700
Subject: [PATCH 0509/2303] Applied black, refs #811

---
 datasette/views/special.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/datasette/views/special.py b/datasette/views/special.py
index b8bd57c6..7a5fbe21 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -85,9 +85,7 @@ class PermissionsDebugView(BaseView):
         self.ds = datasette
 
     async def get(self, request):
-        if not await self.ds.permission_allowed(
-            request.actor, "permissions-debug"
-        ):
+        if not await self.ds.permission_allowed(request.actor, "permissions-debug"):
             return Response("Permission denied", status=403)
         return await self.render(
             ["permissions_debug.html"],

From aa420009c08921d0c9a68cf60a57959be0e8a2e5 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:07:11 -0700
Subject: [PATCH 0510/2303] Show padlock on private table page, refs #811

---
 datasette/templates/table.html | 2 +-
 datasette/views/table.py       | 5 +++++
 tests/test_permissions.py      | 5 +++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index fa6766a8..1289e125 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -26,7 +26,7 @@
 
 {% block content %}
 
-
      {{ metadata.title or table }}{% if is_view %} (view){% endif %}
      
+
      {{ metadata.title or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}
      
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 935fed3d..cd952568 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -271,6 +271,10 @@ class TableView(RowTableShared):
         await self.check_permission(request, "view-database", "database", database)
         await self.check_permission(request, "view-table", "table", (database, table))
 
+        private = not await self.ds.permission_allowed(
+            None, "view-table", "table", (database, table), default=True
+        )
+
         pks = await db.primary_keys(table)
         table_columns = await db.table_columns(table)
 
@@ -834,6 +838,7 @@ class TableView(RowTableShared):
                 "suggested_facets": suggested_facets,
                 "next": next_value and str(next_value) or None,
                 "next_url": next_url,
+                "private": private,
             },
             extra_template,
             (
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index d76d1e15..733afd5f 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -90,11 +90,16 @@ def test_view_table(allow, expected_anon, expected_auth):
     ) as client:
         anon_response = client.get("/fixtures/compound_three_primary_keys")
         assert expected_anon == anon_response.status
+        if allow and anon_response.status == 200:
+            # Should be no padlock
+            assert ">compound_three_primary_keys 🔒" not in anon_response.text
         auth_response = client.get(
             "/fixtures/compound_three_primary_keys",
             cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
         )
         assert expected_auth == auth_response.status
+        if allow and expected_anon == 403 and expected_auth == 200:
+            assert ">compound_three_primary_keys 🔒" in auth_response.text
 
 
 def test_table_list_respects_view_table():

From 9ac27f67fe346e753b562b711a2086e4c616d51d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:13:32 -0700
Subject: [PATCH 0511/2303] Show padlock on private query page, refs #811

---
 datasette/templates/query.html | 2 +-
 datasette/views/database.py    | 6 ++++++
 tests/test_permissions.py      | 5 +++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/query.html b/datasette/templates/query.html
index a7cb6647..7771b101 100644
--- a/datasette/templates/query.html
+++ b/datasette/templates/query.html
@@ -28,7 +28,7 @@
 
 {% block content %}
 
-
      {{ metadata.title or database }}
      
+
      {{ metadata.title or database }}{% if private %} 🔒{% endif %}
      
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 6f6404a7..30817106 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -147,10 +147,14 @@ class QueryView(DataView):
         # Respect canned query permissions
         await self.check_permission(request, "view-instance")
         await self.check_permission(request, "view-database", "database", database)
+        private = False
         if canned_query:
             await self.check_permission(
                 request, "view-query", "query", (database, canned_query)
             )
+            private = not await self.ds.permission_allowed(
+                None, "view-query", "query", (database, canned_query), default=True
+            )
         else:
             await self.check_permission(request, "execute-sql", "database", database)
         # Extract any :named parameters
@@ -214,6 +218,7 @@ class QueryView(DataView):
                         "truncated": False,
                         "columns": [],
                         "query": {"sql": sql, "params": params},
+                        "private": private,
                     },
                     extra_template,
                     templates,
@@ -282,6 +287,7 @@ class QueryView(DataView):
                 "truncated": results.truncated,
                 "columns": columns,
                 "query": {"sql": sql, "params": params},
+                "private": private,
             },
             extra_template,
             templates,
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 733afd5f..55b2d673 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -136,10 +136,15 @@ def test_view_query(allow, expected_anon, expected_auth):
     ) as client:
         anon_response = client.get("/fixtures/q")
         assert expected_anon == anon_response.status
+        if allow and anon_response.status == 200:
+            # Should be no padlock
+            assert ">fixtures 🔒" not in anon_response.text
         auth_response = client.get(
             "/fixtures/q", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
         )
         assert expected_auth == auth_response.status
+        if allow and expected_anon == 403 and expected_auth == 200:
+            assert ">fixtures 🔒" in auth_response.text
 
 
 def test_query_list_respects_view_query():

From dcec89270a2e3b9fabed93f1d7b9be3ef86e9ed2 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:20:21 -0700
Subject: [PATCH 0512/2303] View list respects view-table permission, refs #811

Also makes a small change to the /fixtures.json JSON:

    "views": ["view_name"]

Is now:

    "views": [{"name": "view_name", "private": true}]
---
 datasette/templates/database.html |  2 +-
 datasette/views/database.py       | 11 ++++++++++-
 tests/test_permissions.py         | 18 +++++++++++++-----
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index 089142e2..100faee4 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -51,7 +51,7 @@
     
      Views
      
     
 {% endif %}
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 30817106..824cb632 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -37,10 +37,19 @@ class DatabaseView(DataView):
         db = self.ds.databases[database]
 
         table_counts = await db.table_counts(5)
-        views = await db.view_names()
         hidden_table_names = set(await db.hidden_table_names())
         all_foreign_keys = await db.get_all_foreign_keys()
 
+        views = []
+        for view_name in await db.view_names():
+            visible, private = await check_visibility(
+                self.ds, request.actor, "view-table", "table", (database, view_name),
+            )
+            if visible:
+                views.append(
+                    {"name": view_name, "private": private,}
+                )
+
         tables = []
         for table in table_counts:
             visible, private = await check_visibility(
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 55b2d673..5c338e04 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -107,19 +107,27 @@ def test_table_list_respects_view_table():
         metadata={
             "databases": {
                 "fixtures": {
-                    "tables": {"compound_three_primary_keys": {"allow": {"id": "root"}}}
+                    "tables": {
+                        "compound_three_primary_keys": {"allow": {"id": "root"}},
+                        # And a SQL view too:
+                        "paginated_view": {"allow": {"id": "root"}},
+                    }
                 }
             }
         }
     ) as client:
-        html_fragment = 'compound_three_primary_keys 🔒'
+        html_fragments = [
+            ">compound_three_primary_keys 🔒",
+            ">paginated_view 🔒",
+        ]
         anon_response = client.get("/fixtures")
-        assert html_fragment not in anon_response.text
-        assert '"/fixtures/compound_three_primary_keys"' not in anon_response.text
+        for html_fragment in html_fragments:
+            assert html_fragment not in anon_response.text
         auth_response = client.get(
             "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
         )
-        assert html_fragment in auth_response.text
+        for html_fragment in html_fragments:
+            assert html_fragment in auth_response.text
 
 
 @pytest.mark.parametrize(

From 5598c5de011db95396b65b5c8c251cbe6884d6ae Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:34:14 -0700
Subject: [PATCH 0513/2303] Database list on index page respects table/view
 permissions, refs #811

---
 datasette/templates/index.html |  2 +-
 datasette/views/index.py       | 25 ++++++++++++++++++++-----
 tests/test_permissions.py      | 31 +++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/datasette/templates/index.html b/datasette/templates/index.html
index 5a8dccae..c1adfc59 100644
--- a/datasette/templates/index.html
+++ b/datasette/templates/index.html
@@ -22,7 +22,7 @@
         {% endif %}
     
      
     
      {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}
      
+    }}"{% if table.count %} title="{{ table.count }} rows"{% endif %}>{{ table.name }}{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}
      
 {% endfor %}
 
 {% endblock %}
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 59d3e042..a3e8388c 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -25,14 +25,22 @@ class IndexView(BaseView):
         await self.check_permission(request, "view-instance")
         databases = []
         for name, db in self.ds.databases.items():
-            visible, private = await check_visibility(
+            visible, database_private = await check_visibility(
                 self.ds, request.actor, "view-database", "database", name,
             )
             if not visible:
                 continue
             table_names = await db.table_names()
             hidden_table_names = set(await db.hidden_table_names())
-            views = await db.view_names()
+
+            views = []
+            for view_name in await db.view_names():
+                visible, private = await check_visibility(
+                    self.ds, request.actor, "view-table", "table", (name, view_name),
+                )
+                if visible:
+                    views.append({"name": view_name, "private": private})
+
             # Perform counts only for immutable or DBS with <= COUNT_TABLE_LIMIT tables
             table_counts = {}
             if not db.is_mutable or db.size < COUNT_DB_SIZE_LIMIT:
@@ -40,8 +48,14 @@ class IndexView(BaseView):
                 # If any of these are None it means at least one timed out - ignore them all
                 if any(v is None for v in table_counts.values()):
                     table_counts = {}
+
             tables = {}
             for table in table_names:
+                visible, private = await check_visibility(
+                    self.ds, request.actor, "view-table", "table", (name, table),
+                )
+                if not visible:
+                    continue
                 table_columns = await db.table_columns(table)
                 tables[table] = {
                     "name": table,
@@ -51,6 +65,7 @@ class IndexView(BaseView):
                     "hidden": table in hidden_table_names,
                     "fts_table": await db.fts_table(table),
                     "num_relationships_for_sorting": 0,
+                    "private": private,
                 }
 
             if request.args.get("_sort") == "relationships" or not table_counts:
@@ -78,8 +93,8 @@ class IndexView(BaseView):
             # Only add views if this is less than TRUNCATE_AT
             if len(tables_and_views_truncated) < TRUNCATE_AT:
                 num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated)
-                for view_name in views[:num_views_to_add]:
-                    tables_and_views_truncated.append({"name": view_name})
+                for view in views[:num_views_to_add]:
+                    tables_and_views_truncated.append(view)
 
             databases.append(
                 {
@@ -100,7 +115,7 @@ class IndexView(BaseView):
                     ),
                     "hidden_tables_count": len(hidden_tables),
                     "views_count": len(views),
-                    "private": private,
+                    "private": database_private,
                 }
             )
 
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 5c338e04..475f93dd 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -74,6 +74,37 @@ def test_database_list_respects_view_database():
         assert 'fixtures 🔒' in auth_response.text
 
 
+def test_database_list_respects_view_table():
+    with make_app_client(
+        metadata={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "names": {"allow": {"id": "root"}},
+                        "v": {"allow": {"id": "root"}},
+                    }
+                }
+            }
+        },
+        extra_databases={
+            "data.db": "create table names (name text); create view v as select * from names"
+        },
+    ) as client:
+        html_fragments = [
+            ">names 🔒",
+            ">v 🔒",
+        ]
+        anon_response_text = client.get("/").text
+        assert "0 rows in 0 tables" in anon_response_text
+        for html_fragment in html_fragments:
+            assert html_fragment not in anon_response_text
+        auth_response_text = client.get(
+            "/", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+        ).text
+        for html_fragment in html_fragments:
+            assert html_fragment in auth_response_text
+
+
 @pytest.mark.parametrize(
     "allow,expected_anon,expected_auth",
     [(None, 200, 200), ({}, 403, 403), ({"id": "root"}, 403, 200),],

From c9f1ec616e5a8c83f554baaedd38663569fb9b91 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:51:03 -0700
Subject: [PATCH 0514/2303] Removed resource_type from permissions system,
 closes #817

Refs #811, #699
---
 datasette/app.py                           |  4 +---
 datasette/default_permissions.py           |  5 +---
 datasette/hookspecs.py                     |  2 +-
 datasette/templates/permissions_debug.html |  4 ++--
 datasette/utils/__init__.py                | 16 +++----------
 datasette/views/base.py                    |  5 +---
 datasette/views/database.py                | 28 ++++++++--------------
 datasette/views/index.py                   |  6 ++---
 datasette/views/table.py                   | 10 ++++----
 docs/authentication.rst                    | 19 ++-------------
 docs/internals.rst                         |  7 ++----
 docs/plugins.rst                           |  9 +++----
 tests/conftest.py                          |  4 ++--
 tests/fixtures.py                          |  9 +++----
 14 files changed, 39 insertions(+), 89 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 87e542c1..c12e0af0 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -465,7 +465,7 @@ class Datasette:
             return []
 
     async def permission_allowed(
-        self, actor, action, resource_type=None, resource_identifier=None, default=False
+        self, actor, action, resource_identifier=None, default=False
     ):
         "Check permissions using the permissions_allowed plugin hook"
         result = None
@@ -473,7 +473,6 @@ class Datasette:
             datasette=self,
             actor=actor,
             action=action,
-            resource_type=resource_type,
             resource_identifier=resource_identifier,
         ):
             if callable(check):
@@ -491,7 +490,6 @@ class Datasette:
                 "when": datetime.datetime.utcnow().isoformat(),
                 "actor": actor,
                 "action": action,
-                "resource_type": resource_type,
                 "resource_identifier": resource_identifier,
                 "used_default": used_default,
                 "result": result,
diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index dd1770a3..d27704aa 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -3,7 +3,7 @@ from datasette.utils import actor_matches_allow
 
 
 @hookimpl
-def permission_allowed(datasette, actor, action, resource_type, resource_identifier):
+def permission_allowed(datasette, actor, action, resource_identifier):
     if action == "permissions-debug":
         if actor and actor.get("id") == "root":
             return True
@@ -12,13 +12,11 @@ def permission_allowed(datasette, actor, action, resource_type, resource_identif
         if allow is not None:
             return actor_matches_allow(actor, allow)
     elif action == "view-database":
-        assert resource_type == "database"
         database_allow = datasette.metadata("allow", database=resource_identifier)
         if database_allow is None:
             return True
         return actor_matches_allow(actor, database_allow)
     elif action == "view-table":
-        assert resource_type == "table"
         database, table = resource_identifier
         tables = datasette.metadata("tables", database=database) or {}
         table_allow = (tables.get(table) or {}).get("allow")
@@ -27,7 +25,6 @@ def permission_allowed(datasette, actor, action, resource_type, resource_identif
         return actor_matches_allow(actor, table_allow)
     elif action == "view-query":
         # Check if this query has a "allow" block in metadata
-        assert resource_type == "query"
         database, query_name = resource_identifier
         queries_metadata = datasette.metadata("queries", database=database)
         assert query_name in queries_metadata
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 71d06661..3c202553 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -66,5 +66,5 @@ def actor_from_request(datasette, request):
 
 
 @hookspec
-def permission_allowed(datasette, actor, action, resource_type, resource_identifier):
+def permission_allowed(datasette, actor, action, resource_identifier):
     "Check if actor is allowed to perfom this action - return True, False or None"
diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html
index dda57dfa..7d3ee712 100644
--- a/datasette/templates/permissions_debug.html
+++ b/datasette/templates/permissions_debug.html
@@ -46,8 +46,8 @@
             {% endif %}
         
         
      Actor: {{ check.actor|tojson }}
      
-        {% if check.resource_type %}
-            
      Resource: {{ check.resource_type }} = {{ check.resource_identifier }}
      
+        {% if check.resource_identifier %}
+            
      Resource: {{ check.resource_identifier }}
      
         {% endif %}
     
 {% endfor %}
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 3d964049..257d1285 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -876,24 +876,14 @@ def actor_matches_allow(actor, allow):
     return False
 
 
-async def check_visibility(
-    datasette, actor, action, resource_type, resource_identifier, default=True
-):
+async def check_visibility(datasette, actor, action, resource_identifier, default=True):
     "Returns (visible, private) - visible = can you see it, private = can others see it too"
     visible = await datasette.permission_allowed(
-        actor,
-        action,
-        resource_type=resource_type,
-        resource_identifier=resource_identifier,
-        default=default,
+        actor, action, resource_identifier=resource_identifier, default=default,
     )
     if not visible:
         return (False, False)
     private = not await datasette.permission_allowed(
-        None,
-        action,
-        resource_type=resource_type,
-        resource_identifier=resource_identifier,
-        default=default,
+        None, action, resource_identifier=resource_identifier, default=default,
     )
     return visible, private
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 000d354b..2ca5e86a 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -64,13 +64,10 @@ class BaseView(AsgiView):
         response.body = b""
         return response
 
-    async def check_permission(
-        self, request, action, resource_type=None, resource_identifier=None
-    ):
+    async def check_permission(self, request, action, resource_identifier=None):
         ok = await self.ds.permission_allowed(
             request.actor,
             action,
-            resource_type=resource_type,
             resource_identifier=resource_identifier,
             default=True,
         )
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 824cb632..d562ecb1 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -21,7 +21,7 @@ class DatabaseView(DataView):
 
     async def data(self, request, database, hash, default_labels=False, _size=None):
         await self.check_permission(request, "view-instance")
-        await self.check_permission(request, "view-database", "database", database)
+        await self.check_permission(request, "view-database", database)
         metadata = (self.ds.metadata("databases") or {}).get(database, {})
         self.ds.update_with_inherited_metadata(metadata)
 
@@ -43,7 +43,7 @@ class DatabaseView(DataView):
         views = []
         for view_name in await db.view_names():
             visible, private = await check_visibility(
-                self.ds, request.actor, "view-table", "table", (database, view_name),
+                self.ds, request.actor, "view-table", (database, view_name),
             )
             if visible:
                 views.append(
@@ -53,7 +53,7 @@ class DatabaseView(DataView):
         tables = []
         for table in table_counts:
             visible, private = await check_visibility(
-                self.ds, request.actor, "view-table", "table", (database, table),
+                self.ds, request.actor, "view-table", (database, table),
             )
             if not visible:
                 continue
@@ -75,11 +75,7 @@ class DatabaseView(DataView):
         canned_queries = []
         for query in self.ds.get_canned_queries(database):
             visible, private = await check_visibility(
-                self.ds,
-                request.actor,
-                "view-query",
-                "query",
-                (database, query["name"]),
+                self.ds, request.actor, "view-query", (database, query["name"]),
             )
             if visible:
                 canned_queries.append(dict(query, private=private))
@@ -112,10 +108,8 @@ class DatabaseDownload(DataView):
 
     async def view_get(self, request, database, hash, correct_hash_present, **kwargs):
         await self.check_permission(request, "view-instance")
-        await self.check_permission(request, "view-database", "database", database)
-        await self.check_permission(
-            request, "view-database-download", "database", database
-        )
+        await self.check_permission(request, "view-database", database)
+        await self.check_permission(request, "view-database-download", database)
         if database not in self.ds.databases:
             raise DatasetteError("Invalid database", status=404)
         db = self.ds.databases[database]
@@ -155,17 +149,15 @@ class QueryView(DataView):
 
         # Respect canned query permissions
         await self.check_permission(request, "view-instance")
-        await self.check_permission(request, "view-database", "database", database)
+        await self.check_permission(request, "view-database", database)
         private = False
         if canned_query:
-            await self.check_permission(
-                request, "view-query", "query", (database, canned_query)
-            )
+            await self.check_permission(request, "view-query", (database, canned_query))
             private = not await self.ds.permission_allowed(
-                None, "view-query", "query", (database, canned_query), default=True
+                None, "view-query", (database, canned_query), default=True
             )
         else:
-            await self.check_permission(request, "execute-sql", "database", database)
+            await self.check_permission(request, "execute-sql", database)
         # Extract any :named parameters
         named_parameters = named_parameters or self.re_named_parameter.findall(sql)
         named_parameter_values = {
diff --git a/datasette/views/index.py b/datasette/views/index.py
index a3e8388c..b2706251 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -26,7 +26,7 @@ class IndexView(BaseView):
         databases = []
         for name, db in self.ds.databases.items():
             visible, database_private = await check_visibility(
-                self.ds, request.actor, "view-database", "database", name,
+                self.ds, request.actor, "view-database", name,
             )
             if not visible:
                 continue
@@ -36,7 +36,7 @@ class IndexView(BaseView):
             views = []
             for view_name in await db.view_names():
                 visible, private = await check_visibility(
-                    self.ds, request.actor, "view-table", "table", (name, view_name),
+                    self.ds, request.actor, "view-table", (name, view_name),
                 )
                 if visible:
                     views.append({"name": view_name, "private": private})
@@ -52,7 +52,7 @@ class IndexView(BaseView):
             tables = {}
             for table in table_names:
                 visible, private = await check_visibility(
-                    self.ds, request.actor, "view-table", "table", (name, table),
+                    self.ds, request.actor, "view-table", (name, table),
                 )
                 if not visible:
                     continue
diff --git a/datasette/views/table.py b/datasette/views/table.py
index cd952568..4cec0cda 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -268,11 +268,11 @@ class TableView(RowTableShared):
             raise NotFound("Table not found: {}".format(table))
 
         await self.check_permission(request, "view-instance")
-        await self.check_permission(request, "view-database", "database", database)
-        await self.check_permission(request, "view-table", "table", (database, table))
+        await self.check_permission(request, "view-database", database)
+        await self.check_permission(request, "view-table", (database, table))
 
         private = not await self.ds.permission_allowed(
-            None, "view-table", "table", (database, table), default=True
+            None, "view-table", (database, table), default=True
         )
 
         pks = await db.primary_keys(table)
@@ -854,8 +854,8 @@ class RowView(RowTableShared):
     async def data(self, request, database, hash, table, pk_path, default_labels=False):
         pk_values = urlsafe_components(pk_path)
         await self.check_permission(request, "view-instance")
-        await self.check_permission(request, "view-database", "database", database)
-        await self.check_permission(request, "view-table", "table", (database, table))
+        await self.check_permission(request, "view-database", database)
+        await self.check_permission(request, "view-table", (database, table))
         db = self.ds.databases[database]
         pks = await db.primary_keys(table)
         use_rowid = not pks
diff --git a/docs/authentication.rst b/docs/authentication.rst
index bda6a0b7..67112969 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -52,7 +52,7 @@ The URL on the first line includes a one-use token which can be used to sign in
 Permissions
 ===========
 
-Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method. This method is also used by Datasette core code itself, which allows plugins to help make decisions on which actions are allowed by implementing the :ref:`permission_allowed(...) ` plugin hook.
+Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method. This method is also used by Datasette core code itself, which allows plugins to help make decisions on which actions are allowed by implementing the :ref:`plugin_permission_allowed` plugin hook.
 
 .. _authentication_permissions_canned_queries:
 
@@ -159,7 +159,7 @@ This is designed to help administrators and plugin authors understand exactly ho
 Permissions
 ===========
 
-This section lists all of the permission checks that are carried out by Datasette core, along with their ``resource_type`` and ``resource_identifier`` if those are passed.
+This section lists all of the permission checks that are carried out by Datasette core, along with the ``resource_identifier`` if it was passed.
 
 .. _permissions_view_instance:
 
@@ -176,9 +176,6 @@ view-database
 
 Actor is allowed to view a database page, e.g. https://latest.datasette.io/fixtures
 
-``resource_type`` - string
-    "database"
-
 ``resource_identifier`` - string
     The name of the database
 
@@ -189,9 +186,6 @@ view-database-download
 
 Actor is allowed to download a database, e.g. https://latest.datasette.io/fixtures.db
 
-``resource_type`` - string
-    "database"
-
 ``resource_identifier`` - string
     The name of the database
 
@@ -202,9 +196,6 @@ view-table
 
 Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys
 
-``resource_type`` - string
-    "table" - even if this is actually a SQL view
-
 ``resource_identifier`` - tuple: (string, string)
     The name of the database, then the name of the table
 
@@ -215,9 +206,6 @@ view-query
 
 Actor is allowed to view a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size
 
-``resource_type`` - string
-    "query"
-
 ``resource_identifier`` - string
     The name of the canned query
 
@@ -228,9 +216,6 @@ execute-sql
 
 Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures?sql=select+100
 
-``resource_type`` - string
-    "database"
-
 ``resource_identifier`` - string
     The name of the database
 
diff --git a/docs/internals.rst b/docs/internals.rst
index 7498f017..1d61b6cb 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -121,8 +121,8 @@ Renders a `Jinja template `__ usin
 
 .. _datasette_permission_allowed:
 
-await .permission_allowed(actor, action, resource_type=None, resource_identifier=None, default=False)
------------------------------------------------------------------------------------------------------
+await .permission_allowed(actor, action, resource_identifier=None, default=False)
+---------------------------------------------------------------------------------
 
 ``actor`` - dictionary
     The authenticated actor. This is usually ``request.actor``.
@@ -130,9 +130,6 @@ await .permission_allowed(actor, action, resource_type=None, resource_identifier
 ``action`` - string
     The name of the action that is being permission checked.
 
-``resource_type`` - string, optional
-    The type of resource being checked, e.g. ``"table"``.
-
 ``resource_identifier`` - string, optional
     The resource identifier, e.g. the name of the table.
 
diff --git a/docs/plugins.rst b/docs/plugins.rst
index ecc7cbf1..118fab84 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1005,8 +1005,8 @@ Instead of returning a dictionary, this function can return an awaitable functio
 
 .. _plugin_permission_allowed:
 
-permission_allowed(datasette, actor, action, resource_type, resource_identifier)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+permission_allowed(datasette, actor, action, resource_identifier)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
@@ -1017,10 +1017,7 @@ permission_allowed(datasette, actor, action, resource_type, resource_identifier)
 ``action`` - string
     The action to be performed, e.g. ``"edit-table"``.
 
-``resource_type`` - string
-    The type of resource being acted on, e.g. ``"table"``.
-
-``resource`` - string
+``resource_identifier`` - string
     An identifier for the individual resource, e.g. the name of the table.
 
 Called to check that an actor has permission to perform an action on a resource. Can return ``True`` if the action is allowed, ``False`` if the action is not allowed or ``None`` if the plugin does not have an opinion one way or the other.
diff --git a/tests/conftest.py b/tests/conftest.py
index 1921ae3a..7f1e9387 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -70,8 +70,8 @@ def check_permission_actions_are_documented():
             action = kwargs.get("action").replace("-", "_")
             assert (
                 action in documented_permission_actions
-            ), "Undocumented permission action: {}, resource_type: {}, resource_identifier: {}".format(
-                action, kwargs["resource_type"], kwargs["resource_identifier"]
+            ), "Undocumented permission action: {}, resource_identifier: {}".format(
+                action, kwargs["resource_identifier"]
             )
 
     pm.add_hookcall_monitoring(
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 2ac73fb1..8210d34f 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -857,24 +857,21 @@ if __name__ == "__main__":
 
 
 def assert_permissions_checked(datasette, actions):
-    # actions is a list of "action" or (action, resource_type, resource_identifier) tuples
+    # actions is a list of "action" or (action, resource_identifier) tuples
     for action in actions:
         if isinstance(action, str):
-            resource_type = None
             resource_identifier = None
         else:
-            action, resource_type, resource_identifier = action
+            action, resource_identifier = action
         assert [
             pc
             for pc in datasette._permission_checks
             if pc["action"] == action
-            and pc["resource_type"] == resource_type
             and pc["resource_identifier"] == resource_identifier
-        ], """Missing expected permission check: action={}, resource_type={}, resource_identifier={}
+        ], """Missing expected permission check: action={}, resource_identifier={}
         Permission checks seen: {}
         """.format(
             action,
-            resource_type,
             resource_identifier,
             json.dumps(list(datasette._permission_checks), indent=4),
         )

From 799c5d53570d773203527f19530cf772dc2eeb24 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 11:59:11 -0700
Subject: [PATCH 0515/2303] Renamed resource_identifier to resource, refs #817

---
 datasette/app.py                           | 11 +++--------
 datasette/default_permissions.py           |  8 ++++----
 datasette/hookspecs.py                     |  2 +-
 datasette/templates/permissions_debug.html |  4 ++--
 datasette/utils/__init__.py                |  6 +++---
 datasette/views/base.py                    |  7 ++-----
 datasette/views/database.py                |  2 +-
 docs/authentication.rst                    | 12 ++++++------
 docs/internals.rst                         | 10 ++++++----
 docs/plugins.rst                           |  6 ++++--
 tests/conftest.py                          |  4 ++--
 tests/fixtures.py                          | 15 ++++++---------
 12 files changed, 40 insertions(+), 47 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index c12e0af0..2f89d17c 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -464,16 +464,11 @@ class Datasette:
         else:
             return []
 
-    async def permission_allowed(
-        self, actor, action, resource_identifier=None, default=False
-    ):
+    async def permission_allowed(self, actor, action, resource=None, default=False):
         "Check permissions using the permissions_allowed plugin hook"
         result = None
         for check in pm.hook.permission_allowed(
-            datasette=self,
-            actor=actor,
-            action=action,
-            resource_identifier=resource_identifier,
+            datasette=self, actor=actor, action=action, resource=resource,
         ):
             if callable(check):
                 check = check()
@@ -490,7 +485,7 @@ class Datasette:
                 "when": datetime.datetime.utcnow().isoformat(),
                 "actor": actor,
                 "action": action,
-                "resource_identifier": resource_identifier,
+                "resource": resource,
                 "used_default": used_default,
                 "result": result,
             }
diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index d27704aa..e989c0fa 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -3,7 +3,7 @@ from datasette.utils import actor_matches_allow
 
 
 @hookimpl
-def permission_allowed(datasette, actor, action, resource_identifier):
+def permission_allowed(datasette, actor, action, resource):
     if action == "permissions-debug":
         if actor and actor.get("id") == "root":
             return True
@@ -12,12 +12,12 @@ def permission_allowed(datasette, actor, action, resource_identifier):
         if allow is not None:
             return actor_matches_allow(actor, allow)
     elif action == "view-database":
-        database_allow = datasette.metadata("allow", database=resource_identifier)
+        database_allow = datasette.metadata("allow", database=resource)
         if database_allow is None:
             return True
         return actor_matches_allow(actor, database_allow)
     elif action == "view-table":
-        database, table = resource_identifier
+        database, table = resource
         tables = datasette.metadata("tables", database=database) or {}
         table_allow = (tables.get(table) or {}).get("allow")
         if table_allow is None:
@@ -25,7 +25,7 @@ def permission_allowed(datasette, actor, action, resource_identifier):
         return actor_matches_allow(actor, table_allow)
     elif action == "view-query":
         # Check if this query has a "allow" block in metadata
-        database, query_name = resource_identifier
+        database, query_name = resource
         queries_metadata = datasette.metadata("queries", database=database)
         assert query_name in queries_metadata
         if isinstance(queries_metadata[query_name], str):
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 3c202553..d5fd232f 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -66,5 +66,5 @@ def actor_from_request(datasette, request):
 
 
 @hookspec
-def permission_allowed(datasette, actor, action, resource_identifier):
+def permission_allowed(datasette, actor, action, resource):
     "Check if actor is allowed to perfom this action - return True, False or None"
diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html
index 7d3ee712..d898ea8c 100644
--- a/datasette/templates/permissions_debug.html
+++ b/datasette/templates/permissions_debug.html
@@ -46,8 +46,8 @@
             {% endif %}
         
         
      Actor: {{ check.actor|tojson }}
      
-        {% if check.resource_identifier %}
-            
      Resource: {{ check.resource_identifier }}
      
+        {% if check.resource %}
+            
      Resource: {{ check.resource }}
      
         {% endif %}
     
 {% endfor %}
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 257d1285..7c1f34e0 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -876,14 +876,14 @@ def actor_matches_allow(actor, allow):
     return False
 
 
-async def check_visibility(datasette, actor, action, resource_identifier, default=True):
+async def check_visibility(datasette, actor, action, resource, default=True):
     "Returns (visible, private) - visible = can you see it, private = can others see it too"
     visible = await datasette.permission_allowed(
-        actor, action, resource_identifier=resource_identifier, default=default,
+        actor, action, resource=resource, default=default,
     )
     if not visible:
         return (False, False)
     private = not await datasette.permission_allowed(
-        None, action, resource_identifier=resource_identifier, default=default,
+        None, action, resource=resource, default=default,
     )
     return visible, private
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 2ca5e86a..f327c6cd 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -64,12 +64,9 @@ class BaseView(AsgiView):
         response.body = b""
         return response
 
-    async def check_permission(self, request, action, resource_identifier=None):
+    async def check_permission(self, request, action, resource=None):
         ok = await self.ds.permission_allowed(
-            request.actor,
-            action,
-            resource_identifier=resource_identifier,
-            default=True,
+            request.actor, action, resource=resource, default=True,
         )
         if not ok:
             raise Forbidden(action)
diff --git a/datasette/views/database.py b/datasette/views/database.py
index d562ecb1..e1b29c27 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -88,7 +88,7 @@ class DatabaseView(DataView):
                 "views": views,
                 "queries": canned_queries,
                 "private": not await self.ds.permission_allowed(
-                    None, "view-database", "database", database
+                    None, "view-database", database
                 ),
             },
             {
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 67112969..f5209dfc 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -159,7 +159,7 @@ This is designed to help administrators and plugin authors understand exactly ho
 Permissions
 ===========
 
-This section lists all of the permission checks that are carried out by Datasette core, along with the ``resource_identifier`` if it was passed.
+This section lists all of the permission checks that are carried out by Datasette core, along with the ``resource`` if it was passed.
 
 .. _permissions_view_instance:
 
@@ -176,7 +176,7 @@ view-database
 
 Actor is allowed to view a database page, e.g. https://latest.datasette.io/fixtures
 
-``resource_identifier`` - string
+``resource`` - string
     The name of the database
 
 .. _permissions_view_database_download:
@@ -186,7 +186,7 @@ view-database-download
 
 Actor is allowed to download a database, e.g. https://latest.datasette.io/fixtures.db
 
-``resource_identifier`` - string
+``resource`` - string
     The name of the database
 
 .. _permissions_view_table:
@@ -196,7 +196,7 @@ view-table
 
 Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys
 
-``resource_identifier`` - tuple: (string, string)
+``resource`` - tuple: (string, string)
     The name of the database, then the name of the table
 
 .. _permissions_view_query:
@@ -206,7 +206,7 @@ view-query
 
 Actor is allowed to view a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size
 
-``resource_identifier`` - string
+``resource`` - string
     The name of the canned query
 
 .. _permissions_execute_sql:
@@ -216,7 +216,7 @@ execute-sql
 
 Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures?sql=select+100
 
-``resource_identifier`` - string
+``resource`` - string
     The name of the database
 
 .. _permissions_permissions_debug:
diff --git a/docs/internals.rst b/docs/internals.rst
index 1d61b6cb..83dbd897 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -121,8 +121,8 @@ Renders a `Jinja template `__ usin
 
 .. _datasette_permission_allowed:
 
-await .permission_allowed(actor, action, resource_identifier=None, default=False)
----------------------------------------------------------------------------------
+await .permission_allowed(actor, action, resource=None, default=False)
+----------------------------------------------------------------------
 
 ``actor`` - dictionary
     The authenticated actor. This is usually ``request.actor``.
@@ -130,13 +130,15 @@ await .permission_allowed(actor, action, resource_identifier=None, default=False
 ``action`` - string
     The name of the action that is being permission checked.
 
-``resource_identifier`` - string, optional
-    The resource identifier, e.g. the name of the table.
+``resource`` - string, optional
+    The resource, e.g. the name of the table. Only some permissions apply to a resource.
 
 Check if the given actor has permission to perform the given action on the given resource. This uses plugins that implement the :ref:`plugin_permission_allowed` plugin hook to decide if the action is allowed or not.
 
 If none of the plugins express an opinion, the return value will be the ``default`` argument. This is deny, but you can pass ``default=True`` to default allow instead.
 
+See :ref:`permissions` for a full list of permissions included in Datasette core.
+
 .. _datasette_get_database:
 
 .get_database(name)
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 118fab84..56041d0c 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1005,7 +1005,7 @@ Instead of returning a dictionary, this function can return an awaitable functio
 
 .. _plugin_permission_allowed:
 
-permission_allowed(datasette, actor, action, resource_identifier)
+permission_allowed(datasette, actor, action, resource)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``datasette`` - :ref:`internals_datasette`
@@ -1017,7 +1017,9 @@ permission_allowed(datasette, actor, action, resource_identifier)
 ``action`` - string
     The action to be performed, e.g. ``"edit-table"``.
 
-``resource_identifier`` - string
+``resource`` - string or None
     An identifier for the individual resource, e.g. the name of the table.
 
 Called to check that an actor has permission to perform an action on a resource. Can return ``True`` if the action is allowed, ``False`` if the action is not allowed or ``None`` if the plugin does not have an opinion one way or the other.
+
+See :ref:`permissions` for a full list of permissions included in Datasette core.
diff --git a/tests/conftest.py b/tests/conftest.py
index 7f1e9387..320aa45b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -70,8 +70,8 @@ def check_permission_actions_are_documented():
             action = kwargs.get("action").replace("-", "_")
             assert (
                 action in documented_permission_actions
-            ), "Undocumented permission action: {}, resource_identifier: {}".format(
-                action, kwargs["resource_identifier"]
+            ), "Undocumented permission action: {}, resource: {}".format(
+                action, kwargs["resource"]
             )
 
     pm.add_hookcall_monitoring(
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 8210d34f..e9175b57 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -857,21 +857,18 @@ if __name__ == "__main__":
 
 
 def assert_permissions_checked(datasette, actions):
-    # actions is a list of "action" or (action, resource_identifier) tuples
+    # actions is a list of "action" or (action, resource) tuples
     for action in actions:
         if isinstance(action, str):
-            resource_identifier = None
+            resource = None
         else:
-            action, resource_identifier = action
+            action, resource = action
         assert [
             pc
             for pc in datasette._permission_checks
-            if pc["action"] == action
-            and pc["resource_identifier"] == resource_identifier
-        ], """Missing expected permission check: action={}, resource_identifier={}
+            if pc["action"] == action and pc["resource"] == resource
+        ], """Missing expected permission check: action={}, resource={}
         Permission checks seen: {}
         """.format(
-            action,
-            resource_identifier,
-            json.dumps(list(datasette._permission_checks), indent=4),
+            action, resource, json.dumps(list(datasette._permission_checks), indent=4),
         )

From 040fc0546f1ad602125ecdc27d9d013d830aa808 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 12:02:56 -0700
Subject: [PATCH 0516/2303] Updated tests, refs #817

---
 tests/test_permissions.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 475f93dd..90ba1494 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -210,41 +210,41 @@ def test_query_list_respects_view_query():
     "path,permissions",
     [
         ("/", ["view-instance"]),
-        ("/fixtures", ["view-instance", ("view-database", "database", "fixtures")]),
+        ("/fixtures", ["view-instance", ("view-database", "fixtures")]),
         (
             "/fixtures/facetable/1",
-            ["view-instance", ("view-table", "table", ("fixtures", "facetable"))],
+            ["view-instance", ("view-table", ("fixtures", "facetable"))],
         ),
         (
             "/fixtures/simple_primary_key",
             [
                 "view-instance",
-                ("view-database", "database", "fixtures"),
-                ("view-table", "table", ("fixtures", "simple_primary_key")),
+                ("view-database", "fixtures"),
+                ("view-table", ("fixtures", "simple_primary_key")),
             ],
         ),
         (
             "/fixtures?sql=select+1",
             [
                 "view-instance",
-                ("view-database", "database", "fixtures"),
-                ("execute-sql", "database", "fixtures"),
+                ("view-database", "fixtures"),
+                ("execute-sql", "fixtures"),
             ],
         ),
         (
             "/fixtures.db",
             [
                 "view-instance",
-                ("view-database", "database", "fixtures"),
-                ("view-database-download", "database", "fixtures"),
+                ("view-database", "fixtures"),
+                ("view-database-download", "fixtures"),
             ],
         ),
         (
             "/fixtures/neighborhood_search",
             [
                 "view-instance",
-                ("view-database", "database", "fixtures"),
-                ("view-query", "query", ("fixtures", "neighborhood_search")),
+                ("view-database", "fixtures"),
+                ("view-query", ("fixtures", "neighborhood_search")),
             ],
         ),
     ],

From c7d145e016522dd6ee229d4d0b3ba79a7a8877c1 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 12:06:05 -0700
Subject: [PATCH 0517/2303] Updated example for extra_template_vars hook,
 closes #816

---
 docs/plugins.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/plugins.rst b/docs/plugins.rst
index 56041d0c..6b1e60f2 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -689,14 +689,14 @@ Function that returns an awaitable function that returns a dictionary
 
 Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template.
 
-Here's an example plugin that returns an authentication object from the ASGI scope:
+Here's an example plugin that adds a ``"user_agent"`` variable to the template context containing the current request's User-Agent header:
 
 .. code-block:: python
 
     @hookimpl
     def extra_template_vars(request):
         return {
-            "auth": request.scope.get("auth")
+            "user_agent": request.headers.get("user-agent")
         }
 
 This example returns an awaitable function which adds a list of ``hidden_table_names`` to the context:

From 54370853828bdf87ca844fd0fc00900e0e2e659d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 12:32:27 -0700
Subject: [PATCH 0518/2303] Documentation for allow blocks on more stuff,
 closes #811

---
 docs/authentication.rst | 121 ++++++++++++++++++++++++++++++++--------
 docs/sql_queries.rst    |   2 +-
 2 files changed, 100 insertions(+), 23 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index f5209dfc..a6c4ee79 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -15,7 +15,7 @@ Actors
 
 Through plugins, Datasette can support both authenticated users (with cookies) and authenticated API agents (via authentication tokens). The word "actor" is used to cover both of these cases.
 
-Every request to Datasette has an associated actor value. This can be ``None`` for unauthenticated requests, or a JSON compatible Python dictionary for authenticated users or API agents.
+Every request to Datasette has an associated actor value, available in the code as ``request.actor``. This can be ``None`` for unauthenticated requests, or a JSON compatible Python dictionary for authenticated users or API agents.
 
 The only required field in an actor is ``"id"``, which must be a string. Plugins may decide to add any other fields to the actor dictionary.
 
@@ -24,7 +24,7 @@ Plugins can use the :ref:`plugin_actor_from_request` hook to implement custom lo
 .. _authentication_root:
 
 Using the "root" actor
-======================
+----------------------
 
 Datasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github `__ for example.
 
@@ -49,37 +49,40 @@ The URL on the first line includes a one-use token which can be used to sign in
 
 .. _authentication_permissions:
 
-Permissions
-===========
+Checking permission
+===================
 
 Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method. This method is also used by Datasette core code itself, which allows plugins to help make decisions on which actions are allowed by implementing the :ref:`plugin_permission_allowed` plugin hook.
 
-.. _authentication_permissions_canned_queries:
+.. _authentication_permissions_metadata:
 
-Permissions for canned queries
-==============================
+Configuring permissions in metadata.json
+========================================
 
-Datasette's :ref:`canned queries ` default to allowing any user to execute them.
+You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`metadata` configuration.
 
-You can limit who is allowed to execute a specific query with the ``"allow"`` key in the :ref:`metadata` configuration for that query.
+You can control the following:
 
-Here's how to restrict access to a write query to just the "root" user:
+* Access to the entire Datasette instance
+* Access to specific databases
+* Access to specific tables and views
+* Access to specific :ref:`canned_queries`
+
+If a user cannot access a specific database, they will not be able to access tables, views or queries within that database. If a user cannot access the instance they will not be able to access any of the databases, tables, views or queries.
+
+.. _authentication_permissions_instance:
+
+Controlling access to an instance
+---------------------------------
+
+Here's how to restrict access to your entire Datasette instance to just the ``"id": "root"`` user:
 
 .. code-block:: json
 
     {
-        "databases": {
-            "mydatabase": {
-                "queries": {
-                    "add_name": {
-                        "sql": "INSERT INTO names (name) VALUES (:name)",
-                        "write": true,
-                        "allow": {
-                            "id": ["root"]
-                        }
-                    }
-                }
-            }
+        "title": "My private Datasette instance",
+        "allow": {
+            "id": "root"
         }
     }
 
@@ -126,6 +129,80 @@ If you want to provide access to any actor with a value for a specific key, use
 
 These keys act as an "or" mechanism. A actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block.
 
+.. _authentication_permissions_database:
+
+Controlling access to specific databases
+----------------------------------------
+
+To limit access to a specific ``private.db`` database to just authenticated users, use the ``"allow"`` block like this:
+
+.. code-block:: json
+
+    {
+        "databases": {
+            "private": {
+                "allow": {
+                    "id": "*"
+                }
+            }
+        }
+    }
+
+.. _authentication_permissions_table:
+
+Controlling access to specific tables and views
+-----------------------------------------------
+
+To limit access to the ``users`` table in your ``bakery.db`` database:
+
+.. code-block:: json
+
+    {
+        "databases": {
+            "bakery": {
+                "tables": {
+                    "users": {
+                        "allow": {
+                            "id": "*"
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+This works for SQL views as well - you can treat them as if they are tables.
+
+.. warning::
+    Restricting access to tables and views in this way will NOT prevent users from querying them using arbitrary SQL queries.
+
+    If you are restricting access to specific tables you should also use the ``"allow_sql"`` block to prevent users from accessing 
+
+.. _authentication_permissions_table:
+
+Controlling access to specific canned queries
+---------------------------------------------
+
+To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`:
+
+.. code-block:: json
+
+    {
+        "databases": {
+            "dogs": {
+                "queries": {
+                    "add_name": {
+                        "sql": "INSERT INTO names (name) VALUES (:name)",
+                        "write": true,
+                        "allow": {
+                            "id": ["root"]
+                        }
+                    }
+                }
+            }
+        }
+    }
+
 .. _authentication_actor_matches_allow:
 
 actor_matches_allow()
diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index 5df8bdb0..5295a2e0 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -217,7 +217,7 @@ Writable canned queries
 
 Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database.
 
-See :ref:`authentication_permissions_canned_queries` for details on how to add permission checks to canned queries, using the ``"allow"`` key.
+See :ref:`authentication_permissions_metadata` for details on how to add permission checks to canned queries, using the ``"allow"`` key.
 
 .. code-block:: json
 

From 8205d58316ced1d5ae589b29a5a1b5ecb6257ab0 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 13:10:40 -0700
Subject: [PATCH 0519/2303] Corrected documentation for resource in view-query

---
 docs/authentication.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index a6c4ee79..88808428 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -245,7 +245,6 @@ view-instance
 
 Top level permission - Actor is allowed to view any pages within this instance, starting at https://latest.datasette.io/
 
-
 .. _permissions_view_database:
 
 view-database
@@ -283,8 +282,8 @@ view-query
 
 Actor is allowed to view a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size
 
-``resource`` - string
-    The name of the canned query
+``resource`` - tuple: (string, string)
+    The name of the database, then the name of the canned query
 
 .. _permissions_execute_sql:
 

From e0a4664fbab5556454dac7f3c798253a34db2928 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 15:09:57 -0700
Subject: [PATCH 0520/2303] Better example plugin for permission_allowed

Also fixed it so default permission checks run after plugin permission checks, refs #818
---
 datasette/default_permissions.py |  2 +-
 docs/authentication.rst          |  4 ++--
 docs/plugins.rst                 | 40 ++++++++++++++++++++++++++++++--
 3 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index e989c0fa..a2f4a315 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -2,7 +2,7 @@ from datasette import hookimpl
 from datasette.utils import actor_matches_allow
 
 
-@hookimpl
+@hookimpl(tryfirst=True)
 def permission_allowed(datasette, actor, action, resource):
     if action == "permissions-debug":
         if actor and actor.get("id") == "root":
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 88808428..34d46511 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -174,11 +174,11 @@ To limit access to the ``users`` table in your ``bakery.db`` database:
 This works for SQL views as well - you can treat them as if they are tables.
 
 .. warning::
-    Restricting access to tables and views in this way will NOT prevent users from querying them using arbitrary SQL queries.
+    Restricting access to tables and views in this way will NOT prevent users from querying them using arbitrary SQL queries, `like this `__ for example.
 
     If you are restricting access to specific tables you should also use the ``"allow_sql"`` block to prevent users from accessing 
 
-.. _authentication_permissions_table:
+.. _authentication_permissions_query:
 
 Controlling access to specific canned queries
 ---------------------------------------------
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 6b1e60f2..73d2eabd 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1006,7 +1006,7 @@ Instead of returning a dictionary, this function can return an awaitable functio
 .. _plugin_permission_allowed:
 
 permission_allowed(datasette, actor, action, resource)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
@@ -1022,4 +1022,40 @@ permission_allowed(datasette, actor, action, resource)
 
 Called to check that an actor has permission to perform an action on a resource. Can return ``True`` if the action is allowed, ``False`` if the action is not allowed or ``None`` if the plugin does not have an opinion one way or the other.
 
-See :ref:`permissions` for a full list of permissions included in Datasette core.
+Here's an example plugin which randomly selects if a permission should be allowed or denied, except for ``view-instance`` which always uses the default permission scheme instead.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import random
+
+    @hookimpl
+    def permission_allowed(action):
+        if action != "view-instance":
+            # Return True or False at random
+            return random.random() > 0.5
+        # Returning None falls back to default permissions
+
+This function can alternatively return an awaitable function which itself returns ``True``, ``False`` or ``None``. You can use this option if you need to execute additional database queries using ``await datasette.execute(...)``.
+
+Here's an example that allows users to view the ``admin_log`` table only if their actor ``id`` is present in the ``admin_users`` table. It aso disallows arbitrary SQL queries for the ``staff.db`` database for all users.
+
+.. code-block:: python
+
+    @hookimpl
+    def permission_allowed(datasette, actor, action, resource):
+        async def inner():
+            if action == "execute-sql" and resource == "staff":
+                return False
+            if action == "view-table" and resource == ("staff", "admin_log"):
+                if not actor:
+                    return False
+                user_id = actor["id"]
+                return await datasette.get_database("staff").execute(
+                    "select count(*) from admin_users where user_id = :user_id",
+                    {"user_id": user_id},
+                )
+
+        return inner
+
+See :ref:`permissions` for a full list of permissions that are included in Datasette core.

From 49d6d2f7b0f6cb02e25022e1c9403811f1fa0a7c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 17:05:44 -0700
Subject: [PATCH 0521/2303] allow_sql block to control execute-sql upermission
 in metadata.json, closes #813

Also removed the --config allow_sql:0 mechanism in favour of the new allow_sql block.
---
 datasette/app.py                  |  1 -
 datasette/default_permissions.py  |  8 ++++++++
 datasette/templates/database.html |  2 +-
 datasette/templates/query.html    |  2 +-
 datasette/templates/table.html    |  2 +-
 datasette/views/database.py       |  8 ++++++--
 datasette/views/table.py          |  9 +++++++--
 docs/authentication.rst           | 33 ++++++++++++++++++++++++++++++-
 docs/config.rst                   |  9 ---------
 docs/json_api.rst                 |  2 +-
 docs/pages.rst                    |  2 +-
 docs/sql_queries.rst              |  4 ++--
 tests/test_api.py                 | 12 ++---------
 tests/test_config_dir.py          |  3 ---
 tests/test_html.py                | 10 +---------
 tests/test_permissions.py         | 29 +++++++++++++++++++++++++++
 16 files changed, 92 insertions(+), 44 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 2f89d17c..a7c3c66a 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -110,7 +110,6 @@ CONFIG_OPTIONS = (
         "Allow users to download the original SQLite database files",
     ),
     ConfigOption("suggest_facets", True, "Calculate and display suggested facets"),
-    ConfigOption("allow_sql", True, "Allow arbitrary SQL queries via ?sql= parameter"),
     ConfigOption(
         "default_cache_ttl",
         5,
diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index a2f4a315..e750acbf 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -34,3 +34,11 @@ def permission_allowed(datasette, actor, action, resource):
         if allow is None:
             return True
         return actor_matches_allow(actor, allow)
+    elif action == "execute-sql":
+        # Use allow_sql block from database block, or from top-level
+        database_allow_sql = datasette.metadata("allow_sql", database=resource)
+        if database_allow_sql is None:
+            database_allow_sql = datasette.metadata("allow_sql")
+        if database_allow_sql is None:
+            return True
+        return actor_matches_allow(actor, database_allow_sql)
diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index 100faee4..5ae51ef7 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -22,7 +22,7 @@
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
-{% if config.allow_sql %}
+{% if allow_execute_sql %}
     
      
         
      Custom SQL query
      
         
      
diff --git a/datasette/templates/query.html b/datasette/templates/query.html
index 7771b101..c65953fb 100644
--- a/datasette/templates/query.html
+++ b/datasette/templates/query.html
@@ -35,7 +35,7 @@
 
     
      Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %} {% if hide_sql %}(show){% else %}(hide){% endif %}
      
     {% if not hide_sql %}
-        {% if editable and config.allow_sql %}
+        {% if editable and allow_execute_sql %}
             
      
         {% else %}
             
      {% if query %}{{ query.sql }}{% endif %}
      
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index 1289e125..373fd576 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -109,7 +109,7 @@
 
 {% endif %}
 
-{% if query.sql and config.allow_sql %}
+{% if query.sql and allow_execute_sql %}
     
      ✎ View and edit SQL
      
 {% endif %}
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index e1b29c27..ee99bc2d 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -26,8 +26,6 @@ class DatabaseView(DataView):
         self.ds.update_with_inherited_metadata(metadata)
 
         if request.args.get("sql"):
-            if not self.ds.config("allow_sql"):
-                raise DatasetteError("sql= is not allowed", status=400)
             sql = request.args.get("sql")
             validate_sql_select(sql)
             return await QueryView(self.ds).data(
@@ -90,6 +88,9 @@ class DatabaseView(DataView):
                 "private": not await self.ds.permission_allowed(
                     None, "view-database", database
                 ),
+                "allow_execute_sql": await self.ds.permission_allowed(
+                    request.actor, "execute-sql", database, default=True
+                ),
             },
             {
                 "show_hidden": request.args.get("_show_hidden"),
@@ -289,6 +290,9 @@ class QueryView(DataView):
                 "columns": columns,
                 "query": {"sql": sql, "params": params},
                 "private": private,
+                "allow_execute_sql": await self.ds.permission_allowed(
+                    request.actor, "execute-sql", database, default=True
+                ),
             },
             extra_template,
             templates,
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 4cec0cda..91245293 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -342,8 +342,10 @@ class TableView(RowTableShared):
         extra_wheres_for_ui = []
         # Add _where= from querystring
         if "_where" in request.args:
-            if not self.ds.config("allow_sql"):
-                raise DatasetteError("_where= is not allowed", status=400)
+            if not await self.ds.permission_allowed(
+                request.actor, "execute-sql", resource=database, default=True,
+            ):
+                raise DatasetteError("_where= is not allowed", status=403)
             else:
                 where_clauses.extend(request.args.getlist("_where"))
                 extra_wheres_for_ui = [
@@ -839,6 +841,9 @@ class TableView(RowTableShared):
                 "next": next_value and str(next_value) or None,
                 "next_url": next_url,
                 "private": private,
+                "allow_execute_sql": await self.ds.permission_allowed(
+                    request.actor, "execute-sql", database, default=True
+                ),
             },
             extra_template,
             (
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 34d46511..f7281db4 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -176,7 +176,7 @@ This works for SQL views as well - you can treat them as if they are tables.
 .. warning::
     Restricting access to tables and views in this way will NOT prevent users from querying them using arbitrary SQL queries, `like this `__ for example.
 
-    If you are restricting access to specific tables you should also use the ``"allow_sql"`` block to prevent users from accessing 
+    If you are restricting access to specific tables you should also use the ``"allow_sql"`` block to prevent users from bypassing the limit with their own SQL queries - see :ref:`authentication_permissions_execute_sql`.
 
 .. _authentication_permissions_query:
 
@@ -203,6 +203,37 @@ To limit access to the ``add_name`` canned query in your ``dogs.db`` database to
         }
     }
 
+.. _authentication_permissions_execute_sql:
+
+Controlling the ability to execute arbitrary SQL
+------------------------------------------------
+
+The ``"allow_sql"`` block can be used to control who is allowed to execute arbitrary SQL queries, both using the form on the database page e.g. https://latest.datasette.io/fixtures or by appending a ``?_where=`` parameter to the table page as seen on https://latest.datasette.io/fixtures/facetable?_where=city_id=1.
+
+To enable just the :ref:`root user` to execute SQL for all databases in your instance, use the following:
+
+.. code-block:: json
+
+    {
+        "allow_sql": {
+            "id": "root"
+        }
+    }
+
+To limit this ability for just one specific database, use this:
+
+.. code-block:: json
+
+    {
+        "databases": {
+            "mydatabase": {
+                "allow_sql": {
+                    "id": "root"
+                }
+            }
+        }
+    }
+
 .. _authentication_actor_matches_allow:
 
 actor_matches_allow()
diff --git a/docs/config.rst b/docs/config.rst
index da93e40a..56b38613 100644
--- a/docs/config.rst
+++ b/docs/config.rst
@@ -150,15 +150,6 @@ Should users be able to download the original SQLite database using a link on th
 
     datasette mydatabase.db --config allow_download:off
 
-.. _config_allow_sql:
-
-allow_sql
-~~~~~~~~~
-
-Enable/disable the ability for users to run custom SQL directly against a database. To disable this feature, run::
-
-    datasette mydatabase.db --config allow_sql:off
-
 .. _config_default_cache_ttl:
 
 default_cache_ttl
diff --git a/docs/json_api.rst b/docs/json_api.rst
index 7d37d425..af98eecd 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -291,7 +291,7 @@ Special table arguments
     though this could potentially result in errors if the wrong syntax is used.
 
 ``?_where=SQL-fragment``
-    If the :ref:`config_allow_sql` config option is enabled, this parameter
+    If the :ref:`permissions_execute_sql` permission is enabled, this parameter
     can be used to pass one or more additional SQL fragments to be used in the
     `WHERE` clause of the SQL used to query the table.
 
diff --git a/docs/pages.rst b/docs/pages.rst
index f220f94d..ce8f5d06 100644
--- a/docs/pages.rst
+++ b/docs/pages.rst
@@ -29,7 +29,7 @@ Database
 ========
 
 Each database has a page listing the tables, views and canned queries
-available for that database. If the :ref:`config_allow_sql` config option is enabled (it's turned on by default) there will also be an interface for executing arbitrary SQL select queries against the data.
+available for that database. If the :ref:`permissions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data.
 
 Examples:
 
diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index 5295a2e0..db72deb7 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -12,8 +12,8 @@ you like. You can also construct queries using the filter interface on the
 tables page, then click "View and edit SQL" to open that query in the custom
 SQL editor.
 
-Note that this interface is only available if the :ref:`config_allow_sql` option
-has not been disabled.
+Note that this interface is only available if the :ref:`permissions_execute_sql`
+permission is allowed.
 
 Any Datasette SQL query is reflected in the URL of the page, allowing you to
 bookmark them, share them with others and navigate through previous queries
diff --git a/tests/test_api.py b/tests/test_api.py
index 13a98b6a..1a54edec 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -634,13 +634,6 @@ def test_invalid_custom_sql(app_client):
     assert "Statement must be a SELECT" == response.json["error"]
 
 
-def test_allow_sql_off():
-    with make_app_client(config={"allow_sql": False}) as client:
-        response = client.get("/fixtures.json?sql=select+sleep(0.01)")
-        assert 400 == response.status
-        assert "sql= is not allowed" == response.json["error"]
-
-
 def test_table_json(app_client):
     response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects")
     assert response.status == 200
@@ -1137,9 +1130,9 @@ def test_table_filter_extra_where_invalid(app_client):
 
 
 def test_table_filter_extra_where_disabled_if_no_sql_allowed():
-    with make_app_client(config={"allow_sql": False}) as client:
+    with make_app_client(metadata={"allow_sql": {}}) as client:
         response = client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'")
-        assert 400 == response.status
+        assert 403 == response.status
         assert "_where= is not allowed" == response.json["error"]
 
 
@@ -1325,7 +1318,6 @@ def test_config_json(app_client):
         "allow_download": True,
         "allow_facet": True,
         "suggest_facets": True,
-        "allow_sql": True,
         "default_cache_ttl": 5,
         "default_cache_ttl_hashed": 365 * 24 * 60 * 60,
         "num_sql_threads": 3,
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index 490b1f1d..b1f6994f 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -10,7 +10,6 @@ from datasette import hookimpl
 
 @hookimpl
 def extra_template_vars():
-    print("this is template vars")
     return {
         "from_plugin": "hooray"
     }
@@ -18,7 +17,6 @@ def extra_template_vars():
 METADATA = {"title": "This is from metadata"}
 CONFIG = {
     "default_cache_ttl": 60,
-    "allow_sql": False,
 }
 CSS = """
 body { margin-top: 3em}
@@ -91,7 +89,6 @@ def test_config(config_dir_client):
     response = config_dir_client.get("/-/config.json")
     assert 200 == response.status
     assert 60 == response.json["default_cache_ttl"]
-    assert not response.json["allow_sql"]
 
 
 def test_plugins(config_dir_client):
diff --git a/tests/test_html.py b/tests/test_html.py
index cb0e0c90..e6933dfe 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -924,16 +924,8 @@ def test_allow_download_off():
         assert 403 == response.status
 
 
-def test_allow_sql_on(app_client):
-    response = app_client.get("/fixtures")
-    soup = Soup(response.body, "html.parser")
-    assert len(soup.findAll("textarea", {"name": "sql"}))
-    response = app_client.get("/fixtures/sortable")
-    assert b"View and edit SQL" in response.body
-
-
 def test_allow_sql_off():
-    with make_app_client(config={"allow_sql": False}) as client:
+    with make_app_client(metadata={"allow_sql": {}}) as client:
         response = client.get("/fixtures")
         soup = Soup(response.body, "html.parser")
         assert not len(soup.findAll("textarea", {"name": "sql"}))
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 90ba1494..d8c98825 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -186,6 +186,35 @@ def test_view_query(allow, expected_anon, expected_auth):
             assert ">fixtures 🔒" in auth_response.text
 
 
+@pytest.mark.parametrize(
+    "metadata",
+    [
+        {"allow_sql": {"id": "root"}},
+        {"databases": {"fixtures": {"allow_sql": {"id": "root"}}}},
+    ],
+)
+def test_execute_sql(metadata):
+    with make_app_client(metadata=metadata) as client:
+        form_fragment = '
Date: Mon, 8 Jun 2020 17:35:23 -0700
Subject: [PATCH 0522/2303] Fixed broken CSS on 404 page, closes #777

---
 datasette/app.py   | 11 ++++++++++-
 tests/test_html.py | 12 ++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index a7c3c66a..d562e611 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1015,7 +1015,16 @@ class DatasetteRouter(AsgiRouter):
         templates = ["500.html"]
         if status != 500:
             templates = ["{}.html".format(status)] + templates
-        info.update({"ok": False, "error": message, "status": status, "title": title})
+        info.update(
+            {
+                "ok": False,
+                "error": message,
+                "status": status,
+                "title": title,
+                "base_url": self.ds.config("base_url"),
+                "app_css_hash": self.ds.app_css_hash(),
+            }
+        )
         headers = {}
         if self.ds.cors:
             headers["Access-Control-Allow-Origin"] = "*"
diff --git a/tests/test_html.py b/tests/test_html.py
index e6933dfe..f9b18daa 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -965,6 +965,18 @@ def inner_html(soup):
     return inner_html.strip()
 
 
+@pytest.mark.parametrize("path", ["/404", "/fixtures/404"])
+def test_404(app_client, path):
+    response = app_client.get(path)
+    assert 404 == response.status
+    assert (
+        '
Date: Mon, 8 Jun 2020 19:22:40 -0700
Subject: [PATCH 0523/2303] Fixed test_table_not_exists_json test

---
 datasette/app.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index d562e611..79f52a54 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1016,14 +1016,7 @@ class DatasetteRouter(AsgiRouter):
         if status != 500:
             templates = ["{}.html".format(status)] + templates
         info.update(
-            {
-                "ok": False,
-                "error": message,
-                "status": status,
-                "title": title,
-                "base_url": self.ds.config("base_url"),
-                "app_css_hash": self.ds.app_css_hash(),
-            }
+            {"ok": False, "error": message, "status": status, "title": title,}
         )
         headers = {}
         if self.ds.cors:
@@ -1033,7 +1026,16 @@ class DatasetteRouter(AsgiRouter):
         else:
             template = self.ds.jinja_env.select_template(templates)
             await asgi_send_html(
-                send, await template.render_async(info), status=status, headers=headers
+                send,
+                await template.render_async(
+                    dict(
+                        info,
+                        base_url=self.ds.config("base_url"),
+                        app_css_hash=self.ds.app_css_hash(),
+                    )
+                ),
+                status=status,
+                headers=headers,
             )
 
 

From f5e79adf26d0daa3831e3fba022f1b749a9efdee Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 20:12:06 -0700
Subject: [PATCH 0524/2303] register_routes() plugin hook (#819)

Fixes #215
---
 datasette/app.py            | 21 ++++++++++++++++
 datasette/hookspecs.py      |  5 ++++
 datasette/utils/__init__.py | 12 ++++++++-
 datasette/utils/asgi.py     |  2 +-
 docs/index.rst              |  2 +-
 docs/plugins.rst            | 50 ++++++++++++++++++++++++++++++++++++-
 tests/fixtures.py           |  1 +
 tests/plugins/my_plugin.py  | 25 +++++++++++++++++++
 tests/test_plugins.py       | 15 +++++++++++
 9 files changed, 129 insertions(+), 4 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 79f52a54..120091f7 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -39,6 +39,7 @@ from .renderer import json_renderer
 from .database import Database, QueryInterrupted
 
 from .utils import (
+    async_call_with_supported_arguments,
     escape_css_string,
     escape_sqlite,
     format_bytes,
@@ -783,6 +784,10 @@ class Datasette:
         "Returns an ASGI app function that serves the whole of Datasette"
         routes = []
 
+        for routes_to_add in pm.hook.register_routes():
+            for regex, view_fn in routes_to_add:
+                routes.append((regex, wrap_view(view_fn, self)))
+
         def add_route(view, regex):
             routes.append((regex, view))
 
@@ -1048,3 +1053,19 @@ def _cleaner_task_str(task):
     # running at /Users/simonw/Dropbox/Development/datasette/venv-3.7.5/lib/python3.7/site-packages/uvicorn/main.py:361>
     # Clean up everything up to and including site-packages
     return _cleaner_task_str_re.sub("", s)
+
+
+def wrap_view(view_fn, datasette):
+    async def asgi_view_fn(scope, receive, send):
+        response = await async_call_with_supported_arguments(
+            view_fn,
+            scope=scope,
+            receive=receive,
+            send=send,
+            request=Request(scope, receive),
+            datasette=datasette,
+        )
+        if response is not None:
+            await response.asgi_send(send)
+
+    return asgi_view_fn
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index d5fd232f..ab3e131c 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -60,6 +60,11 @@ def register_facet_classes():
     "Register Facet subclasses"
 
 
+@hookspec
+def register_routes():
+    "Register URL routes: return a list of (regex, view_function) pairs"
+
+
 @hookspec
 def actor_from_request(datasette, request):
     "Return an actor dictionary based on the incoming request"
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 7c1f34e0..49268638 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -842,7 +842,7 @@ def parse_metadata(content):
             raise BadMetadataError("Metadata is not valid JSON or YAML")
 
 
-def call_with_supported_arguments(fn, **kwargs):
+def _gather_arguments(fn, kwargs):
     parameters = inspect.signature(fn).parameters.keys()
     call_with = []
     for parameter in parameters:
@@ -853,9 +853,19 @@ def call_with_supported_arguments(fn, **kwargs):
                 )
             )
         call_with.append(kwargs[parameter])
+    return call_with
+
+
+def call_with_supported_arguments(fn, **kwargs):
+    call_with = _gather_arguments(fn, kwargs)
     return fn(*call_with)
 
 
+async def async_call_with_supported_arguments(fn, **kwargs):
+    call_with = _gather_arguments(fn, kwargs)
+    return await fn(*call_with)
+
+
 def actor_matches_allow(actor, allow):
     actor = actor or {}
     if allow is None:
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index bca9c9ab..349f2a0a 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -399,7 +399,7 @@ class Response:
     @classmethod
     def text(cls, body, status=200, headers=None):
         return cls(
-            body,
+            str(body),
             status=status,
             headers=headers,
             content_type="text/plain; charset=utf-8",
diff --git a/docs/index.rst b/docs/index.rst
index 03988c8e..5334386f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -31,7 +31,7 @@ Contents
 --------
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
 
    getting_started
    installation
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 73d2eabd..caca0019 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -835,6 +835,55 @@ And here is an example ``can_render`` function which returns ``True`` only if th
 
 Examples: `datasette-atom `_, `datasette-ics `_
 
+.. _plugin_register_routes:
+
+register_routes()
+~~~~~~~~~~~~~~~~~
+
+Register additional view functions to execute for specified URL routes.
+
+Return a list of ``(regex, async_view_function)`` pairs, something like this:
+
+.. code-block:: python
+
+    from datasette.utils.asgi import Response
+    import html
+
+
+    async def hello_from(scope):
+        name = scope["url_route"]["kwargs"]["name"]
+        return Response.html("Hello from {}".format(
+            html.escape(name)
+        ))
+
+
+    @hookimpl
+    def register_routes():
+        return [
+            (r"^/hello-from/(?P.*)$"), hello_from)
+        ]
+
+The view functions can take a number of different optional arguments. The corresponding argument will be passed to your function depending on its named parameters - a form of dependency injection.
+
+The optional view function arguments are as follows:
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
+
+``request`` - Request object
+    The current HTTP :ref:`internals_request`.
+
+``scope`` - dictionary
+    The incoming ASGI scope dictionary.
+
+``send`` - function
+    The ASGI send function.
+
+``receive`` - function
+    The ASGI receive function.
+
+The function can either return a ``Response`` or it can return nothing and instead respond directly to the request using the ASGI ``receive`` function (for advanced uses only).
+
 .. _plugin_register_facet_classes:
 
 register_facet_classes()
@@ -901,7 +950,6 @@ The plugin hook can then be used to register the new facet class like this:
     def register_facet_classes():
         return [SpecialFacet]
 
-
 .. _plugin_asgi_wrapper:
 
 asgi_wrapper(datasette)
diff --git a/tests/fixtures.py b/tests/fixtures.py
index e9175b57..a51a869d 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -46,6 +46,7 @@ EXPECTED_PLUGINS = [
             "prepare_connection",
             "prepare_jinja2_environment",
             "register_facet_classes",
+            "register_routes",
             "render_cell",
         ],
     },
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 46893710..57803178 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -1,6 +1,7 @@
 from datasette import hookimpl
 from datasette.facets import Facet
 from datasette.utils import path_with_added_args
+from datasette.utils.asgi import asgi_send_json, Response
 import base64
 import pint
 import json
@@ -142,3 +143,27 @@ def permission_allowed(actor, action):
         return True
     elif action == "this_is_denied":
         return False
+
+
+@hookimpl
+def register_routes():
+    async def one(datasette):
+        return Response.text(
+            (await datasette.get_database().execute("select 1 + 1")).first()[0]
+        )
+
+    async def two(request, scope):
+        name = scope["url_route"]["kwargs"]["name"]
+        greeting = request.args.get("greeting")
+        return Response.text("{} {}".format(greeting, name))
+
+    async def three(scope, send):
+        await asgi_send_json(
+            send, {"hello": "world"}, status=200, headers={"x-three": "1"}
+        )
+
+    return [
+        (r"/one/$", one),
+        (r"/two/(?P.*)$", two),
+        (r"/three/$", three),
+    ]
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index c782b87b..c7bb4859 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -544,3 +544,18 @@ def test_actor_json(app_client):
     assert {"actor": {"id": "bot2", "1+1": 2}} == app_client.get(
         "/-/actor.json/?_bot2=1"
     ).json
+
+
+@pytest.mark.parametrize(
+    "path,body", [("/one/", "2"), ("/two/Ray?greeting=Hail", "Hail Ray"),]
+)
+def test_register_routes(app_client, path, body):
+    response = app_client.get(path)
+    assert 200 == response.status
+    assert body == response.text
+
+
+def test_register_routes_asgi(app_client):
+    response = app_client.get("/three/")
+    assert {"hello": "world"} == response.json
+    assert "1" == response.headers["x-three"]

From db660db4632409334e646237c3dd214764729cd4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 20:32:10 -0700
Subject: [PATCH 0525/2303] Docs + unit tests for Response, closes #821

---
 datasette/utils/asgi.py          |  9 ++++++
 docs/internals.rst               | 48 ++++++++++++++++++++++++++++++++
 docs/plugins.rst                 |  2 +-
 tests/test_internals_response.py | 28 +++++++++++++++++++
 4 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_internals_response.py

diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 349f2a0a..9e6c82dd 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -405,6 +405,15 @@ class Response:
             content_type="text/plain; charset=utf-8",
         )
 
+    @classmethod
+    def json(cls, body, status=200, headers=None):
+        return cls(
+            json.dumps(body),
+            status=status,
+            headers=headers,
+            content_type="application/json; charset=utf-8",
+        )
+
     @classmethod
     def redirect(cls, path, status=302, headers=None):
         headers = headers or {}
diff --git a/docs/internals.rst b/docs/internals.rst
index 83dbd897..b0096cfa 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -80,6 +80,54 @@ Consider the querystring ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` an
 ``len(request.args)`` - integer
     Returns the number of keys.
 
+.. _internals_response:
+
+Response class
+~~~~~~~~~~~~~~
+
+The ``Response`` class can be returned from view functions that have been registered using the :ref:`plugin_register_routes` hook.
+
+The ``Response()`` constructor takes the following arguments:
+
+``body`` - string
+    The body of the response.
+
+``status`` - integer (optional)
+    The HTTP status - defaults to 200.
+
+``headers`` - dictionary (optional)
+    A dictionary of extra HTTP headers, e.g. ``{"x-hello": "world"}``.
+
+``content_type`` - string (optional)
+    The content-type for the response. Defaults to ``text/plain``.
+
+For example:
+
+.. code-block:: python
+
+    from datasette.utils.asgi import Response
+
+    response = Response(
+        "This is XML",
+        content_type="application/xml; charset=utf-8"
+    )
+
+The easiest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods:
+
+.. code-block:: python
+
+    from datasette.utils.asgi import Response
+
+    html_response = Response.html("This is HTML")
+    json_response = Response.json({"this_is": "json"})
+    text_response = Response.text("This will become utf-8 encoded text")
+    # Redirects are served as 302, unless you pass status=301:
+    redirect_response = Response.redirect("https://latest.datasette.io/")
+
+Each of these responses will use the correct corresponding content-type - ``text/html; charset=utf-8``, ``application/json; charset=utf-8`` or ``text/plain; charset=utf-8`` respectively.
+
+Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above.
+
 .. _internals_datasette:
 
 Datasette class
diff --git a/docs/plugins.rst b/docs/plugins.rst
index caca0019..465fcd52 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -882,7 +882,7 @@ The optional view function arguments are as follows:
 ``receive`` - function
     The ASGI receive function.
 
-The function can either return a ``Response`` or it can return nothing and instead respond directly to the request using the ASGI ``receive`` function (for advanced uses only).
+The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only).
 
 .. _plugin_register_facet_classes:
 
diff --git a/tests/test_internals_response.py b/tests/test_internals_response.py
new file mode 100644
index 00000000..7c11f858
--- /dev/null
+++ b/tests/test_internals_response.py
@@ -0,0 +1,28 @@
+from datasette.utils.asgi import Response
+
+
+def test_response_html():
+    response = Response.html("Hello from HTML")
+    assert 200 == response.status
+    assert "Hello from HTML" == response.body
+    assert "text/html; charset=utf-8" == response.content_type
+
+
+def test_response_text():
+    response = Response.text("Hello from text")
+    assert 200 == response.status
+    assert "Hello from text" == response.body
+    assert "text/plain; charset=utf-8" == response.content_type
+
+
+def test_response_json():
+    response = Response.json({"this_is": "json"})
+    assert 200 == response.status
+    assert '{"this_is": "json"}' == response.body
+    assert "application/json; charset=utf-8" == response.content_type
+
+
+def test_response_redirect():
+    response = Response.redirect("/foo")
+    assert 302 == response.status
+    assert "/foo" == response.headers["Location"]

From fac8e9381500fc02cec99281122ee8e0c72fabe1 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 20:40:00 -0700
Subject: [PATCH 0526/2303] request.url_vars property, closes #822

---
 datasette/utils/asgi.py         |  4 ++++
 docs/internals.rst              |  3 +++
 docs/plugins.rst                |  4 ++--
 tests/plugins/my_plugin.py      |  4 ++--
 tests/test_internals_request.py | 17 +++++++++++++++++
 5 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 9e6c82dd..cdd6b148 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -32,6 +32,10 @@ class Request:
             (self.scheme, self.host, self.path, None, self.query_string, None)
         )
 
+    @property
+    def url_vars(self):
+        return (self.scope.get("url_route") or {}).get("kwargs") or {}
+
     @property
     def scheme(self):
         return self.scope.get("scheme") or "http"
diff --git a/docs/internals.rst b/docs/internals.rst
index b0096cfa..df21eb09 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -42,6 +42,9 @@ The request object is passed to various plugin hooks. It represents an incoming
 ``.args`` - MultiParams
     An object representing the parsed querystring parameters, see below.
 
+``.url_vars`` - dictionary (str -> str)
+    Variables extracted from the URL path, if that path was defined using a regular expression. See :ref:`plugin_register_routes`.
+
 ``.actor`` - dictionary (str -> Any) or None
     The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated.
 
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 465fcd52..17fd64df 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -850,8 +850,8 @@ Return a list of ``(regex, async_view_function)`` pairs, something like this:
     import html
 
 
-    async def hello_from(scope):
-        name = scope["url_route"]["kwargs"]["name"]
+    async def hello_from(request):
+        name = request.url_vars["name"]
         return Response.html("Hello from {}".format(
             html.escape(name)
         ))
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 57803178..a0f7441b 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -152,8 +152,8 @@ def register_routes():
             (await datasette.get_database().execute("select 1 + 1")).first()[0]
         )
 
-    async def two(request, scope):
-        name = scope["url_route"]["kwargs"]["name"]
+    async def two(request):
+        name = request.url_vars["name"]
         greeting = request.args.get("greeting")
         return Response.text("{} {}".format(greeting, name))
 
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index 433b23d5..8367a693 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -44,3 +44,20 @@ def test_request_args():
     assert 2 == len(request.args)
     with pytest.raises(KeyError):
         request.args["missing"]
+
+
+def test_request_url_vars():
+    scope = {
+        "http_version": "1.1",
+        "method": "POST",
+        "path": "/",
+        "raw_path": b"/",
+        "query_string": b"",
+        "scheme": "http",
+        "type": "http",
+        "headers": [[b"content-type", b"application/x-www-form-urlencoded"]],
+    }
+    assert {} == Request(scope, None).url_vars
+    assert {"name": "cleo"} == Request(
+        dict(scope, url_route={"kwargs": {"name": "cleo"}}), None
+    ).url_vars

From 5a6a73e3190cac103906b479d56129413e5ef190 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 21:37:35 -0700
Subject: [PATCH 0527/2303] Replace os.urandom(32).hex() with
 secrets.token_hex(32)

---
 datasette/app.py | 5 +++--
 docs/config.rst  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 120091f7..633ca4fe 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -8,6 +8,7 @@ import itertools
 import json
 import os
 import re
+import secrets
 import sys
 import threading
 import traceback
@@ -186,7 +187,7 @@ class Datasette:
         assert config_dir is None or isinstance(
             config_dir, Path
         ), "config_dir= should be a pathlib.Path"
-        self._secret = secret or os.urandom(32).hex()
+        self._secret = secret or secrets.token_hex(32)
         self.files = tuple(files) + tuple(immutables or [])
         if config_dir:
             self.files += tuple([str(p) for p in config_dir.glob("*.db")])
@@ -299,7 +300,7 @@ class Datasette:
 
         self._register_renderers()
         self._permission_checks = collections.deque(maxlen=200)
-        self._root_token = os.urandom(32).hex()
+        self._root_token = secrets.token_hex(32)
 
     def sign(self, value, namespace="default"):
         return URLSafeSerializer(self._secret, namespace).dumps(value)
diff --git a/docs/config.rst b/docs/config.rst
index 56b38613..ab14ea7b 100644
--- a/docs/config.rst
+++ b/docs/config.rst
@@ -302,7 +302,7 @@ Or::
 
 One way to generate a secure random secret is to use Python like this::
 
-    $ python3 -c 'import os; print(os.urandom(32).hex())'
+    $ python3 -c 'import secrets; print(secrets.token_hex(32))'
     cdb19e94283a20f9d42cca50c5a4871c0aa07392db308755d60a1a5b9bb0fa52
 
 Plugin authors make use of this signing mechanism in their plugins using :ref:`datasette_sign` and :ref:`datasette_unsign`.

From eb3ec279becd3b81e5fa509244711548c86f434f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 8 Jun 2020 23:33:06 -0700
Subject: [PATCH 0528/2303] Test for anonymous: true, refs #825

---
 tests/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_utils.py b/tests/test_utils.py
index 975ed0fd..4bade18b 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -466,6 +466,7 @@ def test_multi_params(data, should_raise):
     [
         ({"id": "root"}, None, True),
         ({"id": "root"}, {}, False),
+        ({"anonymous": True}, {"anonymous": True}, True),
         (None, None, True),
         (None, {}, False),
         (None, {"id": "root"}, False),

From fec750435d405ac06cb61a5ddeda7317ef24843a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 07:01:23 -0700
Subject: [PATCH 0529/2303] Support anonymous: true in actor_matches_allow,
 refs #825

---
 datasette/utils/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 49268638..d8cde95a 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -873,12 +873,12 @@ def actor_matches_allow(actor, allow):
     for key, values in allow.items():
         if values == "*" and key in actor:
             return True
-        if isinstance(values, str):
+        if not isinstance(values, list):
             values = [values]
         actor_values = actor.get(key)
         if actor_values is None:
             return False
-        if isinstance(actor_values, str):
+        if not isinstance(actor_values, list):
             actor_values = [actor_values]
         actor_values = set(actor_values)
         if actor_values.intersection(values):

From eefeafaa27a16af3bcb3150b4fe1ef6ee8d5c19f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 07:09:39 -0700
Subject: [PATCH 0530/2303] Removed unused import

---
 datasette/views/database.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index ee99bc2d..4fab2cfb 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -2,7 +2,6 @@ import os
 import jinja2
 
 from datasette.utils import (
-    actor_matches_allow,
     check_visibility,
     to_css_class,
     validate_sql_select,

From fa87d16612ff671683f35ecc5f5e36af007599e4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 07:10:46 -0700
Subject: [PATCH 0531/2303] Clearer docs for actor_matches_allow

---
 datasette/utils/__init__.py | 3 ++-
 docs/authentication.rst     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index d8cde95a..5873fcaa 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -867,7 +867,8 @@ async def async_call_with_supported_arguments(fn, **kwargs):
 
 
 def actor_matches_allow(actor, allow):
-    actor = actor or {}
+    if actor is None:
+        actor = {"anonymous": True}
     if allow is None:
         return True
     for key, values in allow.items():
diff --git a/docs/authentication.rst b/docs/authentication.rst
index f7281db4..04564886 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -239,7 +239,7 @@ To limit this ability for just one specific database, use this:
 actor_matches_allow()
 =====================
 
-Plugins that wish to implement the same permissions scheme as canned queries can take advantage of the ``datasette.utils.actor_matches_allow(actor, allow)`` function:
+Plugins that wish to implement this same ``"allow"`` block permissions scheme can take advantage of the ``datasette.utils.actor_matches_allow(actor, allow)`` function:
 
 .. code-block:: python
 

From 3aa87eeaf21083e32d9e02bd857fd44707dc4113 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 07:58:12 -0700
Subject: [PATCH 0532/2303] Documentation no loger suggests that actor["id"] is
 required, closes #823

---
 docs/authentication.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index 04564886..153466ad 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -17,7 +17,7 @@ Through plugins, Datasette can support both authenticated users (with cookies) a
 
 Every request to Datasette has an associated actor value, available in the code as ``request.actor``. This can be ``None`` for unauthenticated requests, or a JSON compatible Python dictionary for authenticated users or API agents.
 
-The only required field in an actor is ``"id"``, which must be a string. Plugins may decide to add any other fields to the actor dictionary.
+The actor dictionary can be any shape - the design of that data structure is left up to the plugins. A useful convention is to include an ``"id"`` string, as demonstrated by the "root" actor below.
 
 Plugins can use the :ref:`plugin_actor_from_request` hook to implement custom logic for authenticating an actor based on the incoming HTTP request.
 

From 70dd14876e305ddb15263ec0687e23bef5b1ab78 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 09:04:46 -0700
Subject: [PATCH 0533/2303] Improved documentation for permissions, refs #699

---
 docs/authentication.rst | 37 +++++++++++++++++++++++++++++--------
 docs/sql_queries.rst    |  6 ++++++
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index 153466ad..e26c8fc5 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -4,7 +4,7 @@
  Authentication and permissions
 ================================
 
-Datasette does not require authentication by default. Any visitor to a Datasette instance can explore the full data and execute SQL queries.
+Datasette does not require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries.
 
 Datasette's plugin system can be used to add many different styles of authentication, such as user accounts, single sign-on or API keys.
 
@@ -49,10 +49,20 @@ The URL on the first line includes a one-use token which can be used to sign in
 
 .. _authentication_permissions:
 
-Checking permission
-===================
+Permissions
+===========
 
-Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method. This method is also used by Datasette core code itself, which allows plugins to help make decisions on which actions are allowed by implementing the :ref:`plugin_permission_allowed` plugin hook.
+Datasette has an extensive permissions system built-in, which can be further extended and customized by plugins.
+
+The key question the permissions system answers is this:
+
+    Is this **actor** allowed to perform this **action**, optionally against this particular **resource**?
+
+**Actors** are :ref:`described above `.
+
+An **action** is a string describing the action the actor would like to perfom. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``.
+
+A **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource.
 
 .. _authentication_permissions_metadata:
 
@@ -115,7 +125,7 @@ You can provide access to any user that has "developer" as one of their roles li
         }
     }
 
-Note that "roles" is not a concept that is baked into Datasette - it's more of a convention that plugins can choose to implement and act on.
+Note that "roles" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on.
 
 If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to spceify that a query can be accessed by any logged-in user use this:
 
@@ -171,7 +181,7 @@ To limit access to the ``users`` table in your ``bakery.db`` database:
         }
     }
 
-This works for SQL views as well - you can treat them as if they are tables.
+This works for SQL views as well - you can list their names in the ``"tables"`` block above in the same way as regular tables.
 
 .. warning::
     Restricting access to tables and views in this way will NOT prevent users from querying them using arbitrary SQL queries, `like this `__ for example.
@@ -183,6 +193,8 @@ This works for SQL views as well - you can treat them as if they are tables.
 Controlling access to specific canned queries
 ---------------------------------------------
 
+:ref:`canned_queries` allow you to configure named SQL queries in your ``metadata.json`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important.
+
 To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`:
 
 .. code-block:: json
@@ -234,6 +246,15 @@ To limit this ability for just one specific database, use this:
         }
     }
 
+.. _permissions_plugins:
+
+Checking permissions in plugins
+===============================
+
+Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method.
+
+Datasette core performs a number of permission checks, :ref:`documented below `. Plugins can implement the :ref:`plugin_permission_allowed` plugin hook to participate in decisions about whether an actor should be able to perform a specified action.
+
 .. _authentication_actor_matches_allow:
 
 actor_matches_allow()
@@ -264,8 +285,8 @@ This is designed to help administrators and plugin authors understand exactly ho
 
 .. _permissions:
 
-Permissions
-===========
+Built-in permissions
+====================
 
 This section lists all of the permission checks that are carried out by Datasette core, along with the ``resource`` if it was passed.
 
diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index db72deb7..a73f6bc2 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -1,3 +1,5 @@
+.. _sql:
+
 Running SQL queries
 ===================
 
@@ -22,6 +24,8 @@ using your browser back button.
 You can also retrieve the results of any query as JSON by adding ``.json`` to
 the base URL.
 
+.. _sql_parameters:
+
 Named parameters
 ----------------
 
@@ -51,6 +55,8 @@ statements can be used to change database settings at runtime. If you need to
 include the string "pragma" in a query you can do so safely using a named
 parameter.
 
+.. _sql_views:
+
 Views
 -----
 

From 7633b9ab249b2dce5ee0b4fcf9542c13a1703ef0 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 10:01:03 -0700
Subject: [PATCH 0534/2303] unauthenticated: true method plus allow block docs,
 closes #825

---
 datasette/utils/__init__.py |   5 +-
 docs/authentication.rst     | 142 +++++++++++++++++++++++++-----------
 docs/internals.rst          |  11 ++-
 tests/test_auth.py          |  24 ------
 tests/test_permissions.py   |  37 ++++++++++
 tests/test_utils.py         |  10 ++-
 6 files changed, 154 insertions(+), 75 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 5873fcaa..51373c46 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -867,10 +867,11 @@ async def async_call_with_supported_arguments(fn, **kwargs):
 
 
 def actor_matches_allow(actor, allow):
-    if actor is None:
-        actor = {"anonymous": True}
+    if actor is None and allow and allow.get("unauthenticated") is True:
+        return True
     if allow is None:
         return True
+    actor = actor or {}
     for key, values in allow.items():
         if values == "*" and key in actor:
             return True
diff --git a/docs/authentication.rst b/docs/authentication.rst
index e26c8fc5..a9537a20 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -64,6 +64,91 @@ An **action** is a string describing the action the actor would like to perfom.
 
 A **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource.
 
+Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content.
+
+Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file.
+
+.. _authentication_permissions_allow:
+
+Defining permissions with "allow" blocks
+----------------------------------------
+
+The standard way to define permissions in Datasette is to use an ``"allow"`` block. This is a JSON document describing which actors are allowed to perfom a permission.
+
+The most basic form of allow block is this:
+
+.. code-block:: json
+
+    {
+        "allow": {
+            "id": "root"
+        }
+    }
+
+This will match any actors with an ``"id"`` property of ``"root"`` - for example, an actor that looks like this:
+
+.. code-block:: json
+
+    {
+        "id": "root",
+        "name": "Root User"
+    }
+
+Allow keys can provide a list of values. These will match any actor that has any of those values.
+
+.. code-block:: json
+
+    {
+        "allow": {
+            "id": ["simon", "cleopaws"]
+        }
+    }
+
+This will match any actor with an ``"id"`` of either ``"simon"`` or ``"cleopaws"``.
+
+Actors can have properties that feature a list of values. These will be matched against the list of values in an allow block. Consider the following actor:
+
+.. code-block:: json
+
+    {
+        "id": "simon",
+        "roles": ["staff", "developer"]
+    }
+
+This allow block will provide access to any actor that has ``"developer"`` as one of their roles:
+
+.. code-block:: json
+
+    {
+        "allow": {
+            "roles": ["developer"]
+        }
+    }
+
+Note that "roles" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on.
+
+If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to match any logged-in user specify the following:
+
+.. code-block:: json
+
+    {
+        "allow": {
+            "id": "*"
+        }
+    }
+
+You can specify that unauthenticated actors (from anynomous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block:
+
+.. code-block:: json
+
+    {
+        "allow": {
+            "unauthenticated": true
+        }
+    }
+
+Allow keys act as an "or" mechanism. An actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block.
+
 .. _authentication_permissions_metadata:
 
 Configuring permissions in metadata.json
@@ -96,49 +181,6 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i
         }
     }
 
-To allow any of the actors with an ``id`` matching a specific list of values, use this:
-
-.. code-block:: json
-
-    {
-        "allow": {
-            "id": ["simon", "cleopaws"]
-        }
-    }
-
-This works for other keys as well. Imagine an actor that looks like this:
-
-.. code-block:: json
-
-    {
-        "id": "simon",
-        "roles": ["staff", "developer"]
-    }
-
-You can provide access to any user that has "developer" as one of their roles like so:
-
-.. code-block:: json
-
-    {
-        "allow": {
-            "roles": ["developer"]
-        }
-    }
-
-Note that "roles" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on.
-
-If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to spceify that a query can be accessed by any logged-in user use this:
-
-.. code-block:: json
-
-    {
-        "allow": {
-            "id": "*"
-        }
-    }
-
-These keys act as an "or" mechanism. A actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block.
-
 .. _authentication_permissions_database:
 
 Controlling access to specific databases
@@ -297,6 +339,8 @@ view-instance
 
 Top level permission - Actor is allowed to view any pages within this instance, starting at https://latest.datasette.io/
 
+Default *allow*.
+
 .. _permissions_view_database:
 
 view-database
@@ -307,6 +351,8 @@ Actor is allowed to view a database page, e.g. https://latest.datasette.io/fixtu
 ``resource`` - string
     The name of the database
 
+Default *allow*.
+
 .. _permissions_view_database_download:
 
 view-database-download
@@ -317,6 +363,8 @@ Actor is allowed to download a database, e.g. https://latest.datasette.io/fixtur
 ``resource`` - string
     The name of the database
 
+Default *allow*.
+
 .. _permissions_view_table:
 
 view-table
@@ -327,6 +375,8 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i
 ``resource`` - tuple: (string, string)
     The name of the database, then the name of the table
 
+Default *allow*.
+
 .. _permissions_view_query:
 
 view-query
@@ -337,6 +387,8 @@ Actor is allowed to view a :ref:`canned query ` page, e.g. https
 ``resource`` - tuple: (string, string)
     The name of the database, then the name of the canned query
 
+Default *allow*.
+
 .. _permissions_execute_sql:
 
 execute-sql
@@ -347,9 +399,13 @@ Actor is allowed to run arbitrary SQL queries against a specific database, e.g.
 ``resource`` - string
     The name of the database
 
+Default *allow*.
+
 .. _permissions_permissions_debug:
 
 permissions-debug
 -----------------
 
 Actor is allowed to view the ``/-/permissions`` debug page.
+
+Default *deny*.
\ No newline at end of file
diff --git a/docs/internals.rst b/docs/internals.rst
index df21eb09..8136d8ac 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -184,11 +184,16 @@ await .permission_allowed(actor, action, resource=None, default=False)
 ``resource`` - string, optional
     The resource, e.g. the name of the table. Only some permissions apply to a resource.
 
-Check if the given actor has permission to perform the given action on the given resource. This uses plugins that implement the :ref:`plugin_permission_allowed` plugin hook to decide if the action is allowed or not.
+``default`` - optional, True or False
+    Should this permission check be default allow or default deny.
 
-If none of the plugins express an opinion, the return value will be the ``default`` argument. This is deny, but you can pass ``default=True`` to default allow instead.
+Check if the given actor has :ref:`permission ` to perform the given action on the given resource.
 
-See :ref:`permissions` for a full list of permissions included in Datasette core.
+Some permission checks are carried out against :ref:`rules defined in metadata.json `, while other custom permissions may be decided by plugins that implement the :ref:`plugin_permission_allowed` plugin hook.
+
+If neither ``metadata.json`` nor any of the plugins provide an answer to the permission query the ``default`` argument will be returned.
+
+See :ref:`permissions` for a full list of permission actions included in Datasette core.
 
 .. _datasette_get_database:
 
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 40dc2587..0e5563a3 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -1,5 +1,4 @@
 from .fixtures import app_client
-from bs4 import BeautifulSoup as Soup
 
 
 def test_auth_token(app_client):
@@ -20,26 +19,3 @@ def test_actor_cookie(app_client):
     cookie = app_client.ds.sign({"id": "test"}, "actor")
     response = app_client.get("/", cookies={"ds_actor": cookie})
     assert {"id": "test"} == app_client.ds._last_request.scope["actor"]
-
-
-def test_permissions_debug(app_client):
-    app_client.ds._permission_checks.clear()
-    assert 403 == app_client.get("/-/permissions").status
-    # With the cookie it should work
-    cookie = app_client.ds.sign({"id": "root"}, "actor")
-    response = app_client.get("/-/permissions", cookies={"ds_actor": cookie})
-    # Should show one failure and one success
-    soup = Soup(response.body, "html.parser")
-    check_divs = soup.findAll("div", {"class": "check"})
-    checks = [
-        {
-            "action": div.select_one(".check-action").text,
-            "result": bool(div.select(".check-result-true")),
-            "used_default": bool(div.select(".check-used-default")),
-        }
-        for div in check_divs
-    ]
-    assert [
-        {"action": "permissions-debug", "result": True, "used_default": False},
-        {"action": "permissions-debug", "result": False, "used_default": True},
-    ] == checks
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index d8c98825..c088facd 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -1,4 +1,5 @@
 from .fixtures import app_client, assert_permissions_checked, make_app_client
+from bs4 import BeautifulSoup as Soup
 import pytest
 
 
@@ -283,3 +284,39 @@ def test_permissions_checked(app_client, path, permissions):
     response = app_client.get(path)
     assert response.status in (200, 403)
     assert_permissions_checked(app_client.ds, permissions)
+
+
+def test_permissions_debug(app_client):
+    app_client.ds._permission_checks.clear()
+    assert 403 == app_client.get("/-/permissions").status
+    # With the cookie it should work
+    cookie = app_client.ds.sign({"id": "root"}, "actor")
+    response = app_client.get("/-/permissions", cookies={"ds_actor": cookie})
+    # Should show one failure and one success
+    soup = Soup(response.body, "html.parser")
+    check_divs = soup.findAll("div", {"class": "check"})
+    checks = [
+        {
+            "action": div.select_one(".check-action").text,
+            "result": bool(div.select(".check-result-true")),
+            "used_default": bool(div.select(".check-used-default")),
+        }
+        for div in check_divs
+    ]
+    assert [
+        {"action": "permissions-debug", "result": True, "used_default": False},
+        {"action": "permissions-debug", "result": False, "used_default": True},
+    ] == checks
+
+
+@pytest.mark.parametrize("allow,expected", [
+    ({"id": "root"}, 403),
+    ({"id": "root", "unauthenticated": True}, 200),
+])
+def test_allow_unauthenticated(allow, expected):
+    with make_app_client(
+        metadata={
+            "allow": allow
+        }
+    ) as client:
+        assert expected == client.get("/").status
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 4bade18b..0ffe8ae6 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -464,12 +464,16 @@ def test_multi_params(data, should_raise):
 @pytest.mark.parametrize(
     "actor,allow,expected",
     [
-        ({"id": "root"}, None, True),
-        ({"id": "root"}, {}, False),
-        ({"anonymous": True}, {"anonymous": True}, True),
         (None, None, True),
         (None, {}, False),
         (None, {"id": "root"}, False),
+        ({"id": "root"}, None, True),
+        ({"id": "root"}, {}, False),
+        ({"id": "simon", "staff": True}, {"staff": True}, True),
+        ({"id": "simon", "staff": False}, {"staff": True}, False),
+        # Special case for "unauthenticated": true
+        (None, {"unauthenticated": True}, True),
+        (None, {"unauthenticated": False}, False),
         # Special "*" value for any key:
         ({"id": "root"}, {"id": "*"}, True),
         ({}, {"id": "*"}, False),

From 5ef3b7b0c9b9e318af711bbd03e84af2abffdc29 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 12:25:44 -0700
Subject: [PATCH 0535/2303] Applied Black

Refs #825
---
 tests/test_permissions.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index c088facd..477b8160 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -309,14 +309,10 @@ def test_permissions_debug(app_client):
     ] == checks
 
 
-@pytest.mark.parametrize("allow,expected", [
-    ({"id": "root"}, 403),
-    ({"id": "root", "unauthenticated": True}, 200),
-])
+@pytest.mark.parametrize(
+    "allow,expected",
+    [({"id": "root"}, 403), ({"id": "root", "unauthenticated": True}, 200),],
+)
 def test_allow_unauthenticated(allow, expected):
-    with make_app_client(
-        metadata={
-            "allow": allow
-        }
-    ) as client:
+    with make_app_client(metadata={"allow": allow}) as client:
         assert expected == client.get("/").status

From 56eb80a45925d804b443701e2c86315f194b5f7d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 12:32:52 -0700
Subject: [PATCH 0536/2303] Documented CSRF protection, closes #827

---
 docs/internals.rst | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index 8136d8ac..d92c985f 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -51,7 +51,7 @@ The request object is passed to various plugin hooks. It represents an incoming
 The object also has one awaitable method:
 
 ``await request.post_vars()`` - dictionary
-    Returns a dictionary of form variables that were submitted in the request body via ``POST``.
+    Returns a dictionary of form variables that were submitted in the request body via ``POST``. Don't forget to read about :ref:`internals_csrf`!
 
 .. _internals_multiparams:
 
@@ -500,3 +500,17 @@ The ``Database`` class also provides properties and methods for introspecting th
                 }
             ]
         }
+
+
+.. _internals_csrf:
+
+CSRF protection
+~~~~~~~~~~~~~~~
+
+Datasette uses `asgi-csrf `__ to guard against CSRF attacks on form POST submissions. Users receive a ``ds_csrftoken`` cookie which is compared against the ``csrftoken`` form field (or ``x-csrftoken`` HTTP header) for every incoming request.
+
+If your plugin implements a ```` anywhere you will need to include that token. You can do so with the following template snippet:
+
+.. code-block:: html
+
+    

From f240970b834d595947c8d27d46d1f19b9119376d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 12:57:54 -0700
Subject: [PATCH 0537/2303] Fixed tests/fixtures.py, closes #804

---
 docs/contributing.rst | 13 +++++-
 tests/fixtures.py     | 97 ++++++++++++++++++++++++-------------------
 2 files changed, 65 insertions(+), 45 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index da4dc35a..9c44d177 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -70,11 +70,20 @@ You can also use the ``fixtures.py`` script to recreate the testing version of `
 
     python tests/fixtures.py fixtures.db fixtures-metadata.json
 
-(You may need to delete ``fixtures.db`` before running this command.)
+Or to output the plugins used by the tests, run this::
+
+    python tests/fixtures.py fixtures.db fixtures-metadata.json fixtures-plugins
+    Test tables written to fixtures.db
+    - metadata written to fixtures-metadata.json
+    Wrote plugin: fixtures-plugins/register_output_renderer.py
+    Wrote plugin: fixtures-plugins/view_name.py
+    Wrote plugin: fixtures-plugins/my_plugin.py
+    Wrote plugin: fixtures-plugins/messages_output_renderer.py
+    Wrote plugin: fixtures-plugins/my_plugin_2.py
 
 Then run Datasette like this::
 
-    datasette fixtures.db -m fixtures-metadata.json
+    datasette fixtures.db -m fixtures-metadata.json --plugins-dir=fixtures-plugins/
 
 .. _contributing_documentation:
 
diff --git a/tests/fixtures.py b/tests/fixtures.py
index a51a869d..1eb1bb6e 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -2,6 +2,7 @@ from datasette.app import Datasette
 from datasette.utils import sqlite3, MultiParams
 from asgiref.testing import ApplicationCommunicator
 from asgiref.sync import async_to_sync
+import click
 import contextlib
 from http.cookies import SimpleCookie
 import itertools
@@ -813,49 +814,6 @@ INSERT INTO "searchable_fts" (rowid, text1, text2)
     SELECT rowid, text1, text2 FROM searchable;
 """
 
-if __name__ == "__main__":
-    # Can be called with data.db OR data.db metadata.json
-    arg_index = -1
-    db_filename = sys.argv[arg_index]
-    metadata_filename = None
-    plugins_path = None
-    if db_filename.endswith("/"):
-        # It's the plugins dir
-        plugins_path = db_filename
-        arg_index -= 1
-        db_filename = sys.argv[arg_index]
-    if db_filename.endswith(".json"):
-        metadata_filename = db_filename
-        arg_index -= 1
-        db_filename = sys.argv[arg_index]
-    if db_filename.endswith(".db"):
-        conn = sqlite3.connect(db_filename)
-        conn.executescript(TABLES)
-        for sql, params in TABLE_PARAMETERIZED_SQL:
-            with conn:
-                conn.execute(sql, params)
-        print("Test tables written to {}".format(db_filename))
-        if metadata_filename:
-            open(metadata_filename, "w").write(json.dumps(METADATA))
-            print("- metadata written to {}".format(metadata_filename))
-        if plugins_path:
-            path = pathlib.Path(plugins_path)
-            if not path.exists():
-                path.mkdir()
-                for filename, content in (
-                    ("my_plugin.py", PLUGIN1),
-                    ("my_plugin_2.py", PLUGIN2),
-                ):
-                    filepath = path / filename
-                    filepath.write_text(content)
-                    print("  Wrote plugin: {}".format(filepath))
-    else:
-        print(
-            "Usage: {} db_to_write.db [metadata_to_write.json] [plugins-dir/]".format(
-                sys.argv[0]
-            )
-        )
-
 
 def assert_permissions_checked(datasette, actions):
     # actions is a list of "action" or (action, resource) tuples
@@ -873,3 +831,56 @@ def assert_permissions_checked(datasette, actions):
         """.format(
             action, resource, json.dumps(list(datasette._permission_checks), indent=4),
         )
+
+
+@click.command()
+@click.argument(
+    "db_filename",
+    default="fixtures.db",
+    type=click.Path(file_okay=True, dir_okay=False),
+)
+@click.argument("metadata", required=False)
+@click.argument(
+    "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False
+)
+@click.option(
+    "--recreate",
+    is_flag=True,
+    default=False,
+    help="Delete and recreate database if it exists",
+)
+def cli(db_filename, metadata, plugins_path, recreate):
+    "Write out the fixtures database used by Datasette's test suite"
+    if metadata and not metadata.endswith(".json"):
+        raise click.ClickException("Metadata should end with .json")
+    if not db_filename.endswith(".db"):
+        raise click.ClickException("Database file should end with .db")
+    if pathlib.Path(db_filename).exists():
+        if not recreate:
+            raise click.ClickException(
+                "{} already exists, use --recreate to reset it".format(db_filename)
+            )
+        else:
+            pathlib.Path(db_filename).unlink()
+    conn = sqlite3.connect(db_filename)
+    conn.executescript(TABLES)
+    for sql, params in TABLE_PARAMETERIZED_SQL:
+        with conn:
+            conn.execute(sql, params)
+    print("Test tables written to {}".format(db_filename))
+    if metadata:
+        open(metadata, "w").write(json.dumps(METADATA, indent=4))
+        print("- metadata written to {}".format(metadata))
+    if plugins_path:
+        path = pathlib.Path(plugins_path)
+        if not path.exists():
+            path.mkdir()
+        test_plugins = pathlib.Path(__file__).parent / "plugins"
+        for filepath in test_plugins.glob("*.py"):
+            newpath = path / filepath.name
+            newpath.write_text(filepath.open().read())
+            print("  Wrote plugin: {}".format(newpath))
+
+
+if __name__ == "__main__":
+    cli()

From 008e2f63c217aa066027a872ee706b07bd084857 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 15:19:37 -0700
Subject: [PATCH 0538/2303] response.set_cookie(), closes #795

---
 datasette/actor_auth_cookie.py   |  1 -
 datasette/app.py                 | 15 ++-------
 datasette/utils/asgi.py          | 53 +++++++++++++++++++++++++++++---
 datasette/views/special.py       | 14 ++-------
 docs/internals.rst               | 30 ++++++++++++++++++
 tests/test_internals_response.py | 26 ++++++++++++++++
 6 files changed, 108 insertions(+), 31 deletions(-)

diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py
index f3a0f306..a2aa6889 100644
--- a/datasette/actor_auth_cookie.py
+++ b/datasette/actor_auth_cookie.py
@@ -1,6 +1,5 @@
 from datasette import hookimpl
 from itsdangerous import BadSignature
-from http.cookies import SimpleCookie
 
 
 @hookimpl
diff --git a/datasette/app.py b/datasette/app.py
index 633ca4fe..71fa9afb 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -3,7 +3,6 @@ import asgi_csrf
 import collections
 import datetime
 import hashlib
-from http.cookies import SimpleCookie
 import itertools
 import json
 import os
@@ -442,19 +441,9 @@ class Datasette:
     def _write_messages_to_response(self, request, response):
         if getattr(request, "_messages", None):
             # Set those messages
-            cookie = SimpleCookie()
-            cookie["ds_messages"] = self.sign(request._messages, "messages")
-            cookie["ds_messages"]["path"] = "/"
-            # TODO: Co-exist with existing set-cookie headers
-            assert "set-cookie" not in response.headers
-            response.headers["set-cookie"] = cookie.output(header="").lstrip()
+            response.set_cookie("ds_messages", self.sign(request._messages, "messages"))
         elif getattr(request, "_messages_should_clear", False):
-            cookie = SimpleCookie()
-            cookie["ds_messages"] = ""
-            cookie["ds_messages"]["path"] = "/"
-            # TODO: Co-exist with existing set-cookie headers
-            assert "set-cookie" not in response.headers
-            response.headers["set-cookie"] = cookie.output(header="").lstrip()
+            response.set_cookie("ds_messages", "", expires=0, max_age=0)
 
     def _show_messages(self, request):
         if getattr(request, "_messages", None):
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index cdd6b148..5a152570 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -4,10 +4,15 @@ from mimetypes import guess_type
 from urllib.parse import parse_qs, urlunparse, parse_qsl
 from pathlib import Path
 from html import escape
-from http.cookies import SimpleCookie
+from http.cookies import SimpleCookie, Morsel
 import re
 import aiofiles
 
+# Workaround for adding samesite support to pre 3.8 python
+Morsel._reserved["samesite"] = "SameSite"
+# Thanks, Starlette:
+# https://github.com/encode/starlette/blob/519f575/starlette/responses.py#L17
+
 
 class NotFound(Exception):
     pass
@@ -17,6 +22,9 @@ class Forbidden(Exception):
     pass
 
 
+SAMESITE_VALUES = ("strict", "lax", "none")
+
+
 class Request:
     def __init__(self, scope, receive):
         self.scope = scope
@@ -370,20 +378,24 @@ class Response:
         self.body = body
         self.status = status
         self.headers = headers or {}
+        self._set_cookie_headers = []
         self.content_type = content_type
 
     async def asgi_send(self, send):
         headers = {}
         headers.update(self.headers)
         headers["content-type"] = self.content_type
+        raw_headers = [
+            [key.encode("utf-8"), value.encode("utf-8")]
+            for key, value in headers.items()
+        ]
+        for set_cookie in self._set_cookie_headers:
+            raw_headers.append([b"set-cookie", set_cookie.encode("utf-8")])
         await send(
             {
                 "type": "http.response.start",
                 "status": self.status,
-                "headers": [
-                    [key.encode("utf-8"), value.encode("utf-8")]
-                    for key, value in headers.items()
-                ],
+                "headers": raw_headers,
             }
         )
         body = self.body
@@ -391,6 +403,37 @@ class Response:
             body = body.encode("utf-8")
         await send({"type": "http.response.body", "body": body})
 
+    def set_cookie(
+        self,
+        key,
+        value="",
+        max_age=None,
+        expires=None,
+        path="/",
+        domain=None,
+        secure=False,
+        httponly=False,
+        samesite="lax",
+    ):
+        assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format(
+            SAMESITE_VALUES
+        )
+        cookie = SimpleCookie()
+        cookie[key] = value
+        for prop_name, prop_value in (
+            ("max_age", max_age),
+            ("expires", expires),
+            ("path", path),
+            ("domain", domain),
+            ("samesite", samesite),
+        ):
+            if prop_value is not None:
+                cookie[key][prop_name.replace("_", "-")] = prop_value
+        for prop_name, prop_value in (("secure", secure), ("httponly", httponly)):
+            if prop_value:
+                cookie[key][prop_name] = True
+        self._set_cookie_headers.append(cookie.output(header="").strip())
+
     @classmethod
     def html(cls, body, status=200, headers=None):
         return cls(
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 7a5fbe21..7f4284a1 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -1,7 +1,6 @@
 import json
 from datasette.utils.asgi import Response
 from .base import BaseView
-from http.cookies import SimpleCookie
 import secrets
 
 
@@ -62,17 +61,8 @@ class AuthTokenView(BaseView):
             return Response("Root token has already been used", status=403)
         if secrets.compare_digest(token, self.ds._root_token):
             self.ds._root_token = None
-            cookie = SimpleCookie()
-            cookie["ds_actor"] = self.ds.sign({"id": "root"}, "actor")
-            cookie["ds_actor"]["path"] = "/"
-            response = Response(
-                body="",
-                status=302,
-                headers={
-                    "Location": "/",
-                    "set-cookie": cookie.output(header="").lstrip(),
-                },
-            )
+            response = Response.redirect("/")
+            response.set_cookie("ds_actor", self.ds.sign({"id": "root"}, "actor"))
             return response
         else:
             return Response("Invalid token", status=403)
diff --git a/docs/internals.rst b/docs/internals.rst
index d92c985f..7978e3d7 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -131,6 +131,36 @@ Each of these responses will use the correct corresponding content-type - ``text
 
 Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above.
 
+.. _internals_response_set_cookie:
+
+Setting cookies with response.set_cookie()
+------------------------------------------
+
+To set cookies on the response, use the ``response.set_cookie(...)`` method. The method signature looks like this:
+
+.. code-block:: python
+
+    def set_cookie(
+        self,
+        key,
+        value="",
+        max_age=None,
+        expires=None,
+        path="/",
+        domain=None,
+        secure=False,
+        httponly=False,
+        samesite="lax",
+    ):
+
+You can use this with :ref:`datasette.sign() ` to set signed cookies. Here's how you would set the ``ds_actor`` cookie for use with Datasette :ref:`authentication `:
+
+.. code-block:: python
+
+    response = Response.redirect("/")
+    response.set_cookie("ds_actor", datasette.sign({"id": "cleopaws"}, "actor"))
+    return response
+
 .. _internals_datasette:
 
 Datasette class
diff --git a/tests/test_internals_response.py b/tests/test_internals_response.py
index 7c11f858..820b20b2 100644
--- a/tests/test_internals_response.py
+++ b/tests/test_internals_response.py
@@ -1,4 +1,5 @@
 from datasette.utils.asgi import Response
+import pytest
 
 
 def test_response_html():
@@ -26,3 +27,28 @@ def test_response_redirect():
     response = Response.redirect("/foo")
     assert 302 == response.status
     assert "/foo" == response.headers["Location"]
+
+
+@pytest.mark.asyncio
+async def test_response_set_cookie():
+    events = []
+
+    async def send(event):
+        events.append(event)
+
+    response = Response.redirect("/foo")
+    response.set_cookie("foo", "bar", max_age=10, httponly=True)
+    await response.asgi_send(send)
+
+    assert [
+        {
+            "type": "http.response.start",
+            "status": 302,
+            "headers": [
+                [b"Location", b"/foo"],
+                [b"content-type", b"text/plain"],
+                [b"set-cookie", b"foo=bar; HttpOnly; Max-Age=10; Path=/; SameSite=lax"],
+            ],
+        },
+        {"type": "http.response.body", "body": b""},
+    ] == events

From b5f04f42ab56be90735e1df9660e334089fbd6aa Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 15:32:24 -0700
Subject: [PATCH 0539/2303] ds_actor cookie documentation, closes #826

---
 docs/authentication.rst | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index a9537a20..f511e373 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -315,8 +315,8 @@ The currently authenticated actor is made available to plugins as ``request.acto
 
 .. _PermissionsDebugView:
 
-Permissions Debug
-=================
+The permissions debug tool
+==========================
 
 The debug tool at ``/-/permissions`` is only available to the :ref:`authenticated root user ` (or any actor granted the ``permissions-debug`` action according to a plugin).
 
@@ -324,6 +324,22 @@ It shows the thirty most recent permission checks that have been carried out by
 
 This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system.
 
+.. _authentication_ds_actor:
+
+The ds_actor cookie
+===================
+
+Datasette includes a default authentication plugin which looks for a signed ``ds_actor`` cookie containing a JSON actor dictionary. This is how the :ref:`root actor ` mechanism works.
+
+Authentication plugins can set signed ``ds_actor`` cookies themselves like so:
+
+.. code-block:: python
+
+    response = Response.redirect("/")
+    response.set_cookie("ds_actor", datasette.sign({"id": "cleopaws"}, "actor"))
+    return response
+
+Note that you need to pass ``"actor"`` as the namespace to :ref:`datasette_sign`.
 
 .. _permissions:
 

From b3919d8059a519eb7709f0b4fa1561fec219bc98 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 16:03:42 -0700
Subject: [PATCH 0540/2303] Mostly complete release notes for 0.44, refs #806

---
 docs/changelog.rst | 140 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 140 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 8b6272cb..e4e6057b 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,146 @@
 Changelog
 =========
 
+.. _v0_44:
+
+0.44 (2020-06-??)
+-----------------
+
+Authentication and permissions, writable canned queries, flash messages, new plugin hooks and more.
+
+Authentication
+~~~~~~~~~~~~~~
+
+Prior to this release the Datasette ecosystem has treated authentication as exclusively the realm of plugins, most notably through `datasette-auth-github `__.
+
+0.44 introduces :ref:`authentication` as core Datasette concepts (`#699 `__). This makes it easier for different plugins can share responsibility for authenticating requests - you might have one plugin that handles user accounts and another one that allows automated access via API keys, for example.
+
+You'll need to install plugins if you want full user accounts, but default Datasette can now authenticate a single root user with the new ``--root`` command-line option, which outputs a one-time use URL to :ref:`authenticate as a root actor ` (`#784 `__)::
+
+    $ datasette fixtures.db --root
+    http://127.0.0.1:8001/-/auth-token?token=5b632f8cd44b868df625f5a6e2185d88eea5b22237fd3cc8773f107cc4fd6477
+    INFO:     Started server process [14973]
+    INFO:     Waiting for application startup.
+    INFO:     Application startup complete.
+    INFO:     Uvicorn running on http://127.0.0.1:8001 (Press CTRL+C to quit)
+
+Plugins can implement new ways of authenticating users using the new :ref:`plugin_actor_from_request` hook.
+
+Permissions
+~~~~~~~~~~~
+
+Datasette also now has a built-in concept of :ref:`authentication_permissions`. The permissions system answers the following question:
+
+    Is this **actor** allowed to perform this **action**, optionally against this particular **resource**?
+
+You can use the new ``"allow"`` block syntax in ``metadata.json`` (or ``metadata.yaml``) to set required permissions at the instance, database, table or canned query level. For example, to restrict access to the ``fixtures.db`` database to the ``"root"`` user:
+
+.. code-block:: json
+
+    {
+        "databases": {
+            "fixtures": {
+                "allow": {
+                    "id" "root"
+                }
+            }
+        }
+    }
+
+See :ref:`authentication_permissions_allow` for more details.
+
+Plugins can implement their own custom permission checks using the new :ref:`plugin_permission_allowed` hook.
+
+A new debug page at ``/-/permissions`` shows recent permission checks, to help administrators and plugin authors understand exactly what checks are being performed. This tool defaults to only being available to the root user, but can be exposed to other users by plugins that respond to the ``permissions-debug`` permission. (`#788 `__)
+
+Writable canned queries
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Datasette's :ref:`canned_queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example.
+
+Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 introduces the ability for canned queries to execute ``INSERT`` or ``UPDATE`` queries as well, using the new ``"write": true`` property (`#800 `__):
+
+.. code-block:: json
+
+    {
+        "databases": {
+            "dogs": {
+                "queries": {
+                    "add_name": {
+                        "sql": "INSERT INTO names (name) VALUES (:name)",
+                        "write": true
+                    }
+                }
+            }
+        }
+    }
+
+See :ref:`canned_queries_writable` for more details.
+
+Flash messages
+~~~~~~~~~~~~~~
+
+Writable canned queries needed a mechanism to let the user know that the query has been successfully executed. The new flash messaging system (`#790 `__) allows messages to persist in signed cookies which are then displayed to the user on the next page that they visit. Plugins can use this mechanism to display their own messages, see :ref:`datasette_add_message` for details.
+
+You can try out the new messages using the ``/-/messages`` debug tool, for example at https://latest.datasette.io/-/messages
+
+Signed values and secrets
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Both flash messages and user authentication needed a way to sign values and set signed cookies. Two new methods are now available for plugins to take advantage of this mechanism: :ref:`datasette_sign` and :ref:`datasette_unsign`.
+
+Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`config_secret` for more details.
+
+Plugins can now sign value and verify their signatures using the :ref:`datasette.sign() ` and :ref:`datasette.unsign() ` methods.
+
+CSRF protection
+~~~~~~~~~~~~~~~
+
+Since writable canned queries are built using POST forms, Datasette now ships with :ref:`internals_csrf` (`#798 `__). This applies automatically to any POST request, which means plugins need to include a ``csrftoken`` in any POST forms that they render. They can do that like so:
+
+.. code-block:: html
+
+    
+
+register_routes() plugin hooks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Plugins can now register new views and routes via the :ref:`plugin_register_routes` plugin hook (`#819 `__). View functions can be defined that accept any of the current ``datasette`` object, the current ``request``, or the ASGI ``scope``, ``send`` and ``receive`` objects.
+
+Smaller changes
+~~~~~~~~~~~~~~~
+
+- New internals documentation for :ref:`internals_request` and :ref:`internals_response`. (`#706 `__)
+- ``request.url`` now respects the ``force_https_urls`` config setting. closes (`#781 `__)
+- ``request.args.getlist()`` returns ``[]`` if missing. Removed ``request.raw_args`` entirely. (`#774 `__)
+- New :ref:`datasette.get_database() ` method.
+- Added ``_`` prefix to many private, undocumented methods of the Datasette class. (`#576 `__)
+- Removed the ``db.get_outbound_foreign_keys()`` method which duplicated the behaviour of ``db.foreign_keys_for_table()``.
+- New :ref:`await datasette.permission_allowed() ` method.
+- ``/-/actor`` debugging endpoint for viewing the currently authenticated actor.
+- New ``request.cookies`` property.
+- ``/-/plugins`` endpoint now shows a list of hooks implemented by each plugin, e.g. https://latest.datasette.io/-/plugins?all=1
+- ``request.post_vars()`` method no longer discards empty values.
+- New "params" canned query key for explicitly setting named parameters, see :ref:`canned_queries_named_parameters`. (`#797 `__)
+- ``request.args`` is now a :ref:`MultiParams ` object.
+- Fixed a bug with the ``datasette plugins`` command. (`#802 `__)
+- Nicer pattern for using ``make_app_client()`` in tests. (`#395 `__)
+- New ``request.actor`` property.
+- Fixed broken CSS on nested 404 pages. (`#777 `__)
+- New ``request.url_vars`` property. (`#822 `__)
+- Fixed a bug with the ``python tests/fixtures.py`` command for outputting Datasette's testing fixtures database and plugins. (`#804 `__)
+
+The road to Datasette 1.0
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+I've assembled a `milestone for Datasette 1.0 `__. The focus of the 1.0 release will be the following:
+
+- Signify confidence in the quality/stability of Datasette
+- Give plugin authors confidence that their plugins will work for the whole 1.x release cycle
+- Provide the same confidence to developers building against Datasette JSON APIs
+
+If you have thoughts about what you would like to see for Datasette 1.0 you can join `the conversation on issue #519 `__.
+
 .. _v0_43:
 
 0.43 (2020-05-28)

From d94fc39e33b5eccae853e62f54bd8cc8e74688ff Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 16:43:58 -0700
Subject: [PATCH 0541/2303] Crafty JavaScript trick for generating commit
 references

---
 docs/contributing.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 9c44d177..6562afc8 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -156,6 +156,18 @@ To release a new version, first create a commit that updates :ref:`the changelog
 
 Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__.
 
+You can generate the list of issue references for a specific release by pasting the following into the browser devtools while looking at the :ref:`changelog` page (replace ``v0-44`` with the most recent version):
+
+.. code-block:: javascript
+
+    [
+        ...new Set(
+            Array.from(
+                document.getElementById("v0-44").querySelectorAll("a[href*=issues]")
+            ).map((a) => "#" + a.href.split("/issues/")[1])
+        ),
+    ].sort().join(", ");
+
 For non-bugfix releases you may want to update the news section of ``README.md`` as part of the same commit.
 
 To tag and push the releaes, run the following::

From f3951539f1750698976359411e19c1ccb79210ed Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 18:19:11 -0700
Subject: [PATCH 0542/2303] Hopefully fix horizontal scroll with changelog on
 mobile

---
 docs/changelog.rst | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index e4e6057b..911fb1b6 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1051,9 +1051,7 @@ request all rows where that column is less than 50 meters or more than 20 feet f
 - Fix SQLite error when loading rows with no incoming FKs. [Russ
   Garrett]
 
-  This fixes ``ERROR: conn=, sql
-  = 'select ', params = {'id': '1'}`` caused by an invalid query when
-  loading incoming FKs.
+  This fixes an error caused by an invalid query when loading incoming FKs.
 
   The error was ignored due to async but it still got printed to the
   console.

From d828abaddec0dce3ec4b4eeddc3a74384e52cf34 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 9 Jun 2020 21:20:07 -0700
Subject: [PATCH 0543/2303] Fix horizontal scrollbar on changelog, refs #828

---
 docs/_static/css/custom.css | 3 +++
 docs/conf.py                | 5 +++++
 2 files changed, 8 insertions(+)
 create mode 100644 docs/_static/css/custom.css

diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css
new file mode 100644
index 00000000..d7c2f164
--- /dev/null
+++ b/docs/_static/css/custom.css
@@ -0,0 +1,3 @@
+a.external {
+    overflow-wrap: anywhere;
+}
diff --git a/docs/conf.py b/docs/conf.py
index 5e0bb328..b273afca 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -99,6 +99,11 @@ html_theme = "sphinx_rtd_theme"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
+html_css_files = [
+    "css/custom.css",
+]
+
+
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
 #

From 57e812d5de9663a3c177e0344f4d1e552a74d484 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 10 Jun 2020 12:39:54 -0700
Subject: [PATCH 0544/2303] ds_author cookie can now expire, closes #829

Refs https://github.com/simonw/datasette-auth-github/issues/62#issuecomment-642152076
---
 datasette/actor_auth_cookie.py | 13 ++++++++-
 datasette/views/special.py     |  4 ++-
 docs/authentication.rst        | 48 ++++++++++++++++++++++++++++++++--
 docs/internals.rst             |  4 +--
 setup.py                       |  1 +
 tests/fixtures.py              |  3 +++
 tests/test_auth.py             | 21 +++++++++++++--
 tests/test_canned_write.py     |  6 ++---
 tests/test_permissions.py      | 20 +++++++-------
 9 files changed, 99 insertions(+), 21 deletions(-)

diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py
index a2aa6889..15ecd331 100644
--- a/datasette/actor_auth_cookie.py
+++ b/datasette/actor_auth_cookie.py
@@ -1,5 +1,7 @@
 from datasette import hookimpl
 from itsdangerous import BadSignature
+import baseconv
+import time
 
 
 @hookimpl
@@ -7,6 +9,15 @@ def actor_from_request(datasette, request):
     if "ds_actor" not in request.cookies:
         return None
     try:
-        return datasette.unsign(request.cookies["ds_actor"], "actor")
+        decoded = datasette.unsign(request.cookies["ds_actor"], "actor")
+        # If it has "e" and "a" keys process the "e" expiry
+        if not isinstance(decoded, dict) or "a" not in decoded:
+            return None
+        expires_at = decoded.get("e")
+        if expires_at:
+            timestamp = int(baseconv.base62.decode(expires_at))
+            if time.time() > timestamp:
+                return None
+        return decoded["a"]
     except BadSignature:
         return None
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 7f4284a1..dc6a25dc 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -62,7 +62,9 @@ class AuthTokenView(BaseView):
         if secrets.compare_digest(token, self.ds._root_token):
             self.ds._root_token = None
             response = Response.redirect("/")
-            response.set_cookie("ds_actor", self.ds.sign({"id": "root"}, "actor"))
+            response.set_cookie(
+                "ds_actor", self.ds.sign({"a": {"id": "root"}}, "actor")
+            )
             return response
         else:
             return Response("Invalid token", status=403)
diff --git a/docs/authentication.rst b/docs/authentication.rst
index f511e373..9b66132a 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -336,11 +336,55 @@ Authentication plugins can set signed ``ds_actor`` cookies themselves like so:
 .. code-block:: python
 
     response = Response.redirect("/")
-    response.set_cookie("ds_actor", datasette.sign({"id": "cleopaws"}, "actor"))
-    return response
+    response.set_cookie("ds_actor", datasette.sign({
+        "a": {
+            "id": "cleopaws"
+        }
+    }, "actor"))
 
 Note that you need to pass ``"actor"`` as the namespace to :ref:`datasette_sign`.
 
+The shape of data encoded in the cookie is as follows::
+
+    {
+        "a": {... actor ...}
+    }
+
+.. _authentication_ds_actor_expiry:
+
+Including an expiry time
+------------------------
+
+``ds_actor`` cookies can optionally include a signed expiry timestamp, after which the cookies will no longer be valid. Authentication plugins may chose to use this mechanism to limit the lifetime of the cookie. For example, if a plugin implements single-sign-on against another source it may decide to set short-lived cookies so that if the user is removed from the SSO system their existing Datasette cookies will stop working shortly afterwards.
+
+To include an expiry, add a ``"e"`` key to the cookie value containing a `base62-encoded integer `__ representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours:
+
+.. code-block:: python
+
+    import time
+    import baseconv
+
+    expires_at = int(time.time()) + (24 * 60 * 60)
+
+    response = Response.redirect("/")
+    response.set_cookie("ds_actor", datasette.sign({
+        "a": {
+            "id": "cleopaws"
+        },
+        "e": baseconv.base62.encode(expires_at),
+    }, "actor"))
+
+The resulting cookie will encode data that looks something like this:
+
+.. code-block:: json
+
+    {
+        "a": {
+            "id": "cleopaws"
+        },
+        "e": "1jjSji"
+    }
+
 .. _permissions:
 
 Built-in permissions
diff --git a/docs/internals.rst b/docs/internals.rst
index 7978e3d7..d75544e1 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -153,12 +153,12 @@ To set cookies on the response, use the ``response.set_cookie(...)`` method. The
         samesite="lax",
     ):
 
-You can use this with :ref:`datasette.sign() ` to set signed cookies. Here's how you would set the ``ds_actor`` cookie for use with Datasette :ref:`authentication `:
+You can use this with :ref:`datasette.sign() ` to set signed cookies. Here's how you would set the :ref:`ds_actor cookie ` for use with Datasette :ref:`authentication `:
 
 .. code-block:: python
 
     response = Response.redirect("/")
-    response.set_cookie("ds_actor", datasette.sign({"id": "cleopaws"}, "actor"))
+    response.set_cookie("ds_actor", datasette.sign({"a": {"id": "cleopaws"}}, "actor"))
     return response
 
 .. _internals_datasette:
diff --git a/setup.py b/setup.py
index 678a022f..45af0253 100644
--- a/setup.py
+++ b/setup.py
@@ -57,6 +57,7 @@ setup(
         "PyYAML~=5.3",
         "mergedeep>=1.1.1,<1.4.0",
         "itsdangerous~=1.1",
+        "python-baseconv==1.2.2",
     ],
     entry_points="""
         [console_scripts]
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 1eb1bb6e..a846999b 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -109,6 +109,9 @@ class TestClient:
     def __init__(self, asgi_app):
         self.asgi_app = asgi_app
 
+    def actor_cookie(self, actor):
+        return self.ds.sign({"a": actor}, "actor")
+
     @async_to_sync
     async def get(
         self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 0e5563a3..5e847445 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -1,4 +1,7 @@
 from .fixtures import app_client
+import baseconv
+import pytest
+import time
 
 
 def test_auth_token(app_client):
@@ -8,7 +11,9 @@ def test_auth_token(app_client):
     response = app_client.get(path, allow_redirects=False,)
     assert 302 == response.status
     assert "/" == response.headers["Location"]
-    assert {"id": "root"} == app_client.ds.unsign(response.cookies["ds_actor"], "actor")
+    assert {"a": {"id": "root"}} == app_client.ds.unsign(
+        response.cookies["ds_actor"], "actor"
+    )
     # Check that a second with same token fails
     assert app_client.ds._root_token is None
     assert 403 == app_client.get(path, allow_redirects=False,).status
@@ -16,6 +21,18 @@ def test_auth_token(app_client):
 
 def test_actor_cookie(app_client):
     "A valid actor cookie sets request.scope['actor']"
-    cookie = app_client.ds.sign({"id": "test"}, "actor")
+    cookie = app_client.actor_cookie({"id": "test"})
     response = app_client.get("/", cookies={"ds_actor": cookie})
     assert {"id": "test"} == app_client.ds._last_request.scope["actor"]
+
+
+@pytest.mark.parametrize(
+    "offset,expected", [((24 * 60 * 60), {"id": "test"}), (-(24 * 60 * 60), None),]
+)
+def test_actor_cookie_that_expires(app_client, offset, expected):
+    expires_at = int(time.time()) + offset
+    cookie = app_client.ds.sign(
+        {"a": {"id": "test"}, "e": baseconv.base62.encode(expires_at)}, "actor"
+    )
+    response = app_client.get("/", cookies={"ds_actor": cookie})
+    assert expected == app_client.ds._last_request.scope["actor"]
diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py
index dc3fba3f..4257806e 100644
--- a/tests/test_canned_write.py
+++ b/tests/test_canned_write.py
@@ -55,7 +55,7 @@ def test_custom_success_message(canned_write_client):
     response = canned_write_client.post(
         "/data/delete_name",
         {"rowid": 1},
-        cookies={"ds_actor": canned_write_client.ds.sign({"id": "root"}, "actor")},
+        cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})},
         allow_redirects=False,
         csrftoken_from=True,
     )
@@ -116,7 +116,7 @@ def test_canned_query_permissions_on_database_page(canned_write_client):
     # With auth shows four
     response = canned_write_client.get(
         "/data.json",
-        cookies={"ds_actor": canned_write_client.ds.sign({"id": "root"}, "actor")},
+        cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})},
     )
     assert 200 == response.status
     assert [
@@ -132,6 +132,6 @@ def test_canned_query_permissions_on_database_page(canned_write_client):
 def test_canned_query_permissions(canned_write_client):
     assert 403 == canned_write_client.get("/data/delete_name").status
     assert 200 == canned_write_client.get("/data/update_name").status
-    cookies = {"ds_actor": canned_write_client.ds.sign({"id": "root"}, "actor")}
+    cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})}
     assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status
     assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 477b8160..1be9529a 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -21,7 +21,7 @@ def test_view_instance(allow, expected_anon, expected_auth):
                 # Should be no padlock
                 assert "
      Datasette 🔒
      " not in anon_response.text
             auth_response = client.get(
-                path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+                path, cookies={"ds_actor": client.actor_cookie({"id": "root"})},
             )
             assert expected_auth == auth_response.status
             # Check for the padlock
@@ -48,7 +48,7 @@ def test_view_database(allow, expected_anon, expected_auth):
                 # Should be no padlock
                 assert ">fixtures 🔒" not in anon_response.text
             auth_response = client.get(
-                path, cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+                path, cookies={"ds_actor": client.actor_cookie({"id": "root"})},
             )
             assert expected_auth == auth_response.status
             if (
@@ -69,7 +69,7 @@ def test_database_list_respects_view_database():
         assert 'data' in anon_response.text
         assert 'fixtures' not in anon_response.text
         auth_response = client.get(
-            "/", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+            "/", cookies={"ds_actor": client.actor_cookie({"id": "root"})},
         )
         assert 'data' in auth_response.text
         assert 'fixtures 🔒' in auth_response.text
@@ -100,7 +100,7 @@ def test_database_list_respects_view_table():
         for html_fragment in html_fragments:
             assert html_fragment not in anon_response_text
         auth_response_text = client.get(
-            "/", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+            "/", cookies={"ds_actor": client.actor_cookie({"id": "root"})},
         ).text
         for html_fragment in html_fragments:
             assert html_fragment in auth_response_text
@@ -127,7 +127,7 @@ def test_view_table(allow, expected_anon, expected_auth):
             assert ">compound_three_primary_keys 🔒" not in anon_response.text
         auth_response = client.get(
             "/fixtures/compound_three_primary_keys",
-            cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")},
+            cookies={"ds_actor": client.actor_cookie({"id": "root"})},
         )
         assert expected_auth == auth_response.status
         if allow and expected_anon == 403 and expected_auth == 200:
@@ -156,7 +156,7 @@ def test_table_list_respects_view_table():
         for html_fragment in html_fragments:
             assert html_fragment not in anon_response.text
         auth_response = client.get(
-            "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+            "/fixtures", cookies={"ds_actor": client.actor_cookie({"id": "root"})}
         )
         for html_fragment in html_fragments:
             assert html_fragment in auth_response.text
@@ -180,7 +180,7 @@ def test_view_query(allow, expected_anon, expected_auth):
             # Should be no padlock
             assert ">fixtures 🔒" not in anon_response.text
         auth_response = client.get(
-            "/fixtures/q", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+            "/fixtures/q", cookies={"ds_actor": client.actor_cookie({"id": "root"})}
         )
         assert expected_auth == auth_response.status
         if allow and expected_anon == 403 and expected_auth == 200:
@@ -206,7 +206,7 @@ def test_execute_sql(metadata):
         assert 403 == client.get("/fixtures/facet_cities?_where=id=3").status
 
         # But for logged in user all of these should work:
-        cookies = {"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+        cookies = {"ds_actor": client.actor_cookie({"id": "root"})}
         response_text = client.get("/fixtures", cookies=cookies).text
         assert form_fragment in response_text
         assert 200 == client.get("/fixtures?sql=select+1", cookies=cookies).status
@@ -231,7 +231,7 @@ def test_query_list_respects_view_query():
         assert html_fragment not in anon_response.text
         assert '"/fixtures/q"' not in anon_response.text
         auth_response = client.get(
-            "/fixtures", cookies={"ds_actor": client.ds.sign({"id": "root"}, "actor")}
+            "/fixtures", cookies={"ds_actor": client.actor_cookie({"id": "root"})}
         )
         assert html_fragment in auth_response.text
 
@@ -290,7 +290,7 @@ def test_permissions_debug(app_client):
     app_client.ds._permission_checks.clear()
     assert 403 == app_client.get("/-/permissions").status
     # With the cookie it should work
-    cookie = app_client.ds.sign({"id": "root"}, "actor")
+    cookie = app_client.actor_cookie({"id": "root"})
     response = app_client.get("/-/permissions", cookies={"ds_actor": cookie})
     # Should show one failure and one success
     soup = Soup(response.body, "html.parser")

From 9f236c4c00689a022fd1d508f2b809ee2305927f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 10 Jun 2020 13:06:46 -0700
Subject: [PATCH 0545/2303] Warn that register_facet_classes may change, refs
 #830

Also documented policy that plugin hooks should not be shipped without a real example. Refs #818
---
 docs/contributing.rst | 1 +
 docs/plugins.rst      | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 6562afc8..ba52839c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -12,6 +12,7 @@ General guidelines
 
 * **master should always be releasable**. Incomplete features should live in branches. This ensures that any small bug fixes can be quickly released.
 * **The ideal commit** should bundle together the implementation, unit tests and associated documentation updates. The commit message should link to an associated issue.
+* **New plugin hooks** should only be shipped if accompanied by a separate release of a non-demo plugin that uses them.
 
 .. _devenvironment:
 
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 17fd64df..a28092a3 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -891,6 +891,9 @@ register_facet_classes()
 
 Return a list of additional Facet subclasses to be registered.
 
+.. warning::
+    The design of this plugin hook is unstable and may change. See `issue 830 `__.
+
 Each Facet subclass implements a new type of facet operation. The class should look like this:
 
 .. code-block:: python

From 198545733b7a34d7b36ab6510ed30fb7687bcc7e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 10 Jun 2020 16:56:53 -0700
Subject: [PATCH 0546/2303] Document that "allow": {} denies all

https://github.com/simonw/datasette/issues/831#issuecomment-642324847
---
 docs/authentication.rst | 19 +++++++++++++++++++
 tests/test_utils.py     | 11 +++++++----
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index 9b66132a..0da5a38b 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -94,6 +94,14 @@ This will match any actors with an ``"id"`` property of ``"root"`` - for example
         "name": "Root User"
     }
 
+An allow block can specify "no-one is allowed to do this" using an empty ``{}``:
+
+.. code-block:: json
+
+    {
+        "allow": {}
+    }
+
 Allow keys can provide a list of values. These will match any actor that has any of those values.
 
 .. code-block:: json
@@ -181,6 +189,17 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i
         }
     }
 
+To deny access to all users, you can use ``"allow": {}``:
+
+.. code-block:: json
+
+    {
+        "title": "My entirely inaccessible instance",
+        "allow": {}
+    }
+
+One reason to do this is if you are using a Datasette plugin - such as `datasette-permissions-sql `__ - to control permissions instead.
+
 .. _authentication_permissions_database:
 
 Controlling access to specific databases
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 0ffe8ae6..b490953f 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -464,16 +464,19 @@ def test_multi_params(data, should_raise):
 @pytest.mark.parametrize(
     "actor,allow,expected",
     [
+        # Default is to allow:
         (None, None, True),
+        # {} means deny-all:
         (None, {}, False),
-        (None, {"id": "root"}, False),
-        ({"id": "root"}, None, True),
         ({"id": "root"}, {}, False),
-        ({"id": "simon", "staff": True}, {"staff": True}, True),
-        ({"id": "simon", "staff": False}, {"staff": True}, False),
         # Special case for "unauthenticated": true
         (None, {"unauthenticated": True}, True),
         (None, {"unauthenticated": False}, False),
+        # Match on just one property:
+        (None, {"id": "root"}, False),
+        ({"id": "root"}, None, True),
+        ({"id": "simon", "staff": True}, {"staff": True}, True),
+        ({"id": "simon", "staff": False}, {"staff": True}, False),
         # Special "*" value for any key:
         ({"id": "root"}, {"id": "*"}, True),
         ({}, {"id": "*"}, False),

From ce4958018ede00fbdadf0c37a99889b6901bfb9b Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 10 Jun 2020 17:10:28 -0700
Subject: [PATCH 0547/2303] Clarify that view-query also lets you execute
 writable queries

---
 docs/authentication.rst | 2 +-
 docs/sql_queries.rst    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index 0da5a38b..6a526f34 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -461,7 +461,7 @@ Default *allow*.
 view-query
 ----------
 
-Actor is allowed to view a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size
+Actor is allowed to view (and execute) a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`.
 
 ``resource`` - tuple: (string, string)
     The name of the database, then the name of the canned query
diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index a73f6bc2..6cc32da1 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -223,7 +223,7 @@ Writable canned queries
 
 Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database.
 
-See :ref:`authentication_permissions_metadata` for details on how to add permission checks to canned queries, using the ``"allow"`` key.
+See :ref:`authentication_permissions_query` for details on how to add permission checks to canned queries, using the ``"allow"`` key.
 
 .. code-block:: json
 

From 371170eee8d1659437e42c8ee267cb4b2abcffb5 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 08:44:44 -0700
Subject: [PATCH 0548/2303] publish heroku now deploys with Python 3.8.3

---
 datasette/publish/heroku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py
index 4db81d8e..7adf9d92 100644
--- a/datasette/publish/heroku.py
+++ b/datasette/publish/heroku.py
@@ -167,7 +167,7 @@ def temporary_heroku_directory(
         if metadata_content:
             open("metadata.json", "w").write(json.dumps(metadata_content, indent=2))
 
-        open("runtime.txt", "w").write("python-3.8.0")
+        open("runtime.txt", "w").write("python-3.8.3")
 
         if branch:
             install = [

From 98632f0a874b7b9dac6abf0abb9fdb7e2839a4d3 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 09:02:03 -0700
Subject: [PATCH 0549/2303] --secret command for datasette publish

Closes #787
---
 datasette/cli.py                         | 28 +++++++++++++++---------
 datasette/publish/cloudrun.py            |  2 ++
 datasette/publish/common.py              |  7 ++++++
 datasette/publish/heroku.py              |  3 +++
 datasette/utils/__init__.py              |  7 +++++-
 docs/datasette-package-help.txt          |  3 +++
 docs/datasette-publish-cloudrun-help.txt |  3 +++
 docs/datasette-publish-heroku-help.txt   |  3 +++
 docs/plugins.rst                         |  1 +
 tests/test_package.py                    |  8 ++++---
 tests/test_publish_cloudrun.py           |  3 +++
 tests/test_utils.py                      |  4 ++++
 12 files changed, 58 insertions(+), 14 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 2e3c8e36..ff9a2d5c 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -165,6 +165,12 @@ def plugins(all, plugins_dir):
 )
 @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
 @click.option("--version-note", help="Additional note to show on /-/versions")
+@click.option(
+    "--secret",
+    help="Secret used for signing secure values, such as signed cookies",
+    envvar="DATASETTE_PUBLISH_SECRET",
+    default=lambda: os.urandom(32).hex(),
+)
 @click.option(
     "-p", "--port", default=8001, help="Port to run the server on, defaults to 8001",
 )
@@ -187,6 +193,7 @@ def package(
     install,
     spatialite,
     version_note,
+    secret,
     port,
     **extra_metadata
 ):
@@ -203,16 +210,17 @@ def package(
     with temporary_docker_directory(
         files,
         "datasette",
-        metadata,
-        extra_options,
-        branch,
-        template_dir,
-        plugins_dir,
-        static,
-        install,
-        spatialite,
-        version_note,
-        extra_metadata,
+        metadata=metadata,
+        extra_options=extra_options,
+        branch=branch,
+        template_dir=template_dir,
+        plugins_dir=plugins_dir,
+        static=static,
+        install=install,
+        spatialite=spatialite,
+        version_note=version_note,
+        secret=secret,
+        extra_metadata=extra_metadata,
         port=port,
     ):
         args = ["docker", "build"]
diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py
index 8271209a..8f99dc2e 100644
--- a/datasette/publish/cloudrun.py
+++ b/datasette/publish/cloudrun.py
@@ -47,6 +47,7 @@ def publish_subcommand(publish):
         install,
         plugin_secret,
         version_note,
+        secret,
         title,
         license,
         license_url,
@@ -120,6 +121,7 @@ def publish_subcommand(publish):
             install,
             spatialite,
             version_note,
+            secret,
             extra_metadata,
             environment_variables,
         ):
diff --git a/datasette/publish/common.py b/datasette/publish/common.py
index 2911029d..49a4798e 100644
--- a/datasette/publish/common.py
+++ b/datasette/publish/common.py
@@ -1,5 +1,6 @@
 from ..utils import StaticMount
 import click
+import os
 import shutil
 import sys
 
@@ -52,6 +53,12 @@ def add_common_publish_arguments_and_options(subcommand):
             click.option(
                 "--version-note", help="Additional note to show on /-/versions"
             ),
+            click.option(
+                "--secret",
+                help="Secret used for signing secure values, such as signed cookies",
+                envvar="DATASETTE_PUBLISH_SECRET",
+                default=lambda: os.urandom(32).hex(),
+            ),
             click.option("--title", help="Title for metadata"),
             click.option("--license", help="License label for metadata"),
             click.option("--license_url", help="License URL for metadata"),
diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py
index 7adf9d92..6cda68da 100644
--- a/datasette/publish/heroku.py
+++ b/datasette/publish/heroku.py
@@ -35,6 +35,7 @@ def publish_subcommand(publish):
         install,
         plugin_secret,
         version_note,
+        secret,
         title,
         license,
         license_url,
@@ -100,6 +101,7 @@ def publish_subcommand(publish):
             static,
             install,
             version_note,
+            secret,
             extra_metadata,
         ):
             app_name = None
@@ -144,6 +146,7 @@ def temporary_heroku_directory(
     static,
     install,
     version_note,
+    secret,
     extra_metadata=None,
 ):
     extra_metadata = extra_metadata or {}
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 51373c46..5090f67e 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -278,10 +278,13 @@ def make_dockerfile(
     install,
     spatialite,
     version_note,
+    secret,
     environment_variables=None,
     port=8001,
 ):
     cmd = ["datasette", "serve", "--host", "0.0.0.0"]
+    environment_variables = environment_variables or {}
+    environment_variables["DATASETTE_SECRET"] = secret
     for filename in files:
         cmd.extend(["-i", filename])
     cmd.extend(["--cors", "--inspect-file", "inspect-data.json"])
@@ -324,7 +327,7 @@ CMD {cmd}""".format(
         environment_variables="\n".join(
             [
                 "ENV {} '{}'".format(key, value)
-                for key, value in (environment_variables or {}).items()
+                for key, value in environment_variables.items()
             ]
         ),
         files=" ".join(files),
@@ -348,6 +351,7 @@ def temporary_docker_directory(
     install,
     spatialite,
     version_note,
+    secret,
     extra_metadata=None,
     environment_variables=None,
     port=8001,
@@ -381,6 +385,7 @@ def temporary_docker_directory(
             install,
             spatialite,
             version_note,
+            secret,
             environment_variables,
             port=port,
         )
diff --git a/docs/datasette-package-help.txt b/docs/datasette-package-help.txt
index 326b66cb..1b14f908 100644
--- a/docs/datasette-package-help.txt
+++ b/docs/datasette-package-help.txt
@@ -17,6 +17,9 @@ Options:
   --install TEXT            Additional packages (e.g. plugins) to install
   --spatialite              Enable SpatialLite extension
   --version-note TEXT       Additional note to show on /-/versions
+  --secret TEXT             Secret used for signing secure values, such as signed
+                            cookies
+
   -p, --port INTEGER        Port to run the server on, defaults to 8001
   --title TEXT              Title for metadata
   --license TEXT            License label for metadata
diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt
index 98fc9c71..a625bd10 100644
--- a/docs/datasette-publish-cloudrun-help.txt
+++ b/docs/datasette-publish-cloudrun-help.txt
@@ -15,6 +15,9 @@ Options:
                                   datasette-auth-github client_id xxx
 
   --version-note TEXT             Additional note to show on /-/versions
+  --secret TEXT                   Secret used for signing secure values, such as signed
+                                  cookies
+
   --title TEXT                    Title for metadata
   --license TEXT                  License label for metadata
   --license_url TEXT              License URL for metadata
diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt
index ec157753..b2caa2cc 100644
--- a/docs/datasette-publish-heroku-help.txt
+++ b/docs/datasette-publish-heroku-help.txt
@@ -15,6 +15,9 @@ Options:
                                   datasette-auth-github client_id xxx
 
   --version-note TEXT             Additional note to show on /-/versions
+  --secret TEXT                   Secret used for signing secure values, such as signed
+                                  cookies
+
   --title TEXT                    Title for metadata
   --license TEXT                  License label for metadata
   --license_url TEXT              License URL for metadata
diff --git a/docs/plugins.rst b/docs/plugins.rst
index a28092a3..989cf672 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -536,6 +536,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_
             install,
             plugin_secret,
             version_note,
+            secret,
             title,
             license,
             license_url,
diff --git a/tests/test_package.py b/tests/test_package.py
index f0cbe88f..3248b3a4 100644
--- a/tests/test_package.py
+++ b/tests/test_package.py
@@ -15,7 +15,7 @@ FROM python:3.8
 COPY . /app
 WORKDIR /app
 
-
+ENV DATASETTE_SECRET 'sekrit'
 RUN pip install -U datasette
 RUN datasette inspect test.db --inspect-file inspect-data.json
 ENV PORT {port}
@@ -33,7 +33,7 @@ def test_package(mock_call, mock_which):
     mock_call.side_effect = capture
     with runner.isolated_filesystem():
         open("test.db", "w").write("data")
-        result = runner.invoke(cli.cli, ["package", "test.db"])
+        result = runner.invoke(cli.cli, ["package", "test.db", "--secret", "sekrit"])
         assert 0 == result.exit_code
         mock_call.assert_has_calls([mock.call(["docker", "build", "."])])
     assert EXPECTED_DOCKERFILE.format(port=8001) == capture.captured
@@ -48,6 +48,8 @@ def test_package_with_port(mock_call, mock_which):
     runner = CliRunner()
     with runner.isolated_filesystem():
         open("test.db", "w").write("data")
-        result = runner.invoke(cli.cli, ["package", "test.db", "-p", "8080"])
+        result = runner.invoke(
+            cli.cli, ["package", "test.db", "-p", "8080", "--secret", "sekrit"]
+        )
         assert 0 == result.exit_code
     assert EXPECTED_DOCKERFILE.format(port=8080) == capture.captured
diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py
index 55c207c7..c3ed1f90 100644
--- a/tests/test_publish_cloudrun.py
+++ b/tests/test_publish_cloudrun.py
@@ -172,6 +172,8 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which):
                 "client_id",
                 "x-client-id",
                 "--show-files",
+                "--secret",
+                "x-secret",
             ],
         )
         dockerfile = (
@@ -184,6 +186,7 @@ COPY . /app
 WORKDIR /app
 
 ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id'
+ENV DATASETTE_SECRET 'x-secret'
 RUN pip install -U datasette
 RUN datasette inspect test.db --inspect-file inspect-data.json
 ENV PORT 8001
diff --git a/tests/test_utils.py b/tests/test_utils.py
index b490953f..d613e999 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -247,6 +247,7 @@ def test_temporary_docker_directory_uses_hard_link():
             install=[],
             spatialite=False,
             version_note=None,
+            secret="secret",
         ) as temp_docker:
             hello = os.path.join(temp_docker, "hello")
             assert "world" == open(hello).read()
@@ -274,6 +275,7 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link):
             install=[],
             spatialite=False,
             version_note=None,
+            secret=None,
         ) as temp_docker:
             hello = os.path.join(temp_docker, "hello")
             assert "world" == open(hello).read()
@@ -297,11 +299,13 @@ def test_temporary_docker_directory_quotes_args():
             install=[],
             spatialite=False,
             version_note="$PWD",
+            secret="secret",
         ) as temp_docker:
             df = os.path.join(temp_docker, "Dockerfile")
             df_contents = open(df).read()
             assert "'$PWD'" in df_contents
             assert "'--$HOME'" in df_contents
+            assert "ENV DATASETTE_SECRET 'secret'" in df_contents
 
 
 def test_compound_keys_after_sql():

From fcc7cd6379ab62b5c2440d26935659a797133030 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 09:04:32 -0700
Subject: [PATCH 0550/2303] rST formatting

---
 docs/publish.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/publish.rst b/docs/publish.rst
index c1024bd7..6eff74d0 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -139,7 +139,7 @@ You can now run the resulting container like so::
 
 This exposes port 8001 inside the container as port 8081 on your host machine, so you can access the application at ``http://localhost:8081/``
 
-You can customize the port that is exposed by the countainer using the ``--port`` option:
+You can customize the port that is exposed by the countainer using the ``--port`` option::
 
     datasette package mydatabase.db --port 8080
 

From 09bf3c63225babe8e28cde880ca4399ca7dbd78b Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 09:14:30 -0700
Subject: [PATCH 0551/2303] Documentation for publish --secret, refs #787

---
 docs/config.rst  | 13 +++++++++++++
 docs/publish.rst |  2 ++
 2 files changed, 15 insertions(+)

diff --git a/docs/config.rst b/docs/config.rst
index ab14ea7b..bbbea822 100644
--- a/docs/config.rst
+++ b/docs/config.rst
@@ -306,3 +306,16 @@ One way to generate a secure random secret is to use Python like this::
     cdb19e94283a20f9d42cca50c5a4871c0aa07392db308755d60a1a5b9bb0fa52
 
 Plugin authors make use of this signing mechanism in their plugins using :ref:`datasette_sign` and :ref:`datasette_unsign`.
+
+.. _config_publish_secrets:
+
+Using secrets with datasette publish
+------------------------------------
+
+The :ref:`cli_publish` and :ref:`cli_package` commands both generate a secret for you automatically when Datasette is deployed.
+
+This means that every time you deploy a new version of a Datasette project, a new secret will be generated. This will cause signed cookies to become inalid on every fresh deploy.
+
+You can fix this by creating a secret that will be used for multiple deploys and passing it using the ``--secret`` option::
+
+    datasette publish cloudrun mydb.db --service=my-service --secret=cdb19e94283a20f9d42cca5
diff --git a/docs/publish.rst b/docs/publish.rst
index 6eff74d0..ebaf826a 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -100,6 +100,8 @@ If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plug
         --plugin-secret datasette-auth-github client_id your_client_id \
         --plugin-secret datasette-auth-github client_secret your_client_secret
 
+.. _cli_package:
+
 datasette package
 =================
 

From 29c5ff493ad7918b8fc44ea7920b41530e56dd5d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 15:14:51 -0700
Subject: [PATCH 0552/2303] view-instance permission for debug URLs, closes
 #833

---
 datasette/views/special.py |  8 ++++++--
 tests/test_permissions.py  | 30 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/datasette/views/special.py b/datasette/views/special.py
index dc6a25dc..6fcb6b5e 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -14,6 +14,7 @@ class JsonDataView(BaseView):
         self.needs_request = needs_request
 
     async def get(self, request, as_format):
+        await self.check_permission(request, "view-instance")
         if self.needs_request:
             data = self.data_callback(request)
         else:
@@ -46,6 +47,7 @@ class PatternPortfolioView(BaseView):
         self.ds = datasette
 
     async def get(self, request):
+        await self.check_permission(request, "view-instance")
         return await self.render(["patterns.html"], request=request)
 
 
@@ -77,8 +79,8 @@ class PermissionsDebugView(BaseView):
         self.ds = datasette
 
     async def get(self, request):
-        if not await self.ds.permission_allowed(request.actor, "permissions-debug"):
-            return Response("Permission denied", status=403)
+        await self.check_permission(request, "view-instance")
+        await self.check_permission(request, "permissions-debug")
         return await self.render(
             ["permissions_debug.html"],
             request,
@@ -93,9 +95,11 @@ class MessagesDebugView(BaseView):
         self.ds = datasette
 
     async def get(self, request):
+        await self.check_permission(request, "view-instance")
         return await self.render(["messages_debug.html"], request)
 
     async def post(self, request):
+        await self.check_permission(request, "view-instance")
         post = await request.post_vars()
         message = post.get("message", "")
         message_type = post.get("message_type") or "INFO"
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 1be9529a..fcc1b5ed 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -316,3 +316,33 @@ def test_permissions_debug(app_client):
 def test_allow_unauthenticated(allow, expected):
     with make_app_client(metadata={"allow": allow}) as client:
         assert expected == client.get("/").status
+
+
+@pytest.fixture(scope="session")
+def view_instance_client():
+    with make_app_client(metadata={"allow": {}}) as client:
+        yield client
+
+
+@pytest.mark.parametrize(
+    "path",
+    [
+        "/",
+        "/fixtures",
+        "/fixtures/facetable",
+        "/-/metadata",
+        "/-/versions",
+        "/-/plugins",
+        "/-/config",
+        "/-/threads",
+        "/-/databases",
+        "/-/actor",
+        "/-/permissions",
+        "/-/messages",
+        "/-/patterns",
+    ],
+)
+def test_view_instance(path, view_instance_client):
+    assert 403 == view_instance_client.get(path).status
+    if path not in ("/-/permissions", "/-/messages", "/-/patterns"):
+        assert 403 == view_instance_client.get(path + ".json").status

From f39f11133126158e28780dee91bb9c7719ef5875 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 15:47:19 -0700
Subject: [PATCH 0553/2303] Fixed actor_matches_allow bug, closes #836

---
 datasette/utils/__init__.py | 2 +-
 tests/test_utils.py         | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 5090f67e..69cfa400 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -884,7 +884,7 @@ def actor_matches_allow(actor, allow):
             values = [values]
         actor_values = actor.get(key)
         if actor_values is None:
-            return False
+            continue
         if not isinstance(actor_values, list):
             actor_values = [actor_values]
         actor_values = set(actor_values)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index d613e999..da1d298b 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -497,6 +497,8 @@ def test_multi_params(data, should_raise):
         ({"id": "garry", "roles": ["staff", "dev"]}, {"roles": ["dev", "otter"]}, True),
         ({"id": "garry", "roles": []}, {"roles": ["staff"]}, False),
         ({"id": "garry"}, {"roles": ["staff"]}, False),
+        # Any single matching key works:
+        ({"id": "root"}, {"bot_id": "my-bot", "id": ["root"]}, True),
     ],
 )
 def test_actor_matches_allow(actor, allow, expected):

From fba8ff6e76253af2b03749ed8dd6e28985a7fb8f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 17:21:48 -0700
Subject: [PATCH 0554/2303] "$env": "X" mechanism now works with nested lists,
 closes #837

---
 datasette/app.py            | 14 ++------------
 datasette/utils/__init__.py | 16 ++++++++++++++++
 docs/changelog.rst          |  2 ++
 tests/fixtures.py           |  1 +
 tests/test_plugins.py       | 13 +++++++++++++
 tests/test_utils.py         | 14 ++++++++++++++
 6 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 71fa9afb..ebab3bee 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -45,6 +45,7 @@ from .utils import (
     format_bytes,
     module_from_path,
     parse_metadata,
+    resolve_env_secrets,
     sqlite3,
     to_css_class,
 )
@@ -367,18 +368,7 @@ class Datasette:
             return None
         plugin_config = plugins.get(plugin_name)
         # Resolve any $file and $env keys
-        if isinstance(plugin_config, dict):
-            # Create a copy so we don't mutate the version visible at /-/metadata.json
-            plugin_config_copy = dict(plugin_config)
-            for key, value in plugin_config_copy.items():
-                if isinstance(value, dict):
-                    if list(value.keys()) == ["$env"]:
-                        plugin_config_copy[key] = os.environ.get(
-                            list(value.values())[0]
-                        )
-                    elif list(value.keys()) == ["$file"]:
-                        plugin_config_copy[key] = open(list(value.values())[0]).read()
-            return plugin_config_copy
+        plugin_config = resolve_env_secrets(plugin_config, os.environ)
         return plugin_config
 
     def app_css_hash(self):
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 69cfa400..ae7bbdb5 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -904,3 +904,19 @@ async def check_visibility(datasette, actor, action, resource, default=True):
         None, action, resource=resource, default=default,
     )
     return visible, private
+
+
+def resolve_env_secrets(config, environ):
+    'Create copy that recursively replaces {"$env": "NAME"} with values from environ'
+    if isinstance(config, dict):
+        if list(config.keys()) == ["$env"]:
+            return environ.get(list(config.values())[0])
+        else:
+            return {
+                key: resolve_env_secrets(value, environ)
+                for key, value in config.items()
+            }
+    elif isinstance(config, list):
+        return [resolve_env_secrets(value, environ) for value in config]
+    else:
+        return config
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 911fb1b6..3a01d05e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -94,6 +94,8 @@ Both flash messages and user authentication needed a way to sign values and set
 
 Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`config_secret` for more details.
 
+You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`config_publish_secrets`.
+
 Plugins can now sign value and verify their signatures using the :ref:`datasette.sign() ` and :ref:`datasette.unsign() ` methods.
 
 CSRF protection
diff --git a/tests/fixtures.py b/tests/fixtures.py
index a846999b..907bf895 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -408,6 +408,7 @@ METADATA = {
     "plugins": {
         "name-of-plugin": {"depth": "root"},
         "env-plugin": {"foo": {"$env": "FOO_ENV"}},
+        "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}],
         "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}},
     },
     "databases": {
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index c7bb4859..0fae3740 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -173,6 +173,19 @@ def test_plugin_config_env(app_client):
     del os.environ["FOO_ENV"]
 
 
+def test_plugin_config_env_from_list(app_client):
+    os.environ["FOO_ENV"] = "FROM_ENVIRONMENT"
+    assert [{"in_a_list": "FROM_ENVIRONMENT"}] == app_client.ds.plugin_config(
+        "env-plugin-list"
+    )
+    # Ensure secrets aren't visible in /-/metadata.json
+    metadata = app_client.get("/-/metadata.json")
+    assert [{"in_a_list": {"$env": "FOO_ENV"}}] == metadata.json["plugins"][
+        "env-plugin-list"
+    ]
+    del os.environ["FOO_ENV"]
+
+
 def test_plugin_config_file(app_client):
     open(TEMP_PLUGIN_SECRET_FILE, "w").write("FROM_FILE")
     assert {"foo": "FROM_FILE"} == app_client.ds.plugin_config("file-plugin")
diff --git a/tests/test_utils.py b/tests/test_utils.py
index da1d298b..80c6f223 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -503,3 +503,17 @@ def test_multi_params(data, should_raise):
 )
 def test_actor_matches_allow(actor, allow, expected):
     assert expected == utils.actor_matches_allow(actor, allow)
+
+
+@pytest.mark.parametrize(
+    "config,expected",
+    [
+        ({"foo": "bar"}, {"foo": "bar"}),
+        ({"$env": "FOO"}, "x"),
+        ({"k": {"$env": "FOO"}}, {"k": "x"}),
+        ([{"k": {"$env": "FOO"}}, {"z": {"$env": "FOO"}}], [{"k": "x"}, {"z": "x"}]),
+        ({"k": [{"in_a_list": {"$env": "FOO"}}]}, {"k": [{"in_a_list": "x"}]}),
+    ],
+)
+def test_resolve_env_secrets(config, expected):
+    assert expected == utils.resolve_env_secrets(config, {"FOO": "x"})

From 308bcc8805236b8eb5a08d8045c84f68bd0ddf0e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 17:25:12 -0700
Subject: [PATCH 0555/2303] Fixed test_permissions_debug

---
 datasette/views/special.py | 3 ++-
 tests/test_permissions.py  | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/datasette/views/special.py b/datasette/views/special.py
index 6fcb6b5e..6c378995 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -80,7 +80,8 @@ class PermissionsDebugView(BaseView):
 
     async def get(self, request):
         await self.check_permission(request, "view-instance")
-        await self.check_permission(request, "permissions-debug")
+        if not await self.ds.permission_allowed(request.actor, "permissions-debug"):
+            return Response("Permission denied", status=403)
         return await self.render(
             ["permissions_debug.html"],
             request,
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index fcc1b5ed..241dd2e5 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -305,7 +305,9 @@ def test_permissions_debug(app_client):
     ]
     assert [
         {"action": "permissions-debug", "result": True, "used_default": False},
+        {"action": "view-instance", "result": True, "used_default": True},
         {"action": "permissions-debug", "result": False, "used_default": True},
+        {"action": "view-instance", "result": True, "used_default": True},
     ] == checks
 
 

From 1d2e8e09a00a4b695317627483f352464ea8a105 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 17:33:16 -0700
Subject: [PATCH 0556/2303] Some last touches to the 0.44 release notes, refs
 #806

---
 docs/changelog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 3a01d05e..aca8f8c2 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -107,6 +107,13 @@ Since writable canned queries are built using POST forms, Datasette now ships wi
 
     
 
+Cookie methods
+~~~~~~~~~~~~~~
+
+Plugins can now use the new :ref:`response.set_cookie() ` method to set cookies.
+
+A new ``request.cookies`` method on the :ref:internals_request` can be used to read incoming cookies.
+
 register_routes() plugin hooks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -134,6 +141,9 @@ Smaller changes
 - Fixed broken CSS on nested 404 pages. (`#777 `__)
 - New ``request.url_vars`` property. (`#822 `__)
 - Fixed a bug with the ``python tests/fixtures.py`` command for outputting Datasette's testing fixtures database and plugins. (`#804 `__)
+- ``datasette publish heroku`` now deploys using Python 3.8.3.
+- Added a warning that the :ref:`plugin_register_facet_classes` hook is unstable and may change in the future. (`#830 `__)
+- The ``{"$env": "ENVIRONMENT_VARIBALE"}`` mechanism (see :ref:`plugins_configuration_secret`) now works with variables inside nested lists. (`#837 `__)
 
 The road to Datasette 1.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~

From 793a52b31771280a6c8660efb9e48b9b763477ff Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 17:43:51 -0700
Subject: [PATCH 0557/2303] Link to datasett-auth-tokens and
 datasette-permissions-sql in docs, refs #806

---
 docs/authentication.rst |  4 ++--
 docs/changelog.rst      |  4 ++--
 docs/ecosystem.rst      | 10 ++++++++++
 docs/internals.rst      |  2 +-
 docs/plugins.rst        | 17 ++++++++++-------
 5 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index 6a526f34..2a6fa9bc 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -19,7 +19,7 @@ Every request to Datasette has an associated actor value, available in the code
 
 The actor dictionary can be any shape - the design of that data structure is left up to the plugins. A useful convention is to include an ``"id"`` string, as demonstrated by the "root" actor below.
 
-Plugins can use the :ref:`plugin_actor_from_request` hook to implement custom logic for authenticating an actor based on the incoming HTTP request.
+Plugins can use the :ref:`plugin_hook_actor_from_request` hook to implement custom logic for authenticating an actor based on the incoming HTTP request.
 
 .. _authentication_root:
 
@@ -314,7 +314,7 @@ Checking permissions in plugins
 
 Datasette plugins can check if an actor has permission to perform an action using the :ref:`datasette.permission_allowed(...)` method.
 
-Datasette core performs a number of permission checks, :ref:`documented below `. Plugins can implement the :ref:`plugin_permission_allowed` plugin hook to participate in decisions about whether an actor should be able to perform a specified action.
+Datasette core performs a number of permission checks, :ref:`documented below `. Plugins can implement the :ref:`plugin_hook_permission_allowed` plugin hook to participate in decisions about whether an actor should be able to perform a specified action.
 
 .. _authentication_actor_matches_allow:
 
diff --git a/docs/changelog.rst b/docs/changelog.rst
index aca8f8c2..3a7f9562 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -27,7 +27,7 @@ You'll need to install plugins if you want full user accounts, but default Datas
     INFO:     Application startup complete.
     INFO:     Uvicorn running on http://127.0.0.1:8001 (Press CTRL+C to quit)
 
-Plugins can implement new ways of authenticating users using the new :ref:`plugin_actor_from_request` hook.
+Plugins can implement new ways of authenticating users using the new :ref:`plugin_hook_actor_from_request` hook.
 
 Permissions
 ~~~~~~~~~~~
@@ -52,7 +52,7 @@ You can use the new ``"allow"`` block syntax in ``metadata.json`` (or ``metadata
 
 See :ref:`authentication_permissions_allow` for more details.
 
-Plugins can implement their own custom permission checks using the new :ref:`plugin_permission_allowed` hook.
+Plugins can implement their own custom permission checks using the new :ref:`plugin_hook_permission_allowed` hook.
 
 A new debug page at ``/-/permissions`` shows recent permission checks, to help administrators and plugin authors understand exactly what checks are being performed. This tool defaults to only being available to the root user, but can be exposed to other users by plugins that respond to the ``permissions-debug`` permission. (`#788 `__)
 
diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst
index 4777cc16..dcb5a887 100644
--- a/docs/ecosystem.rst
+++ b/docs/ecosystem.rst
@@ -87,6 +87,16 @@ datasette-auth-github
 
 `datasette-auth-github `__ adds an authentication layer to Datasette. Users will have to sign in using their GitHub account before they can view data or interact with Datasette. You can also use it to restrict access to specific GitHub users, or to members of specified GitHub `organizations `__ or `teams `__.
 
+datasette-auth-tokens
+---------------------
+
+`datasette-auth-tokens `__ provides a mechanism for creating secret API tokens that can then be used with Datasette's :ref:`authentication` system.
+
+datasette-permissions-sql
+---------------------
+
+`datasette-permissions-sql `__ lets you configure Datasette permissions checks to use custom SQL queries, which means you can make permisison decisions based on data contained within your databases.
+
 datasette-upload-csvs
 ---------------------
 
diff --git a/docs/internals.rst b/docs/internals.rst
index d75544e1..ab9da410 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -219,7 +219,7 @@ await .permission_allowed(actor, action, resource=None, default=False)
 
 Check if the given actor has :ref:`permission ` to perform the given action on the given resource.
 
-Some permission checks are carried out against :ref:`rules defined in metadata.json `, while other custom permissions may be decided by plugins that implement the :ref:`plugin_permission_allowed` plugin hook.
+Some permission checks are carried out against :ref:`rules defined in metadata.json `, while other custom permissions may be decided by plugins that implement the :ref:`plugin_hook_permission_allowed` plugin hook.
 
 If neither ``metadata.json`` nor any of the plugins provide an answer to the permission query the ``default`` argument will be returned.
 
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 989cf672..608f93da 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -25,9 +25,8 @@ Things you can do with plugins include:
 * Customize how database values are rendered in the Datasette interface, for example
   `datasette-render-binary `__ and
   `datasette-pretty-json `__.
-* Wrap the entire Datasette application in custom ASGI middleware to add new pages
-  or implement authentication, for example
-  `datasette-auth-github `__.
+* Customize how Datasette's authentication and permissions systems work, for example `datasette-auth-tokens `__ and
+  `datasette-permissions-sql `__.
 
 .. _plugins_installing:
 
@@ -996,7 +995,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
 
 Examples: `datasette-auth-github `_, `datasette-search-all `_, `datasette-media `_
 
-.. _plugin_actor_from_request:
+.. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1055,7 +1054,9 @@ Instead of returning a dictionary, this function can return an awaitable functio
 
         return inner
 
-.. _plugin_permission_allowed:
+Example: `datasette-auth-tokens `_
+
+.. _plugin_hook_permission_allowed:
 
 permission_allowed(datasette, actor, action, resource)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1064,7 +1065,7 @@ permission_allowed(datasette, actor, action, resource)
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
 
 ``actor`` - dictionary
-    The current actor, as decided by :ref:`plugin_actor_from_request`.
+    The current actor, as decided by :ref:`plugin_hook_actor_from_request`.
 
 ``action`` - string
     The action to be performed, e.g. ``"edit-table"``.
@@ -1110,4 +1111,6 @@ Here's an example that allows users to view the ``admin_log`` table only if thei
 
         return inner
 
-See :ref:`permissions` for a full list of permissions that are included in Datasette core.
+See :ref:`built-in permissions ` for a full list of permissions that are included in Datasette core.
+
+Example: `datasette-permissions-sql `_

From 9ae0d483ead93c0832142e5dc85959ae3c8f73ea Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 17:48:20 -0700
Subject: [PATCH 0558/2303] Get "$file": "../path"  mechanism working again,
 closes #839

---
 datasette/utils/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index ae7bbdb5..14060669 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -911,6 +911,8 @@ def resolve_env_secrets(config, environ):
     if isinstance(config, dict):
         if list(config.keys()) == ["$env"]:
             return environ.get(list(config.values())[0])
+        elif list(config.keys()) == ["$file"]:
+            return open(list(config.values())[0]).read()
         else:
             return {
                 key: resolve_env_secrets(value, environ)

From b906030235efbdff536405d66078f4868ce0d3bd Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 11 Jun 2020 18:19:30 -0700
Subject: [PATCH 0559/2303] Release Datasette 0.44

Refs #395, #519, #576, #699, #706, #774, #777, #781, #784, #788, #790, #797,
#798, #800, #802, #804, #819, #822, #825, #826, #827, #828, #829, #830,
#833, #836, #837, #839

Closes #806.
---
 README.md          | 1 +
 docs/changelog.rst | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 90df75de..925d68d2 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 
 ## News
 
+ * 11th June 2020: [Datasette 0.44](http://datasette.readthedocs.io/en/latest/changelog.html#v0-44) - [Authentication and permissions](https://datasette.readthedocs.io/en/latest/authentication.html), [writable canned queries](https://datasette.readthedocs.io/en/latest/sql_queries.html#writable-canned-queries), flash messages, new plugin hooks and much, much more.
  * 28th May 2020: [Datasette 0.43](http://datasette.readthedocs.io/en/latest/changelog.html#v0-43) - Redesigned [register_output_renderer](https://datasette.readthedocs.io/en/latest/plugins.html#plugin-register-output-renderer) plugin hook and various small improvements and fixes.
  * 8th May 2020: [Datasette 0.42](http://datasette.readthedocs.io/en/latest/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database.
  * 6th May 2020: [Datasette 0.41](http://datasette.readthedocs.io/en/latest/changelog.html#v0-41) - New mechanism for [creating custom pages](https://datasette.readthedocs.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://datasette.readthedocs.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements.
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 3a7f9562..b1e95bb7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,7 +6,7 @@ Changelog
 
 .. _v0_44:
 
-0.44 (2020-06-??)
+0.44 (2020-06-11)
 -----------------
 
 Authentication and permissions, writable canned queries, flash messages, new plugin hooks and more.

From 09a3479a5402df96489ed6cab6cc9fd674bf3433 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 10:55:41 -0700
Subject: [PATCH 0560/2303] New "startup" plugin hook, closes #834

---
 datasette/app.py           |  7 +++++++
 datasette/cli.py           |  3 +++
 datasette/hookspecs.py     |  5 +++++
 docs/plugins.rst           | 33 +++++++++++++++++++++++++++++++++
 tests/fixtures.py          |  1 +
 tests/plugins/my_plugin.py |  5 +++++
 tests/test_cli.py          |  1 +
 tests/test_plugins.py      |  6 ++++++
 8 files changed, 61 insertions(+)

diff --git a/datasette/app.py b/datasette/app.py
index ebab3bee..ca2efa91 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -302,6 +302,13 @@ class Datasette:
         self._permission_checks = collections.deque(maxlen=200)
         self._root_token = secrets.token_hex(32)
 
+    async def invoke_startup(self):
+        for hook in pm.hook.startup(datasette=self):
+            if callable(hook):
+                hook = hook()
+            if asyncio.iscoroutine(hook):
+                hook = await hook
+
     def sign(self, value, namespace="default"):
         return URLSafeSerializer(self._secret, namespace).dumps(value)
 
diff --git a/datasette/cli.py b/datasette/cli.py
index ff9a2d5c..bba72484 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -397,6 +397,9 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
+    # Run the "startup" plugin hooks
+    asyncio.get_event_loop().run_until_complete(ds.invoke_startup())
+
     # Run async sanity checks - but only if we're not under pytest
     asyncio.get_event_loop().run_until_complete(check_databases(ds))
 
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index ab3e131c..9fceee41 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -5,6 +5,11 @@ hookspec = HookspecMarker("datasette")
 hookimpl = HookimplMarker("datasette")
 
 
+@hookspec
+def startup(datasette):
+    "Fires directly after Datasette first starts running"
+
+
 @hookspec
 def asgi_wrapper(datasette):
     "Returns an ASGI middleware callable to wrap our ASGI application with"
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 608f93da..289be649 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -995,6 +995,39 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
 
 Examples: `datasette-auth-github `_, `datasette-search-all `_, `datasette-media `_
 
+.. _plugin_hook_startup:
+
+startup(datasette)
+~~~~~~~~~~~~~~~~~~
+
+This hook fires when the Datasette application server first starts up. You can implement a regular function, for example to validate required plugin configuration:
+
+.. code-block:: python
+
+    @hookimpl
+    def startup(datasette):
+        config = datasette.plugin_config("my-plugin") or {}
+        assert "required-setting" in config, "my-plugin requires setting required-setting"
+
+Or you can return an async function which will be awaited on startup. Use this option if you need to make any database queries:
+
+    @hookimpl
+    def startup(datasette):
+        async def inner():
+            db = datasette.get_database()
+            if "my_table" not in await db.table_names():
+                await db.execute_write("""
+                    create table my_table (mycol text)
+                """, block=True)
+        return inner
+
+
+Potential use-cases:
+
+* Run some initialization code for the plugin
+* Create database tables that a plugin needs
+* Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid
+
 .. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 907bf895..09819575 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -49,6 +49,7 @@ EXPECTED_PLUGINS = [
             "register_facet_classes",
             "register_routes",
             "render_cell",
+            "startup",
         ],
     },
     {
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index a0f7441b..3f019a84 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -167,3 +167,8 @@ def register_routes():
         (r"/two/(?P.*)$", two),
         (r"/three/$", three),
     ]
+
+
+@hookimpl
+def startup(datasette):
+    datasette._startup_hook_fired = True
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 6939fe57..90aa990d 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -10,6 +10,7 @@ from click.testing import CliRunner
 import io
 import json
 import pathlib
+import pytest
 import textwrap
 
 
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 0fae3740..c0a7438f 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -572,3 +572,9 @@ def test_register_routes_asgi(app_client):
     response = app_client.get("/three/")
     assert {"hello": "world"} == response.json
     assert "1" == response.headers["x-three"]
+
+
+@pytest.mark.asyncio
+async def test_startup(app_client):
+    await app_client.ds.invoke_startup()
+    assert app_client.ds._startup_hook_fired

From 72ae975156a09619a808cdd03fddddcf62e6f533 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 10:58:32 -0700
Subject: [PATCH 0561/2303] Added test for async startup hook, refs #834

---
 tests/plugins/my_plugin_2.py | 8 ++++++++
 tests/test_plugins.py        | 1 +
 2 files changed, 9 insertions(+)

diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index 039112f4..bdfaea8d 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -120,3 +120,11 @@ def permission_allowed(datasette, actor, action):
             return False
 
     return inner
+
+
+@hookimpl
+def startup(datasette):
+    async def inner():
+        result = await datasette.get_database().execute("select 1 + 1")
+        datasette._startup_hook_calculation = result.first()[0]
+    return inner
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index c0a7438f..bc759385 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -578,3 +578,4 @@ def test_register_routes_asgi(app_client):
 async def test_startup(app_client):
     await app_client.ds.invoke_startup()
     assert app_client.ds._startup_hook_fired
+    assert 2 == app_client.ds._startup_hook_calculation

From ae99af25361c9248c721153922c623bd5f440159 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 10:59:35 -0700
Subject: [PATCH 0562/2303] Fixed rST code formatting, refs #834

---
 docs/plugins.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/plugins.rst b/docs/plugins.rst
index 289be649..8add7352 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1011,6 +1011,8 @@ This hook fires when the Datasette application server first starts up. You can i
 
 Or you can return an async function which will be awaited on startup. Use this option if you need to make any database queries:
 
+.. code-block:: python
+
     @hookimpl
     def startup(datasette):
         async def inner():
@@ -1021,7 +1023,6 @@ Or you can return an async function which will be awaited on startup. Use this o
                 """, block=True)
         return inner
 
-
 Potential use-cases:
 
 * Run some initialization code for the plugin

From d60bd6ad13ef908d7e66a677caee20536f3fb277 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 11:15:33 -0700
Subject: [PATCH 0563/2303] Update plugin tests, refs #834

---
 tests/fixtures.py            | 1 +
 tests/plugins/my_plugin_2.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/fixtures.py b/tests/fixtures.py
index 09819575..e2f90f09 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -64,6 +64,7 @@ EXPECTED_PLUGINS = [
             "extra_template_vars",
             "permission_allowed",
             "render_cell",
+            "startup",
         ],
     },
     {
diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index bdfaea8d..f4a082a0 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -127,4 +127,5 @@ def startup(datasette):
     async def inner():
         result = await datasette.get_database().execute("select 1 + 1")
         datasette._startup_hook_calculation = result.first()[0]
+
     return inner

From 0e49842e227a0f1f69d48108c87d17fe0379e548 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 11:29:14 -0700
Subject: [PATCH 0564/2303] datasette/actor_auth_cookie.py coverae to 100%,
 refs #841

---
 tests/test_auth.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_auth.py b/tests/test_auth.py
index 5e847445..bb4bee4b 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -26,6 +26,17 @@ def test_actor_cookie(app_client):
     assert {"id": "test"} == app_client.ds._last_request.scope["actor"]
 
 
+def test_actor_cookie_invalid(app_client):
+    cookie = app_client.actor_cookie({"id": "test"})
+    # Break the signature
+    response = app_client.get("/", cookies={"ds_actor": cookie[:-1] + "."})
+    assert None == app_client.ds._last_request.scope["actor"]
+    # Break the cookie format
+    cookie = app_client.ds.sign({"b": {"id": "test"}}, "actor")
+    response = app_client.get("/", cookies={"ds_actor": cookie})
+    assert None == app_client.ds._last_request.scope["actor"]
+
+
 @pytest.mark.parametrize(
     "offset,expected", [((24 * 60 * 60), {"id": "test"}), (-(24 * 60 * 60), None),]
 )

From 80c18a18fc444b89cc12b73599d56e091f3a3c87 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 13:48:23 -0700
Subject: [PATCH 0565/2303] Configure code coverage, refs #841, #843

---
 .coveragerc | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 .coveragerc

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000..6ca0fac8
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+omit = datasette/_version.py, datasette/utils/shutil_backport.py

From cf7a2bdb404734910ec07abc7571351a2d934828 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 14:36:49 -0700
Subject: [PATCH 0566/2303] Action to run tests and upload coverage to
 codecov.io

Closes #843.
---
 .github/workflows/test-coverage.yml | 41 +++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 .github/workflows/test-coverage.yml

diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml
new file mode 100644
index 00000000..99c0526a
--- /dev/null
+++ b/.github/workflows/test-coverage.yml
@@ -0,0 +1,41 @@
+name: Calculate test coverage
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out datasette
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install Python dependencies
+      run: |
+        python -m pip install -e .[test]
+        python -m pip install pytest-cov
+    - name: Run tests
+      run: |-
+        ls -lah
+        cat .coveragerc
+        pytest --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term
+        ls -lah
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml

From 0c27f10f9d2124f0f534c25612b58be20441c9d8 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 16:41:26 -0700
Subject: [PATCH 0567/2303] Updated plugin examples to include datasette-psutil

---
 docs/plugins.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/plugins.rst b/docs/plugins.rst
index 8add7352..113e6b24 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -884,6 +884,8 @@ The optional view function arguments are as follows:
 
 The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only).
 
+Examples: `datasette-auth-github `__, `datasette-psutil `__
+
 .. _plugin_register_facet_classes:
 
 register_facet_classes()
@@ -993,7 +995,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
             return add_x_databases_header
         return wrap_with_databases_header
 
-Examples: `datasette-auth-github `_, `datasette-search-all `_, `datasette-media `_
+Examples: `datasette-search-all `_, `datasette-media `_
 
 .. _plugin_hook_startup:
 

From a4ad5a504c161bc3b1caaa40b22e46d600f7d4fc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Jun 2020 17:26:02 -0700
Subject: [PATCH 0568/2303] Workaround for 'Too many open files' in test runs,
 refs #846

---
 tests/fixtures.py | 3 +++
 tests/test_api.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/fixtures.py b/tests/fixtures.py
index e2f90f09..a4a96919 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -268,6 +268,9 @@ def make_app_client(
                 "default_page_size": 50,
                 "max_returned_rows": max_returned_rows or 100,
                 "sql_time_limit_ms": sql_time_limit_ms or 200,
+                # Default is 3 but this results in "too many open files"
+                # errors when running the full test suite:
+                "num_sql_threads": 1,
             }
         )
         ds = Datasette(
diff --git a/tests/test_api.py b/tests/test_api.py
index 1a54edec..322a0001 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1320,7 +1320,7 @@ def test_config_json(app_client):
         "suggest_facets": True,
         "default_cache_ttl": 5,
         "default_cache_ttl_hashed": 365 * 24 * 60 * 60,
-        "num_sql_threads": 3,
+        "num_sql_threads": 1,
         "cache_size_kb": 0,
         "allow_csv_stream": True,
         "max_csv_mb": 100,

From d2aef9f7ef30fa20b1450cd181cf803f44fb4e21 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 09:21:15 -0700
Subject: [PATCH 0569/2303] Test illustrating POST against register_routes(),
 closes #853

---
 tests/plugins/my_plugin.py | 7 +++++++
 tests/test_plugins.py      | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 3f019a84..72736e84 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -162,10 +162,17 @@ def register_routes():
             send, {"hello": "world"}, status=200, headers={"x-three": "1"}
         )
 
+    async def post(request):
+        if request.method == "GET":
+            return Response.html(request.scope["csrftoken"]())
+        else:
+            return Response.json(await request.post_vars())
+
     return [
         (r"/one/$", one),
         (r"/two/(?P.*)$", two),
         (r"/three/$", three),
+        (r"/post/$", post),
     ]
 
 
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index bc759385..e3a234f2 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -568,6 +568,13 @@ def test_register_routes(app_client, path, body):
     assert body == response.text
 
 
+def test_register_routes_post(app_client):
+    response = app_client.post("/post/", {"this is": "post data"}, csrftoken_from=True)
+    assert 200 == response.status
+    assert "csrftoken" in response.json
+    assert "post data" == response.json["this is"]
+
+
 def test_register_routes_asgi(app_client):
     response = app_client.get("/three/")
     assert {"hello": "world"} == response.json

From 6151c25a5a8d566c109af296244b9267c536bd9a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 11:37:28 -0700
Subject: [PATCH 0570/2303] Respect existing scope["actor"] if set, closes #854

---
 datasette/app.py           |  3 ++-
 tests/fixtures.py          |  1 +
 tests/plugins/my_plugin.py | 14 ++++++++++++++
 tests/test_plugins.py      |  5 +++++
 4 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index ca2efa91..c684eabc 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -908,6 +908,7 @@ class DatasetteRouter(AsgiRouter):
         ):
             scope_modifications["scheme"] = "https"
         # Handle authentication
+        default_actor = scope.get("actor") or None
         actor = None
         for actor in pm.hook.actor_from_request(
             datasette=self.ds, request=Request(scope, receive)
@@ -918,7 +919,7 @@ class DatasetteRouter(AsgiRouter):
                 actor = await actor
             if actor:
                 break
-        scope_modifications["actor"] = actor
+        scope_modifications["actor"] = actor or default_actor
         return await super().route_path(
             dict(scope, **scope_modifications), receive, send, path
         )
diff --git a/tests/fixtures.py b/tests/fixtures.py
index a4a96919..612bee99 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -39,6 +39,7 @@ EXPECTED_PLUGINS = [
         "version": None,
         "hooks": [
             "actor_from_request",
+            "asgi_wrapper",
             "extra_body_script",
             "extra_css_urls",
             "extra_js_urls",
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 72736e84..a86e3cbf 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -137,6 +137,20 @@ def actor_from_request(datasette, request):
         return None
 
 
+@hookimpl
+def asgi_wrapper():
+    def wrap(app):
+        async def maybe_set_actor_in_scope(scope, recieve, send):
+            if b"_actor_in_scope" in scope["query_string"]:
+                scope = dict(scope, actor={"id": "from-scope"})
+                print(scope)
+            await app(scope, recieve, send)
+
+        return maybe_set_actor_in_scope
+
+    return wrap
+
+
 @hookimpl
 def permission_allowed(actor, action):
     if action == "this_is_allowed":
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index e3a234f2..245c60f7 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -534,6 +534,11 @@ def test_actor_from_request_async(app_client):
     assert {"id": "bot2", "1+1": 2} == app_client.ds._last_request.scope["actor"]
 
 
+def test_existing_scope_actor_respected(app_client):
+    app_client.get("/?_actor_in_scope=1")
+    assert {"id": "from-scope"} == app_client.ds._last_request.scope["actor"]
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "action,expected",

From 13216cb6bd715b3068b917bdeb1f1f24d159c34c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 13:40:33 -0700
Subject: [PATCH 0571/2303] Don't push alpha/beta tagged releases to Docker Hub

Refs #807
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 5e328d7a..5aafe398 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,7 +32,7 @@ jobs:
             branch: master
             tags: true
     - stage: publish docker image
-      if: tag IS present
+      if: (tag IS present) AND NOT (tag =~ [ab])
       python: 3.6
       script:
         # Build and release to Docker Hub

From c81f637d862a6b13ac4b07cef5a493b62e079c81 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 13:49:52 -0700
Subject: [PATCH 0572/2303] Documentation for alpha/beta release process, refs
 #807

---
 docs/contributing.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index ba52839c..75c1c3b2 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -147,6 +147,8 @@ We increment ``minor`` for new features.
 
 We increment ``patch`` for bugfix releass.
 
+:ref:`contributing_release_alpha_beta` may have an additional ``a0`` or ``b0`` prefix - the integer component will be incremented with each subsequent alpha or beta.
+
 To release a new version, first create a commit that updates :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__::
 
     # Update changelog
@@ -180,3 +182,14 @@ Final steps once the release has deployed to https://pypi.org/project/datasette/
 
 * Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases - you can convert the release notes to Markdown by copying and pasting the rendered HTML into this tool: https://euangoddard.github.io/clipboard2markdown/
 * Manually kick off a build of the `stable` branch on Read The Docs: https://readthedocs.org/projects/datasette/builds/
+
+.. _contributing_release_alpha_beta:
+
+Alpha and beta releases
+-----------------------
+
+Alpha and beta releases are published to preview upcoming features that may not yet be stable - in particular to preview new plugin hooks.
+
+You are welcome to try these out, but please be aware that details may change before the final release.
+
+Please join `discussions on the issue tracker `__ to share your thoughts and experiences with on alpha and beta features that you try out.

From dda932d818b34ccab11730a76554f0a3748d8348 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 13:58:09 -0700
Subject: [PATCH 0573/2303] Release notes for 0.45a0

Refs #834 #846 #854 #807
---
 docs/changelog.rst    | 12 ++++++++++++
 docs/contributing.rst |  4 ++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b1e95bb7..705ba4d4 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,18 @@
 Changelog
 =========
 
+.. _v0_45 alpha:
+
+0.45a0 (2020-06-18)
+-------------------
+
+.. warning:: This is an **alpha** release. See :ref:`contributing_alpha_beta`.
+
+- New :ref:`plugin_hook_startup` plugin hook. (`#834 `__)
+- Workaround for "Too many open files" error in test runs. (`#846 `__)
+- Respect existing ``scope["actor"]`` if already set by ASGI middleware. (`#854 `__)
+- New process for shipping :ref:`contributing_alpha_beta`. (`#807 `__)
+
 .. _v0_44:
 
 0.44 (2020-06-11)
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 75c1c3b2..03af7644 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -147,7 +147,7 @@ We increment ``minor`` for new features.
 
 We increment ``patch`` for bugfix releass.
 
-:ref:`contributing_release_alpha_beta` may have an additional ``a0`` or ``b0`` prefix - the integer component will be incremented with each subsequent alpha or beta.
+:ref:`contributing_alpha_beta` may have an additional ``a0`` or ``b0`` prefix - the integer component will be incremented with each subsequent alpha or beta.
 
 To release a new version, first create a commit that updates :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__::
 
@@ -183,7 +183,7 @@ Final steps once the release has deployed to https://pypi.org/project/datasette/
 * Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases - you can convert the release notes to Markdown by copying and pasting the rendered HTML into this tool: https://euangoddard.github.io/clipboard2markdown/
 * Manually kick off a build of the `stable` branch on Read The Docs: https://readthedocs.org/projects/datasette/builds/
 
-.. _contributing_release_alpha_beta:
+.. _contributing_alpha_beta:
 
 Alpha and beta releases
 -----------------------

From d2f387591bdda3949162e1802816be6ca1bb777a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 14:01:36 -0700
Subject: [PATCH 0574/2303] Better rST label for alpha release, refs #807

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 705ba4d4..e117663f 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,7 +4,7 @@
 Changelog
 =========
 
-.. _v0_45 alpha:
+.. _v0_45a0:
 
 0.45a0 (2020-06-18)
 -------------------

From 6c2634583627bfab750c115cb13850252821d637 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 16:22:33 -0700
Subject: [PATCH 0575/2303] New plugin hook: canned_queries(), refs #852

---
 datasette/app.py                 | 26 +++++++----
 datasette/default_permissions.py | 75 ++++++++++++++++----------------
 datasette/hookspecs.py           |  5 +++
 datasette/views/database.py      |  4 +-
 datasette/views/table.py         |  6 ++-
 docs/plugins.rst                 | 67 ++++++++++++++++++++++++++++
 tests/fixtures.py                |  2 +
 tests/plugins/my_plugin.py       |  9 ++++
 tests/plugins/my_plugin_2.py     | 14 ++++++
 tests/test_canned_write.py       | 10 ++++-
 tests/test_html.py               |  2 +
 tests/test_plugins.py            | 31 +++++++++++++
 12 files changed, 202 insertions(+), 49 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index c684eabc..e131ba46 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -387,18 +387,28 @@ class Datasette:
             ).hexdigest()[:6]
         return self._app_css_hash
 
-    def get_canned_queries(self, database_name):
+    async def get_canned_queries(self, database_name, actor):
         queries = self.metadata("queries", database=database_name, fallback=False) or {}
-        names = queries.keys()
-        return [self.get_canned_query(database_name, name) for name in names]
+        for more_queries in pm.hook.canned_queries(
+            datasette=self, database=database_name, actor=actor,
+        ):
+            if callable(more_queries):
+                more_queries = more_queries()
+            if asyncio.iscoroutine(more_queries):
+                more_queries = await more_queries
+            queries.update(more_queries or {})
+        # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}}
+        for key in queries:
+            if not isinstance(queries[key], dict):
+                queries[key] = {"sql": queries[key]}
+            # Also make sure "name" is available:
+            queries[key]["name"] = key
+        return queries
 
-    def get_canned_query(self, database_name, query_name):
-        queries = self.metadata("queries", database=database_name, fallback=False) or {}
+    async def get_canned_query(self, database_name, query_name, actor):
+        queries = await self.get_canned_queries(database_name, actor)
         query = queries.get(query_name)
         if query:
-            if not isinstance(query, dict):
-                query = {"sql": query}
-            query["name"] = query_name
             return query
 
     def update_with_inherited_metadata(self, metadata):
diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py
index e750acbf..0929a17a 100644
--- a/datasette/default_permissions.py
+++ b/datasette/default_permissions.py
@@ -4,41 +4,42 @@ from datasette.utils import actor_matches_allow
 
 @hookimpl(tryfirst=True)
 def permission_allowed(datasette, actor, action, resource):
-    if action == "permissions-debug":
-        if actor and actor.get("id") == "root":
-            return True
-    elif action == "view-instance":
-        allow = datasette.metadata("allow")
-        if allow is not None:
+    async def inner():
+        if action == "permissions-debug":
+            if actor and actor.get("id") == "root":
+                return True
+        elif action == "view-instance":
+            allow = datasette.metadata("allow")
+            if allow is not None:
+                return actor_matches_allow(actor, allow)
+        elif action == "view-database":
+            database_allow = datasette.metadata("allow", database=resource)
+            if database_allow is None:
+                return True
+            return actor_matches_allow(actor, database_allow)
+        elif action == "view-table":
+            database, table = resource
+            tables = datasette.metadata("tables", database=database) or {}
+            table_allow = (tables.get(table) or {}).get("allow")
+            if table_allow is None:
+                return True
+            return actor_matches_allow(actor, table_allow)
+        elif action == "view-query":
+            # Check if this query has a "allow" block in metadata
+            database, query_name = resource
+            query = await datasette.get_canned_query(database, query_name, actor)
+            assert query is not None
+            allow = query.get("allow")
+            if allow is None:
+                return True
             return actor_matches_allow(actor, allow)
-    elif action == "view-database":
-        database_allow = datasette.metadata("allow", database=resource)
-        if database_allow is None:
-            return True
-        return actor_matches_allow(actor, database_allow)
-    elif action == "view-table":
-        database, table = resource
-        tables = datasette.metadata("tables", database=database) or {}
-        table_allow = (tables.get(table) or {}).get("allow")
-        if table_allow is None:
-            return True
-        return actor_matches_allow(actor, table_allow)
-    elif action == "view-query":
-        # Check if this query has a "allow" block in metadata
-        database, query_name = resource
-        queries_metadata = datasette.metadata("queries", database=database)
-        assert query_name in queries_metadata
-        if isinstance(queries_metadata[query_name], str):
-            return True
-        allow = queries_metadata[query_name].get("allow")
-        if allow is None:
-            return True
-        return actor_matches_allow(actor, allow)
-    elif action == "execute-sql":
-        # Use allow_sql block from database block, or from top-level
-        database_allow_sql = datasette.metadata("allow_sql", database=resource)
-        if database_allow_sql is None:
-            database_allow_sql = datasette.metadata("allow_sql")
-        if database_allow_sql is None:
-            return True
-        return actor_matches_allow(actor, database_allow_sql)
+        elif action == "execute-sql":
+            # Use allow_sql block from database block, or from top-level
+            database_allow_sql = datasette.metadata("allow_sql", database=resource)
+            if database_allow_sql is None:
+                database_allow_sql = datasette.metadata("allow_sql")
+            if database_allow_sql is None:
+                return True
+            return actor_matches_allow(actor, database_allow_sql)
+
+    return inner
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 9fceee41..91feb49b 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -78,3 +78,8 @@ def actor_from_request(datasette, request):
 @hookspec
 def permission_allowed(datasette, actor, action, resource):
     "Check if actor is allowed to perfom this action - return True, False or None"
+
+
+@hookspec
+def canned_queries(datasette, database, actor):
+    "Return a dictonary of canned query definitions or an awaitable function that returns them"
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 4fab2cfb..ad28fb63 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -70,7 +70,9 @@ class DatabaseView(DataView):
 
         tables.sort(key=lambda t: (t["hidden"], t["name"]))
         canned_queries = []
-        for query in self.ds.get_canned_queries(database):
+        for query in (
+            await self.ds.get_canned_queries(database, request.actor)
+        ).values():
             visible, private = await check_visibility(
                 self.ds, request.actor, "view-query", (database, query["name"]),
             )
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 91245293..1a55a495 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -223,7 +223,9 @@ class TableView(RowTableShared):
 
     async def post(self, request, db_name, table_and_format):
         # Handle POST to a canned query
-        canned_query = self.ds.get_canned_query(db_name, table_and_format)
+        canned_query = await self.ds.get_canned_query(
+            db_name, table_and_format, request.actor
+        )
         assert canned_query, "You may only POST to a canned query"
         return await QueryView(self.ds).data(
             request,
@@ -247,7 +249,7 @@ class TableView(RowTableShared):
         _next=None,
         _size=None,
     ):
-        canned_query = self.ds.get_canned_query(database, table)
+        canned_query = await self.ds.get_canned_query(database, table, request.actor)
         if canned_query:
             return await QueryView(self.ds).data(
                 request,
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 113e6b24..8444516c 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1031,6 +1031,73 @@ Potential use-cases:
 * Create database tables that a plugin needs
 * Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid
 
+.. _plugin_hook_canned_queries:
+
+canned_queries(datasette, database, actor)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
+
+``database`` - string
+    The name of the database.
+
+``actor`` - dictionary or None
+    The currently authenticated :ref:`authentication_actor`.
+
+Ues this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def canned_queries(datasette, database):
+        if database == "mydb":
+            return {
+                "my_query": {
+                    "sql": "select * from my_table where id > :min_id"
+                }
+            }
+
+The hook can alternatively return an awaitable function that returns a list. Here's an example that returns queries that have been stored in the ``saved_queries`` database table, if one exists:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def canned_queries(datasette, database):
+        async def inner():
+            db = datasette.get_database(database)
+            if await db.table_exists("saved_queries"):
+                results = await db.execute("select name, sql from saved_queries")
+                return {result["name"]: {
+                    "sql": result["sql"]
+                } for result in results}
+        return inner
+
+The actor parameter can be used to include the currently authenticated actor in your decision. Here's an example that returns saved queries that were saved by that actor:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def canned_queries(datasette, database, actor):
+        async def inner():
+            db = datasette.get_database(database)
+            if actor is not None and await db.table_exists("saved_queries"):
+                results = await db.execute(
+                    "select name, sql from saved_queries where actor_id = :id", {
+                        "id": actor["id"]
+                    }
+                )
+                return {result["name"]: {
+                    "sql": result["sql"]
+                } for result in results}
+        return inner
+
 .. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 612bee99..9b28c283 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -40,6 +40,7 @@ EXPECTED_PLUGINS = [
         "hooks": [
             "actor_from_request",
             "asgi_wrapper",
+            "canned_queries",
             "extra_body_script",
             "extra_css_urls",
             "extra_js_urls",
@@ -61,6 +62,7 @@ EXPECTED_PLUGINS = [
         "hooks": [
             "actor_from_request",
             "asgi_wrapper",
+            "canned_queries",
             "extra_js_urls",
             "extra_template_vars",
             "permission_allowed",
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index a86e3cbf..7ed26908 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -193,3 +193,12 @@ def register_routes():
 @hookimpl
 def startup(datasette):
     datasette._startup_hook_fired = True
+
+
+@hookimpl
+def canned_queries(datasette, database, actor):
+    return {
+        "from_hook": "select 1, '{}' as actor_id".format(
+            actor["id"] if actor else "null"
+        )
+    }
diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index f4a082a0..556c8090 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -129,3 +129,17 @@ def startup(datasette):
         datasette._startup_hook_calculation = result.first()[0]
 
     return inner
+
+
+@hookimpl
+def canned_queries(datasette, database):
+    async def inner():
+        return {
+            "from_async_hook": "select {}".format(
+                (
+                    await datasette.get_database(database).execute("select 1 + 1")
+                ).first()[0]
+            )
+        }
+
+    return inner
diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py
index 4257806e..c36baa09 100644
--- a/tests/test_canned_write.py
+++ b/tests/test_canned_write.py
@@ -111,7 +111,13 @@ def test_canned_query_permissions_on_database_page(canned_write_client):
     query_names = [
         q["name"] for q in canned_write_client.get("/data.json").json["queries"]
     ]
-    assert ["add_name", "add_name_specify_id", "update_name"] == query_names
+    assert [
+        "add_name",
+        "add_name_specify_id",
+        "update_name",
+        "from_async_hook",
+        "from_hook",
+    ] == query_names
 
     # With auth shows four
     response = canned_write_client.get(
@@ -124,6 +130,8 @@ def test_canned_query_permissions_on_database_page(canned_write_client):
         {"name": "add_name_specify_id", "private": False},
         {"name": "delete_name", "private": True},
         {"name": "update_name", "private": False},
+        {"name": "from_async_hook", "private": False},
+        {"name": "from_hook", "private": False},
     ] == [
         {"name": q["name"], "private": q["private"]} for q in response.json["queries"]
     ]
diff --git a/tests/test_html.py b/tests/test_html.py
index f9b18daa..7bc935b0 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -97,6 +97,8 @@ def test_database_page(app_client):
         ),
         ("/fixtures/pragma_cache_size", "pragma_cache_size"),
         ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"),
+        ("/fixtures/from_async_hook", "from_async_hook"),
+        ("/fixtures/from_hook", "from_hook"),
     ] == [(a["href"], a.text) for a in queries_ul.find_all("a")]
 
 
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 245c60f7..4f44430e 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -591,3 +591,34 @@ async def test_startup(app_client):
     await app_client.ds.invoke_startup()
     assert app_client.ds._startup_hook_fired
     assert 2 == app_client.ds._startup_hook_calculation
+
+
+def test_canned_queries(app_client):
+    queries = app_client.get("/fixtures.json").json["queries"]
+    queries_by_name = {q["name"]: q for q in queries}
+    assert {
+        "sql": "select 2",
+        "name": "from_async_hook",
+        "private": False,
+    } == queries_by_name["from_async_hook"]
+    assert {
+        "sql": "select 1, 'null' as actor_id",
+        "name": "from_hook",
+        "private": False,
+    } == queries_by_name["from_hook"]
+
+
+def test_canned_queries_non_async(app_client):
+    response = app_client.get("/fixtures/from_hook.json?_shape=array")
+    assert [{"1": 1, "actor_id": "null"}] == response.json
+
+
+def test_canned_queries_async(app_client):
+    response = app_client.get("/fixtures/from_async_hook.json?_shape=array")
+    assert [{"2": 2}] == response.json
+
+
+def test_canned_queries_actor(app_client):
+    assert [{"1": 1, "actor_id": "bot"}] == app_client.get(
+        "/fixtures/from_hook.json?_bot=1&_shape=array"
+    ).json

From 9216127ace8d80493f743a4ef4c469f83a3b81ce Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 16:39:43 -0700
Subject: [PATCH 0576/2303] Documentation tweak, refs #852

---
 docs/plugins.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugins.rst b/docs/plugins.rst
index 8444516c..dce1bdf0 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1043,7 +1043,7 @@ canned_queries(datasette, database, actor)
     The name of the database.
 
 ``actor`` - dictionary or None
-    The currently authenticated :ref:`authentication_actor`.
+    The currently authenticated :ref:`actor `.
 
 Ues this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation.
 

From 0807c4200f6b31c804c476eb546ead3f875a2ecc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 16:40:45 -0700
Subject: [PATCH 0577/2303] Release notes for 0.45a1, refs #852

---
 docs/changelog.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index e117663f..6f3af8ce 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,14 +4,15 @@
 Changelog
 =========
 
-.. _v0_45a0:
+.. _v0_45a1:
 
-0.45a0 (2020-06-18)
+0.45a1 (2020-06-18)
 -------------------
 
 .. warning:: This is an **alpha** release. See :ref:`contributing_alpha_beta`.
 
 - New :ref:`plugin_hook_startup` plugin hook. (`#834 `__)
+- New :ref:`plugin_hook_canned_queries` plugin hook. (`#852 `__)
 - Workaround for "Too many open files" error in test runs. (`#846 `__)
 - Respect existing ``scope["actor"]`` if already set by ASGI middleware. (`#854 `__)
 - New process for shipping :ref:`contributing_alpha_beta`. (`#807 `__)

From b59b92b1b0517cf18fa748ff9d0a0bf86298dd43 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 16:52:06 -0700
Subject: [PATCH 0578/2303] Fix for tests - order was inconsistent, refs #852

---
 tests/test_canned_write.py | 20 ++++++++++++--------
 tests/test_html.py         |  8 +++++---
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/tests/test_canned_write.py b/tests/test_canned_write.py
index c36baa09..e33eed69 100644
--- a/tests/test_canned_write.py
+++ b/tests/test_canned_write.py
@@ -108,16 +108,16 @@ def test_vary_header(canned_write_client):
 
 def test_canned_query_permissions_on_database_page(canned_write_client):
     # Without auth only shows three queries
-    query_names = [
+    query_names = {
         q["name"] for q in canned_write_client.get("/data.json").json["queries"]
-    ]
-    assert [
+    }
+    assert {
         "add_name",
         "add_name_specify_id",
         "update_name",
         "from_async_hook",
         "from_hook",
-    ] == query_names
+    } == query_names
 
     # With auth shows four
     response = canned_write_client.get(
@@ -129,12 +129,16 @@ def test_canned_query_permissions_on_database_page(canned_write_client):
         {"name": "add_name", "private": False},
         {"name": "add_name_specify_id", "private": False},
         {"name": "delete_name", "private": True},
-        {"name": "update_name", "private": False},
         {"name": "from_async_hook", "private": False},
         {"name": "from_hook", "private": False},
-    ] == [
-        {"name": q["name"], "private": q["private"]} for q in response.json["queries"]
-    ]
+        {"name": "update_name", "private": False},
+    ] == sorted(
+        [
+            {"name": q["name"], "private": q["private"]}
+            for q in response.json["queries"]
+        ],
+        key=lambda q: q["name"],
+    )
 
 
 def test_canned_query_permissions(canned_write_client):
diff --git a/tests/test_html.py b/tests/test_html.py
index 7bc935b0..1c7dce90 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -95,11 +95,13 @@ def test_database_page(app_client):
             "/fixtures/%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC",
             "𝐜𝐢𝐭𝐢𝐞𝐬",
         ),
-        ("/fixtures/pragma_cache_size", "pragma_cache_size"),
-        ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"),
         ("/fixtures/from_async_hook", "from_async_hook"),
         ("/fixtures/from_hook", "from_hook"),
-    ] == [(a["href"], a.text) for a in queries_ul.find_all("a")]
+        ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"),
+        ("/fixtures/pragma_cache_size", "pragma_cache_size"),
+    ] == sorted(
+        [(a["href"], a.text) for a in queries_ul.find_all("a")], key=lambda p: p[0]
+    )
 
 
 def test_invalid_custom_sql(app_client):

From 64cc536b89b988b17e3ab853e4c64d9706543116 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Jun 2020 17:03:23 -0700
Subject: [PATCH 0579/2303] Don't include prereleases in changelog badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 925d68d2..42eaaa81 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # Datasette
 
 [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/)
-[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?include_prereleases&label=changelog)](https://datasette.readthedocs.io/en/stable/changelog.html)
+[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://datasette.readthedocs.io/en/stable/changelog.html)
 [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/)
 [![Travis CI](https://travis-ci.org/simonw/datasette.svg?branch=master)](https://travis-ci.org/simonw/datasette)
 [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest)

From 55a6ffb93c57680e71a070416baae1129a0243b8 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Jun 2020 20:08:30 -0700
Subject: [PATCH 0580/2303] Link to datasette-saved-queries plugin, closes #852

---
 docs/changelog.rst | 2 +-
 docs/ecosystem.rst | 6 +++++-
 docs/plugins.rst   | 6 +++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 6f3af8ce..d580f03e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,7 +12,7 @@ Changelog
 .. warning:: This is an **alpha** release. See :ref:`contributing_alpha_beta`.
 
 - New :ref:`plugin_hook_startup` plugin hook. (`#834 `__)
-- New :ref:`plugin_hook_canned_queries` plugin hook. (`#852 `__)
+- New :ref:`plugin_hook_canned_queries` plugin hook. See `datasette-saved-queries `__ for an example of this hook in action. (`#852 `__)
 - Workaround for "Too many open files" error in test runs. (`#846 `__)
 - Respect existing ``scope["actor"]`` if already set by ASGI middleware. (`#854 `__)
 - New process for shipping :ref:`contributing_alpha_beta`. (`#807 `__)
diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst
index dcb5a887..f2da885c 100644
--- a/docs/ecosystem.rst
+++ b/docs/ecosystem.rst
@@ -157,12 +157,16 @@ datasette-leaflet-geojson
 
 `datasette-leaflet-geojson `__ looks out for columns containing GeoJSON formatted geographical information and displays them on a `Leaflet-powered `__ map.
 
-
 datasette-pretty-json
 ---------------------
 
 `datasette-pretty-json `__ seeks out JSON values in Datasette's table browsing interface and pretty-prints them, making them easier to read.
 
+datasette-saved-queries
+-----------------------
+
+`datasette-saved-queries `__ lets users interactively save queries to a ``saved_queries`` table. They are then made available as additional :ref:`canned queries `.
+
 datasette-haversine
 -------------------
 
diff --git a/docs/plugins.rst b/docs/plugins.rst
index dce1bdf0..d2743419 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1028,9 +1028,11 @@ Or you can return an async function which will be awaited on startup. Use this o
 Potential use-cases:
 
 * Run some initialization code for the plugin
-* Create database tables that a plugin needs
+* Create database tables that a plugin needs on startup
 * Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid
 
+Example: `datasette-saved-queries `__
+
 .. _plugin_hook_canned_queries:
 
 canned_queries(datasette, database, actor)
@@ -1098,6 +1100,8 @@ The actor parameter can be used to include the currently authenticated actor in
                 } for result in results}
         return inner
 
+Example: `datasette-saved-queries `__
+
 .. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)

From d1640ba76b8f10830c56d8289f476fefde3bd1fb Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Jun 2020 08:48:39 -0700
Subject: [PATCH 0581/2303] Don't show prereleases on changelog badge

---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index 5334386f..fa5d7f87 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -6,7 +6,7 @@ datasette|
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/datasette.svg
    :target: https://pypi.org/project/datasette/
-.. |Changelog| image:: https://img.shields.io/github/v/release/simonw/datasette?include_prereleases&label=changelog
+.. |Changelog| image:: https://img.shields.io/github/v/release/simonw/datasette?label=changelog
    :target: https://datasette.readthedocs.io/en/stable/changelog.html
 .. |Python 3.x| image:: https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white
    :target: https://pypi.org/project/datasette/

From 84cbf1766083a785f5ce5154d0805654a5314d10 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Jun 2020 10:40:05 -0700
Subject: [PATCH 0582/2303] News: A cookiecutter template for writing Datasette
 plugins

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 42eaaa81..84d1dcd4 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 
 ## News
 
+ * 20th June 2020: [A cookiecutter template for writing Datasette plugins](https://simonwillison.net/2020/Jun/20/cookiecutter-plugins/)
  * 11th June 2020: [Datasette 0.44](http://datasette.readthedocs.io/en/latest/changelog.html#v0-44) - [Authentication and permissions](https://datasette.readthedocs.io/en/latest/authentication.html), [writable canned queries](https://datasette.readthedocs.io/en/latest/sql_queries.html#writable-canned-queries), flash messages, new plugin hooks and much, much more.
  * 28th May 2020: [Datasette 0.43](http://datasette.readthedocs.io/en/latest/changelog.html#v0-43) - Redesigned [register_output_renderer](https://datasette.readthedocs.io/en/latest/plugins.html#plugin-register-output-renderer) plugin hook and various small improvements and fixes.
  * 8th May 2020: [Datasette 0.42](http://datasette.readthedocs.io/en/latest/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database.

From e4216ff5035f57f2fb66031f105e41c3b9728bc1 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 21 Jun 2020 14:55:17 -0700
Subject: [PATCH 0583/2303] Fixed rST warning

---
 docs/ecosystem.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst
index f2da885c..7c8959dd 100644
--- a/docs/ecosystem.rst
+++ b/docs/ecosystem.rst
@@ -93,7 +93,7 @@ datasette-auth-tokens
 `datasette-auth-tokens `__ provides a mechanism for creating secret API tokens that can then be used with Datasette's :ref:`authentication` system.
 
 datasette-permissions-sql
----------------------
+-------------------------
 
 `datasette-permissions-sql `__ lets you configure Datasette permissions checks to use custom SQL queries, which means you can make permisison decisions based on data contained within your databases.
 

From 36e77e100632573e1cf907aba9462debac7928e9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 21 Jun 2020 17:33:48 -0700
Subject: [PATCH 0584/2303] Move plugin hooks docs to plugin_hooks.rst, refs
 #687

---
 docs/index.rst        |   1 +
 docs/plugin_hooks.rst | 888 +++++++++++++++++++++++++++++++++++++++++
 docs/plugins.rst      | 889 ------------------------------------------
 3 files changed, 889 insertions(+), 889 deletions(-)
 create mode 100644 docs/plugin_hooks.rst

diff --git a/docs/index.rst b/docs/index.rst
index fa5d7f87..20a55b2c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -51,6 +51,7 @@ Contents
    introspection
    custom_templates
    plugins
+   plugin_hooks
    internals
    contributing
    changelog
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
new file mode 100644
index 00000000..19f076b9
--- /dev/null
+++ b/docs/plugin_hooks.rst
@@ -0,0 +1,888 @@
+.. _plugin_hooks:
+
+Plugin hooks
+============
+
+When you implement a plugin hook you can accept any or all of the parameters that are documented as being passed to that hook. For example, you can implement a ``render_cell`` plugin hook like this even though the hook definition defines more parameters than just ``value`` and ``column``:
+
+.. code-block:: python
+
+    @hookimpl
+    def render_cell(value, column):
+        if column == "stars":
+            return "*" * int(value)
+
+The full list of available plugin hooks is as follows.
+
+.. _plugin_hook_prepare_connection:
+
+prepare_connection(conn, database, datasette)
+---------------------------------------------
+
+``conn`` - sqlite3 connection object
+    The connection that is being opened
+
+``database`` - string
+    The name of the database
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
+
+This hook is called when a new SQLite database connection is created. You can
+use it to `register custom SQL functions `_,
+aggregates and collations. For example:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import random
+
+    @hookimpl
+    def prepare_connection(conn):
+        conn.create_function('random_integer', 2, random.randint)
+
+This registers a SQL function called ``random_integer`` which takes two
+arguments and can be called like this::
+
+    select random_integer(1, 10);
+
+Examples: `datasette-jellyfish `_, `datasette-jq `_, `datasette-haversine `__, `datasette-rure `__
+
+.. _plugin_hook_prepare_jinja2_environment:
+
+prepare_jinja2_environment(env)
+-------------------------------
+
+``env`` - jinja2 Environment
+    The template environment that is being prepared
+
+This hook is called with the Jinja2 environment that is used to evaluate
+Datasette HTML templates. You can use it to do things like `register custom
+template filters `_, for
+example:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def prepare_jinja2_environment(env):
+        env.filters['uppercase'] = lambda u: u.upper()
+
+You can now use this filter in your custom templates like so::
+
+    Table name: {{ table|uppercase }}
+
+.. _plugin_hook_extra_css_urls:
+
+extra_css_urls(template, database, table, datasette)
+----------------------------------------------------
+
+``template`` - string
+    The template that is being rendered, e.g. ``database.html``
+
+``database`` - string or None
+    The name of the database
+
+``table`` - string or None
+    The name of the table
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
+
+Return a list of extra CSS URLs that should be included on the page. These can
+take advantage of the CSS class hooks described in :ref:`customization`.
+
+This can be a list of URLs:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def extra_css_urls():
+        return [
+            'https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css'
+        ]
+
+Or a list of dictionaries defining both a URL and an
+`SRI hash `_:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def extra_css_urls():
+        return [{
+            'url': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css',
+            'sri': 'sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4',
+        }]
+
+Examples: `datasette-cluster-map `_, `datasette-vega `_
+
+.. _plugin_hook_extra_js_urls:
+
+extra_js_urls(template, database, table, datasette)
+---------------------------------------------------
+
+Same arguments as ``extra_css_urls``.
+
+This works in the same way as ``extra_css_urls()`` but for JavaScript. You can
+return either a list of URLs or a list of dictionaries:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def extra_js_urls():
+        return [{
+            'url': 'https://code.jquery.com/jquery-3.3.1.slim.min.js',
+            'sri': 'sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo',
+        }]
+
+You can also return URLs to files from your plugin's ``static/`` directory, if
+you have one:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def extra_js_urls():
+        return [
+            '/-/static-plugins/your-plugin/app.js'
+        ]
+
+Examples: `datasette-cluster-map `_, `datasette-vega `_
+
+.. _plugin_hook_publish_subcommand:
+
+publish_subcommand(publish)
+---------------------------
+
+``publish`` - Click publish command group
+    The Click command group for the ``datasette publish`` subcommand
+
+This hook allows you to create new providers for the ``datasette publish``
+command. Datasette uses this hook internally to implement the default ``now``
+and ``heroku`` subcommands, so you can read
+`their source `_
+to see examples of this hook in action.
+
+Let's say you want to build a plugin that adds a ``datasette publish my_hosting_provider --api_key=xxx mydatabase.db`` publish command. Your implementation would start like this:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette.publish.common import add_common_publish_arguments_and_options
+    import click
+
+
+    @hookimpl
+    def publish_subcommand(publish):
+        @publish.command()
+        @add_common_publish_arguments_and_options
+        @click.option(
+            "-k",
+            "--api_key",
+            help="API key for talking to my hosting provider",
+        )
+        def my_hosting_provider(
+            files,
+            metadata,
+            extra_options,
+            branch,
+            template_dir,
+            plugins_dir,
+            static,
+            install,
+            plugin_secret,
+            version_note,
+            secret,
+            title,
+            license,
+            license_url,
+            source,
+            source_url,
+            about,
+            about_url,
+            api_key,
+        ):
+            # Your implementation goes here
+
+Examples: `datasette-publish-fly `_, `datasette-publish-now `_
+
+.. _plugin_hook_render_cell:
+
+render_cell(value, column, table, database, datasette)
+------------------------------------------------------
+
+Lets you customize the display of values within table cells in the HTML table view.
+
+``value`` - string, integer or None
+    The value that was loaded from the database
+
+``column`` - string
+    The name of the column being rendered
+
+``table`` - string or None
+    The name of the table - or ``None`` if this is a custom SQL query
+
+``database`` - string
+    The name of the database
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
+
+If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value.
+
+If the hook returns a string, that string will be rendered in the table cell.
+
+If you want to return HTML markup you can do so by returning a ``jinja2.Markup`` object.
+
+Datasette will loop through all available ``render_cell`` hooks and display the value returned by the first one that does not return ``None``.
+
+Here is an example of a custom ``render_cell()`` plugin which looks for values that are a JSON string matching the following format::
+
+    {"href": "https://www.example.com/", "label": "Name"}
+
+If the value matches that pattern, the plugin returns an HTML link element:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import jinja2
+    import json
+
+
+    @hookimpl
+    def render_cell(value):
+        # Render {"href": "...", "label": "..."} as link
+        if not isinstance(value, str):
+            return None
+        stripped = value.strip()
+        if not stripped.startswith("{") and stripped.endswith("}"):
+            return None
+        try:
+            data = json.loads(value)
+        except ValueError:
+            return None
+        if not isinstance(data, dict):
+            return None
+        if set(data.keys()) != {"href", "label"}:
+            return None
+        href = data["href"]
+        if not (
+            href.startswith("/") or href.startswith("http://")
+            or href.startswith("https://")
+        ):
+            return None
+        return jinja2.Markup('{label}'.format(
+            href=jinja2.escape(data["href"]),
+            label=jinja2.escape(data["label"] or "") or " "
+        ))
+
+Examples: `datasette-render-binary `_, `datasette-render-markdown `_
+
+.. _plugin_hook_extra_body_script:
+
+extra_body_script(template, database, table, view_name, datasette)
+------------------------------------------------------------------
+
+Extra JavaScript to be added to a ``")
     json_data = r.search(app_client.get(path).text).group(1)
     actual_data = json.loads(json_data)
     assert expected_extra_body_script == actual_data
 
 
-def test_plugins_asgi_wrapper(app_client):
+def test_hook_asgi_wrapper(app_client):
     response = app_client.get("/fixtures")
     assert "fixtures" == response.headers["x-databases"]
 
 
-def test_plugins_extra_template_vars(restore_working_directory):
+def test_hook_extra_template_vars(restore_working_directory):
     with make_app_client(
         template_dir=str(pathlib.Path(__file__).parent / "test_templates")
     ) as client:
@@ -380,13 +380,13 @@ def test_view_names(view_names_client, path, view_name):
     assert "view_name:{}".format(view_name) == response.text
 
 
-def test_register_output_renderer_no_parameters(app_client):
+def test_hook_register_output_renderer_no_parameters(app_client):
     response = app_client.get("/fixtures/facetable.testnone")
     assert 200 == response.status
     assert b"Hello" == response.body
 
 
-def test_register_output_renderer_all_parameters(app_client):
+def test_hook_register_output_renderer_all_parameters(app_client):
     response = app_client.get("/fixtures/facetable.testall")
     assert 200 == response.status
     # Lots of 'at 0x103a4a690' in here - replace those so we can do
@@ -436,19 +436,19 @@ def test_register_output_renderer_all_parameters(app_client):
     assert "pragma_cache_size" == json.loads(query_response.body)["query_name"]
 
 
-def test_register_output_renderer_custom_status_code(app_client):
+def test_hook_register_output_renderer_custom_status_code(app_client):
     response = app_client.get("/fixtures/pragma_cache_size.testall?status_code=202")
     assert 202 == response.status
 
 
-def test_register_output_renderer_custom_content_type(app_client):
+def test_hook_register_output_renderer_custom_content_type(app_client):
     response = app_client.get(
         "/fixtures/pragma_cache_size.testall?content_type=text/blah"
     )
     assert "text/blah" == response.headers["content-type"]
 
 
-def test_register_output_renderer_custom_headers(app_client):
+def test_hook_register_output_renderer_custom_headers(app_client):
     response = app_client.get(
         "/fixtures/pragma_cache_size.testall?header=x-wow:1&header=x-gosh:2"
     )
@@ -456,7 +456,7 @@ def test_register_output_renderer_custom_headers(app_client):
     assert "2" == response.headers["x-gosh"]
 
 
-def test_register_output_renderer_can_render(app_client):
+def test_hook_register_output_renderer_can_render(app_client):
     response = app_client.get("/fixtures/facetable?_no_can_render=1")
     assert response.status == 200
     links = (
@@ -492,7 +492,7 @@ def test_register_output_renderer_can_render(app_client):
 
 
 @pytest.mark.asyncio
-async def test_prepare_jinja2_environment(app_client):
+async def test_hook_prepare_jinja2_environment(app_client):
     template = app_client.ds.jinja_env.from_string(
         "Hello there, {{ a|format_numeric }}", {"a": 3412341}
     )
@@ -500,7 +500,7 @@ async def test_prepare_jinja2_environment(app_client):
     assert "Hello there, 3,412,341" == rendered
 
 
-def test_publish_subcommand():
+def test_hook_publish_subcommand():
     # This is hard to test properly, because publish subcommand plugins
     # cannot be loaded using the --plugins-dir mechanism - they need
     # to be installed using "pip install". So I'm cheating and taking
@@ -509,7 +509,7 @@ def test_publish_subcommand():
     assert ["cloudrun", "heroku"] == cli.publish.list_commands({})
 
 
-def test_register_facet_classes(app_client):
+def test_hook_register_facet_classes(app_client):
     response = app_client.get(
         "/fixtures/compound_three_primary_keys.json?_dummy_facet=1"
     )
@@ -549,7 +549,7 @@ def test_register_facet_classes(app_client):
     ] == response.json["suggested_facets"]
 
 
-def test_actor_from_request(app_client):
+def test_hook_actor_from_request(app_client):
     app_client.get("/")
     # Should have no actor
     assert None == app_client.ds._last_request.scope["actor"]
@@ -558,7 +558,7 @@ def test_actor_from_request(app_client):
     assert {"id": "bot"} == app_client.ds._last_request.scope["actor"]
 
 
-def test_actor_from_request_async(app_client):
+def test_hook_actor_from_request_async(app_client):
     app_client.get("/")
     # Should have no actor
     assert None == app_client.ds._last_request.scope["actor"]
@@ -583,7 +583,7 @@ def test_existing_scope_actor_respected(app_client):
         ("no_match", None),
     ],
 )
-async def test_permission_allowed(app_client, action, expected):
+async def test_hook_permission_allowed(app_client, action, expected):
     actual = await app_client.ds.permission_allowed(
         {"id": "actor"}, action, default=None
     )
@@ -605,20 +605,20 @@ def test_actor_json(app_client):
         ("/not-async/", "This was not async"),
     ],
 )
-def test_register_routes(app_client, path, body):
+def test_hook_register_routes(app_client, path, body):
     response = app_client.get(path)
     assert 200 == response.status
     assert body == response.text
 
 
-def test_register_routes_post(app_client):
+def test_hook_register_routes_post(app_client):
     response = app_client.post("/post/", {"this is": "post data"}, csrftoken_from=True)
     assert 200 == response.status
     assert "csrftoken" in response.json
     assert "post data" == response.json["this is"]
 
 
-def test_register_routes_csrftoken(restore_working_directory, tmpdir_factory):
+def test_hook_register_routes_csrftoken(restore_working_directory, tmpdir_factory):
     templates = tmpdir_factory.mktemp("templates")
     (templates / "csrftoken_form.html").write_text(
         "CSRFTOKEN: {{ csrftoken() }}", "utf-8"
@@ -629,13 +629,13 @@ def test_register_routes_csrftoken(restore_working_directory, tmpdir_factory):
         assert "CSRFTOKEN: {}".format(expected_token) == response.text
 
 
-def test_register_routes_asgi(app_client):
+def test_hook_register_routes_asgi(app_client):
     response = app_client.get("/three/")
     assert {"hello": "world"} == response.json
     assert "1" == response.headers["x-three"]
 
 
-def test_register_routes_add_message(app_client):
+def test_hook_register_routes_add_message(app_client):
     response = app_client.get("/add-message/")
     assert 200 == response.status
     assert "Added message" == response.text
@@ -643,7 +643,7 @@ def test_register_routes_add_message(app_client):
     assert [["Hello from messages", 1]] == decoded
 
 
-def test_register_routes_render_message(restore_working_directory, tmpdir_factory):
+def test_hook_register_routes_render_message(restore_working_directory, tmpdir_factory):
     templates = tmpdir_factory.mktemp("templates")
     (templates / "render_message.html").write_text('{% extends "base.html" %}', "utf-8")
     with make_app_client(template_dir=templates) as client:
@@ -654,13 +654,13 @@ def test_register_routes_render_message(restore_working_directory, tmpdir_factor
 
 
 @pytest.mark.asyncio
-async def test_startup(app_client):
+async def test_hook_startup(app_client):
     await app_client.ds.invoke_startup()
     assert app_client.ds._startup_hook_fired
     assert 2 == app_client.ds._startup_hook_calculation
 
 
-def test_canned_queries(app_client):
+def test_hook_canned_queries(app_client):
     queries = app_client.get("/fixtures.json").json["queries"]
     queries_by_name = {q["name"]: q for q in queries}
     assert {
@@ -675,23 +675,23 @@ def test_canned_queries(app_client):
     } == queries_by_name["from_hook"]
 
 
-def test_canned_queries_non_async(app_client):
+def test_hook_canned_queries_non_async(app_client):
     response = app_client.get("/fixtures/from_hook.json?_shape=array")
     assert [{"1": 1, "actor_id": "null"}] == response.json
 
 
-def test_canned_queries_async(app_client):
+def test_hook_canned_queries_async(app_client):
     response = app_client.get("/fixtures/from_async_hook.json?_shape=array")
     assert [{"2": 2}] == response.json
 
 
-def test_canned_queries_actor(app_client):
+def test_hook_canned_queries_actor(app_client):
     assert [{"1": 1, "actor_id": "bot"}] == app_client.get(
         "/fixtures/from_hook.json?_bot=1&_shape=array"
     ).json
 
 
-def test_register_magic_parameters(restore_working_directory):
+def test_hook_register_magic_parameters(restore_working_directory):
     with make_app_client(
         extra_databases={"data.db": "create table logs (line text)"},
         metadata={
@@ -719,7 +719,7 @@ def test_register_magic_parameters(restore_working_directory):
         assert 4 == new_uuid.count("-")
 
 
-def test_forbidden(restore_working_directory):
+def test_hook_forbidden(restore_working_directory):
     with make_app_client(
         extra_databases={"data2.db": "create table logs (line text)"},
         metadata={"allow": {}},

From 3a4c8ed36aa97211e46849d32a09f2f386f342dd Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 16 Aug 2020 11:09:53 -0700
Subject: [PATCH 0695/2303] Added columns argument to various extra_ plugin
 hooks, closes #938

---
 datasette/app.py           |   5 +-
 datasette/hookspecs.py     |  12 +-
 docs/plugin_hooks.rst      | 254 +++++++++++++++++--------------------
 tests/plugins/my_plugin.py |  13 +-
 tests/test_plugins.py      |  25 +++-
 5 files changed, 159 insertions(+), 150 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 180ba246..2185a3ab 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -713,6 +713,7 @@ class Datasette:
             template=template.name,
             database=context.get("database"),
             table=context.get("table"),
+            columns=context.get("columns"),
             view_name=view_name,
             request=request,
             datasette=self,
@@ -729,6 +730,7 @@ class Datasette:
             template=template.name,
             database=context.get("database"),
             table=context.get("table"),
+            columns=context.get("columns"),
             view_name=view_name,
             request=request,
             datasette=self,
@@ -779,9 +781,10 @@ class Datasette:
             template=template.name,
             database=context.get("database"),
             table=context.get("table"),
-            datasette=self,
+            columns=context.get("columns"),
             view_name=view_name,
             request=request,
+            datasette=self,
         ):
             if callable(hook):
                 hook = hook()
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 0e9c20cf..f7e90e4e 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -26,22 +26,26 @@ def prepare_jinja2_environment(env):
 
 
 @hookspec
-def extra_css_urls(template, database, table, view_name, request, datasette):
+def extra_css_urls(template, database, table, columns, view_name, request, datasette):
     "Extra CSS URLs added by this plugin"
 
 
 @hookspec
-def extra_js_urls(template, database, table, view_name, request, datasette):
+def extra_js_urls(template, database, table, columns, view_name, request, datasette):
     "Extra JavaScript URLs added by this plugin"
 
 
 @hookspec
-def extra_body_script(template, database, table, view_name, request, datasette):
+def extra_body_script(
+    template, database, table, columns, view_name, request, datasette
+):
     "Extra JavaScript code to be included in ")
     json_data = r.search(app_client.get(path).text).group(1)
     actual_data = json.loads(json_data)
@@ -286,6 +308,7 @@ def test_hook_extra_template_vars(restore_working_directory):
         assert {
             "template": "show_json.html",
             "scope_path": "/-/metadata",
+            "columns": None,
         } == extra_template_vars
         extra_template_vars_from_awaitable = json.loads(
             Soup(response.body, "html.parser")

From 8e7e6458a6787a06a4488798bd643dd7728b8a5b Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 16 Aug 2020 11:24:39 -0700
Subject: [PATCH 0696/2303] Fix bug with ?_nl=on and binary data, closes #914

---
 datasette/renderer.py |  2 +-
 tests/fixtures.py     |  3 ++-
 tests/test_api.py     | 31 ++++++++++++++++++++++++++++++-
 tests/test_html.py    |  9 +++++++--
 4 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/datasette/renderer.py b/datasette/renderer.py
index 3f921fe7..27a5092f 100644
--- a/datasette/renderer.py
+++ b/datasette/renderer.py
@@ -84,7 +84,7 @@ def json_renderer(args, data, view_name):
     # Handle _nl option for _shape=array
     nl = args.get("_nl", "")
     if nl and shape == "array":
-        body = "\n".join(json.dumps(item) for item in data)
+        body = "\n".join(json.dumps(item, cls=CustomJSONEncoder) for item in data)
         content_type = "text/plain"
     else:
         body = json.dumps(data, cls=CustomJSONEncoder)
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 139eff83..5bd063d9 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -663,7 +663,8 @@ CREATE VIEW searchable_view_configured_by_metadata AS
     )
 )
 TABLE_PARAMETERIZED_SQL = [
-    ("insert into binary_data (data) values (?);", [b"this is binary data"])
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]),
 ]
 
 EXTRA_DATABASE_SQL = """
diff --git a/tests/test_api.py b/tests/test_api.py
index 1f93c1a7..22fa87d4 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -105,7 +105,7 @@ def test_database_page(app_client):
             "name": "binary_data",
             "columns": ["data"],
             "primary_keys": [],
-            "count": 1,
+            "count": 2,
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
@@ -1793,3 +1793,32 @@ def test_null_foreign_keys_are_not_expanded(app_client):
 def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file):
     response = app_client_immutable_and_inspect_file.get("/fixtures/sortable.json")
     assert response.json["filtered_table_rows_count"] == 100
+
+
+@pytest.mark.parametrize(
+    "path,expected_json,expected_text",
+    [
+        (
+            "/fixtures/binary_data.json?_shape=array",
+            [
+                {"rowid": 1, "data": {"$base64": True, "encoded": "FRwCx60F/g=="}},
+                {"rowid": 2, "data": {"$base64": True, "encoded": "FRwDx60F/g=="}},
+            ],
+            None,
+        ),
+        (
+            "/fixtures/binary_data.json?_shape=array&_nl=on",
+            None,
+            (
+                '{"rowid": 1, "data": {"$base64": true, "encoded": "FRwCx60F/g=="}}\n'
+                '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}'
+            ),
+        ),
+    ],
+)
+def test_binary_data_in_json(app_client, path, expected_json, expected_text):
+    response = app_client.get(path)
+    if expected_json:
+        assert response.json == expected_json
+    else:
+        assert response.text == expected_text
diff --git a/tests/test_html.py b/tests/test_html.py
index 89aa4d06..1a12b3ce 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1134,8 +1134,13 @@ def test_binary_data_display(app_client):
         [
             '1',
             '1',
-            '<Binary\xa0data:\xa019\xa0bytes>',
-        ]
+            '<Binary\xa0data:\xa07\xa0bytes>',
+        ],
+        [
+            '2',
+            '2',
+            '<Binary\xa0data:\xa07\xa0bytes>',
+        ],
     ]
     assert expected_tds == [
         [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")

From 52eabb019d4051084b21524bd0fd9c2731126985 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 16 Aug 2020 11:56:31 -0700
Subject: [PATCH 0697/2303] Release 0.48

Refs #939, #938, #935, #914
---
 README.md          |  1 +
 docs/changelog.rst | 12 ++++++++++++
 docs/internals.rst |  2 ++
 3 files changed, 15 insertions(+)

diff --git a/README.md b/README.md
index 9b49cc14..ee3246a5 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 
 ## News
 
+ * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks.
  * 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line.
  * 9th August 2020: [Datasette 0.46](https://docs.datasette.io/en/stable/changelog.html#v0-46) - security fix relating to CSRF protection for writable canned queries, a new logo, new debugging tools, improved file downloads and more.
  * 6th August 2020: [GraphQL in Datasette with the new datasette-graphql plugin](https://simonwillison.net/2020/Aug/7/datasette-graphql/)
diff --git a/docs/changelog.rst b/docs/changelog.rst
index bf53b6f3..d18dae80 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,18 @@
 Changelog
 =========
 
+.. _v0_48:
+
+0.48 (2020-08-16)
+-----------------
+
+- Datasette documentation now lives at `docs.datasette.io `__.
+- ``db.is_mutable`` property is now documented and tested, see :ref:`internals_database_introspection`.
+- The ``extra_template_vars``, ``extra_css_urls``, ``extra_js_urls`` and ``extra_body_script`` plugin hooks now all accept the same arguments. See :ref:`plugin_hook_extra_template_vars` for details. (`#939 `__)
+- Those hooks now accept a new ``columns`` argument detailing the table columns that will be rendered on that page. (`#938 `__)
+- Fixed bug where plugins calling ``db.execute_write_fn()`` could hang Datasette if the connection failed. (`#935 `__)
+- Fixed bug with the ``?_nl=on`` output option and binary data. (`#914 `__)
+
 .. _v0_47_3:
 
 0.47.3 (2020-08-15)
diff --git a/docs/internals.rst b/docs/internals.rst
index f8d4a136..ff7e883c 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -466,6 +466,8 @@ Here's an example of ``block=True`` in action:
     except Exception as e:
         print("An error occurred:", e)
 
+.. _internals_database_introspection:
+
 Database introspection
 ----------------------
 

From 5e0b72247ecab4ce0fcec599b77a83d73a480872 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 17 Aug 2020 22:09:34 -0700
Subject: [PATCH 0698/2303] Run CI on GitHub Actions, not Travis

* Run CI on GitHub Actions, not Travis - refs #940
* Update documentation refs to Travis
* Release action now runs parallel tests, then pushes to PyPI, then Docker Hub
---
 .dockerignore                 |  1 -
 .github/workflows/publish.yml | 72 +++++++++++++++++++++++++++++++++++
 .github/workflows/test.yml    | 29 ++++++++++++++
 .travis.yml                   | 47 -----------------------
 README.md                     |  2 +-
 docs/contributing.rst         |  2 +-
 docs/index.rst                |  4 +-
 setup.py                      |  2 +-
 8 files changed, 106 insertions(+), 53 deletions(-)
 create mode 100644 .github/workflows/publish.yml
 create mode 100644 .github/workflows/test.yml
 delete mode 100644 .travis.yml

diff --git a/.dockerignore b/.dockerignore
index 938173e9..490f509e 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,7 +3,6 @@
 .eggs
 .gitignore
 .ipynb_checkpoints
-.travis.yml
 build
 *.spec
 *.egg-info
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 00000000..4e554eda
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,72 @@
+name: Publish Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        pip install -e '.[test]'
+    - name: Run tests
+      run: |
+        pytest
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [test]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-publish-pip-
+    - name: Install dependencies
+      run: |
+        pip install setuptools wheel twine
+    - name: Publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*
+  deploy_docker:
+    runs-on: ubuntu-latest
+    needs: [deploy]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build and push to Docker Hub
+      env:
+        DOCKER_USER: ${{ secrets.DOCKER_USER }}
+        DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+      run: |-
+         docker login -u $DOCKER_USER -p $DOCKER_PASS 
+         export REPO=datasetteproject/datasette 
+         docker build -f Dockerfile -t $REPO::${GITHUB_REF#refs/tags/} . 
+         docker tag $REPO::${GITHUB_REF#refs/tags/} $REPO:latest
+         docker push $REPO 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 00000000..74e56e13
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,29 @@
+name: Test
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        pip install -e '.[test]'
+    - name: Run tests
+      run: |
+        pytest
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 181bc3f3..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-language: python
-dist: xenial
-
-branches:
-  except:
-  - master
-
-# 3.6 is listed first so it gets used for the later build stages
-python:
-  - "3.6"
-  - "3.7"
-  - "3.8"
-
-# Executed for 3.5 AND 3.5 as the first "test" stage:
-script:
-  - pip install -U pip wheel
-  - pip install .[test]
-  - pytest
-
-cache:
-    directories:
-        - $HOME/.cache/pip
-
-# This defines further stages that execute after the tests
-jobs:
-  include:
-    - stage: release tagged version
-      if: tag IS present
-      python: 3.6
-      deploy:
-        - provider: pypi
-          user: simonw
-          distributions: "sdist bdist_wheel"
-          password: ${PYPI_PASSWORD}
-          on:
-            branch: master
-            tags: true
-    - stage: publish docker image
-      if: (tag IS present) AND NOT (tag =~ [ab])
-      python: 3.6
-      script:
-        # Build and release to Docker Hub
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - export REPO=datasetteproject/datasette
-        - docker build -f Dockerfile -t $REPO:$TRAVIS_TAG .
-        - docker tag $REPO:$TRAVIS_TAG $REPO:latest
-        - docker push $REPO
diff --git a/README.md b/README.md
index ee3246a5..38ea7f79 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/)
 [![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html)
 [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/)
-[![Travis CI](https://travis-ci.org/simonw/datasette.svg?branch=main)](https://travis-ci.org/simonw/datasette)
+[![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest)
 [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE)
 [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette)
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 118146cf..95663dbc 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -126,7 +126,7 @@ Now browse to ``http://localhost:8000/`` to view the documentation. Any edits yo
 Release process
 ---------------
 
-Datasette releases are performed using tags. When a new version tag is pushed to GitHub, a `Travis CI task `__ will perform the following:
+Datasette releases are performed using tags. When a new release is published on GitHub, a `GitHub Action workflow `__ will perform the following:
 
 * Run the unit tests against all supported Python versions. If the tests pass...
 * Build a Docker image of the release and push a tag to https://hub.docker.com/r/datasetteproject/datasette
diff --git a/docs/index.rst b/docs/index.rst
index f9f2f0bb..946fa542 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -10,8 +10,8 @@ datasette|
    :target: https://docs.datasette.io/en/stable/changelog.html
 .. |Python 3.x| image:: https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white
    :target: https://pypi.org/project/datasette/
-.. |Travis CI| image:: https://travis-ci.org/simonw/datasette.svg?branch=main
-   :target: https://travis-ci.org/simonw/datasette
+.. |Tests| image:: https://github.com/simonw/datasette/workflows/Test/badge.svg
+   :target: https://github.com/simonw/datasette/actions?query=workflow%3ATest
 .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
    :target: https://github.com/simonw/datasette/blob/main/LICENSE
 .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue
diff --git a/setup.py b/setup.py
index bbd0aa8b..d9526149 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@ setup(
         "Live demo": "https://latest.datasette.io/",
         "Source code": "https://github.com/simonw/datasette",
         "Issues": "https://github.com/simonw/datasette/issues",
-        "CI": "https://travis-ci.org/simonw/datasette",
+        "CI": "https://github.com/simonw/datasette/actions?query=workflow%3ATest",
     },
     packages=find_packages(exclude=("tests",)),
     package_data={"datasette": ["templates/*.html"]},

From b21ed237ab940768574c834aa5a7130724bd3a2d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 18 Aug 2020 13:49:13 -0700
Subject: [PATCH 0699/2303] publish heroku now deploys with Python 3.8.5

---
 datasette/publish/heroku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py
index 6cda68da..24393b90 100644
--- a/datasette/publish/heroku.py
+++ b/datasette/publish/heroku.py
@@ -170,7 +170,7 @@ def temporary_heroku_directory(
         if metadata_content:
             open("metadata.json", "w").write(json.dumps(metadata_content, indent=2))
 
-        open("runtime.txt", "w").write("python-3.8.3")
+        open("runtime.txt", "w").write("python-3.8.5")
 
         if branch:
             install = [

From 69033c6ec4a76d720e5c866aaa43b175c5ec1d8b Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 19 Aug 2020 10:20:41 -0700
Subject: [PATCH 0700/2303] datasette install --upgrade option, closes #945

---
 datasette/cli.py  | 11 +++++++++--
 docs/plugins.rst  | 10 +++++++++-
 tests/test_cli.py |  9 +++++++++
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index f3455f72..8dbc97c4 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -235,9 +235,16 @@ def package(
 
 @cli.command()
 @click.argument("packages", nargs=-1, required=True)
-def install(packages):
+@click.option(
+    "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version"
+)
+def install(packages, upgrade):
     "Install Python packages - e.g. Datasette plugins - into the same environment as Datasette"
-    sys.argv = ["pip", "install"] + list(packages)
+    args = ["pip", "install"]
+    if upgrade:
+        args += ["--upgrade"]
+    args += list(packages)
+    sys.argv = args
     run_module("pip", run_name="__main__")
 
 
diff --git a/docs/plugins.rst b/docs/plugins.rst
index e67c77b3..1c0dd588 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -43,7 +43,15 @@ You can uninstall plugins with ``datasette uninstall``::
 
     datasette uninstall datasette-vega
 
-These ommands are thin wrappers around ``pip install`` and ``pip uninstall``, which ensure they run ``pip`` in the same virtual environment as Datasette itself.
+You can upgrade plugins with ``datasette install --upgrade`` or ``datasette install -U``::
+
+    datasette install -U datasette-vega
+
+This command can also be used to upgrade Datasette itself to the latest released version::
+
+    datasette install -U datasette
+
+These commands are thin wrappers around ``pip install`` and ``pip uninstall``, which ensure they run ``pip`` in the same virtual environment as Datasette itself.
 
 One-off plugins using --plugins-dir
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 38bb8834..dc5229cd 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -125,6 +125,15 @@ def test_install(run_module):
     ]
 
 
+@pytest.mark.parametrize("flag", ["-U", "--upgrade"])
+@mock.patch("datasette.cli.run_module")
+def test_install_upgrade(run_module, flag):
+    runner = CliRunner()
+    runner.invoke(cli, ["install", flag, "datasette"])
+    run_module.assert_called_once_with("pip", run_name="__main__")
+    assert sys.argv == ["pip", "install", "--upgrade", "datasette"]
+
+
 @mock.patch("datasette.cli.run_module")
 def test_uninstall(run_module):
     runner = CliRunner()

From 86aefc39c5aca01b00dbc57ba386a6743c21fb46 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 19 Aug 2020 10:22:33 -0700
Subject: [PATCH 0701/2303] Fixed undefined reference in index.rst

---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index 946fa542..db87f029 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,7 +1,7 @@
 Datasette
 =========
 
-|PyPI| |Changelog| |Python 3.x| |Travis CI| |License| |docker:
+|PyPI| |Changelog| |Python 3.x| |Tests| |License| |docker:
 datasette|
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/datasette.svg

From 799ecae94824640bdff21f86997f69844048d5c3 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 27 Aug 2020 21:02:50 -0700
Subject: [PATCH 0702/2303] register_output_renderer can now return Response,
 closes #953

---
 datasette/views/base.py                   | 18 +++++++++++-------
 docs/plugin_hooks.rst                     | 18 +++++++++---------
 tests/plugins/register_output_renderer.py |  8 ++++++++
 tests/test_html.py                        |  2 ++
 tests/test_plugins.py                     | 12 ++++++++++++
 5 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/datasette/views/base.py b/datasette/views/base.py
index a1f38f21..fa730af8 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -455,13 +455,17 @@ class DataView(BaseView):
                 result = await result
             if result is None:
                 raise NotFound("No data")
-
-            r = Response(
-                body=result.get("body"),
-                status=result.get("status_code", 200),
-                content_type=result.get("content_type", "text/plain"),
-                headers=result.get("headers"),
-            )
+            if isinstance(result, dict):
+                r = Response(
+                    body=result.get("body"),
+                    status=result.get("status_code", 200),
+                    content_type=result.get("content_type", "text/plain"),
+                    headers=result.get("headers"),
+                )
+            elif isinstance(result, Response):
+                r = result
+            else:
+                assert False, "{} should be dict or Response".format(result)
         else:
             extras = {}
             if callable(extra_template_data):
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 96a1cd7f..fc710a2b 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -455,7 +455,9 @@ When a request is received, the ``"render"`` callback function is called with ze
 ``view_name`` - string
     The name of the current view being called. ``index``, ``database``, ``table``, and ``row`` are the most important ones.
 
-The callback function can return ``None``, if it is unable to render the data, or a dictionary with the following keys:
+The callback function can return ``None``, if it is unable to render the data, or a :ref:`internals_response` that will be returned to the caller.
+
+It can also return a dictionary with the following keys. This format is **deprecated** as-of Datasette 0.49 and will be removed by Datasette 1.0.
 
 ``body`` - string or bytes, optional
     The response body, default empty
@@ -474,9 +476,7 @@ A simple example of an output renderer callback function:
 .. code-block:: python
 
     def render_demo():
-        return {
-            "body": "Hello World"
-        }
+        return Response.text("Hello World")
 
 Here is a more complex example:
 
@@ -490,11 +490,11 @@ Here is a more complex example:
         lines.append("=" * len(first_row))
         for row in rows:
             lines.append(" | ".join(row))
-        return {
-            "body": "\n".join(lines),
-            "content_type": "text/plain; charset=utf-8",
-            "headers": {"x-sqlite-version": result.first()[0]},
-        }
+        return Response(
+            "\n".join(lines),
+            content_type="text/plain; charset=utf-8",
+            headers={"x-sqlite-version": result.first()[0]}
+        )
 
 And here is an example ``can_render`` function which returns ``True`` only if the query results contain the columns ``atom_id``, ``atom_title`` and ``atom_updated``:
 
diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py
index 82b60d01..cfe15215 100644
--- a/tests/plugins/register_output_renderer.py
+++ b/tests/plugins/register_output_renderer.py
@@ -1,4 +1,5 @@
 from datasette import hookimpl
+from datasette.utils.asgi import Response
 import json
 
 
@@ -56,6 +57,12 @@ def render_test_no_parameters():
     return {"body": "Hello"}
 
 
+async def render_response(request):
+    if request.args.get("_broken"):
+        return "this should break"
+    return Response.json({"this_is": "json"})
+
+
 @hookimpl
 def register_output_renderer(datasette):
     return [
@@ -65,4 +72,5 @@ def register_output_renderer(datasette):
             "can_render": can_render,
         },
         {"extension": "testnone", "callback": render_test_no_parameters},
+        {"extension": "testresponse", "render": render_response},
     ]
diff --git a/tests/test_html.py b/tests/test_html.py
index 1a12b3ce..aec4db1d 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -559,6 +559,7 @@ def test_table_csv_json_export_interface(app_client):
         "simple_primary_key.json?id__gt=2",
         "simple_primary_key.testall?id__gt=2",
         "simple_primary_key.testnone?id__gt=2",
+        "simple_primary_key.testresponse?id__gt=2",
         "simple_primary_key.csv?id__gt=2&_size=max",
         "#export",
     ]
@@ -597,6 +598,7 @@ def test_csv_json_export_links_include_labels_if_foreign_keys(app_client):
         "facetable.json?_labels=on",
         "facetable.testall?_labels=on",
         "facetable.testnone?_labels=on",
+        "facetable.testresponse?_labels=on",
         "facetable.csv?_labels=on&_size=max",
         "#export",
     ]
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index c535810c..f2017f07 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -479,6 +479,18 @@ def test_hook_register_output_renderer_custom_headers(app_client):
     assert "2" == response.headers["x-gosh"]
 
 
+def test_hook_register_output_renderer_returning_response(app_client):
+    response = app_client.get("/fixtures/facetable.testresponse")
+    assert 200 == response.status
+    assert response.json == {"this_is": "json"}
+
+
+def test_hook_register_output_renderer_returning_broken_value(app_client):
+    response = app_client.get("/fixtures/facetable.testresponse?_broken=1")
+    assert 500 == response.status
+    assert "this should break should be dict or Response" in response.text
+
+
 def test_hook_register_output_renderer_can_render(app_client):
     response = app_client.get("/fixtures/facetable?_no_can_render=1")
     assert response.status == 200

From 7178126d902e2cfca606be0b0cff96c6c679c5b8 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 28 Aug 2020 16:12:47 -0700
Subject: [PATCH 0703/2303] Release notes for 0.49a0

Refs #953, #945
---
 docs/changelog.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index d18dae80..74426f52 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,17 @@
 Changelog
 =========
 
+.. _v0_49a0:
+
+0.49a0 (2020-08-28)
+-------------------
+
+.. warning:: This is an **alpha** release. See :ref:`contributing_alpha_beta`.
+
+- ``register_output_renderer()`` render functions can now return a ``Response``. (`#953 `__)
+- New ``--upgrade`` option for ``datasette install``. (`#945 `__)
+- ``datasette publish heroku`` now deploys using Python 3.8.5
+
 .. _v0_48:
 
 0.48 (2020-08-16)

From c36e287d71d68ecb2a45e9808eede15f19f931fb Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 28 Aug 2020 18:18:52 -0700
Subject: [PATCH 0704/2303] Don't deploy alpha/betas to Docker Hub

Refs #956
---
 .github/workflows/publish.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 4e554eda..e538a463 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -58,6 +58,8 @@ jobs:
   deploy_docker:
     runs-on: ubuntu-latest
     needs: [deploy]
+    if: |
+      !(contains(github.ref, "a") || contains(github.ref, "b"))
     steps:
     - uses: actions/checkout@v2
     - name: Build and push to Docker Hub

From 44cf424a94a85b74552075272660bb96a7432661 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 28 Aug 2020 18:33:05 -0700
Subject: [PATCH 0705/2303] Remove double colon, refs #956

---
 .github/workflows/publish.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index e538a463..1a94a6b3 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -69,6 +69,6 @@ jobs:
       run: |-
          docker login -u $DOCKER_USER -p $DOCKER_PASS 
          export REPO=datasetteproject/datasette 
-         docker build -f Dockerfile -t $REPO::${GITHUB_REF#refs/tags/} . 
-         docker tag $REPO::${GITHUB_REF#refs/tags/} $REPO:latest
+         docker build -f Dockerfile -t $REPO:${GITHUB_REF#refs/tags/} . 
+         docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest
          docker push $REPO 

From 9dbbfa1f0b5cf07c91ba4c8d7b0145cf0ed4cf0f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 30 Aug 2020 10:39:16 -0700
Subject: [PATCH 0706/2303] Upgrade CodeMirror to 5.57.0, refs #948

---
 datasette/static/codemirror-5.31.0-min.css    |    2 -
 datasette/static/codemirror-5.31.0-sql.min.js |    1 -
 datasette/static/codemirror-5.31.0.js         | 9659 -----------------
 datasette/static/codemirror-5.57.0-sql.min.js |    5 +
 datasette/static/codemirror-5.57.0.min.css    |    1 +
 datasette/static/codemirror-5.57.0.min.js     |   11 +
 datasette/templates/_codemirror.html          |    6 +-
 7 files changed, 20 insertions(+), 9665 deletions(-)
 delete mode 100644 datasette/static/codemirror-5.31.0-min.css
 delete mode 100644 datasette/static/codemirror-5.31.0-sql.min.js
 delete mode 100644 datasette/static/codemirror-5.31.0.js
 create mode 100644 datasette/static/codemirror-5.57.0-sql.min.js
 create mode 100644 datasette/static/codemirror-5.57.0.min.css
 create mode 100644 datasette/static/codemirror-5.57.0.min.js

diff --git a/datasette/static/codemirror-5.31.0-min.css b/datasette/static/codemirror-5.31.0-min.css
deleted file mode 100644
index 7e162037..00000000
--- a/datasette/static/codemirror-5.31.0-min.css
+++ /dev/null
@@ -1,2 +0,0 @@
-.CodeMirror{font-family:monospace;height:300px;color:#000;direction:ltr}.CodeMirror-lines{padding:4px 0}.CodeMirror pre{padding:0 4px}.CodeMirror-gutter-filler,.CodeMirror-scrollbar-filler{background-color:#fff}.CodeMirror-gutters{border-right:1px solid #ddd;background-color:#f7f7f7;white-space:nowrap}.CodeMirror-linenumber{padding:0 3px 0 5px;min-width:20px;text-align:right;color:#999;white-space:nowrap}.CodeMirror-guttermarker{color:#000}.CodeMirror-guttermarker-subtle{color:#999}.CodeMirror-cursor{border-left:1px solid #000;border-right:none;width:0}.CodeMirror div.CodeMirror-secondarycursor{border-left:1px solid silver}.cm-fat-cursor .CodeMirror-cursor{width:auto;border:0!important;background:#7e7}.cm-fat-cursor div.CodeMirror-cursors{z-index:1}.cm-fat-cursor-mark{background-color:rgba(20,255,20,.5);-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite}.cm-animate-fat-cursor{width:auto;border:0;-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite;background-color:#7e7}@-moz-keyframes blink{50%{background-color:transparent}}@-webkit-keyframes blink{50%{background-color:transparent}}@keyframes blink{50%{background-color:transparent}}.cm-tab{display:inline-block;text-decoration:inherit}.CodeMirror-rulers{position:absolute;left:0;right:0;top:-50px;bottom:-20px;overflow:hidden}.CodeMirror-ruler{border-left:1px solid #ccc;top:0;bottom:0;position:absolute}.cm-s-default .cm-header{color:#00f}.cm-s-default .cm-quote{color:#090}.cm-negative{color:#d44}.cm-positive{color:#292}.cm-header,.cm-strong{font-weight:700}.cm-em{font-style:italic}.cm-link{text-decoration:underline}.cm-strikethrough{text-decoration:line-through}.cm-s-default .cm-keyword{color:#708}.cm-s-default .cm-atom{color:#219}.cm-s-default .cm-number{color:#164}.cm-s-default .cm-def{color:#00f}.cm-s-default .cm-variable-2{color:#05a}.cm-s-default .cm-type,.cm-s-default .cm-variable-3{color:#085}.cm-s-default .cm-comment{color:#a50}.cm-s-default .cm-string{color:#a11}.cm-s-default .cm-string-2{color:#f50}.cm-s-default .cm-meta{color:#555}.cm-s-default .cm-qualifier{color:#555}.cm-s-default .cm-builtin{color:#30a}.cm-s-default .cm-bracket{color:#997}.cm-s-default .cm-tag{color:#170}.cm-s-default .cm-attribute{color:#00c}.cm-s-default .cm-hr{color:#999}.cm-s-default .cm-link{color:#00c}.cm-s-default .cm-error{color:red}.cm-invalidchar{color:red}.CodeMirror-composing{border-bottom:2px solid}div.CodeMirror span.CodeMirror-matchingbracket{color:#0f0}div.CodeMirror span.CodeMirror-nonmatchingbracket{color:#f22}.CodeMirror-matchingtag{background:rgba(255,150,0,.3)}.CodeMirror-activeline-background{background:#e8f2ff}.CodeMirror{position:relative;overflow:hidden;background:#fff}.CodeMirror-scroll{overflow:scroll!important;margin-bottom:-30px;margin-right:-30px;padding-bottom:30px;height:100%;outline:0;position:relative}.CodeMirror-sizer{position:relative;border-right:30px solid transparent}.CodeMirror-gutter-filler,.CodeMirror-hscrollbar,.CodeMirror-scrollbar-filler,.CodeMirror-vscrollbar{position:absolute;z-index:6;display:none}.CodeMirror-vscrollbar{right:0;top:0;overflow-x:hidden;overflow-y:scroll}.CodeMirror-hscrollbar{bottom:0;left:0;overflow-y:hidden;overflow-x:scroll}.CodeMirror-scrollbar-filler{right:0;bottom:0}.CodeMirror-gutter-filler{left:0;bottom:0}.CodeMirror-gutters{position:absolute;left:0;top:0;min-height:100%;z-index:3}.CodeMirror-gutter{white-space:normal;height:100%;display:inline-block;vertical-align:top;margin-bottom:-30px}.CodeMirror-gutter-wrapper{position:absolute;z-index:4;background:0 0!important;border:none!important}.CodeMirror-gutter-background{position:absolute;top:0;bottom:0;z-index:4}.CodeMirror-gutter-elt{position:absolute;cursor:default;z-index:4}.CodeMirror-gutter-wrapper ::selection{background-color:transparent}.CodeMirror-gutter-wrapper ::-moz-selection{background-color:transparent}.CodeMirror-lines{cursor:text;min-height:1px}.CodeMirror pre{-moz-border-radius:0;-webkit-border-radius:0;border-radius:0;border-width:0;background:0 0;font-family:inherit;font-size:inherit;margin:0;white-space:pre;word-wrap:normal;line-height:inherit;color:inherit;z-index:2;position:relative;overflow:visible;-webkit-tap-highlight-color:transparent;-webkit-font-variant-ligatures:contextual;font-variant-ligatures:contextual}.CodeMirror-wrap pre{word-wrap:break-word;white-space:pre-wrap;word-break:normal}.CodeMirror-linebackground{position:absolute;left:0;right:0;top:0;bottom:0;z-index:0}.CodeMirror-linewidget{position:relative;z-index:2;overflow:auto}.CodeMirror-rtl pre{direction:rtl}.CodeMirror-code{outline:0}.CodeMirror-gutter,.CodeMirror-gutters,.CodeMirror-linenumber,.CodeMirror-scroll,.CodeMirror-sizer{-moz-box-sizing:content-box;box-sizing:content-box}.CodeMirror-measure{position:absolute;width:100%;height:0;overflow:hidden;visibility:hidden}.CodeMirror-cursor{position:absolute;pointer-events:none}.CodeMirror-measure pre{position:static}div.CodeMirror-cursors{visibility:hidden;position:relative;z-index:3}div.CodeMirror-dragcursors{visibility:visible}.CodeMirror-focused div.CodeMirror-cursors{visibility:visible}.CodeMirror-selected{background:#d9d9d9}.CodeMirror-focused .CodeMirror-selected{background:#d7d4f0}.CodeMirror-crosshair{cursor:crosshair}.CodeMirror-line::selection,.CodeMirror-line>span::selection,.CodeMirror-line>span>span::selection{background:#d7d4f0}.CodeMirror-line::-moz-selection,.CodeMirror-line>span::-moz-selection,.CodeMirror-line>span>span::-moz-selection{background:#d7d4f0}.cm-searching{background-color:#ffa;background-color:rgba(255,255,0,.4)}.cm-force-border{padding-right:.1px}@media print{.CodeMirror div.CodeMirror-cursors{visibility:hidden}}.cm-tab-wrap-hack:after{content:''}span.CodeMirror-selectedtext{background:0 0}
-/*# sourceMappingURL=codemirror.min.css.map */
\ No newline at end of file
diff --git a/datasette/static/codemirror-5.31.0-sql.min.js b/datasette/static/codemirror-5.31.0-sql.min.js
deleted file mode 100644
index 1f05c0d0..00000000
--- a/datasette/static/codemirror-5.31.0-sql.min.js
+++ /dev/null
@@ -1 +0,0 @@
-!function(e){"object"==typeof exports&&"object"==typeof module?e(require("../../lib/codemirror")):"function"==typeof define&&define.amd?define(["../../lib/codemirror"],e):e(CodeMirror)}(function(e){"use strict";e.defineMode("sql",function(t,r){function a(e,t){var r=e.next();if(g[r]){var a=g[r](e,t);if(!1!==a)return a}if(p.hexNumber&&("0"==r&&e.match(/^[xX][0-9a-fA-F]+/)||("x"==r||"X"==r)&&e.match(/^'[0-9a-fA-F]+'/)))return"number";if(p.binaryNumber&&(("b"==r||"B"==r)&&e.match(/^'[01]+'/)||"0"==r&&e.match(/^b[01]+/)))return"number";if(r.charCodeAt(0)>47&&r.charCodeAt(0)<58)return e.match(/^[0-9]*(\.[0-9]+)?([eE][-+]?[0-9]+)?/),p.decimallessFloat&&e.match(/^\.(?!\.)/),"number";if("?"==r&&(e.eatSpace()||e.eol()||e.eat(";")))return"variable-3";if("'"==r||'"'==r&&p.doubleQuote)return t.tokenize=n(r),t.tokenize(e,t);if((p.nCharCast&&("n"==r||"N"==r)||p.charsetCast&&"_"==r&&e.match(/[a-z][a-z0-9]*/i))&&("'"==e.peek()||'"'==e.peek()))return"keyword";if(/^[\(\),\;\[\]]/.test(r))return null;if(p.commentSlashSlash&&"/"==r&&e.eat("/"))return e.skipToEnd(),"comment";if(p.commentHash&&"#"==r||"-"==r&&e.eat("-")&&(!p.commentSpaceRequired||e.eat(" ")))return e.skipToEnd(),"comment";if("/"==r&&e.eat("*"))return t.tokenize=i(1),t.tokenize(e,t);if("."!=r){if(m.test(r))return e.eatWhile(m),null;if("{"==r&&(e.match(/^( )*(d|D|t|T|ts|TS)( )*'[^']*'( )*}/)||e.match(/^( )*(d|D|t|T|ts|TS)( )*"[^"]*"( )*}/)))return"number";e.eatWhile(/^[_\w\d]/);var o=e.current().toLowerCase();return b.hasOwnProperty(o)&&(e.match(/^( )+'[^']*'/)||e.match(/^( )+"[^"]*"/))?"number":c.hasOwnProperty(o)?"atom":u.hasOwnProperty(o)?"builtin":d.hasOwnProperty(o)?"keyword":l.hasOwnProperty(o)?"string-2":null}return p.zerolessFloat&&e.match(/^(?:\d+(?:e[+-]?\d+)?)/i)?"number":e.match(/^\.+/)?null:p.ODBCdotTable&&e.match(/^[\w\d_]+/)?"variable-2":void 0}function n(e){return function(t,r){for(var n,i=!1;null!=(n=t.next());){if(n==e&&!i){r.tokenize=a;break}i=!i&&"\\"==n}return"string"}}function i(e){return function(t,r){var n=t.match(/^.*?(\/\*|\*\/)/);return n?"/*"==n[1]?r.tokenize=i(e+1):r.tokenize=e>1?i(e-1):a:t.skipToEnd(),"comment"}}function o(e,t,r){t.context={prev:t.context,indent:e.indentation(),col:e.column(),type:r}}function s(e){e.indent=e.context.indent,e.context=e.context.prev}var l=r.client||{},c=r.atoms||{false:!0,true:!0,null:!0},u=r.builtin||{},d=r.keywords||{},m=r.operatorChars||/^[*+\-%<>!=&|~^]/,p=r.support||{},g=r.hooks||{},b=r.dateSQL||{date:!0,time:!0,timestamp:!0};return{startState:function(){return{tokenize:a,context:null}},token:function(e,t){if(e.sol()&&t.context&&null==t.context.align&&(t.context.align=!1),t.tokenize==a&&e.eatSpace())return null;var r=t.tokenize(e,t);if("comment"==r)return r;t.context&&null==t.context.align&&(t.context.align=!0);var n=e.current();return"("==n?o(e,t,")"):"["==n?o(e,t,"]"):t.context&&t.context.type==n&&s(t),r},indent:function(r,a){var n=r.context;if(!n)return e.Pass;var i=a.charAt(0)==n.type;return n.align?n.col+(i?0:1):n.indent+(i?0:t.indentUnit)},blockCommentStart:"/*",blockCommentEnd:"*/",lineComment:p.commentSlashSlash?"//":p.commentHash?"#":"--"}}),function(){function t(e){for(var t;null!=(t=e.next());)if("`"==t&&!e.eat("`"))return"variable-2";return e.backUp(e.current().length-1),e.eatWhile(/\w/)?"variable-2":null}function r(e){return e.eat("@")&&(e.match(/^session\./),e.match(/^local\./),e.match(/^global\./)),e.eat("'")?(e.match(/^.*'/),"variable-2"):e.eat('"')?(e.match(/^.*"/),"variable-2"):e.eat("`")?(e.match(/^.*`/),"variable-2"):e.match(/^[0-9a-zA-Z$\.\_]+/)?"variable-2":null}function a(e){return e.eat("N")?"atom":e.match(/^[a-zA-Z.#!?]/)?"variable-2":null}function n(e){for(var t={},r=e.split(" "),a=0;a!=]/,dateSQL:n("date time timestamp"),support:n("ODBCdotTable doubleQuote binaryNumber hexNumber")}),e.defineMIME("text/x-mssql",{name:"sql",client:n("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),keywords:n(i+"begin trigger proc view index for add constraint key primary foreign collate clustered nonclustered declare exec"),builtin:n("bigint numeric bit smallint decimal smallmoney int tinyint money float real char varchar text nchar nvarchar ntext binary varbinary image cursor timestamp hierarchyid uniqueidentifier sql_variant xml table "),atoms:n("false true null unknown"),operatorChars:/^[*+\-%<>!=]/,dateSQL:n("date datetimeoffset datetime2 smalldatetime datetime time"),hooks:{"@":r}}),e.defineMIME("text/x-mysql",{name:"sql",client:n("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),keywords:n(i+"accessible action add after algorithm all analyze asensitive at authors auto_increment autocommit avg avg_row_length before binary binlog both btree cache call cascade cascaded case catalog_name chain change changed character check checkpoint checksum class_origin client_statistics close coalesce code collate collation collations column columns comment commit committed completion concurrent condition connection consistent constraint contains continue contributors convert cross current current_date current_time current_timestamp current_user cursor data database databases day_hour day_microsecond day_minute day_second deallocate dec declare default delay_key_write delayed delimiter des_key_file describe deterministic dev_pop dev_samp deviance diagnostics directory disable discard distinctrow div dual dumpfile each elseif enable enclosed end ends engine engines enum errors escape escaped even event events every execute exists exit explain extended fast fetch field fields first flush for force foreign found_rows full fulltext function general get global grant grants group group_concat handler hash help high_priority hosts hour_microsecond hour_minute hour_second if ignore ignore_server_ids import index index_statistics infile inner innodb inout insensitive insert_method install interval invoker isolation iterate key keys kill language last leading leave left level limit linear lines list load local localtime localtimestamp lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters match max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modifies modify mutex mysql_errno natural next no no_write_to_binlog offline offset one online open optimize option optionally out outer outfile pack_keys parser partition partitions password phase plugin plugins prepare preserve prev primary privileges procedure processlist profile profiles purge query quick range read read_write reads real rebuild recover references regexp relaylog release remove rename reorganize repair repeatable replace require resignal restrict resume return returns revoke right rlike rollback rollup row row_format rtree savepoint schedule schema schema_name schemas second_microsecond security sensitive separator serializable server session share show signal slave slow smallint snapshot soname spatial specific sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result sqlexception sqlstate sqlwarning ssl start starting starts status std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace temporary terminated to trailing transaction trigger triggers truncate uncommitted undo uninstall unique unlock upgrade usage use use_frm user user_resources user_statistics using utc_date utc_time utc_timestamp value variables varying view views warnings when while with work write xa xor year_month zerofill begin do then else loop repeat"),builtin:n("bool boolean bit blob decimal double float long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision date datetime year unsigned signed numeric"),atoms:n("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^]/,dateSQL:n("date time timestamp"),support:n("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber doubleQuote nCharCast charsetCast commentHash commentSpaceRequired"),hooks:{"@":r,"`":t,"\\":a}}),e.defineMIME("text/x-mariadb",{name:"sql",client:n("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),keywords:n(i+"accessible action add after algorithm all always analyze asensitive at authors auto_increment autocommit avg avg_row_length before binary binlog both btree cache call cascade cascaded case catalog_name chain change changed character check checkpoint checksum class_origin client_statistics close coalesce code collate collation collations column columns comment commit committed completion concurrent condition connection consistent constraint contains continue contributors convert cross current current_date current_time current_timestamp current_user cursor data database databases day_hour day_microsecond day_minute day_second deallocate dec declare default delay_key_write delayed delimiter des_key_file describe deterministic dev_pop dev_samp deviance diagnostics directory disable discard distinctrow div dual dumpfile each elseif enable enclosed end ends engine engines enum errors escape escaped even event events every execute exists exit explain extended fast fetch field fields first flush for force foreign found_rows full fulltext function general generated get global grant grants group groupby_concat handler hard hash help high_priority hosts hour_microsecond hour_minute hour_second if ignore ignore_server_ids import index index_statistics infile inner innodb inout insensitive insert_method install interval invoker isolation iterate key keys kill language last leading leave left level limit linear lines list load local localtime localtimestamp lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters match max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modifies modify mutex mysql_errno natural next no no_write_to_binlog offline offset one online open optimize option optionally out outer outfile pack_keys parser partition partitions password persistent phase plugin plugins prepare preserve prev primary privileges procedure processlist profile profiles purge query quick range read read_write reads real rebuild recover references regexp relaylog release remove rename reorganize repair repeatable replace require resignal restrict resume return returns revoke right rlike rollback rollup row row_format rtree savepoint schedule schema schema_name schemas second_microsecond security sensitive separator serializable server session share show shutdown signal slave slow smallint snapshot soft soname spatial specific sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result sqlexception sqlstate sqlwarning ssl start starting starts status std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace temporary terminated to trailing transaction trigger triggers truncate uncommitted undo uninstall unique unlock upgrade usage use use_frm user user_resources user_statistics using utc_date utc_time utc_timestamp value variables varying view views virtual warnings when while with work write xa xor year_month zerofill begin do then else loop repeat"),builtin:n("bool boolean bit blob decimal double float long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision date datetime year unsigned signed numeric"),atoms:n("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^]/,dateSQL:n("date time timestamp"),support:n("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber doubleQuote nCharCast charsetCast commentHash commentSpaceRequired"),hooks:{"@":r,"`":t,"\\":a}}),e.defineMIME("text/x-sqlite",{name:"sql",client:n("auth backup bail binary changes check clone databases dbinfo dump echo eqp exit explain fullschema headers help import imposter indexes iotrace limit lint load log mode nullvalue once open output print prompt quit read restore save scanstats schema separator session shell show stats system tables testcase timeout timer trace vfsinfo vfslist vfsname width"),keywords:n(i+"abort action add after all analyze attach autoincrement before begin cascade case cast check collate column commit conflict constraint cross current_date current_time current_timestamp database default deferrable deferred detach each else end escape except exclusive exists explain fail for foreign full glob if ignore immediate index indexed initially inner instead intersect isnull key left limit match natural no notnull null of offset outer plan pragma primary query raise recursive references regexp reindex release rename replace restrict right rollback row savepoint temp temporary then to transaction trigger unique using vacuum view virtual when with without"),builtin:n("bool boolean bit blob decimal double float long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text clob bigint int int2 int8 integer float double char varchar date datetime year unsigned signed numeric real"),atoms:n("null current_date current_time current_timestamp"),operatorChars:/^[*+\-%<>!=&|/~]/,dateSQL:n("date time timestamp datetime"),support:n("decimallessFloat zerolessFloat"),identifierQuote:'"',hooks:{"@":r,":":r,"?":r,$:r,'"':function(e){for(var t;null!=(t=e.next());)if('"'==t&&!e.eat('"'))return"variable-2";return e.backUp(e.current().length-1),e.eatWhile(/\w/)?"variable-2":null},"`":t}}),e.defineMIME("text/x-cassandra",{name:"sql",client:{},keywords:n("add all allow alter and any apply as asc authorize batch begin by clustering columnfamily compact consistency count create custom delete desc distinct drop each_quorum exists filtering from grant if in index insert into key keyspace keyspaces level limit local_one local_quorum modify nan norecursive nosuperuser not of on one order password permission permissions primary quorum rename revoke schema select set storage superuser table three to token truncate ttl two type unlogged update use user users using values where with writetime"),builtin:n("ascii bigint blob boolean counter decimal double float frozen inet int list map static text timestamp timeuuid tuple uuid varchar varint"),atoms:n("false true infinity NaN"),operatorChars:/^[<>=]/,dateSQL:{},support:n("commentSlashSlash decimallessFloat"),hooks:{}}),e.defineMIME("text/x-plsql",{name:"sql",client:n("appinfo arraysize autocommit autoprint autorecovery autotrace blockterminator break btitle cmdsep colsep compatibility compute concat copycommit copytypecheck define describe echo editfile embedded escape exec execute feedback flagger flush heading headsep instance linesize lno loboffset logsource long longchunksize markup native newpage numformat numwidth pagesize pause pno recsep recsepchar release repfooter repheader serveroutput shiftinout show showmode size spool sqlblanklines sqlcase sqlcode sqlcontinue sqlnumber sqlpluscompatibility sqlprefix sqlprompt sqlterminator suffix tab term termout time timing trimout trimspool ttitle underline verify version wrap"),keywords:n("abort accept access add all alter and any array arraylen as asc assert assign at attributes audit authorization avg base_table begin between binary_integer body boolean by case cast char char_base check close cluster clusters colauth column comment commit compress connect connected constant constraint crash create current currval cursor data_base database date dba deallocate debugoff debugon decimal declare default definition delay delete desc digits dispose distinct do drop else elseif elsif enable end entry escape exception exception_init exchange exclusive exists exit external fast fetch file for force form from function generic goto grant group having identified if immediate in increment index indexes indicator initial initrans insert interface intersect into is key level library like limited local lock log logging long loop master maxextents maxtrans member minextents minus mislabel mode modify multiset new next no noaudit nocompress nologging noparallel not nowait number_base object of off offline on online only open option or order out package parallel partition pctfree pctincrease pctused pls_integer positive positiven pragma primary prior private privileges procedure public raise range raw read rebuild record ref references refresh release rename replace resource restrict return returning returns reverse revoke rollback row rowid rowlabel rownum rows run savepoint schema segment select separate session set share snapshot some space split sql start statement storage subtype successful synonym tabauth table tables tablespace task terminate then to trigger truncate type union unique unlimited unrecoverable unusable update use using validate value values variable view views when whenever where while with work"),builtin:n("abs acos add_months ascii asin atan atan2 average bfile bfilename bigserial bit blob ceil character chartorowid chr clob concat convert cos cosh count dec decode deref dual dump dup_val_on_index empty error exp false float floor found glb greatest hextoraw initcap instr instrb int integer isopen last_day least length lengthb ln lower lpad ltrim lub make_ref max min mlslabel mod months_between natural naturaln nchar nclob new_time next_day nextval nls_charset_decl_len nls_charset_id nls_charset_name nls_initcap nls_lower nls_sort nls_upper nlssort no_data_found notfound null number numeric nvarchar2 nvl others power rawtohex real reftohex round rowcount rowidtochar rowtype rpad rtrim serial sign signtype sin sinh smallint soundex sqlcode sqlerrm sqrt stddev string substr substrb sum sysdate tan tanh to_char text to_date to_label to_multi_byte to_number to_single_byte translate true trunc uid unlogged upper user userenv varchar varchar2 variance varying vsize xml"),operatorChars:/^[*+\-%<>!=~]/,dateSQL:n("date time timestamp"),support:n("doubleQuote nCharCast zerolessFloat binaryNumber hexNumber")}),e.defineMIME("text/x-hive",{name:"sql",keywords:n("select alter $elem$ $key$ $value$ add after all analyze and archive as asc before between binary both bucket buckets by cascade case cast change cluster clustered clusterstatus collection column columns comment compute concatenate continue create cross cursor data database databases dbproperties deferred delete delimited desc describe directory disable distinct distribute drop else enable end escaped exclusive exists explain export extended external false fetch fields fileformat first format formatted from full function functions grant group having hold_ddltime idxproperties if import in index indexes inpath inputdriver inputformat insert intersect into is items join keys lateral left like limit lines load local location lock locks mapjoin materialized minus msck no_drop nocompress not of offline on option or order out outer outputdriver outputformat overwrite partition partitioned partitions percent plus preserve procedure purge range rcfile read readonly reads rebuild recordreader recordwriter recover reduce regexp rename repair replace restrict revoke right rlike row schema schemas semi sequencefile serde serdeproperties set shared show show_database sort sorted ssl statistics stored streamtable table tables tablesample tblproperties temporary terminated textfile then tmp to touch transform trigger true unarchive undo union uniquejoin unlock update use using utc utc_tmestamp view when where while with"),builtin:n("bool boolean long timestamp tinyint smallint bigint int float double date datetime unsigned string array struct map uniontype"),atoms:n("false true null unknown"),operatorChars:/^[*+\-%<>!=]/,dateSQL:n("date timestamp"),support:n("ODBCdotTable doubleQuote binaryNumber hexNumber")}),e.defineMIME("text/x-pgsql",{name:"sql",client:n("source"),keywords:n(i+"a abort abs absent absolute access according action ada add admin after aggregate all allocate also always analyse analyze any are array array_agg array_max_cardinality asensitive assertion assignment asymmetric at atomic attribute attributes authorization avg backward base64 before begin begin_frame begin_partition bernoulli binary bit_length blob blocked bom both breadth c cache call called cardinality cascade cascaded case cast catalog catalog_name ceil ceiling chain characteristics characters character_length character_set_catalog character_set_name character_set_schema char_length check checkpoint class class_origin clob close cluster coalesce cobol collate collation collation_catalog collation_name collation_schema collect column columns column_name command_function command_function_code comment comments commit committed concurrently condition condition_number configuration conflict connect connection connection_name constraint constraints constraint_catalog constraint_name constraint_schema constructor contains content continue control conversion convert copy corr corresponding cost covar_pop covar_samp cross csv cube cume_dist current current_catalog current_date current_default_transform_group current_path current_role current_row current_schema current_time current_timestamp current_transform_group_for_type current_user cursor cursor_name cycle data database datalink datetime_interval_code datetime_interval_precision day db deallocate dec declare default defaults deferrable deferred defined definer degree delimiter delimiters dense_rank depth deref derived describe descriptor deterministic diagnostics dictionary disable discard disconnect dispatch dlnewcopy dlpreviouscopy dlurlcomplete dlurlcompleteonly dlurlcompletewrite dlurlpath dlurlpathonly dlurlpathwrite dlurlscheme dlurlserver dlvalue do document domain dynamic dynamic_function dynamic_function_code each element else empty enable encoding encrypted end end-exec end_frame end_partition enforced enum equals escape event every except exception exclude excluding exclusive exec execute exists exp explain expression extension external extract false family fetch file filter final first first_value flag float floor following for force foreign fortran forward found frame_row free freeze fs full function functions fusion g general generated get global go goto grant granted greatest grouping groups handler header hex hierarchy hold hour id identity if ignore ilike immediate immediately immutable implementation implicit import including increment indent index indexes indicator inherit inherits initially inline inner inout input insensitive instance instantiable instead integrity intersect intersection invoker isnull isolation k key key_member key_type label lag language large last last_value lateral lead leading leakproof least left length level library like_regex link listen ln load local localtime localtimestamp location locator lock locked logged lower m map mapping match matched materialized max maxvalue max_cardinality member merge message_length message_octet_length message_text method min minute minvalue mod mode modifies module month more move multiset mumps name names namespace national natural nchar nclob nesting new next nfc nfd nfkc nfkd nil no none normalize normalized nothing notify notnull nowait nth_value ntile null nullable nullif nulls number object occurrences_regex octets octet_length of off offset oids old only open operator option options ordering ordinality others out outer output over overlaps overlay overriding owned owner p pad parallel parameter parameter_mode parameter_name parameter_ordinal_position parameter_specific_catalog parameter_specific_name parameter_specific_schema parser partial partition pascal passing passthrough password percent percentile_cont percentile_disc percent_rank period permission placing plans pli policy portion position position_regex power precedes preceding prepare prepared preserve primary prior privileges procedural procedure program public quote range rank read reads reassign recheck recovery recursive ref references referencing refresh regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy regr_syy reindex relative release rename repeatable replace replica requiring reset respect restart restore restrict restricted result return returned_cardinality returned_length returned_octet_length returned_sqlstate returning returns revoke right role rollback rollup routine routine_catalog routine_name routine_schema row rows row_count row_number rule savepoint scale schema schema_name scope scope_catalog scope_name scope_schema scroll search second section security selective self sensitive sequence sequences serializable server server_name session session_user setof sets share show similar simple size skip snapshot some source space specific specifictype specific_name sql sqlcode sqlerror sqlexception sqlstate sqlwarning sqrt stable standalone start state statement static statistics stddev_pop stddev_samp stdin stdout storage strict strip structure style subclass_origin submultiset substring substring_regex succeeds sum symmetric sysid system system_time system_user t tables tablesample tablespace table_name temp template temporary then ties timezone_hour timezone_minute to token top_level_count trailing transaction transactions_committed transactions_rolled_back transaction_active transform transforms translate translate_regex translation treat trigger trigger_catalog trigger_name trigger_schema trim trim_array true truncate trusted type types uescape unbounded uncommitted under unencrypted unique unknown unlink unlisten unlogged unnamed unnest until untyped upper uri usage user user_defined_type_catalog user_defined_type_code user_defined_type_name user_defined_type_schema using vacuum valid validate validator value value_of varbinary variadic var_pop var_samp verbose version versioning view views volatile when whenever whitespace width_bucket window within work wrapper write xmlagg xmlattributes xmlbinary xmlcast xmlcomment xmlconcat xmldeclaration xmldocument xmlelement xmlexists xmlforest xmliterate xmlnamespaces xmlparse xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltext xmlvalidate year yes loop repeat attach path depends detach zone"),builtin:n("bigint int8 bigserial serial8 bit varying varbit boolean bool box bytea character char varchar cidr circle date double precision float8 inet integer int int4 interval json jsonb line lseg macaddr macaddr8 money numeric decimal path pg_lsn point polygon real float4 smallint int2 smallserial serial2 serial serial4 text time without zone with timetz timestamp timestamptz tsquery tsvector txid_snapshot uuid xml"),atoms:n("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^\/#@?~]/,dateSQL:n("date time timestamp"),support:n("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber nCharCast charsetCast")}),e.defineMIME("text/x-gql",{name:"sql",keywords:n("ancestor and asc by contains desc descendant distinct from group has in is limit offset on order select superset where"),atoms:n("false true"),builtin:n("blob datetime first key __key__ string integer double boolean null"),operatorChars:/^[*+\-%<>!=]/}),e.defineMIME("text/x-gpsql",{name:"sql",client:n("source"),keywords:n("abort absolute access action active add admin after aggregate all also alter always analyse analyze and any array as asc assertion assignment asymmetric at authorization backward before begin between bigint binary bit boolean both by cache called cascade cascaded case cast chain char character characteristics check checkpoint class close cluster coalesce codegen collate column comment commit committed concurrency concurrently configuration connection constraint constraints contains content continue conversion copy cost cpu_rate_limit create createdb createexttable createrole createuser cross csv cube current current_catalog current_date current_role current_schema current_time current_timestamp current_user cursor cycle data database day deallocate dec decimal declare decode default defaults deferrable deferred definer delete delimiter delimiters deny desc dictionary disable discard distinct distributed do document domain double drop dxl each else enable encoding encrypted end enum errors escape every except exchange exclude excluding exclusive execute exists explain extension external extract false family fetch fields filespace fill filter first float following for force foreign format forward freeze from full function global grant granted greatest group group_id grouping handler hash having header hold host hour identity if ignore ilike immediate immutable implicit in including inclusive increment index indexes inherit inherits initially inline inner inout input insensitive insert instead int integer intersect interval into invoker is isnull isolation join key language large last leading least left level like limit list listen load local localtime localtimestamp location lock log login mapping master match maxvalue median merge minute minvalue missing mode modifies modify month move name names national natural nchar new newline next no nocreatedb nocreateexttable nocreaterole nocreateuser noinherit nologin none noovercommit nosuperuser not nothing notify notnull nowait null nullif nulls numeric object of off offset oids old on only operator option options or order ordered others out outer over overcommit overlaps overlay owned owner parser partial partition partitions passing password percent percentile_cont percentile_disc placing plans position preceding precision prepare prepared preserve primary prior privileges procedural procedure protocol queue quote randomly range read readable reads real reassign recheck recursive ref references reindex reject relative release rename repeatable replace replica reset resource restart restrict returning returns revoke right role rollback rollup rootpartition row rows rule savepoint scatter schema scroll search second security segment select sequence serializable session session_user set setof sets share show similar simple smallint some split sql stable standalone start statement statistics stdin stdout storage strict strip subpartition subpartitions substring superuser symmetric sysid system table tablespace temp template temporary text then threshold ties time timestamp to trailing transaction treat trigger trim true truncate trusted type unbounded uncommitted unencrypted union unique unknown unlisten until update user using vacuum valid validation validator value values varchar variadic varying verbose version view volatile web when where whitespace window with within without work writable write xml xmlattributes xmlconcat xmlelement xmlexists xmlforest xmlparse xmlpi xmlroot xmlserialize year yes zone"),builtin:n("bigint int8 bigserial serial8 bit varying varbit boolean bool box bytea character char varchar cidr circle date double precision float float8 inet integer int int4 interval json jsonb line lseg macaddr macaddr8 money numeric decimal path pg_lsn point polygon real float4 smallint int2 smallserial serial2 serial serial4 text time without zone with timetz timestamp timestamptz tsquery tsvector txid_snapshot uuid xml"),atoms:n("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^\/#@?~]/,dateSQL:n("date time timestamp"),support:n("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber nCharCast charsetCast")}),e.defineMIME("text/x-sparksql",{name:"sql",keywords:n("add after all alter analyze and anti archive array as asc at between bucket buckets by cache cascade case cast change clear cluster clustered codegen collection column columns comment commit compact compactions compute concatenate cost create cross cube current current_date current_timestamp database databases datata dbproperties defined delete delimited desc describe dfs directories distinct distribute drop else end escaped except exchange exists explain export extended external false fields fileformat first following for format formatted from full function functions global grant group grouping having if ignore import in index indexes inner inpath inputformat insert intersect interval into is items join keys last lateral lazy left like limit lines list load local location lock locks logical macro map minus msck natural no not null nulls of on option options or order out outer outputformat over overwrite partition partitioned partitions percent preceding principals purge range recordreader recordwriter recover reduce refresh regexp rename repair replace reset restrict revoke right rlike role roles rollback rollup row rows schema schemas select semi separated serde serdeproperties set sets show skewed sort sorted start statistics stored stratify struct table tables tablesample tblproperties temp temporary terminated then to touch transaction transactions transform true truncate unarchive unbounded uncache union unlock unset use using values view when where window with"),builtin:n("tinyint smallint int bigint boolean float double string binary timestamp decimal array map struct uniontype delimited serde sequencefile textfile rcfile inputformat outputformat"),atoms:n("false true null"),operatorChars:/^[*+\-%<>!=~&|^]/,dateSQL:n("date time timestamp"),support:n("ODBCdotTable doubleQuote zerolessFloat")}),e.defineMIME("text/x-esper",{name:"sql",client:n("source"),keywords:n("alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit after all and as at asc avedev avg between by case cast coalesce count create current_timestamp day days delete define desc distinct else end escape events every exists false first from full group having hour hours in inner insert instanceof into irstream is istream join last lastweekday left limit like max match_recognize matches median measures metadatasql min minute minutes msec millisecond milliseconds not null offset on or order outer output partition pattern prev prior regexp retain-union retain-intersection right rstream sec second seconds select set some snapshot sql stddev sum then true unidirectional until update variable weekday when where window"),builtin:{},atoms:n("false true null"),operatorChars:/^[*+\-%<>!=&|^\/#@?~]/,dateSQL:n("time"),support:n("decimallessFloat zerolessFloat binaryNumber hexNumber")})}()});
\ No newline at end of file
diff --git a/datasette/static/codemirror-5.31.0.js b/datasette/static/codemirror-5.31.0.js
deleted file mode 100644
index a0d5d688..00000000
--- a/datasette/static/codemirror-5.31.0.js
+++ /dev/null
@@ -1,9659 +0,0 @@
-// CodeMirror, copyright (c) by Marijn Haverbeke and others
-// Distributed under an MIT license: http://codemirror.net/LICENSE
-
-// This is CodeMirror (http://codemirror.net), a code editor
-// implemented in JavaScript on top of the browser's DOM.
-//
-// You can find some technical background for some of the code below
-// at http://marijnhaverbeke.nl/blog/#cm-internals .
-
-(function (global, factory) {
-	typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :
-	typeof define === 'function' && define.amd ? define(factory) :
-	(global.CodeMirror = factory());
-}(this, (function () { 'use strict';
-
-// Kludges for bugs and behavior differences that can't be feature
-// detected are enabled based on userAgent etc sniffing.
-var userAgent = navigator.userAgent;
-var platform = navigator.platform;
-
-var gecko = /gecko\/\d/i.test(userAgent);
-var ie_upto10 = /MSIE \d/.test(userAgent);
-var ie_11up = /Trident\/(?:[7-9]|\d{2,})\..*rv:(\d+)/.exec(userAgent);
-var edge = /Edge\/(\d+)/.exec(userAgent);
-var ie = ie_upto10 || ie_11up || edge;
-var ie_version = ie && (ie_upto10 ? document.documentMode || 6 : +(edge || ie_11up)[1]);
-var webkit = !edge && /WebKit\//.test(userAgent);
-var qtwebkit = webkit && /Qt\/\d+\.\d+/.test(userAgent);
-var chrome = !edge && /Chrome\//.test(userAgent);
-var presto = /Opera\//.test(userAgent);
-var safari = /Apple Computer/.test(navigator.vendor);
-var mac_geMountainLion = /Mac OS X 1\d\D([8-9]|\d\d)\D/.test(userAgent);
-var phantom = /PhantomJS/.test(userAgent);
-
-var ios = !edge && /AppleWebKit/.test(userAgent) && /Mobile\/\w+/.test(userAgent);
-var android = /Android/.test(userAgent);
-// This is woefully incomplete. Suggestions for alternative methods welcome.
-var mobile = ios || android || /webOS|BlackBerry|Opera Mini|Opera Mobi|IEMobile/i.test(userAgent);
-var mac = ios || /Mac/.test(platform);
-var chromeOS = /\bCrOS\b/.test(userAgent);
-var windows = /win/i.test(platform);
-
-var presto_version = presto && userAgent.match(/Version\/(\d*\.\d*)/);
-if (presto_version) { presto_version = Number(presto_version[1]); }
-if (presto_version && presto_version >= 15) { presto = false; webkit = true; }
-// Some browsers use the wrong event properties to signal cmd/ctrl on OS X
-var flipCtrlCmd = mac && (qtwebkit || presto && (presto_version == null || presto_version < 12.11));
-var captureRightClick = gecko || (ie && ie_version >= 9);
-
-function classTest(cls) { return new RegExp("(^|\\s)" + cls + "(?:$|\\s)\\s*") }
-
-var rmClass = function(node, cls) {
-  var current = node.className;
-  var match = classTest(cls).exec(current);
-  if (match) {
-    var after = current.slice(match.index + match[0].length);
-    node.className = current.slice(0, match.index) + (after ? match[1] + after : "");
-  }
-};
-
-function removeChildren(e) {
-  for (var count = e.childNodes.length; count > 0; --count)
-    { e.removeChild(e.firstChild); }
-  return e
-}
-
-function removeChildrenAndAdd(parent, e) {
-  return removeChildren(parent).appendChild(e)
-}
-
-function elt(tag, content, className, style) {
-  var e = document.createElement(tag);
-  if (className) { e.className = className; }
-  if (style) { e.style.cssText = style; }
-  if (typeof content == "string") { e.appendChild(document.createTextNode(content)); }
-  else if (content) { for (var i = 0; i < content.length; ++i) { e.appendChild(content[i]); } }
-  return e
-}
-// wrapper for elt, which removes the elt from the accessibility tree
-function eltP(tag, content, className, style) {
-  var e = elt(tag, content, className, style);
-  e.setAttribute("role", "presentation");
-  return e
-}
-
-var range;
-if (document.createRange) { range = function(node, start, end, endNode) {
-  var r = document.createRange();
-  r.setEnd(endNode || node, end);
-  r.setStart(node, start);
-  return r
-}; }
-else { range = function(node, start, end) {
-  var r = document.body.createTextRange();
-  try { r.moveToElementText(node.parentNode); }
-  catch(e) { return r }
-  r.collapse(true);
-  r.moveEnd("character", end);
-  r.moveStart("character", start);
-  return r
-}; }
-
-function contains(parent, child) {
-  if (child.nodeType == 3) // Android browser always returns false when child is a textnode
-    { child = child.parentNode; }
-  if (parent.contains)
-    { return parent.contains(child) }
-  do {
-    if (child.nodeType == 11) { child = child.host; }
-    if (child == parent) { return true }
-  } while (child = child.parentNode)
-}
-
-function activeElt() {
-  // IE and Edge may throw an "Unspecified Error" when accessing document.activeElement.
-  // IE < 10 will throw when accessed while the page is loading or in an iframe.
-  // IE > 9 and Edge will throw when accessed in an iframe if document.body is unavailable.
-  var activeElement;
-  try {
-    activeElement = document.activeElement;
-  } catch(e) {
-    activeElement = document.body || null;
-  }
-  while (activeElement && activeElement.shadowRoot && activeElement.shadowRoot.activeElement)
-    { activeElement = activeElement.shadowRoot.activeElement; }
-  return activeElement
-}
-
-function addClass(node, cls) {
-  var current = node.className;
-  if (!classTest(cls).test(current)) { node.className += (current ? " " : "") + cls; }
-}
-function joinClasses(a, b) {
-  var as = a.split(" ");
-  for (var i = 0; i < as.length; i++)
-    { if (as[i] && !classTest(as[i]).test(b)) { b += " " + as[i]; } }
-  return b
-}
-
-var selectInput = function(node) { node.select(); };
-if (ios) // Mobile Safari apparently has a bug where select() is broken.
-  { selectInput = function(node) { node.selectionStart = 0; node.selectionEnd = node.value.length; }; }
-else if (ie) // Suppress mysterious IE10 errors
-  { selectInput = function(node) { try { node.select(); } catch(_e) {} }; }
-
-function bind(f) {
-  var args = Array.prototype.slice.call(arguments, 1);
-  return function(){return f.apply(null, args)}
-}
-
-function copyObj(obj, target, overwrite) {
-  if (!target) { target = {}; }
-  for (var prop in obj)
-    { if (obj.hasOwnProperty(prop) && (overwrite !== false || !target.hasOwnProperty(prop)))
-      { target[prop] = obj[prop]; } }
-  return target
-}
-
-// Counts the column offset in a string, taking tabs into account.
-// Used mostly to find indentation.
-function countColumn(string, end, tabSize, startIndex, startValue) {
-  if (end == null) {
-    end = string.search(/[^\s\u00a0]/);
-    if (end == -1) { end = string.length; }
-  }
-  for (var i = startIndex || 0, n = startValue || 0;;) {
-    var nextTab = string.indexOf("\t", i);
-    if (nextTab < 0 || nextTab >= end)
-      { return n + (end - i) }
-    n += nextTab - i;
-    n += tabSize - (n % tabSize);
-    i = nextTab + 1;
-  }
-}
-
-var Delayed = function() {this.id = null;};
-Delayed.prototype.set = function (ms, f) {
-  clearTimeout(this.id);
-  this.id = setTimeout(f, ms);
-};
-
-function indexOf(array, elt) {
-  for (var i = 0; i < array.length; ++i)
-    { if (array[i] == elt) { return i } }
-  return -1
-}
-
-// Number of pixels added to scroller and sizer to hide scrollbar
-var scrollerGap = 30;
-
-// Returned or thrown by various protocols to signal 'I'm not
-// handling this'.
-var Pass = {toString: function(){return "CodeMirror.Pass"}};
-
-// Reused option objects for setSelection & friends
-var sel_dontScroll = {scroll: false};
-var sel_mouse = {origin: "*mouse"};
-var sel_move = {origin: "+move"};
-
-// The inverse of countColumn -- find the offset that corresponds to
-// a particular column.
-function findColumn(string, goal, tabSize) {
-  for (var pos = 0, col = 0;;) {
-    var nextTab = string.indexOf("\t", pos);
-    if (nextTab == -1) { nextTab = string.length; }
-    var skipped = nextTab - pos;
-    if (nextTab == string.length || col + skipped >= goal)
-      { return pos + Math.min(skipped, goal - col) }
-    col += nextTab - pos;
-    col += tabSize - (col % tabSize);
-    pos = nextTab + 1;
-    if (col >= goal) { return pos }
-  }
-}
-
-var spaceStrs = [""];
-function spaceStr(n) {
-  while (spaceStrs.length <= n)
-    { spaceStrs.push(lst(spaceStrs) + " "); }
-  return spaceStrs[n]
-}
-
-function lst(arr) { return arr[arr.length-1] }
-
-function map(array, f) {
-  var out = [];
-  for (var i = 0; i < array.length; i++) { out[i] = f(array[i], i); }
-  return out
-}
-
-function insertSorted(array, value, score) {
-  var pos = 0, priority = score(value);
-  while (pos < array.length && score(array[pos]) <= priority) { pos++; }
-  array.splice(pos, 0, value);
-}
-
-function nothing() {}
-
-function createObj(base, props) {
-  var inst;
-  if (Object.create) {
-    inst = Object.create(base);
-  } else {
-    nothing.prototype = base;
-    inst = new nothing();
-  }
-  if (props) { copyObj(props, inst); }
-  return inst
-}
-
-var nonASCIISingleCaseWordChar = /[\u00df\u0587\u0590-\u05f4\u0600-\u06ff\u3040-\u309f\u30a0-\u30ff\u3400-\u4db5\u4e00-\u9fcc\uac00-\ud7af]/;
-function isWordCharBasic(ch) {
-  return /\w/.test(ch) || ch > "\x80" &&
-    (ch.toUpperCase() != ch.toLowerCase() || nonASCIISingleCaseWordChar.test(ch))
-}
-function isWordChar(ch, helper) {
-  if (!helper) { return isWordCharBasic(ch) }
-  if (helper.source.indexOf("\\w") > -1 && isWordCharBasic(ch)) { return true }
-  return helper.test(ch)
-}
-
-function isEmpty(obj) {
-  for (var n in obj) { if (obj.hasOwnProperty(n) && obj[n]) { return false } }
-  return true
-}
-
-// Extending unicode characters. A series of a non-extending char +
-// any number of extending chars is treated as a single unit as far
-// as editing and measuring is concerned. This is not fully correct,
-// since some scripts/fonts/browsers also treat other configurations
-// of code points as a group.
-var extendingChars = /[\u0300-\u036f\u0483-\u0489\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u064b-\u065e\u0670\u06d6-\u06dc\u06de-\u06e4\u06e7\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0900-\u0902\u093c\u0941-\u0948\u094d\u0951-\u0955\u0962\u0963\u0981\u09bc\u09be\u09c1-\u09c4\u09cd\u09d7\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a70\u0a71\u0a75\u0a81\u0a82\u0abc\u0ac1-\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3e\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b57\u0b62\u0b63\u0b82\u0bbe\u0bc0\u0bcd\u0bd7\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62\u0c63\u0cbc\u0cbf\u0cc2\u0cc6\u0ccc\u0ccd\u0cd5\u0cd6\u0ce2\u0ce3\u0d3e\u0d41-\u0d44\u0d4d\u0d57\u0d62\u0d63\u0dca\u0dcf\u0dd2-\u0dd4\u0dd6\u0ddf\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb\u0ebc\u0ec8-\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86\u0f87\u0f90-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039\u103a\u103d\u103e\u1058\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085\u1086\u108d\u109d\u135f\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927\u1928\u1932\u1939-\u193b\u1a17\u1a18\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80\u1b81\u1ba2-\u1ba5\u1ba8\u1ba9\u1c2c-\u1c33\u1c36\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1dc0-\u1de6\u1dfd-\u1dff\u200c\u200d\u20d0-\u20f0\u2cef-\u2cf1\u2de0-\u2dff\u302a-\u302f\u3099\u309a\ua66f-\ua672\ua67c\ua67d\ua6f0\ua6f1\ua802\ua806\ua80b\ua825\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31\uaa32\uaa35\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7\uaab8\uaabe\uaabf\uaac1\uabe5\uabe8\uabed\udc00-\udfff\ufb1e\ufe00-\ufe0f\ufe20-\ufe26\uff9e\uff9f]/;
-function isExtendingChar(ch) { return ch.charCodeAt(0) >= 768 && extendingChars.test(ch) }
-
-// Returns a number from the range [`0`; `str.length`] unless `pos` is outside that range.
-function skipExtendingChars(str, pos, dir) {
-  while ((dir < 0 ? pos > 0 : pos < str.length) && isExtendingChar(str.charAt(pos))) { pos += dir; }
-  return pos
-}
-
-// Returns the value from the range [`from`; `to`] that satisfies
-// `pred` and is closest to `from`. Assumes that at least `to`
-// satisfies `pred`. Supports `from` being greater than `to`.
-function findFirst(pred, from, to) {
-  // At any point we are certain `to` satisfies `pred`, don't know
-  // whether `from` does.
-  var dir = from > to ? -1 : 1;
-  for (;;) {
-    if (from == to) { return from }
-    var midF = (from + to) / 2, mid = dir < 0 ? Math.ceil(midF) : Math.floor(midF);
-    if (mid == from) { return pred(mid) ? from : to }
-    if (pred(mid)) { to = mid; }
-    else { from = mid + dir; }
-  }
-}
-
-// The display handles the DOM integration, both for input reading
-// and content drawing. It holds references to DOM nodes and
-// display-related state.
-
-function Display(place, doc, input) {
-  var d = this;
-  this.input = input;
-
-  // Covers bottom-right square when both scrollbars are present.
-  d.scrollbarFiller = elt("div", null, "CodeMirror-scrollbar-filler");
-  d.scrollbarFiller.setAttribute("cm-not-content", "true");
-  // Covers bottom of gutter when coverGutterNextToScrollbar is on
-  // and h scrollbar is present.
-  d.gutterFiller = elt("div", null, "CodeMirror-gutter-filler");
-  d.gutterFiller.setAttribute("cm-not-content", "true");
-  // Will contain the actual code, positioned to cover the viewport.
-  d.lineDiv = eltP("div", null, "CodeMirror-code");
-  // Elements are added to these to represent selection and cursors.
-  d.selectionDiv = elt("div", null, null, "position: relative; z-index: 1");
-  d.cursorDiv = elt("div", null, "CodeMirror-cursors");
-  // A visibility: hidden element used to find the size of things.
-  d.measure = elt("div", null, "CodeMirror-measure");
-  // When lines outside of the viewport are measured, they are drawn in this.
-  d.lineMeasure = elt("div", null, "CodeMirror-measure");
-  // Wraps everything that needs to exist inside the vertically-padded coordinate system
-  d.lineSpace = eltP("div", [d.measure, d.lineMeasure, d.selectionDiv, d.cursorDiv, d.lineDiv],
-                    null, "position: relative; outline: none");
-  var lines = eltP("div", [d.lineSpace], "CodeMirror-lines");
-  // Moved around its parent to cover visible view.
-  d.mover = elt("div", [lines], null, "position: relative");
-  // Set to the height of the document, allowing scrolling.
-  d.sizer = elt("div", [d.mover], "CodeMirror-sizer");
-  d.sizerWidth = null;
-  // Behavior of elts with overflow: auto and padding is
-  // inconsistent across browsers. This is used to ensure the
-  // scrollable area is big enough.
-  d.heightForcer = elt("div", null, null, "position: absolute; height: " + scrollerGap + "px; width: 1px;");
-  // Will contain the gutters, if any.
-  d.gutters = elt("div", null, "CodeMirror-gutters");
-  d.lineGutter = null;
-  // Actual scrollable element.
-  d.scroller = elt("div", [d.sizer, d.heightForcer, d.gutters], "CodeMirror-scroll");
-  d.scroller.setAttribute("tabIndex", "-1");
-  // The element in which the editor lives.
-  d.wrapper = elt("div", [d.scrollbarFiller, d.gutterFiller, d.scroller], "CodeMirror");
-
-  // Work around IE7 z-index bug (not perfect, hence IE7 not really being supported)
-  if (ie && ie_version < 8) { d.gutters.style.zIndex = -1; d.scroller.style.paddingRight = 0; }
-  if (!webkit && !(gecko && mobile)) { d.scroller.draggable = true; }
-
-  if (place) {
-    if (place.appendChild) { place.appendChild(d.wrapper); }
-    else { place(d.wrapper); }
-  }
-
-  // Current rendered range (may be bigger than the view window).
-  d.viewFrom = d.viewTo = doc.first;
-  d.reportedViewFrom = d.reportedViewTo = doc.first;
-  // Information about the rendered lines.
-  d.view = [];
-  d.renderedView = null;
-  // Holds info about a single rendered line when it was rendered
-  // for measurement, while not in view.
-  d.externalMeasured = null;
-  // Empty space (in pixels) above the view
-  d.viewOffset = 0;
-  d.lastWrapHeight = d.lastWrapWidth = 0;
-  d.updateLineNumbers = null;
-
-  d.nativeBarWidth = d.barHeight = d.barWidth = 0;
-  d.scrollbarsClipped = false;
-
-  // Used to only resize the line number gutter when necessary (when
-  // the amount of lines crosses a boundary that makes its width change)
-  d.lineNumWidth = d.lineNumInnerWidth = d.lineNumChars = null;
-  // Set to true when a non-horizontal-scrolling line widget is
-  // added. As an optimization, line widget aligning is skipped when
-  // this is false.
-  d.alignWidgets = false;
-
-  d.cachedCharWidth = d.cachedTextHeight = d.cachedPaddingH = null;
-
-  // Tracks the maximum line length so that the horizontal scrollbar
-  // can be kept static when scrolling.
-  d.maxLine = null;
-  d.maxLineLength = 0;
-  d.maxLineChanged = false;
-
-  // Used for measuring wheel scrolling granularity
-  d.wheelDX = d.wheelDY = d.wheelStartX = d.wheelStartY = null;
-
-  // True when shift is held down.
-  d.shift = false;
-
-  // Used to track whether anything happened since the context menu
-  // was opened.
-  d.selForContextMenu = null;
-
-  d.activeTouch = null;
-
-  input.init(d);
-}
-
-// Find the line object corresponding to the given line number.
-function getLine(doc, n) {
-  n -= doc.first;
-  if (n < 0 || n >= doc.size) { throw new Error("There is no line " + (n + doc.first) + " in the document.") }
-  var chunk = doc;
-  while (!chunk.lines) {
-    for (var i = 0;; ++i) {
-      var child = chunk.children[i], sz = child.chunkSize();
-      if (n < sz) { chunk = child; break }
-      n -= sz;
-    }
-  }
-  return chunk.lines[n]
-}
-
-// Get the part of a document between two positions, as an array of
-// strings.
-function getBetween(doc, start, end) {
-  var out = [], n = start.line;
-  doc.iter(start.line, end.line + 1, function (line) {
-    var text = line.text;
-    if (n == end.line) { text = text.slice(0, end.ch); }
-    if (n == start.line) { text = text.slice(start.ch); }
-    out.push(text);
-    ++n;
-  });
-  return out
-}
-// Get the lines between from and to, as array of strings.
-function getLines(doc, from, to) {
-  var out = [];
-  doc.iter(from, to, function (line) { out.push(line.text); }); // iter aborts when callback returns truthy value
-  return out
-}
-
-// Update the height of a line, propagating the height change
-// upwards to parent nodes.
-function updateLineHeight(line, height) {
-  var diff = height - line.height;
-  if (diff) { for (var n = line; n; n = n.parent) { n.height += diff; } }
-}
-
-// Given a line object, find its line number by walking up through
-// its parent links.
-function lineNo(line) {
-  if (line.parent == null) { return null }
-  var cur = line.parent, no = indexOf(cur.lines, line);
-  for (var chunk = cur.parent; chunk; cur = chunk, chunk = chunk.parent) {
-    for (var i = 0;; ++i) {
-      if (chunk.children[i] == cur) { break }
-      no += chunk.children[i].chunkSize();
-    }
-  }
-  return no + cur.first
-}
-
-// Find the line at the given vertical position, using the height
-// information in the document tree.
-function lineAtHeight(chunk, h) {
-  var n = chunk.first;
-  outer: do {
-    for (var i$1 = 0; i$1 < chunk.children.length; ++i$1) {
-      var child = chunk.children[i$1], ch = child.height;
-      if (h < ch) { chunk = child; continue outer }
-      h -= ch;
-      n += child.chunkSize();
-    }
-    return n
-  } while (!chunk.lines)
-  var i = 0;
-  for (; i < chunk.lines.length; ++i) {
-    var line = chunk.lines[i], lh = line.height;
-    if (h < lh) { break }
-    h -= lh;
-  }
-  return n + i
-}
-
-function isLine(doc, l) {return l >= doc.first && l < doc.first + doc.size}
-
-function lineNumberFor(options, i) {
-  return String(options.lineNumberFormatter(i + options.firstLineNumber))
-}
-
-// A Pos instance represents a position within the text.
-function Pos(line, ch, sticky) {
-  if ( sticky === void 0 ) sticky = null;
-
-  if (!(this instanceof Pos)) { return new Pos(line, ch, sticky) }
-  this.line = line;
-  this.ch = ch;
-  this.sticky = sticky;
-}
-
-// Compare two positions, return 0 if they are the same, a negative
-// number when a is less, and a positive number otherwise.
-function cmp(a, b) { return a.line - b.line || a.ch - b.ch }
-
-function equalCursorPos(a, b) { return a.sticky == b.sticky && cmp(a, b) == 0 }
-
-function copyPos(x) {return Pos(x.line, x.ch)}
-function maxPos(a, b) { return cmp(a, b) < 0 ? b : a }
-function minPos(a, b) { return cmp(a, b) < 0 ? a : b }
-
-// Most of the external API clips given positions to make sure they
-// actually exist within the document.
-function clipLine(doc, n) {return Math.max(doc.first, Math.min(n, doc.first + doc.size - 1))}
-function clipPos(doc, pos) {
-  if (pos.line < doc.first) { return Pos(doc.first, 0) }
-  var last = doc.first + doc.size - 1;
-  if (pos.line > last) { return Pos(last, getLine(doc, last).text.length) }
-  return clipToLen(pos, getLine(doc, pos.line).text.length)
-}
-function clipToLen(pos, linelen) {
-  var ch = pos.ch;
-  if (ch == null || ch > linelen) { return Pos(pos.line, linelen) }
-  else if (ch < 0) { return Pos(pos.line, 0) }
-  else { return pos }
-}
-function clipPosArray(doc, array) {
-  var out = [];
-  for (var i = 0; i < array.length; i++) { out[i] = clipPos(doc, array[i]); }
-  return out
-}
-
-// Optimize some code when these features are not used.
-var sawReadOnlySpans = false;
-var sawCollapsedSpans = false;
-
-function seeReadOnlySpans() {
-  sawReadOnlySpans = true;
-}
-
-function seeCollapsedSpans() {
-  sawCollapsedSpans = true;
-}
-
-// TEXTMARKER SPANS
-
-function MarkedSpan(marker, from, to) {
-  this.marker = marker;
-  this.from = from; this.to = to;
-}
-
-// Search an array of spans for a span matching the given marker.
-function getMarkedSpanFor(spans, marker) {
-  if (spans) { for (var i = 0; i < spans.length; ++i) {
-    var span = spans[i];
-    if (span.marker == marker) { return span }
-  } }
-}
-// Remove a span from an array, returning undefined if no spans are
-// left (we don't store arrays for lines without spans).
-function removeMarkedSpan(spans, span) {
-  var r;
-  for (var i = 0; i < spans.length; ++i)
-    { if (spans[i] != span) { (r || (r = [])).push(spans[i]); } }
-  return r
-}
-// Add a span to a line.
-function addMarkedSpan(line, span) {
-  line.markedSpans = line.markedSpans ? line.markedSpans.concat([span]) : [span];
-  span.marker.attachLine(line);
-}
-
-// Used for the algorithm that adjusts markers for a change in the
-// document. These functions cut an array of spans at a given
-// character position, returning an array of remaining chunks (or
-// undefined if nothing remains).
-function markedSpansBefore(old, startCh, isInsert) {
-  var nw;
-  if (old) { for (var i = 0; i < old.length; ++i) {
-    var span = old[i], marker = span.marker;
-    var startsBefore = span.from == null || (marker.inclusiveLeft ? span.from <= startCh : span.from < startCh);
-    if (startsBefore || span.from == startCh && marker.type == "bookmark" && (!isInsert || !span.marker.insertLeft)) {
-      var endsAfter = span.to == null || (marker.inclusiveRight ? span.to >= startCh : span.to > startCh);(nw || (nw = [])).push(new MarkedSpan(marker, span.from, endsAfter ? null : span.to));
-    }
-  } }
-  return nw
-}
-function markedSpansAfter(old, endCh, isInsert) {
-  var nw;
-  if (old) { for (var i = 0; i < old.length; ++i) {
-    var span = old[i], marker = span.marker;
-    var endsAfter = span.to == null || (marker.inclusiveRight ? span.to >= endCh : span.to > endCh);
-    if (endsAfter || span.from == endCh && marker.type == "bookmark" && (!isInsert || span.marker.insertLeft)) {
-      var startsBefore = span.from == null || (marker.inclusiveLeft ? span.from <= endCh : span.from < endCh);(nw || (nw = [])).push(new MarkedSpan(marker, startsBefore ? null : span.from - endCh,
-                                            span.to == null ? null : span.to - endCh));
-    }
-  } }
-  return nw
-}
-
-// Given a change object, compute the new set of marker spans that
-// cover the line in which the change took place. Removes spans
-// entirely within the change, reconnects spans belonging to the
-// same marker that appear on both sides of the change, and cuts off
-// spans partially within the change. Returns an array of span
-// arrays with one element for each line in (after) the change.
-function stretchSpansOverChange(doc, change) {
-  if (change.full) { return null }
-  var oldFirst = isLine(doc, change.from.line) && getLine(doc, change.from.line).markedSpans;
-  var oldLast = isLine(doc, change.to.line) && getLine(doc, change.to.line).markedSpans;
-  if (!oldFirst && !oldLast) { return null }
-
-  var startCh = change.from.ch, endCh = change.to.ch, isInsert = cmp(change.from, change.to) == 0;
-  // Get the spans that 'stick out' on both sides
-  var first = markedSpansBefore(oldFirst, startCh, isInsert);
-  var last = markedSpansAfter(oldLast, endCh, isInsert);
-
-  // Next, merge those two ends
-  var sameLine = change.text.length == 1, offset = lst(change.text).length + (sameLine ? startCh : 0);
-  if (first) {
-    // Fix up .to properties of first
-    for (var i = 0; i < first.length; ++i) {
-      var span = first[i];
-      if (span.to == null) {
-        var found = getMarkedSpanFor(last, span.marker);
-        if (!found) { span.to = startCh; }
-        else if (sameLine) { span.to = found.to == null ? null : found.to + offset; }
-      }
-    }
-  }
-  if (last) {
-    // Fix up .from in last (or move them into first in case of sameLine)
-    for (var i$1 = 0; i$1 < last.length; ++i$1) {
-      var span$1 = last[i$1];
-      if (span$1.to != null) { span$1.to += offset; }
-      if (span$1.from == null) {
-        var found$1 = getMarkedSpanFor(first, span$1.marker);
-        if (!found$1) {
-          span$1.from = offset;
-          if (sameLine) { (first || (first = [])).push(span$1); }
-        }
-      } else {
-        span$1.from += offset;
-        if (sameLine) { (first || (first = [])).push(span$1); }
-      }
-    }
-  }
-  // Make sure we didn't create any zero-length spans
-  if (first) { first = clearEmptySpans(first); }
-  if (last && last != first) { last = clearEmptySpans(last); }
-
-  var newMarkers = [first];
-  if (!sameLine) {
-    // Fill gap with whole-line-spans
-    var gap = change.text.length - 2, gapMarkers;
-    if (gap > 0 && first)
-      { for (var i$2 = 0; i$2 < first.length; ++i$2)
-        { if (first[i$2].to == null)
-          { (gapMarkers || (gapMarkers = [])).push(new MarkedSpan(first[i$2].marker, null, null)); } } }
-    for (var i$3 = 0; i$3 < gap; ++i$3)
-      { newMarkers.push(gapMarkers); }
-    newMarkers.push(last);
-  }
-  return newMarkers
-}
-
-// Remove spans that are empty and don't have a clearWhenEmpty
-// option of false.
-function clearEmptySpans(spans) {
-  for (var i = 0; i < spans.length; ++i) {
-    var span = spans[i];
-    if (span.from != null && span.from == span.to && span.marker.clearWhenEmpty !== false)
-      { spans.splice(i--, 1); }
-  }
-  if (!spans.length) { return null }
-  return spans
-}
-
-// Used to 'clip' out readOnly ranges when making a change.
-function removeReadOnlyRanges(doc, from, to) {
-  var markers = null;
-  doc.iter(from.line, to.line + 1, function (line) {
-    if (line.markedSpans) { for (var i = 0; i < line.markedSpans.length; ++i) {
-      var mark = line.markedSpans[i].marker;
-      if (mark.readOnly && (!markers || indexOf(markers, mark) == -1))
-        { (markers || (markers = [])).push(mark); }
-    } }
-  });
-  if (!markers) { return null }
-  var parts = [{from: from, to: to}];
-  for (var i = 0; i < markers.length; ++i) {
-    var mk = markers[i], m = mk.find(0);
-    for (var j = 0; j < parts.length; ++j) {
-      var p = parts[j];
-      if (cmp(p.to, m.from) < 0 || cmp(p.from, m.to) > 0) { continue }
-      var newParts = [j, 1], dfrom = cmp(p.from, m.from), dto = cmp(p.to, m.to);
-      if (dfrom < 0 || !mk.inclusiveLeft && !dfrom)
-        { newParts.push({from: p.from, to: m.from}); }
-      if (dto > 0 || !mk.inclusiveRight && !dto)
-        { newParts.push({from: m.to, to: p.to}); }
-      parts.splice.apply(parts, newParts);
-      j += newParts.length - 3;
-    }
-  }
-  return parts
-}
-
-// Connect or disconnect spans from a line.
-function detachMarkedSpans(line) {
-  var spans = line.markedSpans;
-  if (!spans) { return }
-  for (var i = 0; i < spans.length; ++i)
-    { spans[i].marker.detachLine(line); }
-  line.markedSpans = null;
-}
-function attachMarkedSpans(line, spans) {
-  if (!spans) { return }
-  for (var i = 0; i < spans.length; ++i)
-    { spans[i].marker.attachLine(line); }
-  line.markedSpans = spans;
-}
-
-// Helpers used when computing which overlapping collapsed span
-// counts as the larger one.
-function extraLeft(marker) { return marker.inclusiveLeft ? -1 : 0 }
-function extraRight(marker) { return marker.inclusiveRight ? 1 : 0 }
-
-// Returns a number indicating which of two overlapping collapsed
-// spans is larger (and thus includes the other). Falls back to
-// comparing ids when the spans cover exactly the same range.
-function compareCollapsedMarkers(a, b) {
-  var lenDiff = a.lines.length - b.lines.length;
-  if (lenDiff != 0) { return lenDiff }
-  var aPos = a.find(), bPos = b.find();
-  var fromCmp = cmp(aPos.from, bPos.from) || extraLeft(a) - extraLeft(b);
-  if (fromCmp) { return -fromCmp }
-  var toCmp = cmp(aPos.to, bPos.to) || extraRight(a) - extraRight(b);
-  if (toCmp) { return toCmp }
-  return b.id - a.id
-}
-
-// Find out whether a line ends or starts in a collapsed span. If
-// so, return the marker for that span.
-function collapsedSpanAtSide(line, start) {
-  var sps = sawCollapsedSpans && line.markedSpans, found;
-  if (sps) { for (var sp = (void 0), i = 0; i < sps.length; ++i) {
-    sp = sps[i];
-    if (sp.marker.collapsed && (start ? sp.from : sp.to) == null &&
-        (!found || compareCollapsedMarkers(found, sp.marker) < 0))
-      { found = sp.marker; }
-  } }
-  return found
-}
-function collapsedSpanAtStart(line) { return collapsedSpanAtSide(line, true) }
-function collapsedSpanAtEnd(line) { return collapsedSpanAtSide(line, false) }
-
-// Test whether there exists a collapsed span that partially
-// overlaps (covers the start or end, but not both) of a new span.
-// Such overlap is not allowed.
-function conflictingCollapsedRange(doc, lineNo$$1, from, to, marker) {
-  var line = getLine(doc, lineNo$$1);
-  var sps = sawCollapsedSpans && line.markedSpans;
-  if (sps) { for (var i = 0; i < sps.length; ++i) {
-    var sp = sps[i];
-    if (!sp.marker.collapsed) { continue }
-    var found = sp.marker.find(0);
-    var fromCmp = cmp(found.from, from) || extraLeft(sp.marker) - extraLeft(marker);
-    var toCmp = cmp(found.to, to) || extraRight(sp.marker) - extraRight(marker);
-    if (fromCmp >= 0 && toCmp <= 0 || fromCmp <= 0 && toCmp >= 0) { continue }
-    if (fromCmp <= 0 && (sp.marker.inclusiveRight && marker.inclusiveLeft ? cmp(found.to, from) >= 0 : cmp(found.to, from) > 0) ||
-        fromCmp >= 0 && (sp.marker.inclusiveRight && marker.inclusiveLeft ? cmp(found.from, to) <= 0 : cmp(found.from, to) < 0))
-      { return true }
-  } }
-}
-
-// A visual line is a line as drawn on the screen. Folding, for
-// example, can cause multiple logical lines to appear on the same
-// visual line. This finds the start of the visual line that the
-// given line is part of (usually that is the line itself).
-function visualLine(line) {
-  var merged;
-  while (merged = collapsedSpanAtStart(line))
-    { line = merged.find(-1, true).line; }
-  return line
-}
-
-function visualLineEnd(line) {
-  var merged;
-  while (merged = collapsedSpanAtEnd(line))
-    { line = merged.find(1, true).line; }
-  return line
-}
-
-// Returns an array of logical lines that continue the visual line
-// started by the argument, or undefined if there are no such lines.
-function visualLineContinued(line) {
-  var merged, lines;
-  while (merged = collapsedSpanAtEnd(line)) {
-    line = merged.find(1, true).line
-    ;(lines || (lines = [])).push(line);
-  }
-  return lines
-}
-
-// Get the line number of the start of the visual line that the
-// given line number is part of.
-function visualLineNo(doc, lineN) {
-  var line = getLine(doc, lineN), vis = visualLine(line);
-  if (line == vis) { return lineN }
-  return lineNo(vis)
-}
-
-// Get the line number of the start of the next visual line after
-// the given line.
-function visualLineEndNo(doc, lineN) {
-  if (lineN > doc.lastLine()) { return lineN }
-  var line = getLine(doc, lineN), merged;
-  if (!lineIsHidden(doc, line)) { return lineN }
-  while (merged = collapsedSpanAtEnd(line))
-    { line = merged.find(1, true).line; }
-  return lineNo(line) + 1
-}
-
-// Compute whether a line is hidden. Lines count as hidden when they
-// are part of a visual line that starts with another line, or when
-// they are entirely covered by collapsed, non-widget span.
-function lineIsHidden(doc, line) {
-  var sps = sawCollapsedSpans && line.markedSpans;
-  if (sps) { for (var sp = (void 0), i = 0; i < sps.length; ++i) {
-    sp = sps[i];
-    if (!sp.marker.collapsed) { continue }
-    if (sp.from == null) { return true }
-    if (sp.marker.widgetNode) { continue }
-    if (sp.from == 0 && sp.marker.inclusiveLeft && lineIsHiddenInner(doc, line, sp))
-      { return true }
-  } }
-}
-function lineIsHiddenInner(doc, line, span) {
-  if (span.to == null) {
-    var end = span.marker.find(1, true);
-    return lineIsHiddenInner(doc, end.line, getMarkedSpanFor(end.line.markedSpans, span.marker))
-  }
-  if (span.marker.inclusiveRight && span.to == line.text.length)
-    { return true }
-  for (var sp = (void 0), i = 0; i < line.markedSpans.length; ++i) {
-    sp = line.markedSpans[i];
-    if (sp.marker.collapsed && !sp.marker.widgetNode && sp.from == span.to &&
-        (sp.to == null || sp.to != span.from) &&
-        (sp.marker.inclusiveLeft || span.marker.inclusiveRight) &&
-        lineIsHiddenInner(doc, line, sp)) { return true }
-  }
-}
-
-// Find the height above the given line.
-function heightAtLine(lineObj) {
-  lineObj = visualLine(lineObj);
-
-  var h = 0, chunk = lineObj.parent;
-  for (var i = 0; i < chunk.lines.length; ++i) {
-    var line = chunk.lines[i];
-    if (line == lineObj) { break }
-    else { h += line.height; }
-  }
-  for (var p = chunk.parent; p; chunk = p, p = chunk.parent) {
-    for (var i$1 = 0; i$1 < p.children.length; ++i$1) {
-      var cur = p.children[i$1];
-      if (cur == chunk) { break }
-      else { h += cur.height; }
-    }
-  }
-  return h
-}
-
-// Compute the character length of a line, taking into account
-// collapsed ranges (see markText) that might hide parts, and join
-// other lines onto it.
-function lineLength(line) {
-  if (line.height == 0) { return 0 }
-  var len = line.text.length, merged, cur = line;
-  while (merged = collapsedSpanAtStart(cur)) {
-    var found = merged.find(0, true);
-    cur = found.from.line;
-    len += found.from.ch - found.to.ch;
-  }
-  cur = line;
-  while (merged = collapsedSpanAtEnd(cur)) {
-    var found$1 = merged.find(0, true);
-    len -= cur.text.length - found$1.from.ch;
-    cur = found$1.to.line;
-    len += cur.text.length - found$1.to.ch;
-  }
-  return len
-}
-
-// Find the longest line in the document.
-function findMaxLine(cm) {
-  var d = cm.display, doc = cm.doc;
-  d.maxLine = getLine(doc, doc.first);
-  d.maxLineLength = lineLength(d.maxLine);
-  d.maxLineChanged = true;
-  doc.iter(function (line) {
-    var len = lineLength(line);
-    if (len > d.maxLineLength) {
-      d.maxLineLength = len;
-      d.maxLine = line;
-    }
-  });
-}
-
-// BIDI HELPERS
-
-function iterateBidiSections(order, from, to, f) {
-  if (!order) { return f(from, to, "ltr", 0) }
-  var found = false;
-  for (var i = 0; i < order.length; ++i) {
-    var part = order[i];
-    if (part.from < to && part.to > from || from == to && part.to == from) {
-      f(Math.max(part.from, from), Math.min(part.to, to), part.level == 1 ? "rtl" : "ltr", i);
-      found = true;
-    }
-  }
-  if (!found) { f(from, to, "ltr"); }
-}
-
-var bidiOther = null;
-function getBidiPartAt(order, ch, sticky) {
-  var found;
-  bidiOther = null;
-  for (var i = 0; i < order.length; ++i) {
-    var cur = order[i];
-    if (cur.from < ch && cur.to > ch) { return i }
-    if (cur.to == ch) {
-      if (cur.from != cur.to && sticky == "before") { found = i; }
-      else { bidiOther = i; }
-    }
-    if (cur.from == ch) {
-      if (cur.from != cur.to && sticky != "before") { found = i; }
-      else { bidiOther = i; }
-    }
-  }
-  return found != null ? found : bidiOther
-}
-
-// Bidirectional ordering algorithm
-// See http://unicode.org/reports/tr9/tr9-13.html for the algorithm
-// that this (partially) implements.
-
-// One-char codes used for character types:
-// L (L):   Left-to-Right
-// R (R):   Right-to-Left
-// r (AL):  Right-to-Left Arabic
-// 1 (EN):  European Number
-// + (ES):  European Number Separator
-// % (ET):  European Number Terminator
-// n (AN):  Arabic Number
-// , (CS):  Common Number Separator
-// m (NSM): Non-Spacing Mark
-// b (BN):  Boundary Neutral
-// s (B):   Paragraph Separator
-// t (S):   Segment Separator
-// w (WS):  Whitespace
-// N (ON):  Other Neutrals
-
-// Returns null if characters are ordered as they appear
-// (left-to-right), or an array of sections ({from, to, level}
-// objects) in the order in which they occur visually.
-var bidiOrdering = (function() {
-  // Character types for codepoints 0 to 0xff
-  var lowTypes = "bbbbbbbbbtstwsbbbbbbbbbbbbbbssstwNN%%%NNNNNN,N,N1111111111NNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNbbbbbbsbbbbbbbbbbbbbbbbbbbbbbbbbb,N%%%%NNNNLNNNNN%%11NLNNN1LNNNNNLLLLLLLLLLLLLLLLLLLLLLLNLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLN";
-  // Character types for codepoints 0x600 to 0x6f9
-  var arabicTypes = "nnnnnnNNr%%r,rNNmmmmmmmmmmmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmmmmmmmmmmmmmmmnnnnnnnnnn%nnrrrmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmnNmmmmmmrrmmNmmmmrr1111111111";
-  function charType(code) {
-    if (code <= 0xf7) { return lowTypes.charAt(code) }
-    else if (0x590 <= code && code <= 0x5f4) { return "R" }
-    else if (0x600 <= code && code <= 0x6f9) { return arabicTypes.charAt(code - 0x600) }
-    else if (0x6ee <= code && code <= 0x8ac) { return "r" }
-    else if (0x2000 <= code && code <= 0x200b) { return "w" }
-    else if (code == 0x200c) { return "b" }
-    else { return "L" }
-  }
-
-  var bidiRE = /[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac]/;
-  var isNeutral = /[stwN]/, isStrong = /[LRr]/, countsAsLeft = /[Lb1n]/, countsAsNum = /[1n]/;
-
-  function BidiSpan(level, from, to) {
-    this.level = level;
-    this.from = from; this.to = to;
-  }
-
-  return function(str, direction) {
-    var outerType = direction == "ltr" ? "L" : "R";
-
-    if (str.length == 0 || direction == "ltr" && !bidiRE.test(str)) { return false }
-    var len = str.length, types = [];
-    for (var i = 0; i < len; ++i)
-      { types.push(charType(str.charCodeAt(i))); }
-
-    // W1. Examine each non-spacing mark (NSM) in the level run, and
-    // change the type of the NSM to the type of the previous
-    // character. If the NSM is at the start of the level run, it will
-    // get the type of sor.
-    for (var i$1 = 0, prev = outerType; i$1 < len; ++i$1) {
-      var type = types[i$1];
-      if (type == "m") { types[i$1] = prev; }
-      else { prev = type; }
-    }
-
-    // W2. Search backwards from each instance of a European number
-    // until the first strong type (R, L, AL, or sor) is found. If an
-    // AL is found, change the type of the European number to Arabic
-    // number.
-    // W3. Change all ALs to R.
-    for (var i$2 = 0, cur = outerType; i$2 < len; ++i$2) {
-      var type$1 = types[i$2];
-      if (type$1 == "1" && cur == "r") { types[i$2] = "n"; }
-      else if (isStrong.test(type$1)) { cur = type$1; if (type$1 == "r") { types[i$2] = "R"; } }
-    }
-
-    // W4. A single European separator between two European numbers
-    // changes to a European number. A single common separator between
-    // two numbers of the same type changes to that type.
-    for (var i$3 = 1, prev$1 = types[0]; i$3 < len - 1; ++i$3) {
-      var type$2 = types[i$3];
-      if (type$2 == "+" && prev$1 == "1" && types[i$3+1] == "1") { types[i$3] = "1"; }
-      else if (type$2 == "," && prev$1 == types[i$3+1] &&
-               (prev$1 == "1" || prev$1 == "n")) { types[i$3] = prev$1; }
-      prev$1 = type$2;
-    }
-
-    // W5. A sequence of European terminators adjacent to European
-    // numbers changes to all European numbers.
-    // W6. Otherwise, separators and terminators change to Other
-    // Neutral.
-    for (var i$4 = 0; i$4 < len; ++i$4) {
-      var type$3 = types[i$4];
-      if (type$3 == ",") { types[i$4] = "N"; }
-      else if (type$3 == "%") {
-        var end = (void 0);
-        for (end = i$4 + 1; end < len && types[end] == "%"; ++end) {}
-        var replace = (i$4 && types[i$4-1] == "!") || (end < len && types[end] == "1") ? "1" : "N";
-        for (var j = i$4; j < end; ++j) { types[j] = replace; }
-        i$4 = end - 1;
-      }
-    }
-
-    // W7. Search backwards from each instance of a European number
-    // until the first strong type (R, L, or sor) is found. If an L is
-    // found, then change the type of the European number to L.
-    for (var i$5 = 0, cur$1 = outerType; i$5 < len; ++i$5) {
-      var type$4 = types[i$5];
-      if (cur$1 == "L" && type$4 == "1") { types[i$5] = "L"; }
-      else if (isStrong.test(type$4)) { cur$1 = type$4; }
-    }
-
-    // N1. A sequence of neutrals takes the direction of the
-    // surrounding strong text if the text on both sides has the same
-    // direction. European and Arabic numbers act as if they were R in
-    // terms of their influence on neutrals. Start-of-level-run (sor)
-    // and end-of-level-run (eor) are used at level run boundaries.
-    // N2. Any remaining neutrals take the embedding direction.
-    for (var i$6 = 0; i$6 < len; ++i$6) {
-      if (isNeutral.test(types[i$6])) {
-        var end$1 = (void 0);
-        for (end$1 = i$6 + 1; end$1 < len && isNeutral.test(types[end$1]); ++end$1) {}
-        var before = (i$6 ? types[i$6-1] : outerType) == "L";
-        var after = (end$1 < len ? types[end$1] : outerType) == "L";
-        var replace$1 = before == after ? (before ? "L" : "R") : outerType;
-        for (var j$1 = i$6; j$1 < end$1; ++j$1) { types[j$1] = replace$1; }
-        i$6 = end$1 - 1;
-      }
-    }
-
-    // Here we depart from the documented algorithm, in order to avoid
-    // building up an actual levels array. Since there are only three
-    // levels (0, 1, 2) in an implementation that doesn't take
-    // explicit embedding into account, we can build up the order on
-    // the fly, without following the level-based algorithm.
-    var order = [], m;
-    for (var i$7 = 0; i$7 < len;) {
-      if (countsAsLeft.test(types[i$7])) {
-        var start = i$7;
-        for (++i$7; i$7 < len && countsAsLeft.test(types[i$7]); ++i$7) {}
-        order.push(new BidiSpan(0, start, i$7));
-      } else {
-        var pos = i$7, at = order.length;
-        for (++i$7; i$7 < len && types[i$7] != "L"; ++i$7) {}
-        for (var j$2 = pos; j$2 < i$7;) {
-          if (countsAsNum.test(types[j$2])) {
-            if (pos < j$2) { order.splice(at, 0, new BidiSpan(1, pos, j$2)); }
-            var nstart = j$2;
-            for (++j$2; j$2 < i$7 && countsAsNum.test(types[j$2]); ++j$2) {}
-            order.splice(at, 0, new BidiSpan(2, nstart, j$2));
-            pos = j$2;
-          } else { ++j$2; }
-        }
-        if (pos < i$7) { order.splice(at, 0, new BidiSpan(1, pos, i$7)); }
-      }
-    }
-    if (direction == "ltr") {
-      if (order[0].level == 1 && (m = str.match(/^\s+/))) {
-        order[0].from = m[0].length;
-        order.unshift(new BidiSpan(0, 0, m[0].length));
-      }
-      if (lst(order).level == 1 && (m = str.match(/\s+$/))) {
-        lst(order).to -= m[0].length;
-        order.push(new BidiSpan(0, len - m[0].length, len));
-      }
-    }
-
-    return direction == "rtl" ? order.reverse() : order
-  }
-})();
-
-// Get the bidi ordering for the given line (and cache it). Returns
-// false for lines that are fully left-to-right, and an array of
-// BidiSpan objects otherwise.
-function getOrder(line, direction) {
-  var order = line.order;
-  if (order == null) { order = line.order = bidiOrdering(line.text, direction); }
-  return order
-}
-
-// EVENT HANDLING
-
-// Lightweight event framework. on/off also work on DOM nodes,
-// registering native DOM handlers.
-
-var noHandlers = [];
-
-var on = function(emitter, type, f) {
-  if (emitter.addEventListener) {
-    emitter.addEventListener(type, f, false);
-  } else if (emitter.attachEvent) {
-    emitter.attachEvent("on" + type, f);
-  } else {
-    var map$$1 = emitter._handlers || (emitter._handlers = {});
-    map$$1[type] = (map$$1[type] || noHandlers).concat(f);
-  }
-};
-
-function getHandlers(emitter, type) {
-  return emitter._handlers && emitter._handlers[type] || noHandlers
-}
-
-function off(emitter, type, f) {
-  if (emitter.removeEventListener) {
-    emitter.removeEventListener(type, f, false);
-  } else if (emitter.detachEvent) {
-    emitter.detachEvent("on" + type, f);
-  } else {
-    var map$$1 = emitter._handlers, arr = map$$1 && map$$1[type];
-    if (arr) {
-      var index = indexOf(arr, f);
-      if (index > -1)
-        { map$$1[type] = arr.slice(0, index).concat(arr.slice(index + 1)); }
-    }
-  }
-}
-
-function signal(emitter, type /*, values...*/) {
-  var handlers = getHandlers(emitter, type);
-  if (!handlers.length) { return }
-  var args = Array.prototype.slice.call(arguments, 2);
-  for (var i = 0; i < handlers.length; ++i) { handlers[i].apply(null, args); }
-}
-
-// The DOM events that CodeMirror handles can be overridden by
-// registering a (non-DOM) handler on the editor for the event name,
-// and preventDefault-ing the event in that handler.
-function signalDOMEvent(cm, e, override) {
-  if (typeof e == "string")
-    { e = {type: e, preventDefault: function() { this.defaultPrevented = true; }}; }
-  signal(cm, override || e.type, cm, e);
-  return e_defaultPrevented(e) || e.codemirrorIgnore
-}
-
-function signalCursorActivity(cm) {
-  var arr = cm._handlers && cm._handlers.cursorActivity;
-  if (!arr) { return }
-  var set = cm.curOp.cursorActivityHandlers || (cm.curOp.cursorActivityHandlers = []);
-  for (var i = 0; i < arr.length; ++i) { if (indexOf(set, arr[i]) == -1)
-    { set.push(arr[i]); } }
-}
-
-function hasHandler(emitter, type) {
-  return getHandlers(emitter, type).length > 0
-}
-
-// Add on and off methods to a constructor's prototype, to make
-// registering events on such objects more convenient.
-function eventMixin(ctor) {
-  ctor.prototype.on = function(type, f) {on(this, type, f);};
-  ctor.prototype.off = function(type, f) {off(this, type, f);};
-}
-
-// Due to the fact that we still support jurassic IE versions, some
-// compatibility wrappers are needed.
-
-function e_preventDefault(e) {
-  if (e.preventDefault) { e.preventDefault(); }
-  else { e.returnValue = false; }
-}
-function e_stopPropagation(e) {
-  if (e.stopPropagation) { e.stopPropagation(); }
-  else { e.cancelBubble = true; }
-}
-function e_defaultPrevented(e) {
-  return e.defaultPrevented != null ? e.defaultPrevented : e.returnValue == false
-}
-function e_stop(e) {e_preventDefault(e); e_stopPropagation(e);}
-
-function e_target(e) {return e.target || e.srcElement}
-function e_button(e) {
-  var b = e.which;
-  if (b == null) {
-    if (e.button & 1) { b = 1; }
-    else if (e.button & 2) { b = 3; }
-    else if (e.button & 4) { b = 2; }
-  }
-  if (mac && e.ctrlKey && b == 1) { b = 3; }
-  return b
-}
-
-// Detect drag-and-drop
-var dragAndDrop = function() {
-  // There is *some* kind of drag-and-drop support in IE6-8, but I
-  // couldn't get it to work yet.
-  if (ie && ie_version < 9) { return false }
-  var div = elt('div');
-  return "draggable" in div || "dragDrop" in div
-}();
-
-var zwspSupported;
-function zeroWidthElement(measure) {
-  if (zwspSupported == null) {
-    var test = elt("span", "\u200b");
-    removeChildrenAndAdd(measure, elt("span", [test, document.createTextNode("x")]));
-    if (measure.firstChild.offsetHeight != 0)
-      { zwspSupported = test.offsetWidth <= 1 && test.offsetHeight > 2 && !(ie && ie_version < 8); }
-  }
-  var node = zwspSupported ? elt("span", "\u200b") :
-    elt("span", "\u00a0", null, "display: inline-block; width: 1px; margin-right: -1px");
-  node.setAttribute("cm-text", "");
-  return node
-}
-
-// Feature-detect IE's crummy client rect reporting for bidi text
-var badBidiRects;
-function hasBadBidiRects(measure) {
-  if (badBidiRects != null) { return badBidiRects }
-  var txt = removeChildrenAndAdd(measure, document.createTextNode("A\u062eA"));
-  var r0 = range(txt, 0, 1).getBoundingClientRect();
-  var r1 = range(txt, 1, 2).getBoundingClientRect();
-  removeChildren(measure);
-  if (!r0 || r0.left == r0.right) { return false } // Safari returns null in some cases (#2780)
-  return badBidiRects = (r1.right - r0.right < 3)
-}
-
-// See if "".split is the broken IE version, if so, provide an
-// alternative way to split lines.
-var splitLinesAuto = "\n\nb".split(/\n/).length != 3 ? function (string) {
-  var pos = 0, result = [], l = string.length;
-  while (pos <= l) {
-    var nl = string.indexOf("\n", pos);
-    if (nl == -1) { nl = string.length; }
-    var line = string.slice(pos, string.charAt(nl - 1) == "\r" ? nl - 1 : nl);
-    var rt = line.indexOf("\r");
-    if (rt != -1) {
-      result.push(line.slice(0, rt));
-      pos += rt + 1;
-    } else {
-      result.push(line);
-      pos = nl + 1;
-    }
-  }
-  return result
-} : function (string) { return string.split(/\r\n?|\n/); };
-
-var hasSelection = window.getSelection ? function (te) {
-  try { return te.selectionStart != te.selectionEnd }
-  catch(e) { return false }
-} : function (te) {
-  var range$$1;
-  try {range$$1 = te.ownerDocument.selection.createRange();}
-  catch(e) {}
-  if (!range$$1 || range$$1.parentElement() != te) { return false }
-  return range$$1.compareEndPoints("StartToEnd", range$$1) != 0
-};
-
-var hasCopyEvent = (function () {
-  var e = elt("div");
-  if ("oncopy" in e) { return true }
-  e.setAttribute("oncopy", "return;");
-  return typeof e.oncopy == "function"
-})();
-
-var badZoomedRects = null;
-function hasBadZoomedRects(measure) {
-  if (badZoomedRects != null) { return badZoomedRects }
-  var node = removeChildrenAndAdd(measure, elt("span", "x"));
-  var normal = node.getBoundingClientRect();
-  var fromRange = range(node, 0, 1).getBoundingClientRect();
-  return badZoomedRects = Math.abs(normal.left - fromRange.left) > 1
-}
-
-// Known modes, by name and by MIME
-var modes = {};
-var mimeModes = {};
-
-// Extra arguments are stored as the mode's dependencies, which is
-// used by (legacy) mechanisms like loadmode.js to automatically
-// load a mode. (Preferred mechanism is the require/define calls.)
-function defineMode(name, mode) {
-  if (arguments.length > 2)
-    { mode.dependencies = Array.prototype.slice.call(arguments, 2); }
-  modes[name] = mode;
-}
-
-function defineMIME(mime, spec) {
-  mimeModes[mime] = spec;
-}
-
-// Given a MIME type, a {name, ...options} config object, or a name
-// string, return a mode config object.
-function resolveMode(spec) {
-  if (typeof spec == "string" && mimeModes.hasOwnProperty(spec)) {
-    spec = mimeModes[spec];
-  } else if (spec && typeof spec.name == "string" && mimeModes.hasOwnProperty(spec.name)) {
-    var found = mimeModes[spec.name];
-    if (typeof found == "string") { found = {name: found}; }
-    spec = createObj(found, spec);
-    spec.name = found.name;
-  } else if (typeof spec == "string" && /^[\w\-]+\/[\w\-]+\+xml$/.test(spec)) {
-    return resolveMode("application/xml")
-  } else if (typeof spec == "string" && /^[\w\-]+\/[\w\-]+\+json$/.test(spec)) {
-    return resolveMode("application/json")
-  }
-  if (typeof spec == "string") { return {name: spec} }
-  else { return spec || {name: "null"} }
-}
-
-// Given a mode spec (anything that resolveMode accepts), find and
-// initialize an actual mode object.
-function getMode(options, spec) {
-  spec = resolveMode(spec);
-  var mfactory = modes[spec.name];
-  if (!mfactory) { return getMode(options, "text/plain") }
-  var modeObj = mfactory(options, spec);
-  if (modeExtensions.hasOwnProperty(spec.name)) {
-    var exts = modeExtensions[spec.name];
-    for (var prop in exts) {
-      if (!exts.hasOwnProperty(prop)) { continue }
-      if (modeObj.hasOwnProperty(prop)) { modeObj["_" + prop] = modeObj[prop]; }
-      modeObj[prop] = exts[prop];
-    }
-  }
-  modeObj.name = spec.name;
-  if (spec.helperType) { modeObj.helperType = spec.helperType; }
-  if (spec.modeProps) { for (var prop$1 in spec.modeProps)
-    { modeObj[prop$1] = spec.modeProps[prop$1]; } }
-
-  return modeObj
-}
-
-// This can be used to attach properties to mode objects from
-// outside the actual mode definition.
-var modeExtensions = {};
-function extendMode(mode, properties) {
-  var exts = modeExtensions.hasOwnProperty(mode) ? modeExtensions[mode] : (modeExtensions[mode] = {});
-  copyObj(properties, exts);
-}
-
-function copyState(mode, state) {
-  if (state === true) { return state }
-  if (mode.copyState) { return mode.copyState(state) }
-  var nstate = {};
-  for (var n in state) {
-    var val = state[n];
-    if (val instanceof Array) { val = val.concat([]); }
-    nstate[n] = val;
-  }
-  return nstate
-}
-
-// Given a mode and a state (for that mode), find the inner mode and
-// state at the position that the state refers to.
-function innerMode(mode, state) {
-  var info;
-  while (mode.innerMode) {
-    info = mode.innerMode(state);
-    if (!info || info.mode == mode) { break }
-    state = info.state;
-    mode = info.mode;
-  }
-  return info || {mode: mode, state: state}
-}
-
-function startState(mode, a1, a2) {
-  return mode.startState ? mode.startState(a1, a2) : true
-}
-
-// STRING STREAM
-
-// Fed to the mode parsers, provides helper functions to make
-// parsers more succinct.
-
-var StringStream = function(string, tabSize, lineOracle) {
-  this.pos = this.start = 0;
-  this.string = string;
-  this.tabSize = tabSize || 8;
-  this.lastColumnPos = this.lastColumnValue = 0;
-  this.lineStart = 0;
-  this.lineOracle = lineOracle;
-};
-
-StringStream.prototype.eol = function () {return this.pos >= this.string.length};
-StringStream.prototype.sol = function () {return this.pos == this.lineStart};
-StringStream.prototype.peek = function () {return this.string.charAt(this.pos) || undefined};
-StringStream.prototype.next = function () {
-  if (this.pos < this.string.length)
-    { return this.string.charAt(this.pos++) }
-};
-StringStream.prototype.eat = function (match) {
-  var ch = this.string.charAt(this.pos);
-  var ok;
-  if (typeof match == "string") { ok = ch == match; }
-  else { ok = ch && (match.test ? match.test(ch) : match(ch)); }
-  if (ok) {++this.pos; return ch}
-};
-StringStream.prototype.eatWhile = function (match) {
-  var start = this.pos;
-  while (this.eat(match)){}
-  return this.pos > start
-};
-StringStream.prototype.eatSpace = function () {
-    var this$1 = this;
-
-  var start = this.pos;
-  while (/[\s\u00a0]/.test(this.string.charAt(this.pos))) { ++this$1.pos; }
-  return this.pos > start
-};
-StringStream.prototype.skipToEnd = function () {this.pos = this.string.length;};
-StringStream.prototype.skipTo = function (ch) {
-  var found = this.string.indexOf(ch, this.pos);
-  if (found > -1) {this.pos = found; return true}
-};
-StringStream.prototype.backUp = function (n) {this.pos -= n;};
-StringStream.prototype.column = function () {
-  if (this.lastColumnPos < this.start) {
-    this.lastColumnValue = countColumn(this.string, this.start, this.tabSize, this.lastColumnPos, this.lastColumnValue);
-    this.lastColumnPos = this.start;
-  }
-  return this.lastColumnValue - (this.lineStart ? countColumn(this.string, this.lineStart, this.tabSize) : 0)
-};
-StringStream.prototype.indentation = function () {
-  return countColumn(this.string, null, this.tabSize) -
-    (this.lineStart ? countColumn(this.string, this.lineStart, this.tabSize) : 0)
-};
-StringStream.prototype.match = function (pattern, consume, caseInsensitive) {
-  if (typeof pattern == "string") {
-    var cased = function (str) { return caseInsensitive ? str.toLowerCase() : str; };
-    var substr = this.string.substr(this.pos, pattern.length);
-    if (cased(substr) == cased(pattern)) {
-      if (consume !== false) { this.pos += pattern.length; }
-      return true
-    }
-  } else {
-    var match = this.string.slice(this.pos).match(pattern);
-    if (match && match.index > 0) { return null }
-    if (match && consume !== false) { this.pos += match[0].length; }
-    return match
-  }
-};
-StringStream.prototype.current = function (){return this.string.slice(this.start, this.pos)};
-StringStream.prototype.hideFirstChars = function (n, inner) {
-  this.lineStart += n;
-  try { return inner() }
-  finally { this.lineStart -= n; }
-};
-StringStream.prototype.lookAhead = function (n) {
-  var oracle = this.lineOracle;
-  return oracle && oracle.lookAhead(n)
-};
-StringStream.prototype.baseToken = function () {
-  var oracle = this.lineOracle;
-  return oracle && oracle.baseToken(this.pos)
-};
-
-var SavedContext = function(state, lookAhead) {
-  this.state = state;
-  this.lookAhead = lookAhead;
-};
-
-var Context = function(doc, state, line, lookAhead) {
-  this.state = state;
-  this.doc = doc;
-  this.line = line;
-  this.maxLookAhead = lookAhead || 0;
-  this.baseTokens = null;
-  this.baseTokenPos = 1;
-};
-
-Context.prototype.lookAhead = function (n) {
-  var line = this.doc.getLine(this.line + n);
-  if (line != null && n > this.maxLookAhead) { this.maxLookAhead = n; }
-  return line
-};
-
-Context.prototype.baseToken = function (n) {
-    var this$1 = this;
-
-  if (!this.baseTokens) { return null }
-  while (this.baseTokens[this.baseTokenPos] <= n)
-    { this$1.baseTokenPos += 2; }
-  var type = this.baseTokens[this.baseTokenPos + 1];
-  return {type: type && type.replace(/( |^)overlay .*/, ""),
-          size: this.baseTokens[this.baseTokenPos] - n}
-};
-
-Context.prototype.nextLine = function () {
-  this.line++;
-  if (this.maxLookAhead > 0) { this.maxLookAhead--; }
-};
-
-Context.fromSaved = function (doc, saved, line) {
-  if (saved instanceof SavedContext)
-    { return new Context(doc, copyState(doc.mode, saved.state), line, saved.lookAhead) }
-  else
-    { return new Context(doc, copyState(doc.mode, saved), line) }
-};
-
-Context.prototype.save = function (copy) {
-  var state = copy !== false ? copyState(this.doc.mode, this.state) : this.state;
-  return this.maxLookAhead > 0 ? new SavedContext(state, this.maxLookAhead) : state
-};
-
-
-// Compute a style array (an array starting with a mode generation
-// -- for invalidation -- followed by pairs of end positions and
-// style strings), which is used to highlight the tokens on the
-// line.
-function highlightLine(cm, line, context, forceToEnd) {
-  // A styles array always starts with a number identifying the
-  // mode/overlays that it is based on (for easy invalidation).
-  var st = [cm.state.modeGen], lineClasses = {};
-  // Compute the base array of styles
-  runMode(cm, line.text, cm.doc.mode, context, function (end, style) { return st.push(end, style); },
-          lineClasses, forceToEnd);
-  var state = context.state;
-
-  // Run overlays, adjust style array.
-  var loop = function ( o ) {
-    context.baseTokens = st;
-    var overlay = cm.state.overlays[o], i = 1, at = 0;
-    context.state = true;
-    runMode(cm, line.text, overlay.mode, context, function (end, style) {
-      var start = i;
-      // Ensure there's a token end at the current position, and that i points at it
-      while (at < end) {
-        var i_end = st[i];
-        if (i_end > end)
-          { st.splice(i, 1, end, st[i+1], i_end); }
-        i += 2;
-        at = Math.min(end, i_end);
-      }
-      if (!style) { return }
-      if (overlay.opaque) {
-        st.splice(start, i - start, end, "overlay " + style);
-        i = start + 2;
-      } else {
-        for (; start < i; start += 2) {
-          var cur = st[start+1];
-          st[start+1] = (cur ? cur + " " : "") + "overlay " + style;
-        }
-      }
-    }, lineClasses);
-    context.state = state;
-    context.baseTokens = null;
-    context.baseTokenPos = 1;
-  };
-
-  for (var o = 0; o < cm.state.overlays.length; ++o) loop( o );
-
-  return {styles: st, classes: lineClasses.bgClass || lineClasses.textClass ? lineClasses : null}
-}
-
-function getLineStyles(cm, line, updateFrontier) {
-  if (!line.styles || line.styles[0] != cm.state.modeGen) {
-    var context = getContextBefore(cm, lineNo(line));
-    var resetState = line.text.length > cm.options.maxHighlightLength && copyState(cm.doc.mode, context.state);
-    var result = highlightLine(cm, line, context);
-    if (resetState) { context.state = resetState; }
-    line.stateAfter = context.save(!resetState);
-    line.styles = result.styles;
-    if (result.classes) { line.styleClasses = result.classes; }
-    else if (line.styleClasses) { line.styleClasses = null; }
-    if (updateFrontier === cm.doc.highlightFrontier)
-      { cm.doc.modeFrontier = Math.max(cm.doc.modeFrontier, ++cm.doc.highlightFrontier); }
-  }
-  return line.styles
-}
-
-function getContextBefore(cm, n, precise) {
-  var doc = cm.doc, display = cm.display;
-  if (!doc.mode.startState) { return new Context(doc, true, n) }
-  var start = findStartLine(cm, n, precise);
-  var saved = start > doc.first && getLine(doc, start - 1).stateAfter;
-  var context = saved ? Context.fromSaved(doc, saved, start) : new Context(doc, startState(doc.mode), start);
-
-  doc.iter(start, n, function (line) {
-    processLine(cm, line.text, context);
-    var pos = context.line;
-    line.stateAfter = pos == n - 1 || pos % 5 == 0 || pos >= display.viewFrom && pos < display.viewTo ? context.save() : null;
-    context.nextLine();
-  });
-  if (precise) { doc.modeFrontier = context.line; }
-  return context
-}
-
-// Lightweight form of highlight -- proceed over this line and
-// update state, but don't save a style array. Used for lines that
-// aren't currently visible.
-function processLine(cm, text, context, startAt) {
-  var mode = cm.doc.mode;
-  var stream = new StringStream(text, cm.options.tabSize, context);
-  stream.start = stream.pos = startAt || 0;
-  if (text == "") { callBlankLine(mode, context.state); }
-  while (!stream.eol()) {
-    readToken(mode, stream, context.state);
-    stream.start = stream.pos;
-  }
-}
-
-function callBlankLine(mode, state) {
-  if (mode.blankLine) { return mode.blankLine(state) }
-  if (!mode.innerMode) { return }
-  var inner = innerMode(mode, state);
-  if (inner.mode.blankLine) { return inner.mode.blankLine(inner.state) }
-}
-
-function readToken(mode, stream, state, inner) {
-  for (var i = 0; i < 10; i++) {
-    if (inner) { inner[0] = innerMode(mode, state).mode; }
-    var style = mode.token(stream, state);
-    if (stream.pos > stream.start) { return style }
-  }
-  throw new Error("Mode " + mode.name + " failed to advance stream.")
-}
-
-var Token = function(stream, type, state) {
-  this.start = stream.start; this.end = stream.pos;
-  this.string = stream.current();
-  this.type = type || null;
-  this.state = state;
-};
-
-// Utility for getTokenAt and getLineTokens
-function takeToken(cm, pos, precise, asArray) {
-  var doc = cm.doc, mode = doc.mode, style;
-  pos = clipPos(doc, pos);
-  var line = getLine(doc, pos.line), context = getContextBefore(cm, pos.line, precise);
-  var stream = new StringStream(line.text, cm.options.tabSize, context), tokens;
-  if (asArray) { tokens = []; }
-  while ((asArray || stream.pos < pos.ch) && !stream.eol()) {
-    stream.start = stream.pos;
-    style = readToken(mode, stream, context.state);
-    if (asArray) { tokens.push(new Token(stream, style, copyState(doc.mode, context.state))); }
-  }
-  return asArray ? tokens : new Token(stream, style, context.state)
-}
-
-function extractLineClasses(type, output) {
-  if (type) { for (;;) {
-    var lineClass = type.match(/(?:^|\s+)line-(background-)?(\S+)/);
-    if (!lineClass) { break }
-    type = type.slice(0, lineClass.index) + type.slice(lineClass.index + lineClass[0].length);
-    var prop = lineClass[1] ? "bgClass" : "textClass";
-    if (output[prop] == null)
-      { output[prop] = lineClass[2]; }
-    else if (!(new RegExp("(?:^|\s)" + lineClass[2] + "(?:$|\s)")).test(output[prop]))
-      { output[prop] += " " + lineClass[2]; }
-  } }
-  return type
-}
-
-// Run the given mode's parser over a line, calling f for each token.
-function runMode(cm, text, mode, context, f, lineClasses, forceToEnd) {
-  var flattenSpans = mode.flattenSpans;
-  if (flattenSpans == null) { flattenSpans = cm.options.flattenSpans; }
-  var curStart = 0, curStyle = null;
-  var stream = new StringStream(text, cm.options.tabSize, context), style;
-  var inner = cm.options.addModeClass && [null];
-  if (text == "") { extractLineClasses(callBlankLine(mode, context.state), lineClasses); }
-  while (!stream.eol()) {
-    if (stream.pos > cm.options.maxHighlightLength) {
-      flattenSpans = false;
-      if (forceToEnd) { processLine(cm, text, context, stream.pos); }
-      stream.pos = text.length;
-      style = null;
-    } else {
-      style = extractLineClasses(readToken(mode, stream, context.state, inner), lineClasses);
-    }
-    if (inner) {
-      var mName = inner[0].name;
-      if (mName) { style = "m-" + (style ? mName + " " + style : mName); }
-    }
-    if (!flattenSpans || curStyle != style) {
-      while (curStart < stream.start) {
-        curStart = Math.min(stream.start, curStart + 5000);
-        f(curStart, curStyle);
-      }
-      curStyle = style;
-    }
-    stream.start = stream.pos;
-  }
-  while (curStart < stream.pos) {
-    // Webkit seems to refuse to render text nodes longer than 57444
-    // characters, and returns inaccurate measurements in nodes
-    // starting around 5000 chars.
-    var pos = Math.min(stream.pos, curStart + 5000);
-    f(pos, curStyle);
-    curStart = pos;
-  }
-}
-
-// Finds the line to start with when starting a parse. Tries to
-// find a line with a stateAfter, so that it can start with a
-// valid state. If that fails, it returns the line with the
-// smallest indentation, which tends to need the least context to
-// parse correctly.
-function findStartLine(cm, n, precise) {
-  var minindent, minline, doc = cm.doc;
-  var lim = precise ? -1 : n - (cm.doc.mode.innerMode ? 1000 : 100);
-  for (var search = n; search > lim; --search) {
-    if (search <= doc.first) { return doc.first }
-    var line = getLine(doc, search - 1), after = line.stateAfter;
-    if (after && (!precise || search + (after instanceof SavedContext ? after.lookAhead : 0) <= doc.modeFrontier))
-      { return search }
-    var indented = countColumn(line.text, null, cm.options.tabSize);
-    if (minline == null || minindent > indented) {
-      minline = search - 1;
-      minindent = indented;
-    }
-  }
-  return minline
-}
-
-function retreatFrontier(doc, n) {
-  doc.modeFrontier = Math.min(doc.modeFrontier, n);
-  if (doc.highlightFrontier < n - 10) { return }
-  var start = doc.first;
-  for (var line = n - 1; line > start; line--) {
-    var saved = getLine(doc, line).stateAfter;
-    // change is on 3
-    // state on line 1 looked ahead 2 -- so saw 3
-    // test 1 + 2 < 3 should cover this
-    if (saved && (!(saved instanceof SavedContext) || line + saved.lookAhead < n)) {
-      start = line + 1;
-      break
-    }
-  }
-  doc.highlightFrontier = Math.min(doc.highlightFrontier, start);
-}
-
-// LINE DATA STRUCTURE
-
-// Line objects. These hold state related to a line, including
-// highlighting info (the styles array).
-var Line = function(text, markedSpans, estimateHeight) {
-  this.text = text;
-  attachMarkedSpans(this, markedSpans);
-  this.height = estimateHeight ? estimateHeight(this) : 1;
-};
-
-Line.prototype.lineNo = function () { return lineNo(this) };
-eventMixin(Line);
-
-// Change the content (text, markers) of a line. Automatically
-// invalidates cached information and tries to re-estimate the
-// line's height.
-function updateLine(line, text, markedSpans, estimateHeight) {
-  line.text = text;
-  if (line.stateAfter) { line.stateAfter = null; }
-  if (line.styles) { line.styles = null; }
-  if (line.order != null) { line.order = null; }
-  detachMarkedSpans(line);
-  attachMarkedSpans(line, markedSpans);
-  var estHeight = estimateHeight ? estimateHeight(line) : 1;
-  if (estHeight != line.height) { updateLineHeight(line, estHeight); }
-}
-
-// Detach a line from the document tree and its markers.
-function cleanUpLine(line) {
-  line.parent = null;
-  detachMarkedSpans(line);
-}
-
-// Convert a style as returned by a mode (either null, or a string
-// containing one or more styles) to a CSS style. This is cached,
-// and also looks for line-wide styles.
-var styleToClassCache = {};
-var styleToClassCacheWithMode = {};
-function interpretTokenStyle(style, options) {
-  if (!style || /^\s*$/.test(style)) { return null }
-  var cache = options.addModeClass ? styleToClassCacheWithMode : styleToClassCache;
-  return cache[style] ||
-    (cache[style] = style.replace(/\S+/g, "cm-$&"))
-}
-
-// Render the DOM representation of the text of a line. Also builds
-// up a 'line map', which points at the DOM nodes that represent
-// specific stretches of text, and is used by the measuring code.
-// The returned object contains the DOM node, this map, and
-// information about line-wide styles that were set by the mode.
-function buildLineContent(cm, lineView) {
-  // The padding-right forces the element to have a 'border', which
-  // is needed on Webkit to be able to get line-level bounding
-  // rectangles for it (in measureChar).
-  var content = eltP("span", null, null, webkit ? "padding-right: .1px" : null);
-  var builder = {pre: eltP("pre", [content], "CodeMirror-line"), content: content,
-                 col: 0, pos: 0, cm: cm,
-                 trailingSpace: false,
-                 splitSpaces: (ie || webkit) && cm.getOption("lineWrapping")};
-  lineView.measure = {};
-
-  // Iterate over the logical lines that make up this visual line.
-  for (var i = 0; i <= (lineView.rest ? lineView.rest.length : 0); i++) {
-    var line = i ? lineView.rest[i - 1] : lineView.line, order = (void 0);
-    builder.pos = 0;
-    builder.addToken = buildToken;
-    // Optionally wire in some hacks into the token-rendering
-    // algorithm, to deal with browser quirks.
-    if (hasBadBidiRects(cm.display.measure) && (order = getOrder(line, cm.doc.direction)))
-      { builder.addToken = buildTokenBadBidi(builder.addToken, order); }
-    builder.map = [];
-    var allowFrontierUpdate = lineView != cm.display.externalMeasured && lineNo(line);
-    insertLineContent(line, builder, getLineStyles(cm, line, allowFrontierUpdate));
-    if (line.styleClasses) {
-      if (line.styleClasses.bgClass)
-        { builder.bgClass = joinClasses(line.styleClasses.bgClass, builder.bgClass || ""); }
-      if (line.styleClasses.textClass)
-        { builder.textClass = joinClasses(line.styleClasses.textClass, builder.textClass || ""); }
-    }
-
-    // Ensure at least a single node is present, for measuring.
-    if (builder.map.length == 0)
-      { builder.map.push(0, 0, builder.content.appendChild(zeroWidthElement(cm.display.measure))); }
-
-    // Store the map and a cache object for the current logical line
-    if (i == 0) {
-      lineView.measure.map = builder.map;
-      lineView.measure.cache = {};
-    } else {
-      (lineView.measure.maps || (lineView.measure.maps = [])).push(builder.map)
-      ;(lineView.measure.caches || (lineView.measure.caches = [])).push({});
-    }
-  }
-
-  // See issue #2901
-  if (webkit) {
-    var last = builder.content.lastChild;
-    if (/\bcm-tab\b/.test(last.className) || (last.querySelector && last.querySelector(".cm-tab")))
-      { builder.content.className = "cm-tab-wrap-hack"; }
-  }
-
-  signal(cm, "renderLine", cm, lineView.line, builder.pre);
-  if (builder.pre.className)
-    { builder.textClass = joinClasses(builder.pre.className, builder.textClass || ""); }
-
-  return builder
-}
-
-function defaultSpecialCharPlaceholder(ch) {
-  var token = elt("span", "\u2022", "cm-invalidchar");
-  token.title = "\\u" + ch.charCodeAt(0).toString(16);
-  token.setAttribute("aria-label", token.title);
-  return token
-}
-
-// Build up the DOM representation for a single token, and add it to
-// the line map. Takes care to render special characters separately.
-function buildToken(builder, text, style, startStyle, endStyle, title, css) {
-  if (!text) { return }
-  var displayText = builder.splitSpaces ? splitSpaces(text, builder.trailingSpace) : text;
-  var special = builder.cm.state.specialChars, mustWrap = false;
-  var content;
-  if (!special.test(text)) {
-    builder.col += text.length;
-    content = document.createTextNode(displayText);
-    builder.map.push(builder.pos, builder.pos + text.length, content);
-    if (ie && ie_version < 9) { mustWrap = true; }
-    builder.pos += text.length;
-  } else {
-    content = document.createDocumentFragment();
-    var pos = 0;
-    while (true) {
-      special.lastIndex = pos;
-      var m = special.exec(text);
-      var skipped = m ? m.index - pos : text.length - pos;
-      if (skipped) {
-        var txt = document.createTextNode(displayText.slice(pos, pos + skipped));
-        if (ie && ie_version < 9) { content.appendChild(elt("span", [txt])); }
-        else { content.appendChild(txt); }
-        builder.map.push(builder.pos, builder.pos + skipped, txt);
-        builder.col += skipped;
-        builder.pos += skipped;
-      }
-      if (!m) { break }
-      pos += skipped + 1;
-      var txt$1 = (void 0);
-      if (m[0] == "\t") {
-        var tabSize = builder.cm.options.tabSize, tabWidth = tabSize - builder.col % tabSize;
-        txt$1 = content.appendChild(elt("span", spaceStr(tabWidth), "cm-tab"));
-        txt$1.setAttribute("role", "presentation");
-        txt$1.setAttribute("cm-text", "\t");
-        builder.col += tabWidth;
-      } else if (m[0] == "\r" || m[0] == "\n") {
-        txt$1 = content.appendChild(elt("span", m[0] == "\r" ? "\u240d" : "\u2424", "cm-invalidchar"));
-        txt$1.setAttribute("cm-text", m[0]);
-        builder.col += 1;
-      } else {
-        txt$1 = builder.cm.options.specialCharPlaceholder(m[0]);
-        txt$1.setAttribute("cm-text", m[0]);
-        if (ie && ie_version < 9) { content.appendChild(elt("span", [txt$1])); }
-        else { content.appendChild(txt$1); }
-        builder.col += 1;
-      }
-      builder.map.push(builder.pos, builder.pos + 1, txt$1);
-      builder.pos++;
-    }
-  }
-  builder.trailingSpace = displayText.charCodeAt(text.length - 1) == 32;
-  if (style || startStyle || endStyle || mustWrap || css) {
-    var fullStyle = style || "";
-    if (startStyle) { fullStyle += startStyle; }
-    if (endStyle) { fullStyle += endStyle; }
-    var token = elt("span", [content], fullStyle, css);
-    if (title) { token.title = title; }
-    return builder.content.appendChild(token)
-  }
-  builder.content.appendChild(content);
-}
-
-function splitSpaces(text, trailingBefore) {
-  if (text.length > 1 && !/  /.test(text)) { return text }
-  var spaceBefore = trailingBefore, result = "";
-  for (var i = 0; i < text.length; i++) {
-    var ch = text.charAt(i);
-    if (ch == " " && spaceBefore && (i == text.length - 1 || text.charCodeAt(i + 1) == 32))
-      { ch = "\u00a0"; }
-    result += ch;
-    spaceBefore = ch == " ";
-  }
-  return result
-}
-
-// Work around nonsense dimensions being reported for stretches of
-// right-to-left text.
-function buildTokenBadBidi(inner, order) {
-  return function (builder, text, style, startStyle, endStyle, title, css) {
-    style = style ? style + " cm-force-border" : "cm-force-border";
-    var start = builder.pos, end = start + text.length;
-    for (;;) {
-      // Find the part that overlaps with the start of this text
-      var part = (void 0);
-      for (var i = 0; i < order.length; i++) {
-        part = order[i];
-        if (part.to > start && part.from <= start) { break }
-      }
-      if (part.to >= end) { return inner(builder, text, style, startStyle, endStyle, title, css) }
-      inner(builder, text.slice(0, part.to - start), style, startStyle, null, title, css);
-      startStyle = null;
-      text = text.slice(part.to - start);
-      start = part.to;
-    }
-  }
-}
-
-function buildCollapsedSpan(builder, size, marker, ignoreWidget) {
-  var widget = !ignoreWidget && marker.widgetNode;
-  if (widget) { builder.map.push(builder.pos, builder.pos + size, widget); }
-  if (!ignoreWidget && builder.cm.display.input.needsContentAttribute) {
-    if (!widget)
-      { widget = builder.content.appendChild(document.createElement("span")); }
-    widget.setAttribute("cm-marker", marker.id);
-  }
-  if (widget) {
-    builder.cm.display.input.setUneditable(widget);
-    builder.content.appendChild(widget);
-  }
-  builder.pos += size;
-  builder.trailingSpace = false;
-}
-
-// Outputs a number of spans to make up a line, taking highlighting
-// and marked text into account.
-function insertLineContent(line, builder, styles) {
-  var spans = line.markedSpans, allText = line.text, at = 0;
-  if (!spans) {
-    for (var i$1 = 1; i$1 < styles.length; i$1+=2)
-      { builder.addToken(builder, allText.slice(at, at = styles[i$1]), interpretTokenStyle(styles[i$1+1], builder.cm.options)); }
-    return
-  }
-
-  var len = allText.length, pos = 0, i = 1, text = "", style, css;
-  var nextChange = 0, spanStyle, spanEndStyle, spanStartStyle, title, collapsed;
-  for (;;) {
-    if (nextChange == pos) { // Update current marker set
-      spanStyle = spanEndStyle = spanStartStyle = title = css = "";
-      collapsed = null; nextChange = Infinity;
-      var foundBookmarks = [], endStyles = (void 0);
-      for (var j = 0; j < spans.length; ++j) {
-        var sp = spans[j], m = sp.marker;
-        if (m.type == "bookmark" && sp.from == pos && m.widgetNode) {
-          foundBookmarks.push(m);
-        } else if (sp.from <= pos && (sp.to == null || sp.to > pos || m.collapsed && sp.to == pos && sp.from == pos)) {
-          if (sp.to != null && sp.to != pos && nextChange > sp.to) {
-            nextChange = sp.to;
-            spanEndStyle = "";
-          }
-          if (m.className) { spanStyle += " " + m.className; }
-          if (m.css) { css = (css ? css + ";" : "") + m.css; }
-          if (m.startStyle && sp.from == pos) { spanStartStyle += " " + m.startStyle; }
-          if (m.endStyle && sp.to == nextChange) { (endStyles || (endStyles = [])).push(m.endStyle, sp.to); }
-          if (m.title && !title) { title = m.title; }
-          if (m.collapsed && (!collapsed || compareCollapsedMarkers(collapsed.marker, m) < 0))
-            { collapsed = sp; }
-        } else if (sp.from > pos && nextChange > sp.from) {
-          nextChange = sp.from;
-        }
-      }
-      if (endStyles) { for (var j$1 = 0; j$1 < endStyles.length; j$1 += 2)
-        { if (endStyles[j$1 + 1] == nextChange) { spanEndStyle += " " + endStyles[j$1]; } } }
-
-      if (!collapsed || collapsed.from == pos) { for (var j$2 = 0; j$2 < foundBookmarks.length; ++j$2)
-        { buildCollapsedSpan(builder, 0, foundBookmarks[j$2]); } }
-      if (collapsed && (collapsed.from || 0) == pos) {
-        buildCollapsedSpan(builder, (collapsed.to == null ? len + 1 : collapsed.to) - pos,
-                           collapsed.marker, collapsed.from == null);
-        if (collapsed.to == null) { return }
-        if (collapsed.to == pos) { collapsed = false; }
-      }
-    }
-    if (pos >= len) { break }
-
-    var upto = Math.min(len, nextChange);
-    while (true) {
-      if (text) {
-        var end = pos + text.length;
-        if (!collapsed) {
-          var tokenText = end > upto ? text.slice(0, upto - pos) : text;
-          builder.addToken(builder, tokenText, style ? style + spanStyle : spanStyle,
-                           spanStartStyle, pos + tokenText.length == nextChange ? spanEndStyle : "", title, css);
-        }
-        if (end >= upto) {text = text.slice(upto - pos); pos = upto; break}
-        pos = end;
-        spanStartStyle = "";
-      }
-      text = allText.slice(at, at = styles[i++]);
-      style = interpretTokenStyle(styles[i++], builder.cm.options);
-    }
-  }
-}
-
-
-// These objects are used to represent the visible (currently drawn)
-// part of the document. A LineView may correspond to multiple
-// logical lines, if those are connected by collapsed ranges.
-function LineView(doc, line, lineN) {
-  // The starting line
-  this.line = line;
-  // Continuing lines, if any
-  this.rest = visualLineContinued(line);
-  // Number of logical lines in this visual line
-  this.size = this.rest ? lineNo(lst(this.rest)) - lineN + 1 : 1;
-  this.node = this.text = null;
-  this.hidden = lineIsHidden(doc, line);
-}
-
-// Create a range of LineView objects for the given lines.
-function buildViewArray(cm, from, to) {
-  var array = [], nextPos;
-  for (var pos = from; pos < to; pos = nextPos) {
-    var view = new LineView(cm.doc, getLine(cm.doc, pos), pos);
-    nextPos = pos + view.size;
-    array.push(view);
-  }
-  return array
-}
-
-var operationGroup = null;
-
-function pushOperation(op) {
-  if (operationGroup) {
-    operationGroup.ops.push(op);
-  } else {
-    op.ownsGroup = operationGroup = {
-      ops: [op],
-      delayedCallbacks: []
-    };
-  }
-}
-
-function fireCallbacksForOps(group) {
-  // Calls delayed callbacks and cursorActivity handlers until no
-  // new ones appear
-  var callbacks = group.delayedCallbacks, i = 0;
-  do {
-    for (; i < callbacks.length; i++)
-      { callbacks[i].call(null); }
-    for (var j = 0; j < group.ops.length; j++) {
-      var op = group.ops[j];
-      if (op.cursorActivityHandlers)
-        { while (op.cursorActivityCalled < op.cursorActivityHandlers.length)
-          { op.cursorActivityHandlers[op.cursorActivityCalled++].call(null, op.cm); } }
-    }
-  } while (i < callbacks.length)
-}
-
-function finishOperation(op, endCb) {
-  var group = op.ownsGroup;
-  if (!group) { return }
-
-  try { fireCallbacksForOps(group); }
-  finally {
-    operationGroup = null;
-    endCb(group);
-  }
-}
-
-var orphanDelayedCallbacks = null;
-
-// Often, we want to signal events at a point where we are in the
-// middle of some work, but don't want the handler to start calling
-// other methods on the editor, which might be in an inconsistent
-// state or simply not expect any other events to happen.
-// signalLater looks whether there are any handlers, and schedules
-// them to be executed when the last operation ends, or, if no
-// operation is active, when a timeout fires.
-function signalLater(emitter, type /*, values...*/) {
-  var arr = getHandlers(emitter, type);
-  if (!arr.length) { return }
-  var args = Array.prototype.slice.call(arguments, 2), list;
-  if (operationGroup) {
-    list = operationGroup.delayedCallbacks;
-  } else if (orphanDelayedCallbacks) {
-    list = orphanDelayedCallbacks;
-  } else {
-    list = orphanDelayedCallbacks = [];
-    setTimeout(fireOrphanDelayed, 0);
-  }
-  var loop = function ( i ) {
-    list.push(function () { return arr[i].apply(null, args); });
-  };
-
-  for (var i = 0; i < arr.length; ++i)
-    loop( i );
-}
-
-function fireOrphanDelayed() {
-  var delayed = orphanDelayedCallbacks;
-  orphanDelayedCallbacks = null;
-  for (var i = 0; i < delayed.length; ++i) { delayed[i](); }
-}
-
-// When an aspect of a line changes, a string is added to
-// lineView.changes. This updates the relevant part of the line's
-// DOM structure.
-function updateLineForChanges(cm, lineView, lineN, dims) {
-  for (var j = 0; j < lineView.changes.length; j++) {
-    var type = lineView.changes[j];
-    if (type == "text") { updateLineText(cm, lineView); }
-    else if (type == "gutter") { updateLineGutter(cm, lineView, lineN, dims); }
-    else if (type == "class") { updateLineClasses(cm, lineView); }
-    else if (type == "widget") { updateLineWidgets(cm, lineView, dims); }
-  }
-  lineView.changes = null;
-}
-
-// Lines with gutter elements, widgets or a background class need to
-// be wrapped, and have the extra elements added to the wrapper div
-function ensureLineWrapped(lineView) {
-  if (lineView.node == lineView.text) {
-    lineView.node = elt("div", null, null, "position: relative");
-    if (lineView.text.parentNode)
-      { lineView.text.parentNode.replaceChild(lineView.node, lineView.text); }
-    lineView.node.appendChild(lineView.text);
-    if (ie && ie_version < 8) { lineView.node.style.zIndex = 2; }
-  }
-  return lineView.node
-}
-
-function updateLineBackground(cm, lineView) {
-  var cls = lineView.bgClass ? lineView.bgClass + " " + (lineView.line.bgClass || "") : lineView.line.bgClass;
-  if (cls) { cls += " CodeMirror-linebackground"; }
-  if (lineView.background) {
-    if (cls) { lineView.background.className = cls; }
-    else { lineView.background.parentNode.removeChild(lineView.background); lineView.background = null; }
-  } else if (cls) {
-    var wrap = ensureLineWrapped(lineView);
-    lineView.background = wrap.insertBefore(elt("div", null, cls), wrap.firstChild);
-    cm.display.input.setUneditable(lineView.background);
-  }
-}
-
-// Wrapper around buildLineContent which will reuse the structure
-// in display.externalMeasured when possible.
-function getLineContent(cm, lineView) {
-  var ext = cm.display.externalMeasured;
-  if (ext && ext.line == lineView.line) {
-    cm.display.externalMeasured = null;
-    lineView.measure = ext.measure;
-    return ext.built
-  }
-  return buildLineContent(cm, lineView)
-}
-
-// Redraw the line's text. Interacts with the background and text
-// classes because the mode may output tokens that influence these
-// classes.
-function updateLineText(cm, lineView) {
-  var cls = lineView.text.className;
-  var built = getLineContent(cm, lineView);
-  if (lineView.text == lineView.node) { lineView.node = built.pre; }
-  lineView.text.parentNode.replaceChild(built.pre, lineView.text);
-  lineView.text = built.pre;
-  if (built.bgClass != lineView.bgClass || built.textClass != lineView.textClass) {
-    lineView.bgClass = built.bgClass;
-    lineView.textClass = built.textClass;
-    updateLineClasses(cm, lineView);
-  } else if (cls) {
-    lineView.text.className = cls;
-  }
-}
-
-function updateLineClasses(cm, lineView) {
-  updateLineBackground(cm, lineView);
-  if (lineView.line.wrapClass)
-    { ensureLineWrapped(lineView).className = lineView.line.wrapClass; }
-  else if (lineView.node != lineView.text)
-    { lineView.node.className = ""; }
-  var textClass = lineView.textClass ? lineView.textClass + " " + (lineView.line.textClass || "") : lineView.line.textClass;
-  lineView.text.className = textClass || "";
-}
-
-function updateLineGutter(cm, lineView, lineN, dims) {
-  if (lineView.gutter) {
-    lineView.node.removeChild(lineView.gutter);
-    lineView.gutter = null;
-  }
-  if (lineView.gutterBackground) {
-    lineView.node.removeChild(lineView.gutterBackground);
-    lineView.gutterBackground = null;
-  }
-  if (lineView.line.gutterClass) {
-    var wrap = ensureLineWrapped(lineView);
-    lineView.gutterBackground = elt("div", null, "CodeMirror-gutter-background " + lineView.line.gutterClass,
-                                    ("left: " + (cm.options.fixedGutter ? dims.fixedPos : -dims.gutterTotalWidth) + "px; width: " + (dims.gutterTotalWidth) + "px"));
-    cm.display.input.setUneditable(lineView.gutterBackground);
-    wrap.insertBefore(lineView.gutterBackground, lineView.text);
-  }
-  var markers = lineView.line.gutterMarkers;
-  if (cm.options.lineNumbers || markers) {
-    var wrap$1 = ensureLineWrapped(lineView);
-    var gutterWrap = lineView.gutter = elt("div", null, "CodeMirror-gutter-wrapper", ("left: " + (cm.options.fixedGutter ? dims.fixedPos : -dims.gutterTotalWidth) + "px"));
-    cm.display.input.setUneditable(gutterWrap);
-    wrap$1.insertBefore(gutterWrap, lineView.text);
-    if (lineView.line.gutterClass)
-      { gutterWrap.className += " " + lineView.line.gutterClass; }
-    if (cm.options.lineNumbers && (!markers || !markers["CodeMirror-linenumbers"]))
-      { lineView.lineNumber = gutterWrap.appendChild(
-        elt("div", lineNumberFor(cm.options, lineN),
-            "CodeMirror-linenumber CodeMirror-gutter-elt",
-            ("left: " + (dims.gutterLeft["CodeMirror-linenumbers"]) + "px; width: " + (cm.display.lineNumInnerWidth) + "px"))); }
-    if (markers) { for (var k = 0; k < cm.options.gutters.length; ++k) {
-      var id = cm.options.gutters[k], found = markers.hasOwnProperty(id) && markers[id];
-      if (found)
-        { gutterWrap.appendChild(elt("div", [found], "CodeMirror-gutter-elt",
-                                   ("left: " + (dims.gutterLeft[id]) + "px; width: " + (dims.gutterWidth[id]) + "px"))); }
-    } }
-  }
-}
-
-function updateLineWidgets(cm, lineView, dims) {
-  if (lineView.alignable) { lineView.alignable = null; }
-  for (var node = lineView.node.firstChild, next = (void 0); node; node = next) {
-    next = node.nextSibling;
-    if (node.className == "CodeMirror-linewidget")
-      { lineView.node.removeChild(node); }
-  }
-  insertLineWidgets(cm, lineView, dims);
-}
-
-// Build a line's DOM representation from scratch
-function buildLineElement(cm, lineView, lineN, dims) {
-  var built = getLineContent(cm, lineView);
-  lineView.text = lineView.node = built.pre;
-  if (built.bgClass) { lineView.bgClass = built.bgClass; }
-  if (built.textClass) { lineView.textClass = built.textClass; }
-
-  updateLineClasses(cm, lineView);
-  updateLineGutter(cm, lineView, lineN, dims);
-  insertLineWidgets(cm, lineView, dims);
-  return lineView.node
-}
-
-// A lineView may contain multiple logical lines (when merged by
-// collapsed spans). The widgets for all of them need to be drawn.
-function insertLineWidgets(cm, lineView, dims) {
-  insertLineWidgetsFor(cm, lineView.line, lineView, dims, true);
-  if (lineView.rest) { for (var i = 0; i < lineView.rest.length; i++)
-    { insertLineWidgetsFor(cm, lineView.rest[i], lineView, dims, false); } }
-}
-
-function insertLineWidgetsFor(cm, line, lineView, dims, allowAbove) {
-  if (!line.widgets) { return }
-  var wrap = ensureLineWrapped(lineView);
-  for (var i = 0, ws = line.widgets; i < ws.length; ++i) {
-    var widget = ws[i], node = elt("div", [widget.node], "CodeMirror-linewidget");
-    if (!widget.handleMouseEvents) { node.setAttribute("cm-ignore-events", "true"); }
-    positionLineWidget(widget, node, lineView, dims);
-    cm.display.input.setUneditable(node);
-    if (allowAbove && widget.above)
-      { wrap.insertBefore(node, lineView.gutter || lineView.text); }
-    else
-      { wrap.appendChild(node); }
-    signalLater(widget, "redraw");
-  }
-}
-
-function positionLineWidget(widget, node, lineView, dims) {
-  if (widget.noHScroll) {
-    (lineView.alignable || (lineView.alignable = [])).push(node);
-    var width = dims.wrapperWidth;
-    node.style.left = dims.fixedPos + "px";
-    if (!widget.coverGutter) {
-      width -= dims.gutterTotalWidth;
-      node.style.paddingLeft = dims.gutterTotalWidth + "px";
-    }
-    node.style.width = width + "px";
-  }
-  if (widget.coverGutter) {
-    node.style.zIndex = 5;
-    node.style.position = "relative";
-    if (!widget.noHScroll) { node.style.marginLeft = -dims.gutterTotalWidth + "px"; }
-  }
-}
-
-function widgetHeight(widget) {
-  if (widget.height != null) { return widget.height }
-  var cm = widget.doc.cm;
-  if (!cm) { return 0 }
-  if (!contains(document.body, widget.node)) {
-    var parentStyle = "position: relative;";
-    if (widget.coverGutter)
-      { parentStyle += "margin-left: -" + cm.display.gutters.offsetWidth + "px;"; }
-    if (widget.noHScroll)
-      { parentStyle += "width: " + cm.display.wrapper.clientWidth + "px;"; }
-    removeChildrenAndAdd(cm.display.measure, elt("div", [widget.node], null, parentStyle));
-  }
-  return widget.height = widget.node.parentNode.offsetHeight
-}
-
-// Return true when the given mouse event happened in a widget
-function eventInWidget(display, e) {
-  for (var n = e_target(e); n != display.wrapper; n = n.parentNode) {
-    if (!n || (n.nodeType == 1 && n.getAttribute("cm-ignore-events") == "true") ||
-        (n.parentNode == display.sizer && n != display.mover))
-      { return true }
-  }
-}
-
-// POSITION MEASUREMENT
-
-function paddingTop(display) {return display.lineSpace.offsetTop}
-function paddingVert(display) {return display.mover.offsetHeight - display.lineSpace.offsetHeight}
-function paddingH(display) {
-  if (display.cachedPaddingH) { return display.cachedPaddingH }
-  var e = removeChildrenAndAdd(display.measure, elt("pre", "x"));
-  var style = window.getComputedStyle ? window.getComputedStyle(e) : e.currentStyle;
-  var data = {left: parseInt(style.paddingLeft), right: parseInt(style.paddingRight)};
-  if (!isNaN(data.left) && !isNaN(data.right)) { display.cachedPaddingH = data; }
-  return data
-}
-
-function scrollGap(cm) { return scrollerGap - cm.display.nativeBarWidth }
-function displayWidth(cm) {
-  return cm.display.scroller.clientWidth - scrollGap(cm) - cm.display.barWidth
-}
-function displayHeight(cm) {
-  return cm.display.scroller.clientHeight - scrollGap(cm) - cm.display.barHeight
-}
-
-// Ensure the lineView.wrapping.heights array is populated. This is
-// an array of bottom offsets for the lines that make up a drawn
-// line. When lineWrapping is on, there might be more than one
-// height.
-function ensureLineHeights(cm, lineView, rect) {
-  var wrapping = cm.options.lineWrapping;
-  var curWidth = wrapping && displayWidth(cm);
-  if (!lineView.measure.heights || wrapping && lineView.measure.width != curWidth) {
-    var heights = lineView.measure.heights = [];
-    if (wrapping) {
-      lineView.measure.width = curWidth;
-      var rects = lineView.text.firstChild.getClientRects();
-      for (var i = 0; i < rects.length - 1; i++) {
-        var cur = rects[i], next = rects[i + 1];
-        if (Math.abs(cur.bottom - next.bottom) > 2)
-          { heights.push((cur.bottom + next.top) / 2 - rect.top); }
-      }
-    }
-    heights.push(rect.bottom - rect.top);
-  }
-}
-
-// Find a line map (mapping character offsets to text nodes) and a
-// measurement cache for the given line number. (A line view might
-// contain multiple lines when collapsed ranges are present.)
-function mapFromLineView(lineView, line, lineN) {
-  if (lineView.line == line)
-    { return {map: lineView.measure.map, cache: lineView.measure.cache} }
-  for (var i = 0; i < lineView.rest.length; i++)
-    { if (lineView.rest[i] == line)
-      { return {map: lineView.measure.maps[i], cache: lineView.measure.caches[i]} } }
-  for (var i$1 = 0; i$1 < lineView.rest.length; i$1++)
-    { if (lineNo(lineView.rest[i$1]) > lineN)
-      { return {map: lineView.measure.maps[i$1], cache: lineView.measure.caches[i$1], before: true} } }
-}
-
-// Render a line into the hidden node display.externalMeasured. Used
-// when measurement is needed for a line that's not in the viewport.
-function updateExternalMeasurement(cm, line) {
-  line = visualLine(line);
-  var lineN = lineNo(line);
-  var view = cm.display.externalMeasured = new LineView(cm.doc, line, lineN);
-  view.lineN = lineN;
-  var built = view.built = buildLineContent(cm, view);
-  view.text = built.pre;
-  removeChildrenAndAdd(cm.display.lineMeasure, built.pre);
-  return view
-}
-
-// Get a {top, bottom, left, right} box (in line-local coordinates)
-// for a given character.
-function measureChar(cm, line, ch, bias) {
-  return measureCharPrepared(cm, prepareMeasureForLine(cm, line), ch, bias)
-}
-
-// Find a line view that corresponds to the given line number.
-function findViewForLine(cm, lineN) {
-  if (lineN >= cm.display.viewFrom && lineN < cm.display.viewTo)
-    { return cm.display.view[findViewIndex(cm, lineN)] }
-  var ext = cm.display.externalMeasured;
-  if (ext && lineN >= ext.lineN && lineN < ext.lineN + ext.size)
-    { return ext }
-}
-
-// Measurement can be split in two steps, the set-up work that
-// applies to the whole line, and the measurement of the actual
-// character. Functions like coordsChar, that need to do a lot of
-// measurements in a row, can thus ensure that the set-up work is
-// only done once.
-function prepareMeasureForLine(cm, line) {
-  var lineN = lineNo(line);
-  var view = findViewForLine(cm, lineN);
-  if (view && !view.text) {
-    view = null;
-  } else if (view && view.changes) {
-    updateLineForChanges(cm, view, lineN, getDimensions(cm));
-    cm.curOp.forceUpdate = true;
-  }
-  if (!view)
-    { view = updateExternalMeasurement(cm, line); }
-
-  var info = mapFromLineView(view, line, lineN);
-  return {
-    line: line, view: view, rect: null,
-    map: info.map, cache: info.cache, before: info.before,
-    hasHeights: false
-  }
-}
-
-// Given a prepared measurement object, measures the position of an
-// actual character (or fetches it from the cache).
-function measureCharPrepared(cm, prepared, ch, bias, varHeight) {
-  if (prepared.before) { ch = -1; }
-  var key = ch + (bias || ""), found;
-  if (prepared.cache.hasOwnProperty(key)) {
-    found = prepared.cache[key];
-  } else {
-    if (!prepared.rect)
-      { prepared.rect = prepared.view.text.getBoundingClientRect(); }
-    if (!prepared.hasHeights) {
-      ensureLineHeights(cm, prepared.view, prepared.rect);
-      prepared.hasHeights = true;
-    }
-    found = measureCharInner(cm, prepared, ch, bias);
-    if (!found.bogus) { prepared.cache[key] = found; }
-  }
-  return {left: found.left, right: found.right,
-          top: varHeight ? found.rtop : found.top,
-          bottom: varHeight ? found.rbottom : found.bottom}
-}
-
-var nullRect = {left: 0, right: 0, top: 0, bottom: 0};
-
-function nodeAndOffsetInLineMap(map$$1, ch, bias) {
-  var node, start, end, collapse, mStart, mEnd;
-  // First, search the line map for the text node corresponding to,
-  // or closest to, the target character.
-  for (var i = 0; i < map$$1.length; i += 3) {
-    mStart = map$$1[i];
-    mEnd = map$$1[i + 1];
-    if (ch < mStart) {
-      start = 0; end = 1;
-      collapse = "left";
-    } else if (ch < mEnd) {
-      start = ch - mStart;
-      end = start + 1;
-    } else if (i == map$$1.length - 3 || ch == mEnd && map$$1[i + 3] > ch) {
-      end = mEnd - mStart;
-      start = end - 1;
-      if (ch >= mEnd) { collapse = "right"; }
-    }
-    if (start != null) {
-      node = map$$1[i + 2];
-      if (mStart == mEnd && bias == (node.insertLeft ? "left" : "right"))
-        { collapse = bias; }
-      if (bias == "left" && start == 0)
-        { while (i && map$$1[i - 2] == map$$1[i - 3] && map$$1[i - 1].insertLeft) {
-          node = map$$1[(i -= 3) + 2];
-          collapse = "left";
-        } }
-      if (bias == "right" && start == mEnd - mStart)
-        { while (i < map$$1.length - 3 && map$$1[i + 3] == map$$1[i + 4] && !map$$1[i + 5].insertLeft) {
-          node = map$$1[(i += 3) + 2];
-          collapse = "right";
-        } }
-      break
-    }
-  }
-  return {node: node, start: start, end: end, collapse: collapse, coverStart: mStart, coverEnd: mEnd}
-}
-
-function getUsefulRect(rects, bias) {
-  var rect = nullRect;
-  if (bias == "left") { for (var i = 0; i < rects.length; i++) {
-    if ((rect = rects[i]).left != rect.right) { break }
-  } } else { for (var i$1 = rects.length - 1; i$1 >= 0; i$1--) {
-    if ((rect = rects[i$1]).left != rect.right) { break }
-  } }
-  return rect
-}
-
-function measureCharInner(cm, prepared, ch, bias) {
-  var place = nodeAndOffsetInLineMap(prepared.map, ch, bias);
-  var node = place.node, start = place.start, end = place.end, collapse = place.collapse;
-
-  var rect;
-  if (node.nodeType == 3) { // If it is a text node, use a range to retrieve the coordinates.
-    for (var i$1 = 0; i$1 < 4; i$1++) { // Retry a maximum of 4 times when nonsense rectangles are returned
-      while (start && isExtendingChar(prepared.line.text.charAt(place.coverStart + start))) { --start; }
-      while (place.coverStart + end < place.coverEnd && isExtendingChar(prepared.line.text.charAt(place.coverStart + end))) { ++end; }
-      if (ie && ie_version < 9 && start == 0 && end == place.coverEnd - place.coverStart)
-        { rect = node.parentNode.getBoundingClientRect(); }
-      else
-        { rect = getUsefulRect(range(node, start, end).getClientRects(), bias); }
-      if (rect.left || rect.right || start == 0) { break }
-      end = start;
-      start = start - 1;
-      collapse = "right";
-    }
-    if (ie && ie_version < 11) { rect = maybeUpdateRectForZooming(cm.display.measure, rect); }
-  } else { // If it is a widget, simply get the box for the whole widget.
-    if (start > 0) { collapse = bias = "right"; }
-    var rects;
-    if (cm.options.lineWrapping && (rects = node.getClientRects()).length > 1)
-      { rect = rects[bias == "right" ? rects.length - 1 : 0]; }
-    else
-      { rect = node.getBoundingClientRect(); }
-  }
-  if (ie && ie_version < 9 && !start && (!rect || !rect.left && !rect.right)) {
-    var rSpan = node.parentNode.getClientRects()[0];
-    if (rSpan)
-      { rect = {left: rSpan.left, right: rSpan.left + charWidth(cm.display), top: rSpan.top, bottom: rSpan.bottom}; }
-    else
-      { rect = nullRect; }
-  }
-
-  var rtop = rect.top - prepared.rect.top, rbot = rect.bottom - prepared.rect.top;
-  var mid = (rtop + rbot) / 2;
-  var heights = prepared.view.measure.heights;
-  var i = 0;
-  for (; i < heights.length - 1; i++)
-    { if (mid < heights[i]) { break } }
-  var top = i ? heights[i - 1] : 0, bot = heights[i];
-  var result = {left: (collapse == "right" ? rect.right : rect.left) - prepared.rect.left,
-                right: (collapse == "left" ? rect.left : rect.right) - prepared.rect.left,
-                top: top, bottom: bot};
-  if (!rect.left && !rect.right) { result.bogus = true; }
-  if (!cm.options.singleCursorHeightPerLine) { result.rtop = rtop; result.rbottom = rbot; }
-
-  return result
-}
-
-// Work around problem with bounding client rects on ranges being
-// returned incorrectly when zoomed on IE10 and below.
-function maybeUpdateRectForZooming(measure, rect) {
-  if (!window.screen || screen.logicalXDPI == null ||
-      screen.logicalXDPI == screen.deviceXDPI || !hasBadZoomedRects(measure))
-    { return rect }
-  var scaleX = screen.logicalXDPI / screen.deviceXDPI;
-  var scaleY = screen.logicalYDPI / screen.deviceYDPI;
-  return {left: rect.left * scaleX, right: rect.right * scaleX,
-          top: rect.top * scaleY, bottom: rect.bottom * scaleY}
-}
-
-function clearLineMeasurementCacheFor(lineView) {
-  if (lineView.measure) {
-    lineView.measure.cache = {};
-    lineView.measure.heights = null;
-    if (lineView.rest) { for (var i = 0; i < lineView.rest.length; i++)
-      { lineView.measure.caches[i] = {}; } }
-  }
-}
-
-function clearLineMeasurementCache(cm) {
-  cm.display.externalMeasure = null;
-  removeChildren(cm.display.lineMeasure);
-  for (var i = 0; i < cm.display.view.length; i++)
-    { clearLineMeasurementCacheFor(cm.display.view[i]); }
-}
-
-function clearCaches(cm) {
-  clearLineMeasurementCache(cm);
-  cm.display.cachedCharWidth = cm.display.cachedTextHeight = cm.display.cachedPaddingH = null;
-  if (!cm.options.lineWrapping) { cm.display.maxLineChanged = true; }
-  cm.display.lineNumChars = null;
-}
-
-function pageScrollX() {
-  // Work around https://bugs.chromium.org/p/chromium/issues/detail?id=489206
-  // which causes page_Offset and bounding client rects to use
-  // different reference viewports and invalidate our calculations.
-  if (chrome && android) { return -(document.body.getBoundingClientRect().left - parseInt(getComputedStyle(document.body).marginLeft)) }
-  return window.pageXOffset || (document.documentElement || document.body).scrollLeft
-}
-function pageScrollY() {
-  if (chrome && android) { return -(document.body.getBoundingClientRect().top - parseInt(getComputedStyle(document.body).marginTop)) }
-  return window.pageYOffset || (document.documentElement || document.body).scrollTop
-}
-
-function widgetTopHeight(lineObj) {
-  var height = 0;
-  if (lineObj.widgets) { for (var i = 0; i < lineObj.widgets.length; ++i) { if (lineObj.widgets[i].above)
-    { height += widgetHeight(lineObj.widgets[i]); } } }
-  return height
-}
-
-// Converts a {top, bottom, left, right} box from line-local
-// coordinates into another coordinate system. Context may be one of
-// "line", "div" (display.lineDiv), "local"./null (editor), "window",
-// or "page".
-function intoCoordSystem(cm, lineObj, rect, context, includeWidgets) {
-  if (!includeWidgets) {
-    var height = widgetTopHeight(lineObj);
-    rect.top += height; rect.bottom += height;
-  }
-  if (context == "line") { return rect }
-  if (!context) { context = "local"; }
-  var yOff = heightAtLine(lineObj);
-  if (context == "local") { yOff += paddingTop(cm.display); }
-  else { yOff -= cm.display.viewOffset; }
-  if (context == "page" || context == "window") {
-    var lOff = cm.display.lineSpace.getBoundingClientRect();
-    yOff += lOff.top + (context == "window" ? 0 : pageScrollY());
-    var xOff = lOff.left + (context == "window" ? 0 : pageScrollX());
-    rect.left += xOff; rect.right += xOff;
-  }
-  rect.top += yOff; rect.bottom += yOff;
-  return rect
-}
-
-// Coverts a box from "div" coords to another coordinate system.
-// Context may be "window", "page", "div", or "local"./null.
-function fromCoordSystem(cm, coords, context) {
-  if (context == "div") { return coords }
-  var left = coords.left, top = coords.top;
-  // First move into "page" coordinate system
-  if (context == "page") {
-    left -= pageScrollX();
-    top -= pageScrollY();
-  } else if (context == "local" || !context) {
-    var localBox = cm.display.sizer.getBoundingClientRect();
-    left += localBox.left;
-    top += localBox.top;
-  }
-
-  var lineSpaceBox = cm.display.lineSpace.getBoundingClientRect();
-  return {left: left - lineSpaceBox.left, top: top - lineSpaceBox.top}
-}
-
-function charCoords(cm, pos, context, lineObj, bias) {
-  if (!lineObj) { lineObj = getLine(cm.doc, pos.line); }
-  return intoCoordSystem(cm, lineObj, measureChar(cm, lineObj, pos.ch, bias), context)
-}
-
-// Returns a box for a given cursor position, which may have an
-// 'other' property containing the position of the secondary cursor
-// on a bidi boundary.
-// A cursor Pos(line, char, "before") is on the same visual line as `char - 1`
-// and after `char - 1` in writing order of `char - 1`
-// A cursor Pos(line, char, "after") is on the same visual line as `char`
-// and before `char` in writing order of `char`
-// Examples (upper-case letters are RTL, lower-case are LTR):
-//     Pos(0, 1, ...)
-//     before   after
-// ab     a|b     a|b
-// aB     a|B     aB|
-// Ab     |Ab     A|b
-// AB     B|A     B|A
-// Every position after the last character on a line is considered to stick
-// to the last character on the line.
-function cursorCoords(cm, pos, context, lineObj, preparedMeasure, varHeight) {
-  lineObj = lineObj || getLine(cm.doc, pos.line);
-  if (!preparedMeasure) { preparedMeasure = prepareMeasureForLine(cm, lineObj); }
-  function get(ch, right) {
-    var m = measureCharPrepared(cm, preparedMeasure, ch, right ? "right" : "left", varHeight);
-    if (right) { m.left = m.right; } else { m.right = m.left; }
-    return intoCoordSystem(cm, lineObj, m, context)
-  }
-  var order = getOrder(lineObj, cm.doc.direction), ch = pos.ch, sticky = pos.sticky;
-  if (ch >= lineObj.text.length) {
-    ch = lineObj.text.length;
-    sticky = "before";
-  } else if (ch <= 0) {
-    ch = 0;
-    sticky = "after";
-  }
-  if (!order) { return get(sticky == "before" ? ch - 1 : ch, sticky == "before") }
-
-  function getBidi(ch, partPos, invert) {
-    var part = order[partPos], right = part.level == 1;
-    return get(invert ? ch - 1 : ch, right != invert)
-  }
-  var partPos = getBidiPartAt(order, ch, sticky);
-  var other = bidiOther;
-  var val = getBidi(ch, partPos, sticky == "before");
-  if (other != null) { val.other = getBidi(ch, other, sticky != "before"); }
-  return val
-}
-
-// Used to cheaply estimate the coordinates for a position. Used for
-// intermediate scroll updates.
-function estimateCoords(cm, pos) {
-  var left = 0;
-  pos = clipPos(cm.doc, pos);
-  if (!cm.options.lineWrapping) { left = charWidth(cm.display) * pos.ch; }
-  var lineObj = getLine(cm.doc, pos.line);
-  var top = heightAtLine(lineObj) + paddingTop(cm.display);
-  return {left: left, right: left, top: top, bottom: top + lineObj.height}
-}
-
-// Positions returned by coordsChar contain some extra information.
-// xRel is the relative x position of the input coordinates compared
-// to the found position (so xRel > 0 means the coordinates are to
-// the right of the character position, for example). When outside
-// is true, that means the coordinates lie outside the line's
-// vertical range.
-function PosWithInfo(line, ch, sticky, outside, xRel) {
-  var pos = Pos(line, ch, sticky);
-  pos.xRel = xRel;
-  if (outside) { pos.outside = true; }
-  return pos
-}
-
-// Compute the character position closest to the given coordinates.
-// Input must be lineSpace-local ("div" coordinate system).
-function coordsChar(cm, x, y) {
-  var doc = cm.doc;
-  y += cm.display.viewOffset;
-  if (y < 0) { return PosWithInfo(doc.first, 0, null, true, -1) }
-  var lineN = lineAtHeight(doc, y), last = doc.first + doc.size - 1;
-  if (lineN > last)
-    { return PosWithInfo(doc.first + doc.size - 1, getLine(doc, last).text.length, null, true, 1) }
-  if (x < 0) { x = 0; }
-
-  var lineObj = getLine(doc, lineN);
-  for (;;) {
-    var found = coordsCharInner(cm, lineObj, lineN, x, y);
-    var merged = collapsedSpanAtEnd(lineObj);
-    var mergedPos = merged && merged.find(0, true);
-    if (merged && (found.ch > mergedPos.from.ch || found.ch == mergedPos.from.ch && found.xRel > 0))
-      { lineN = lineNo(lineObj = mergedPos.to.line); }
-    else
-      { return found }
-  }
-}
-
-function wrappedLineExtent(cm, lineObj, preparedMeasure, y) {
-  y -= widgetTopHeight(lineObj);
-  var end = lineObj.text.length;
-  var begin = findFirst(function (ch) { return measureCharPrepared(cm, preparedMeasure, ch - 1).bottom <= y; }, end, 0);
-  end = findFirst(function (ch) { return measureCharPrepared(cm, preparedMeasure, ch).top > y; }, begin, end);
-  return {begin: begin, end: end}
-}
-
-function wrappedLineExtentChar(cm, lineObj, preparedMeasure, target) {
-  if (!preparedMeasure) { preparedMeasure = prepareMeasureForLine(cm, lineObj); }
-  var targetTop = intoCoordSystem(cm, lineObj, measureCharPrepared(cm, preparedMeasure, target), "line").top;
-  return wrappedLineExtent(cm, lineObj, preparedMeasure, targetTop)
-}
-
-// Returns true if the given side of a box is after the given
-// coordinates, in top-to-bottom, left-to-right order.
-function boxIsAfter(box, x, y, left) {
-  return box.bottom <= y ? false : box.top > y ? true : (left ? box.left : box.right) > x
-}
-
-function coordsCharInner(cm, lineObj, lineNo$$1, x, y) {
-  // Move y into line-local coordinate space
-  y -= heightAtLine(lineObj);
-  var preparedMeasure = prepareMeasureForLine(cm, lineObj);
-  // When directly calling `measureCharPrepared`, we have to adjust
-  // for the widgets at this line.
-  var widgetHeight$$1 = widgetTopHeight(lineObj);
-  var begin = 0, end = lineObj.text.length, ltr = true;
-
-  var order = getOrder(lineObj, cm.doc.direction);
-  // If the line isn't plain left-to-right text, first figure out
-  // which bidi section the coordinates fall into.
-  if (order) {
-    var part = (cm.options.lineWrapping ? coordsBidiPartWrapped : coordsBidiPart)
-                 (cm, lineObj, lineNo$$1, preparedMeasure, order, x, y);
-    ltr = part.level != 1;
-    // The awkward -1 offsets are needed because findFirst (called
-    // on these below) will treat its first bound as inclusive,
-    // second as exclusive, but we want to actually address the
-    // characters in the part's range
-    begin = ltr ? part.from : part.to - 1;
-    end = ltr ? part.to : part.from - 1;
-  }
-
-  // A binary search to find the first character whose bounding box
-  // starts after the coordinates. If we run across any whose box wrap
-  // the coordinates, store that.
-  var chAround = null, boxAround = null;
-  var ch = findFirst(function (ch) {
-    var box = measureCharPrepared(cm, preparedMeasure, ch);
-    box.top += widgetHeight$$1; box.bottom += widgetHeight$$1;
-    if (!boxIsAfter(box, x, y, false)) { return false }
-    if (box.top <= y && box.left <= x) {
-      chAround = ch;
-      boxAround = box;
-    }
-    return true
-  }, begin, end);
-
-  var baseX, sticky, outside = false;
-  // If a box around the coordinates was found, use that
-  if (boxAround) {
-    // Distinguish coordinates nearer to the left or right side of the box
-    var atLeft = x - boxAround.left < boxAround.right - x, atStart = atLeft == ltr;
-    ch = chAround + (atStart ? 0 : 1);
-    sticky = atStart ? "after" : "before";
-    baseX = atLeft ? boxAround.left : boxAround.right;
-  } else {
-    // (Adjust for extended bound, if necessary.)
-    if (!ltr && (ch == end || ch == begin)) { ch++; }
-    // To determine which side to associate with, get the box to the
-    // left of the character and compare it's vertical position to the
-    // coordinates
-    sticky = ch == 0 ? "after" : ch == lineObj.text.length ? "before" :
-      (measureCharPrepared(cm, preparedMeasure, ch - (ltr ? 1 : 0)).bottom + widgetHeight$$1 <= y) == ltr ?
-      "after" : "before";
-    // Now get accurate coordinates for this place, in order to get a
-    // base X position
-    var coords = cursorCoords(cm, Pos(lineNo$$1, ch, sticky), "line", lineObj, preparedMeasure);
-    baseX = coords.left;
-    outside = y < coords.top || y >= coords.bottom;
-  }
-
-  ch = skipExtendingChars(lineObj.text, ch, 1);
-  return PosWithInfo(lineNo$$1, ch, sticky, outside, x - baseX)
-}
-
-function coordsBidiPart(cm, lineObj, lineNo$$1, preparedMeasure, order, x, y) {
-  // Bidi parts are sorted left-to-right, and in a non-line-wrapping
-  // situation, we can take this ordering to correspond to the visual
-  // ordering. This finds the first part whose end is after the given
-  // coordinates.
-  var index = findFirst(function (i) {
-    var part = order[i], ltr = part.level != 1;
-    return boxIsAfter(cursorCoords(cm, Pos(lineNo$$1, ltr ? part.to : part.from, ltr ? "before" : "after"),
-                                   "line", lineObj, preparedMeasure), x, y, true)
-  }, 0, order.length - 1);
-  var part = order[index];
-  // If this isn't the first part, the part's start is also after
-  // the coordinates, and the coordinates aren't on the same line as
-  // that start, move one part back.
-  if (index > 0) {
-    var ltr = part.level != 1;
-    var start = cursorCoords(cm, Pos(lineNo$$1, ltr ? part.from : part.to, ltr ? "after" : "before"),
-                             "line", lineObj, preparedMeasure);
-    if (boxIsAfter(start, x, y, true) && start.top > y)
-      { part = order[index - 1]; }
-  }
-  return part
-}
-
-function coordsBidiPartWrapped(cm, lineObj, _lineNo, preparedMeasure, order, x, y) {
-  // In a wrapped line, rtl text on wrapping boundaries can do things
-  // that don't correspond to the ordering in our `order` array at
-  // all, so a binary search doesn't work, and we want to return a
-  // part that only spans one line so that the binary search in
-  // coordsCharInner is safe. As such, we first find the extent of the
-  // wrapped line, and then do a flat search in which we discard any
-  // spans that aren't on the line.
-  var ref = wrappedLineExtent(cm, lineObj, preparedMeasure, y);
-  var begin = ref.begin;
-  var end = ref.end;
-  if (/\s/.test(lineObj.text.charAt(end - 1))) { end--; }
-  var part = null, closestDist = null;
-  for (var i = 0; i < order.length; i++) {
-    var p = order[i];
-    if (p.from >= end || p.to <= begin) { continue }
-    var ltr = p.level != 1;
-    var endX = measureCharPrepared(cm, preparedMeasure, ltr ? Math.min(end, p.to) - 1 : Math.max(begin, p.from)).right;
-    // Weigh against spans ending before this, so that they are only
-    // picked if nothing ends after
-    var dist = endX < x ? x - endX + 1e9 : endX - x;
-    if (!part || closestDist > dist) {
-      part = p;
-      closestDist = dist;
-    }
-  }
-  if (!part) { part = order[order.length - 1]; }
-  // Clip the part to the wrapped line.
-  if (part.from < begin) { part = {from: begin, to: part.to, level: part.level}; }
-  if (part.to > end) { part = {from: part.from, to: end, level: part.level}; }
-  return part
-}
-
-var measureText;
-// Compute the default text height.
-function textHeight(display) {
-  if (display.cachedTextHeight != null) { return display.cachedTextHeight }
-  if (measureText == null) {
-    measureText = elt("pre");
-    // Measure a bunch of lines, for browsers that compute
-    // fractional heights.
-    for (var i = 0; i < 49; ++i) {
-      measureText.appendChild(document.createTextNode("x"));
-      measureText.appendChild(elt("br"));
-    }
-    measureText.appendChild(document.createTextNode("x"));
-  }
-  removeChildrenAndAdd(display.measure, measureText);
-  var height = measureText.offsetHeight / 50;
-  if (height > 3) { display.cachedTextHeight = height; }
-  removeChildren(display.measure);
-  return height || 1
-}
-
-// Compute the default character width.
-function charWidth(display) {
-  if (display.cachedCharWidth != null) { return display.cachedCharWidth }
-  var anchor = elt("span", "xxxxxxxxxx");
-  var pre = elt("pre", [anchor]);
-  removeChildrenAndAdd(display.measure, pre);
-  var rect = anchor.getBoundingClientRect(), width = (rect.right - rect.left) / 10;
-  if (width > 2) { display.cachedCharWidth = width; }
-  return width || 10
-}
-
-// Do a bulk-read of the DOM positions and sizes needed to draw the
-// view, so that we don't interleave reading and writing to the DOM.
-function getDimensions(cm) {
-  var d = cm.display, left = {}, width = {};
-  var gutterLeft = d.gutters.clientLeft;
-  for (var n = d.gutters.firstChild, i = 0; n; n = n.nextSibling, ++i) {
-    left[cm.options.gutters[i]] = n.offsetLeft + n.clientLeft + gutterLeft;
-    width[cm.options.gutters[i]] = n.clientWidth;
-  }
-  return {fixedPos: compensateForHScroll(d),
-          gutterTotalWidth: d.gutters.offsetWidth,
-          gutterLeft: left,
-          gutterWidth: width,
-          wrapperWidth: d.wrapper.clientWidth}
-}
-
-// Computes display.scroller.scrollLeft + display.gutters.offsetWidth,
-// but using getBoundingClientRect to get a sub-pixel-accurate
-// result.
-function compensateForHScroll(display) {
-  return display.scroller.getBoundingClientRect().left - display.sizer.getBoundingClientRect().left
-}
-
-// Returns a function that estimates the height of a line, to use as
-// first approximation until the line becomes visible (and is thus
-// properly measurable).
-function estimateHeight(cm) {
-  var th = textHeight(cm.display), wrapping = cm.options.lineWrapping;
-  var perLine = wrapping && Math.max(5, cm.display.scroller.clientWidth / charWidth(cm.display) - 3);
-  return function (line) {
-    if (lineIsHidden(cm.doc, line)) { return 0 }
-
-    var widgetsHeight = 0;
-    if (line.widgets) { for (var i = 0; i < line.widgets.length; i++) {
-      if (line.widgets[i].height) { widgetsHeight += line.widgets[i].height; }
-    } }
-
-    if (wrapping)
-      { return widgetsHeight + (Math.ceil(line.text.length / perLine) || 1) * th }
-    else
-      { return widgetsHeight + th }
-  }
-}
-
-function estimateLineHeights(cm) {
-  var doc = cm.doc, est = estimateHeight(cm);
-  doc.iter(function (line) {
-    var estHeight = est(line);
-    if (estHeight != line.height) { updateLineHeight(line, estHeight); }
-  });
-}
-
-// Given a mouse event, find the corresponding position. If liberal
-// is false, it checks whether a gutter or scrollbar was clicked,
-// and returns null if it was. forRect is used by rectangular
-// selections, and tries to estimate a character position even for
-// coordinates beyond the right of the text.
-function posFromMouse(cm, e, liberal, forRect) {
-  var display = cm.display;
-  if (!liberal && e_target(e).getAttribute("cm-not-content") == "true") { return null }
-
-  var x, y, space = display.lineSpace.getBoundingClientRect();
-  // Fails unpredictably on IE[67] when mouse is dragged around quickly.
-  try { x = e.clientX - space.left; y = e.clientY - space.top; }
-  catch (e) { return null }
-  var coords = coordsChar(cm, x, y), line;
-  if (forRect && coords.xRel == 1 && (line = getLine(cm.doc, coords.line).text).length == coords.ch) {
-    var colDiff = countColumn(line, line.length, cm.options.tabSize) - line.length;
-    coords = Pos(coords.line, Math.max(0, Math.round((x - paddingH(cm.display).left) / charWidth(cm.display)) - colDiff));
-  }
-  return coords
-}
-
-// Find the view element corresponding to a given line. Return null
-// when the line isn't visible.
-function findViewIndex(cm, n) {
-  if (n >= cm.display.viewTo) { return null }
-  n -= cm.display.viewFrom;
-  if (n < 0) { return null }
-  var view = cm.display.view;
-  for (var i = 0; i < view.length; i++) {
-    n -= view[i].size;
-    if (n < 0) { return i }
-  }
-}
-
-function updateSelection(cm) {
-  cm.display.input.showSelection(cm.display.input.prepareSelection());
-}
-
-function prepareSelection(cm, primary) {
-  if ( primary === void 0 ) primary = true;
-
-  var doc = cm.doc, result = {};
-  var curFragment = result.cursors = document.createDocumentFragment();
-  var selFragment = result.selection = document.createDocumentFragment();
-
-  for (var i = 0; i < doc.sel.ranges.length; i++) {
-    if (!primary && i == doc.sel.primIndex) { continue }
-    var range$$1 = doc.sel.ranges[i];
-    if (range$$1.from().line >= cm.display.viewTo || range$$1.to().line < cm.display.viewFrom) { continue }
-    var collapsed = range$$1.empty();
-    if (collapsed || cm.options.showCursorWhenSelecting)
-      { drawSelectionCursor(cm, range$$1.head, curFragment); }
-    if (!collapsed)
-      { drawSelectionRange(cm, range$$1, selFragment); }
-  }
-  return result
-}
-
-// Draws a cursor for the given range
-function drawSelectionCursor(cm, head, output) {
-  var pos = cursorCoords(cm, head, "div", null, null, !cm.options.singleCursorHeightPerLine);
-
-  var cursor = output.appendChild(elt("div", "\u00a0", "CodeMirror-cursor"));
-  cursor.style.left = pos.left + "px";
-  cursor.style.top = pos.top + "px";
-  cursor.style.height = Math.max(0, pos.bottom - pos.top) * cm.options.cursorHeight + "px";
-
-  if (pos.other) {
-    // Secondary cursor, shown when on a 'jump' in bi-directional text
-    var otherCursor = output.appendChild(elt("div", "\u00a0", "CodeMirror-cursor CodeMirror-secondarycursor"));
-    otherCursor.style.display = "";
-    otherCursor.style.left = pos.other.left + "px";
-    otherCursor.style.top = pos.other.top + "px";
-    otherCursor.style.height = (pos.other.bottom - pos.other.top) * .85 + "px";
-  }
-}
-
-function cmpCoords(a, b) { return a.top - b.top || a.left - b.left }
-
-// Draws the given range as a highlighted selection
-function drawSelectionRange(cm, range$$1, output) {
-  var display = cm.display, doc = cm.doc;
-  var fragment = document.createDocumentFragment();
-  var padding = paddingH(cm.display), leftSide = padding.left;
-  var rightSide = Math.max(display.sizerWidth, displayWidth(cm) - display.sizer.offsetLeft) - padding.right;
-  var docLTR = doc.direction == "ltr";
-
-  function add(left, top, width, bottom) {
-    if (top < 0) { top = 0; }
-    top = Math.round(top);
-    bottom = Math.round(bottom);
-    fragment.appendChild(elt("div", null, "CodeMirror-selected", ("position: absolute; left: " + left + "px;\n                             top: " + top + "px; width: " + (width == null ? rightSide - left : width) + "px;\n                             height: " + (bottom - top) + "px")));
-  }
-
-  function drawForLine(line, fromArg, toArg) {
-    var lineObj = getLine(doc, line);
-    var lineLen = lineObj.text.length;
-    var start, end;
-    function coords(ch, bias) {
-      return charCoords(cm, Pos(line, ch), "div", lineObj, bias)
-    }
-
-    function wrapX(pos, dir, side) {
-      var extent = wrappedLineExtentChar(cm, lineObj, null, pos);
-      var prop = (dir == "ltr") == (side == "after") ? "left" : "right";
-      var ch = side == "after" ? extent.begin : extent.end - (/\s/.test(lineObj.text.charAt(extent.end - 1)) ? 2 : 1);
-      return coords(ch, prop)[prop]
-    }
-
-    var order = getOrder(lineObj, doc.direction);
-    iterateBidiSections(order, fromArg || 0, toArg == null ? lineLen : toArg, function (from, to, dir, i) {
-      var ltr = dir == "ltr";
-      var fromPos = coords(from, ltr ? "left" : "right");
-      var toPos = coords(to - 1, ltr ? "right" : "left");
-
-      var openStart = fromArg == null && from == 0, openEnd = toArg == null && to == lineLen;
-      var first = i == 0, last = !order || i == order.length - 1;
-      if (toPos.top - fromPos.top <= 3) { // Single line
-        var openLeft = (docLTR ? openStart : openEnd) && first;
-        var openRight = (docLTR ? openEnd : openStart) && last;
-        var left = openLeft ? leftSide : (ltr ? fromPos : toPos).left;
-        var right = openRight ? rightSide : (ltr ? toPos : fromPos).right;
-        add(left, fromPos.top, right - left, fromPos.bottom);
-      } else { // Multiple lines
-        var topLeft, topRight, botLeft, botRight;
-        if (ltr) {
-          topLeft = docLTR && openStart && first ? leftSide : fromPos.left;
-          topRight = docLTR ? rightSide : wrapX(from, dir, "before");
-          botLeft = docLTR ? leftSide : wrapX(to, dir, "after");
-          botRight = docLTR && openEnd && last ? rightSide : toPos.right;
-        } else {
-          topLeft = !docLTR ? leftSide : wrapX(from, dir, "before");
-          topRight = !docLTR && openStart && first ? rightSide : fromPos.right;
-          botLeft = !docLTR && openEnd && last ? leftSide : toPos.left;
-          botRight = !docLTR ? rightSide : wrapX(to, dir, "after");
-        }
-        add(topLeft, fromPos.top, topRight - topLeft, fromPos.bottom);
-        if (fromPos.bottom < toPos.top) { add(leftSide, fromPos.bottom, null, toPos.top); }
-        add(botLeft, toPos.top, botRight - botLeft, toPos.bottom);
-      }
-
-      if (!start || cmpCoords(fromPos, start) < 0) { start = fromPos; }
-      if (cmpCoords(toPos, start) < 0) { start = toPos; }
-      if (!end || cmpCoords(fromPos, end) < 0) { end = fromPos; }
-      if (cmpCoords(toPos, end) < 0) { end = toPos; }
-    });
-    return {start: start, end: end}
-  }
-
-  var sFrom = range$$1.from(), sTo = range$$1.to();
-  if (sFrom.line == sTo.line) {
-    drawForLine(sFrom.line, sFrom.ch, sTo.ch);
-  } else {
-    var fromLine = getLine(doc, sFrom.line), toLine = getLine(doc, sTo.line);
-    var singleVLine = visualLine(fromLine) == visualLine(toLine);
-    var leftEnd = drawForLine(sFrom.line, sFrom.ch, singleVLine ? fromLine.text.length + 1 : null).end;
-    var rightStart = drawForLine(sTo.line, singleVLine ? 0 : null, sTo.ch).start;
-    if (singleVLine) {
-      if (leftEnd.top < rightStart.top - 2) {
-        add(leftEnd.right, leftEnd.top, null, leftEnd.bottom);
-        add(leftSide, rightStart.top, rightStart.left, rightStart.bottom);
-      } else {
-        add(leftEnd.right, leftEnd.top, rightStart.left - leftEnd.right, leftEnd.bottom);
-      }
-    }
-    if (leftEnd.bottom < rightStart.top)
-      { add(leftSide, leftEnd.bottom, null, rightStart.top); }
-  }
-
-  output.appendChild(fragment);
-}
-
-// Cursor-blinking
-function restartBlink(cm) {
-  if (!cm.state.focused) { return }
-  var display = cm.display;
-  clearInterval(display.blinker);
-  var on = true;
-  display.cursorDiv.style.visibility = "";
-  if (cm.options.cursorBlinkRate > 0)
-    { display.blinker = setInterval(function () { return display.cursorDiv.style.visibility = (on = !on) ? "" : "hidden"; },
-      cm.options.cursorBlinkRate); }
-  else if (cm.options.cursorBlinkRate < 0)
-    { display.cursorDiv.style.visibility = "hidden"; }
-}
-
-function ensureFocus(cm) {
-  if (!cm.state.focused) { cm.display.input.focus(); onFocus(cm); }
-}
-
-function delayBlurEvent(cm) {
-  cm.state.delayingBlurEvent = true;
-  setTimeout(function () { if (cm.state.delayingBlurEvent) {
-    cm.state.delayingBlurEvent = false;
-    onBlur(cm);
-  } }, 100);
-}
-
-function onFocus(cm, e) {
-  if (cm.state.delayingBlurEvent) { cm.state.delayingBlurEvent = false; }
-
-  if (cm.options.readOnly == "nocursor") { return }
-  if (!cm.state.focused) {
-    signal(cm, "focus", cm, e);
-    cm.state.focused = true;
-    addClass(cm.display.wrapper, "CodeMirror-focused");
-    // This test prevents this from firing when a context
-    // menu is closed (since the input reset would kill the
-    // select-all detection hack)
-    if (!cm.curOp && cm.display.selForContextMenu != cm.doc.sel) {
-      cm.display.input.reset();
-      if (webkit) { setTimeout(function () { return cm.display.input.reset(true); }, 20); } // Issue #1730
-    }
-    cm.display.input.receivedFocus();
-  }
-  restartBlink(cm);
-}
-function onBlur(cm, e) {
-  if (cm.state.delayingBlurEvent) { return }
-
-  if (cm.state.focused) {
-    signal(cm, "blur", cm, e);
-    cm.state.focused = false;
-    rmClass(cm.display.wrapper, "CodeMirror-focused");
-  }
-  clearInterval(cm.display.blinker);
-  setTimeout(function () { if (!cm.state.focused) { cm.display.shift = false; } }, 150);
-}
-
-// Read the actual heights of the rendered lines, and update their
-// stored heights to match.
-function updateHeightsInViewport(cm) {
-  var display = cm.display;
-  var prevBottom = display.lineDiv.offsetTop;
-  for (var i = 0; i < display.view.length; i++) {
-    var cur = display.view[i], height = (void 0);
-    if (cur.hidden) { continue }
-    if (ie && ie_version < 8) {
-      var bot = cur.node.offsetTop + cur.node.offsetHeight;
-      height = bot - prevBottom;
-      prevBottom = bot;
-    } else {
-      var box = cur.node.getBoundingClientRect();
-      height = box.bottom - box.top;
-    }
-    var diff = cur.line.height - height;
-    if (height < 2) { height = textHeight(display); }
-    if (diff > .005 || diff < -.005) {
-      updateLineHeight(cur.line, height);
-      updateWidgetHeight(cur.line);
-      if (cur.rest) { for (var j = 0; j < cur.rest.length; j++)
-        { updateWidgetHeight(cur.rest[j]); } }
-    }
-  }
-}
-
-// Read and store the height of line widgets associated with the
-// given line.
-function updateWidgetHeight(line) {
-  if (line.widgets) { for (var i = 0; i < line.widgets.length; ++i)
-    { line.widgets[i].height = line.widgets[i].node.parentNode.offsetHeight; } }
-}
-
-// Compute the lines that are visible in a given viewport (defaults
-// the the current scroll position). viewport may contain top,
-// height, and ensure (see op.scrollToPos) properties.
-function visibleLines(display, doc, viewport) {
-  var top = viewport && viewport.top != null ? Math.max(0, viewport.top) : display.scroller.scrollTop;
-  top = Math.floor(top - paddingTop(display));
-  var bottom = viewport && viewport.bottom != null ? viewport.bottom : top + display.wrapper.clientHeight;
-
-  var from = lineAtHeight(doc, top), to = lineAtHeight(doc, bottom);
-  // Ensure is a {from: {line, ch}, to: {line, ch}} object, and
-  // forces those lines into the viewport (if possible).
-  if (viewport && viewport.ensure) {
-    var ensureFrom = viewport.ensure.from.line, ensureTo = viewport.ensure.to.line;
-    if (ensureFrom < from) {
-      from = ensureFrom;
-      to = lineAtHeight(doc, heightAtLine(getLine(doc, ensureFrom)) + display.wrapper.clientHeight);
-    } else if (Math.min(ensureTo, doc.lastLine()) >= to) {
-      from = lineAtHeight(doc, heightAtLine(getLine(doc, ensureTo)) - display.wrapper.clientHeight);
-      to = ensureTo;
-    }
-  }
-  return {from: from, to: Math.max(to, from + 1)}
-}
-
-// Re-align line numbers and gutter marks to compensate for
-// horizontal scrolling.
-function alignHorizontally(cm) {
-  var display = cm.display, view = display.view;
-  if (!display.alignWidgets && (!display.gutters.firstChild || !cm.options.fixedGutter)) { return }
-  var comp = compensateForHScroll(display) - display.scroller.scrollLeft + cm.doc.scrollLeft;
-  var gutterW = display.gutters.offsetWidth, left = comp + "px";
-  for (var i = 0; i < view.length; i++) { if (!view[i].hidden) {
-    if (cm.options.fixedGutter) {
-      if (view[i].gutter)
-        { view[i].gutter.style.left = left; }
-      if (view[i].gutterBackground)
-        { view[i].gutterBackground.style.left = left; }
-    }
-    var align = view[i].alignable;
-    if (align) { for (var j = 0; j < align.length; j++)
-      { align[j].style.left = left; } }
-  } }
-  if (cm.options.fixedGutter)
-    { display.gutters.style.left = (comp + gutterW) + "px"; }
-}
-
-// Used to ensure that the line number gutter is still the right
-// size for the current document size. Returns true when an update
-// is needed.
-function maybeUpdateLineNumberWidth(cm) {
-  if (!cm.options.lineNumbers) { return false }
-  var doc = cm.doc, last = lineNumberFor(cm.options, doc.first + doc.size - 1), display = cm.display;
-  if (last.length != display.lineNumChars) {
-    var test = display.measure.appendChild(elt("div", [elt("div", last)],
-                                               "CodeMirror-linenumber CodeMirror-gutter-elt"));
-    var innerW = test.firstChild.offsetWidth, padding = test.offsetWidth - innerW;
-    display.lineGutter.style.width = "";
-    display.lineNumInnerWidth = Math.max(innerW, display.lineGutter.offsetWidth - padding) + 1;
-    display.lineNumWidth = display.lineNumInnerWidth + padding;
-    display.lineNumChars = display.lineNumInnerWidth ? last.length : -1;
-    display.lineGutter.style.width = display.lineNumWidth + "px";
-    updateGutterSpace(cm);
-    return true
-  }
-  return false
-}
-
-// SCROLLING THINGS INTO VIEW
-
-// If an editor sits on the top or bottom of the window, partially
-// scrolled out of view, this ensures that the cursor is visible.
-function maybeScrollWindow(cm, rect) {
-  if (signalDOMEvent(cm, "scrollCursorIntoView")) { return }
-
-  var display = cm.display, box = display.sizer.getBoundingClientRect(), doScroll = null;
-  if (rect.top + box.top < 0) { doScroll = true; }
-  else if (rect.bottom + box.top > (window.innerHeight || document.documentElement.clientHeight)) { doScroll = false; }
-  if (doScroll != null && !phantom) {
-    var scrollNode = elt("div", "\u200b", null, ("position: absolute;\n                         top: " + (rect.top - display.viewOffset - paddingTop(cm.display)) + "px;\n                         height: " + (rect.bottom - rect.top + scrollGap(cm) + display.barHeight) + "px;\n                         left: " + (rect.left) + "px; width: " + (Math.max(2, rect.right - rect.left)) + "px;"));
-    cm.display.lineSpace.appendChild(scrollNode);
-    scrollNode.scrollIntoView(doScroll);
-    cm.display.lineSpace.removeChild(scrollNode);
-  }
-}
-
-// Scroll a given position into view (immediately), verifying that
-// it actually became visible (as line heights are accurately
-// measured, the position of something may 'drift' during drawing).
-function scrollPosIntoView(cm, pos, end, margin) {
-  if (margin == null) { margin = 0; }
-  var rect;
-  if (!cm.options.lineWrapping && pos == end) {
-    // Set pos and end to the cursor positions around the character pos sticks to
-    // If pos.sticky == "before", that is around pos.ch - 1, otherwise around pos.ch
-    // If pos == Pos(_, 0, "before"), pos and end are unchanged
-    pos = pos.ch ? Pos(pos.line, pos.sticky == "before" ? pos.ch - 1 : pos.ch, "after") : pos;
-    end = pos.sticky == "before" ? Pos(pos.line, pos.ch + 1, "before") : pos;
-  }
-  for (var limit = 0; limit < 5; limit++) {
-    var changed = false;
-    var coords = cursorCoords(cm, pos);
-    var endCoords = !end || end == pos ? coords : cursorCoords(cm, end);
-    rect = {left: Math.min(coords.left, endCoords.left),
-            top: Math.min(coords.top, endCoords.top) - margin,
-            right: Math.max(coords.left, endCoords.left),
-            bottom: Math.max(coords.bottom, endCoords.bottom) + margin};
-    var scrollPos = calculateScrollPos(cm, rect);
-    var startTop = cm.doc.scrollTop, startLeft = cm.doc.scrollLeft;
-    if (scrollPos.scrollTop != null) {
-      updateScrollTop(cm, scrollPos.scrollTop);
-      if (Math.abs(cm.doc.scrollTop - startTop) > 1) { changed = true; }
-    }
-    if (scrollPos.scrollLeft != null) {
-      setScrollLeft(cm, scrollPos.scrollLeft);
-      if (Math.abs(cm.doc.scrollLeft - startLeft) > 1) { changed = true; }
-    }
-    if (!changed) { break }
-  }
-  return rect
-}
-
-// Scroll a given set of coordinates into view (immediately).
-function scrollIntoView(cm, rect) {
-  var scrollPos = calculateScrollPos(cm, rect);
-  if (scrollPos.scrollTop != null) { updateScrollTop(cm, scrollPos.scrollTop); }
-  if (scrollPos.scrollLeft != null) { setScrollLeft(cm, scrollPos.scrollLeft); }
-}
-
-// Calculate a new scroll position needed to scroll the given
-// rectangle into view. Returns an object with scrollTop and
-// scrollLeft properties. When these are undefined, the
-// vertical/horizontal position does not need to be adjusted.
-function calculateScrollPos(cm, rect) {
-  var display = cm.display, snapMargin = textHeight(cm.display);
-  if (rect.top < 0) { rect.top = 0; }
-  var screentop = cm.curOp && cm.curOp.scrollTop != null ? cm.curOp.scrollTop : display.scroller.scrollTop;
-  var screen = displayHeight(cm), result = {};
-  if (rect.bottom - rect.top > screen) { rect.bottom = rect.top + screen; }
-  var docBottom = cm.doc.height + paddingVert(display);
-  var atTop = rect.top < snapMargin, atBottom = rect.bottom > docBottom - snapMargin;
-  if (rect.top < screentop) {
-    result.scrollTop = atTop ? 0 : rect.top;
-  } else if (rect.bottom > screentop + screen) {
-    var newTop = Math.min(rect.top, (atBottom ? docBottom : rect.bottom) - screen);
-    if (newTop != screentop) { result.scrollTop = newTop; }
-  }
-
-  var screenleft = cm.curOp && cm.curOp.scrollLeft != null ? cm.curOp.scrollLeft : display.scroller.scrollLeft;
-  var screenw = displayWidth(cm) - (cm.options.fixedGutter ? display.gutters.offsetWidth : 0);
-  var tooWide = rect.right - rect.left > screenw;
-  if (tooWide) { rect.right = rect.left + screenw; }
-  if (rect.left < 10)
-    { result.scrollLeft = 0; }
-  else if (rect.left < screenleft)
-    { result.scrollLeft = Math.max(0, rect.left - (tooWide ? 0 : 10)); }
-  else if (rect.right > screenw + screenleft - 3)
-    { result.scrollLeft = rect.right + (tooWide ? 0 : 10) - screenw; }
-  return result
-}
-
-// Store a relative adjustment to the scroll position in the current
-// operation (to be applied when the operation finishes).
-function addToScrollTop(cm, top) {
-  if (top == null) { return }
-  resolveScrollToPos(cm);
-  cm.curOp.scrollTop = (cm.curOp.scrollTop == null ? cm.doc.scrollTop : cm.curOp.scrollTop) + top;
-}
-
-// Make sure that at the end of the operation the current cursor is
-// shown.
-function ensureCursorVisible(cm) {
-  resolveScrollToPos(cm);
-  var cur = cm.getCursor();
-  cm.curOp.scrollToPos = {from: cur, to: cur, margin: cm.options.cursorScrollMargin};
-}
-
-function scrollToCoords(cm, x, y) {
-  if (x != null || y != null) { resolveScrollToPos(cm); }
-  if (x != null) { cm.curOp.scrollLeft = x; }
-  if (y != null) { cm.curOp.scrollTop = y; }
-}
-
-function scrollToRange(cm, range$$1) {
-  resolveScrollToPos(cm);
-  cm.curOp.scrollToPos = range$$1;
-}
-
-// When an operation has its scrollToPos property set, and another
-// scroll action is applied before the end of the operation, this
-// 'simulates' scrolling that position into view in a cheap way, so
-// that the effect of intermediate scroll commands is not ignored.
-function resolveScrollToPos(cm) {
-  var range$$1 = cm.curOp.scrollToPos;
-  if (range$$1) {
-    cm.curOp.scrollToPos = null;
-    var from = estimateCoords(cm, range$$1.from), to = estimateCoords(cm, range$$1.to);
-    scrollToCoordsRange(cm, from, to, range$$1.margin);
-  }
-}
-
-function scrollToCoordsRange(cm, from, to, margin) {
-  var sPos = calculateScrollPos(cm, {
-    left: Math.min(from.left, to.left),
-    top: Math.min(from.top, to.top) - margin,
-    right: Math.max(from.right, to.right),
-    bottom: Math.max(from.bottom, to.bottom) + margin
-  });
-  scrollToCoords(cm, sPos.scrollLeft, sPos.scrollTop);
-}
-
-// Sync the scrollable area and scrollbars, ensure the viewport
-// covers the visible area.
-function updateScrollTop(cm, val) {
-  if (Math.abs(cm.doc.scrollTop - val) < 2) { return }
-  if (!gecko) { updateDisplaySimple(cm, {top: val}); }
-  setScrollTop(cm, val, true);
-  if (gecko) { updateDisplaySimple(cm); }
-  startWorker(cm, 100);
-}
-
-function setScrollTop(cm, val, forceScroll) {
-  val = Math.min(cm.display.scroller.scrollHeight - cm.display.scroller.clientHeight, val);
-  if (cm.display.scroller.scrollTop == val && !forceScroll) { return }
-  cm.doc.scrollTop = val;
-  cm.display.scrollbars.setScrollTop(val);
-  if (cm.display.scroller.scrollTop != val) { cm.display.scroller.scrollTop = val; }
-}
-
-// Sync scroller and scrollbar, ensure the gutter elements are
-// aligned.
-function setScrollLeft(cm, val, isScroller, forceScroll) {
-  val = Math.min(val, cm.display.scroller.scrollWidth - cm.display.scroller.clientWidth);
-  if ((isScroller ? val == cm.doc.scrollLeft : Math.abs(cm.doc.scrollLeft - val) < 2) && !forceScroll) { return }
-  cm.doc.scrollLeft = val;
-  alignHorizontally(cm);
-  if (cm.display.scroller.scrollLeft != val) { cm.display.scroller.scrollLeft = val; }
-  cm.display.scrollbars.setScrollLeft(val);
-}
-
-// SCROLLBARS
-
-// Prepare DOM reads needed to update the scrollbars. Done in one
-// shot to minimize update/measure roundtrips.
-function measureForScrollbars(cm) {
-  var d = cm.display, gutterW = d.gutters.offsetWidth;
-  var docH = Math.round(cm.doc.height + paddingVert(cm.display));
-  return {
-    clientHeight: d.scroller.clientHeight,
-    viewHeight: d.wrapper.clientHeight,
-    scrollWidth: d.scroller.scrollWidth, clientWidth: d.scroller.clientWidth,
-    viewWidth: d.wrapper.clientWidth,
-    barLeft: cm.options.fixedGutter ? gutterW : 0,
-    docHeight: docH,
-    scrollHeight: docH + scrollGap(cm) + d.barHeight,
-    nativeBarWidth: d.nativeBarWidth,
-    gutterWidth: gutterW
-  }
-}
-
-var NativeScrollbars = function(place, scroll, cm) {
-  this.cm = cm;
-  var vert = this.vert = elt("div", [elt("div", null, null, "min-width: 1px")], "CodeMirror-vscrollbar");
-  var horiz = this.horiz = elt("div", [elt("div", null, null, "height: 100%; min-height: 1px")], "CodeMirror-hscrollbar");
-  place(vert); place(horiz);
-
-  on(vert, "scroll", function () {
-    if (vert.clientHeight) { scroll(vert.scrollTop, "vertical"); }
-  });
-  on(horiz, "scroll", function () {
-    if (horiz.clientWidth) { scroll(horiz.scrollLeft, "horizontal"); }
-  });
-
-  this.checkedZeroWidth = false;
-  // Need to set a minimum width to see the scrollbar on IE7 (but must not set it on IE8).
-  if (ie && ie_version < 8) { this.horiz.style.minHeight = this.vert.style.minWidth = "18px"; }
-};
-
-NativeScrollbars.prototype.update = function (measure) {
-  var needsH = measure.scrollWidth > measure.clientWidth + 1;
-  var needsV = measure.scrollHeight > measure.clientHeight + 1;
-  var sWidth = measure.nativeBarWidth;
-
-  if (needsV) {
-    this.vert.style.display = "block";
-    this.vert.style.bottom = needsH ? sWidth + "px" : "0";
-    var totalHeight = measure.viewHeight - (needsH ? sWidth : 0);
-    // A bug in IE8 can cause this value to be negative, so guard it.
-    this.vert.firstChild.style.height =
-      Math.max(0, measure.scrollHeight - measure.clientHeight + totalHeight) + "px";
-  } else {
-    this.vert.style.display = "";
-    this.vert.firstChild.style.height = "0";
-  }
-
-  if (needsH) {
-    this.horiz.style.display = "block";
-    this.horiz.style.right = needsV ? sWidth + "px" : "0";
-    this.horiz.style.left = measure.barLeft + "px";
-    var totalWidth = measure.viewWidth - measure.barLeft - (needsV ? sWidth : 0);
-    this.horiz.firstChild.style.width =
-      Math.max(0, measure.scrollWidth - measure.clientWidth + totalWidth) + "px";
-  } else {
-    this.horiz.style.display = "";
-    this.horiz.firstChild.style.width = "0";
-  }
-
-  if (!this.checkedZeroWidth && measure.clientHeight > 0) {
-    if (sWidth == 0) { this.zeroWidthHack(); }
-    this.checkedZeroWidth = true;
-  }
-
-  return {right: needsV ? sWidth : 0, bottom: needsH ? sWidth : 0}
-};
-
-NativeScrollbars.prototype.setScrollLeft = function (pos) {
-  if (this.horiz.scrollLeft != pos) { this.horiz.scrollLeft = pos; }
-  if (this.disableHoriz) { this.enableZeroWidthBar(this.horiz, this.disableHoriz, "horiz"); }
-};
-
-NativeScrollbars.prototype.setScrollTop = function (pos) {
-  if (this.vert.scrollTop != pos) { this.vert.scrollTop = pos; }
-  if (this.disableVert) { this.enableZeroWidthBar(this.vert, this.disableVert, "vert"); }
-};
-
-NativeScrollbars.prototype.zeroWidthHack = function () {
-  var w = mac && !mac_geMountainLion ? "12px" : "18px";
-  this.horiz.style.height = this.vert.style.width = w;
-  this.horiz.style.pointerEvents = this.vert.style.pointerEvents = "none";
-  this.disableHoriz = new Delayed;
-  this.disableVert = new Delayed;
-};
-
-NativeScrollbars.prototype.enableZeroWidthBar = function (bar, delay, type) {
-  bar.style.pointerEvents = "auto";
-  function maybeDisable() {
-    // To find out whether the scrollbar is still visible, we
-    // check whether the element under the pixel in the bottom
-    // right corner of the scrollbar box is the scrollbar box
-    // itself (when the bar is still visible) or its filler child
-    // (when the bar is hidden). If it is still visible, we keep
-    // it enabled, if it's hidden, we disable pointer events.
-    var box = bar.getBoundingClientRect();
-    var elt$$1 = type == "vert" ? document.elementFromPoint(box.right - 1, (box.top + box.bottom) / 2)
-        : document.elementFromPoint((box.right + box.left) / 2, box.bottom - 1);
-    if (elt$$1 != bar) { bar.style.pointerEvents = "none"; }
-    else { delay.set(1000, maybeDisable); }
-  }
-  delay.set(1000, maybeDisable);
-};
-
-NativeScrollbars.prototype.clear = function () {
-  var parent = this.horiz.parentNode;
-  parent.removeChild(this.horiz);
-  parent.removeChild(this.vert);
-};
-
-var NullScrollbars = function () {};
-
-NullScrollbars.prototype.update = function () { return {bottom: 0, right: 0} };
-NullScrollbars.prototype.setScrollLeft = function () {};
-NullScrollbars.prototype.setScrollTop = function () {};
-NullScrollbars.prototype.clear = function () {};
-
-function updateScrollbars(cm, measure) {
-  if (!measure) { measure = measureForScrollbars(cm); }
-  var startWidth = cm.display.barWidth, startHeight = cm.display.barHeight;
-  updateScrollbarsInner(cm, measure);
-  for (var i = 0; i < 4 && startWidth != cm.display.barWidth || startHeight != cm.display.barHeight; i++) {
-    if (startWidth != cm.display.barWidth && cm.options.lineWrapping)
-      { updateHeightsInViewport(cm); }
-    updateScrollbarsInner(cm, measureForScrollbars(cm));
-    startWidth = cm.display.barWidth; startHeight = cm.display.barHeight;
-  }
-}
-
-// Re-synchronize the fake scrollbars with the actual size of the
-// content.
-function updateScrollbarsInner(cm, measure) {
-  var d = cm.display;
-  var sizes = d.scrollbars.update(measure);
-
-  d.sizer.style.paddingRight = (d.barWidth = sizes.right) + "px";
-  d.sizer.style.paddingBottom = (d.barHeight = sizes.bottom) + "px";
-  d.heightForcer.style.borderBottom = sizes.bottom + "px solid transparent";
-
-  if (sizes.right && sizes.bottom) {
-    d.scrollbarFiller.style.display = "block";
-    d.scrollbarFiller.style.height = sizes.bottom + "px";
-    d.scrollbarFiller.style.width = sizes.right + "px";
-  } else { d.scrollbarFiller.style.display = ""; }
-  if (sizes.bottom && cm.options.coverGutterNextToScrollbar && cm.options.fixedGutter) {
-    d.gutterFiller.style.display = "block";
-    d.gutterFiller.style.height = sizes.bottom + "px";
-    d.gutterFiller.style.width = measure.gutterWidth + "px";
-  } else { d.gutterFiller.style.display = ""; }
-}
-
-var scrollbarModel = {"native": NativeScrollbars, "null": NullScrollbars};
-
-function initScrollbars(cm) {
-  if (cm.display.scrollbars) {
-    cm.display.scrollbars.clear();
-    if (cm.display.scrollbars.addClass)
-      { rmClass(cm.display.wrapper, cm.display.scrollbars.addClass); }
-  }
-
-  cm.display.scrollbars = new scrollbarModel[cm.options.scrollbarStyle](function (node) {
-    cm.display.wrapper.insertBefore(node, cm.display.scrollbarFiller);
-    // Prevent clicks in the scrollbars from killing focus
-    on(node, "mousedown", function () {
-      if (cm.state.focused) { setTimeout(function () { return cm.display.input.focus(); }, 0); }
-    });
-    node.setAttribute("cm-not-content", "true");
-  }, function (pos, axis) {
-    if (axis == "horizontal") { setScrollLeft(cm, pos); }
-    else { updateScrollTop(cm, pos); }
-  }, cm);
-  if (cm.display.scrollbars.addClass)
-    { addClass(cm.display.wrapper, cm.display.scrollbars.addClass); }
-}
-
-// Operations are used to wrap a series of changes to the editor
-// state in such a way that each change won't have to update the
-// cursor and display (which would be awkward, slow, and
-// error-prone). Instead, display updates are batched and then all
-// combined and executed at once.
-
-var nextOpId = 0;
-// Start a new operation.
-function startOperation(cm) {
-  cm.curOp = {
-    cm: cm,
-    viewChanged: false,      // Flag that indicates that lines might need to be redrawn
-    startHeight: cm.doc.height, // Used to detect need to update scrollbar
-    forceUpdate: false,      // Used to force a redraw
-    updateInput: null,       // Whether to reset the input textarea
-    typing: false,           // Whether this reset should be careful to leave existing text (for compositing)
-    changeObjs: null,        // Accumulated changes, for firing change events
-    cursorActivityHandlers: null, // Set of handlers to fire cursorActivity on
-    cursorActivityCalled: 0, // Tracks which cursorActivity handlers have been called already
-    selectionChanged: false, // Whether the selection needs to be redrawn
-    updateMaxLine: false,    // Set when the widest line needs to be determined anew
-    scrollLeft: null, scrollTop: null, // Intermediate scroll position, not pushed to DOM yet
-    scrollToPos: null,       // Used to scroll to a specific position
-    focus: false,
-    id: ++nextOpId           // Unique ID
-  };
-  pushOperation(cm.curOp);
-}
-
-// Finish an operation, updating the display and signalling delayed events
-function endOperation(cm) {
-  var op = cm.curOp;
-  finishOperation(op, function (group) {
-    for (var i = 0; i < group.ops.length; i++)
-      { group.ops[i].cm.curOp = null; }
-    endOperations(group);
-  });
-}
-
-// The DOM updates done when an operation finishes are batched so
-// that the minimum number of relayouts are required.
-function endOperations(group) {
-  var ops = group.ops;
-  for (var i = 0; i < ops.length; i++) // Read DOM
-    { endOperation_R1(ops[i]); }
-  for (var i$1 = 0; i$1 < ops.length; i$1++) // Write DOM (maybe)
-    { endOperation_W1(ops[i$1]); }
-  for (var i$2 = 0; i$2 < ops.length; i$2++) // Read DOM
-    { endOperation_R2(ops[i$2]); }
-  for (var i$3 = 0; i$3 < ops.length; i$3++) // Write DOM (maybe)
-    { endOperation_W2(ops[i$3]); }
-  for (var i$4 = 0; i$4 < ops.length; i$4++) // Read DOM
-    { endOperation_finish(ops[i$4]); }
-}
-
-function endOperation_R1(op) {
-  var cm = op.cm, display = cm.display;
-  maybeClipScrollbars(cm);
-  if (op.updateMaxLine) { findMaxLine(cm); }
-
-  op.mustUpdate = op.viewChanged || op.forceUpdate || op.scrollTop != null ||
-    op.scrollToPos && (op.scrollToPos.from.line < display.viewFrom ||
-                       op.scrollToPos.to.line >= display.viewTo) ||
-    display.maxLineChanged && cm.options.lineWrapping;
-  op.update = op.mustUpdate &&
-    new DisplayUpdate(cm, op.mustUpdate && {top: op.scrollTop, ensure: op.scrollToPos}, op.forceUpdate);
-}
-
-function endOperation_W1(op) {
-  op.updatedDisplay = op.mustUpdate && updateDisplayIfNeeded(op.cm, op.update);
-}
-
-function endOperation_R2(op) {
-  var cm = op.cm, display = cm.display;
-  if (op.updatedDisplay) { updateHeightsInViewport(cm); }
-
-  op.barMeasure = measureForScrollbars(cm);
-
-  // If the max line changed since it was last measured, measure it,
-  // and ensure the document's width matches it.
-  // updateDisplay_W2 will use these properties to do the actual resizing
-  if (display.maxLineChanged && !cm.options.lineWrapping) {
-    op.adjustWidthTo = measureChar(cm, display.maxLine, display.maxLine.text.length).left + 3;
-    cm.display.sizerWidth = op.adjustWidthTo;
-    op.barMeasure.scrollWidth =
-      Math.max(display.scroller.clientWidth, display.sizer.offsetLeft + op.adjustWidthTo + scrollGap(cm) + cm.display.barWidth);
-    op.maxScrollLeft = Math.max(0, display.sizer.offsetLeft + op.adjustWidthTo - displayWidth(cm));
-  }
-
-  if (op.updatedDisplay || op.selectionChanged)
-    { op.preparedSelection = display.input.prepareSelection(); }
-}
-
-function endOperation_W2(op) {
-  var cm = op.cm;
-
-  if (op.adjustWidthTo != null) {
-    cm.display.sizer.style.minWidth = op.adjustWidthTo + "px";
-    if (op.maxScrollLeft < cm.doc.scrollLeft)
-      { setScrollLeft(cm, Math.min(cm.display.scroller.scrollLeft, op.maxScrollLeft), true); }
-    cm.display.maxLineChanged = false;
-  }
-
-  var takeFocus = op.focus && op.focus == activeElt();
-  if (op.preparedSelection)
-    { cm.display.input.showSelection(op.preparedSelection, takeFocus); }
-  if (op.updatedDisplay || op.startHeight != cm.doc.height)
-    { updateScrollbars(cm, op.barMeasure); }
-  if (op.updatedDisplay)
-    { setDocumentHeight(cm, op.barMeasure); }
-
-  if (op.selectionChanged) { restartBlink(cm); }
-
-  if (cm.state.focused && op.updateInput)
-    { cm.display.input.reset(op.typing); }
-  if (takeFocus) { ensureFocus(op.cm); }
-}
-
-function endOperation_finish(op) {
-  var cm = op.cm, display = cm.display, doc = cm.doc;
-
-  if (op.updatedDisplay) { postUpdateDisplay(cm, op.update); }
-
-  // Abort mouse wheel delta measurement, when scrolling explicitly
-  if (display.wheelStartX != null && (op.scrollTop != null || op.scrollLeft != null || op.scrollToPos))
-    { display.wheelStartX = display.wheelStartY = null; }
-
-  // Propagate the scroll position to the actual DOM scroller
-  if (op.scrollTop != null) { setScrollTop(cm, op.scrollTop, op.forceScroll); }
-
-  if (op.scrollLeft != null) { setScrollLeft(cm, op.scrollLeft, true, true); }
-  // If we need to scroll a specific position into view, do so.
-  if (op.scrollToPos) {
-    var rect = scrollPosIntoView(cm, clipPos(doc, op.scrollToPos.from),
-                                 clipPos(doc, op.scrollToPos.to), op.scrollToPos.margin);
-    maybeScrollWindow(cm, rect);
-  }
-
-  // Fire events for markers that are hidden/unidden by editing or
-  // undoing
-  var hidden = op.maybeHiddenMarkers, unhidden = op.maybeUnhiddenMarkers;
-  if (hidden) { for (var i = 0; i < hidden.length; ++i)
-    { if (!hidden[i].lines.length) { signal(hidden[i], "hide"); } } }
-  if (unhidden) { for (var i$1 = 0; i$1 < unhidden.length; ++i$1)
-    { if (unhidden[i$1].lines.length) { signal(unhidden[i$1], "unhide"); } } }
-
-  if (display.wrapper.offsetHeight)
-    { doc.scrollTop = cm.display.scroller.scrollTop; }
-
-  // Fire change events, and delayed event handlers
-  if (op.changeObjs)
-    { signal(cm, "changes", cm, op.changeObjs); }
-  if (op.update)
-    { op.update.finish(); }
-}
-
-// Run the given function in an operation
-function runInOp(cm, f) {
-  if (cm.curOp) { return f() }
-  startOperation(cm);
-  try { return f() }
-  finally { endOperation(cm); }
-}
-// Wraps a function in an operation. Returns the wrapped function.
-function operation(cm, f) {
-  return function() {
-    if (cm.curOp) { return f.apply(cm, arguments) }
-    startOperation(cm);
-    try { return f.apply(cm, arguments) }
-    finally { endOperation(cm); }
-  }
-}
-// Used to add methods to editor and doc instances, wrapping them in
-// operations.
-function methodOp(f) {
-  return function() {
-    if (this.curOp) { return f.apply(this, arguments) }
-    startOperation(this);
-    try { return f.apply(this, arguments) }
-    finally { endOperation(this); }
-  }
-}
-function docMethodOp(f) {
-  return function() {
-    var cm = this.cm;
-    if (!cm || cm.curOp) { return f.apply(this, arguments) }
-    startOperation(cm);
-    try { return f.apply(this, arguments) }
-    finally { endOperation(cm); }
-  }
-}
-
-// Updates the display.view data structure for a given change to the
-// document. From and to are in pre-change coordinates. Lendiff is
-// the amount of lines added or subtracted by the change. This is
-// used for changes that span multiple lines, or change the way
-// lines are divided into visual lines. regLineChange (below)
-// registers single-line changes.
-function regChange(cm, from, to, lendiff) {
-  if (from == null) { from = cm.doc.first; }
-  if (to == null) { to = cm.doc.first + cm.doc.size; }
-  if (!lendiff) { lendiff = 0; }
-
-  var display = cm.display;
-  if (lendiff && to < display.viewTo &&
-      (display.updateLineNumbers == null || display.updateLineNumbers > from))
-    { display.updateLineNumbers = from; }
-
-  cm.curOp.viewChanged = true;
-
-  if (from >= display.viewTo) { // Change after
-    if (sawCollapsedSpans && visualLineNo(cm.doc, from) < display.viewTo)
-      { resetView(cm); }
-  } else if (to <= display.viewFrom) { // Change before
-    if (sawCollapsedSpans && visualLineEndNo(cm.doc, to + lendiff) > display.viewFrom) {
-      resetView(cm);
-    } else {
-      display.viewFrom += lendiff;
-      display.viewTo += lendiff;
-    }
-  } else if (from <= display.viewFrom && to >= display.viewTo) { // Full overlap
-    resetView(cm);
-  } else if (from <= display.viewFrom) { // Top overlap
-    var cut = viewCuttingPoint(cm, to, to + lendiff, 1);
-    if (cut) {
-      display.view = display.view.slice(cut.index);
-      display.viewFrom = cut.lineN;
-      display.viewTo += lendiff;
-    } else {
-      resetView(cm);
-    }
-  } else if (to >= display.viewTo) { // Bottom overlap
-    var cut$1 = viewCuttingPoint(cm, from, from, -1);
-    if (cut$1) {
-      display.view = display.view.slice(0, cut$1.index);
-      display.viewTo = cut$1.lineN;
-    } else {
-      resetView(cm);
-    }
-  } else { // Gap in the middle
-    var cutTop = viewCuttingPoint(cm, from, from, -1);
-    var cutBot = viewCuttingPoint(cm, to, to + lendiff, 1);
-    if (cutTop && cutBot) {
-      display.view = display.view.slice(0, cutTop.index)
-        .concat(buildViewArray(cm, cutTop.lineN, cutBot.lineN))
-        .concat(display.view.slice(cutBot.index));
-      display.viewTo += lendiff;
-    } else {
-      resetView(cm);
-    }
-  }
-
-  var ext = display.externalMeasured;
-  if (ext) {
-    if (to < ext.lineN)
-      { ext.lineN += lendiff; }
-    else if (from < ext.lineN + ext.size)
-      { display.externalMeasured = null; }
-  }
-}
-
-// Register a change to a single line. Type must be one of "text",
-// "gutter", "class", "widget"
-function regLineChange(cm, line, type) {
-  cm.curOp.viewChanged = true;
-  var display = cm.display, ext = cm.display.externalMeasured;
-  if (ext && line >= ext.lineN && line < ext.lineN + ext.size)
-    { display.externalMeasured = null; }
-
-  if (line < display.viewFrom || line >= display.viewTo) { return }
-  var lineView = display.view[findViewIndex(cm, line)];
-  if (lineView.node == null) { return }
-  var arr = lineView.changes || (lineView.changes = []);
-  if (indexOf(arr, type) == -1) { arr.push(type); }
-}
-
-// Clear the view.
-function resetView(cm) {
-  cm.display.viewFrom = cm.display.viewTo = cm.doc.first;
-  cm.display.view = [];
-  cm.display.viewOffset = 0;
-}
-
-function viewCuttingPoint(cm, oldN, newN, dir) {
-  var index = findViewIndex(cm, oldN), diff, view = cm.display.view;
-  if (!sawCollapsedSpans || newN == cm.doc.first + cm.doc.size)
-    { return {index: index, lineN: newN} }
-  var n = cm.display.viewFrom;
-  for (var i = 0; i < index; i++)
-    { n += view[i].size; }
-  if (n != oldN) {
-    if (dir > 0) {
-      if (index == view.length - 1) { return null }
-      diff = (n + view[index].size) - oldN;
-      index++;
-    } else {
-      diff = n - oldN;
-    }
-    oldN += diff; newN += diff;
-  }
-  while (visualLineNo(cm.doc, newN) != newN) {
-    if (index == (dir < 0 ? 0 : view.length - 1)) { return null }
-    newN += dir * view[index - (dir < 0 ? 1 : 0)].size;
-    index += dir;
-  }
-  return {index: index, lineN: newN}
-}
-
-// Force the view to cover a given range, adding empty view element
-// or clipping off existing ones as needed.
-function adjustView(cm, from, to) {
-  var display = cm.display, view = display.view;
-  if (view.length == 0 || from >= display.viewTo || to <= display.viewFrom) {
-    display.view = buildViewArray(cm, from, to);
-    display.viewFrom = from;
-  } else {
-    if (display.viewFrom > from)
-      { display.view = buildViewArray(cm, from, display.viewFrom).concat(display.view); }
-    else if (display.viewFrom < from)
-      { display.view = display.view.slice(findViewIndex(cm, from)); }
-    display.viewFrom = from;
-    if (display.viewTo < to)
-      { display.view = display.view.concat(buildViewArray(cm, display.viewTo, to)); }
-    else if (display.viewTo > to)
-      { display.view = display.view.slice(0, findViewIndex(cm, to)); }
-  }
-  display.viewTo = to;
-}
-
-// Count the number of lines in the view whose DOM representation is
-// out of date (or nonexistent).
-function countDirtyView(cm) {
-  var view = cm.display.view, dirty = 0;
-  for (var i = 0; i < view.length; i++) {
-    var lineView = view[i];
-    if (!lineView.hidden && (!lineView.node || lineView.changes)) { ++dirty; }
-  }
-  return dirty
-}
-
-// HIGHLIGHT WORKER
-
-function startWorker(cm, time) {
-  if (cm.doc.highlightFrontier < cm.display.viewTo)
-    { cm.state.highlight.set(time, bind(highlightWorker, cm)); }
-}
-
-function highlightWorker(cm) {
-  var doc = cm.doc;
-  if (doc.highlightFrontier >= cm.display.viewTo) { return }
-  var end = +new Date + cm.options.workTime;
-  var context = getContextBefore(cm, doc.highlightFrontier);
-  var changedLines = [];
-
-  doc.iter(context.line, Math.min(doc.first + doc.size, cm.display.viewTo + 500), function (line) {
-    if (context.line >= cm.display.viewFrom) { // Visible
-      var oldStyles = line.styles;
-      var resetState = line.text.length > cm.options.maxHighlightLength ? copyState(doc.mode, context.state) : null;
-      var highlighted = highlightLine(cm, line, context, true);
-      if (resetState) { context.state = resetState; }
-      line.styles = highlighted.styles;
-      var oldCls = line.styleClasses, newCls = highlighted.classes;
-      if (newCls) { line.styleClasses = newCls; }
-      else if (oldCls) { line.styleClasses = null; }
-      var ischange = !oldStyles || oldStyles.length != line.styles.length ||
-        oldCls != newCls && (!oldCls || !newCls || oldCls.bgClass != newCls.bgClass || oldCls.textClass != newCls.textClass);
-      for (var i = 0; !ischange && i < oldStyles.length; ++i) { ischange = oldStyles[i] != line.styles[i]; }
-      if (ischange) { changedLines.push(context.line); }
-      line.stateAfter = context.save();
-      context.nextLine();
-    } else {
-      if (line.text.length <= cm.options.maxHighlightLength)
-        { processLine(cm, line.text, context); }
-      line.stateAfter = context.line % 5 == 0 ? context.save() : null;
-      context.nextLine();
-    }
-    if (+new Date > end) {
-      startWorker(cm, cm.options.workDelay);
-      return true
-    }
-  });
-  doc.highlightFrontier = context.line;
-  doc.modeFrontier = Math.max(doc.modeFrontier, context.line);
-  if (changedLines.length) { runInOp(cm, function () {
-    for (var i = 0; i < changedLines.length; i++)
-      { regLineChange(cm, changedLines[i], "text"); }
-  }); }
-}
-
-// DISPLAY DRAWING
-
-var DisplayUpdate = function(cm, viewport, force) {
-  var display = cm.display;
-
-  this.viewport = viewport;
-  // Store some values that we'll need later (but don't want to force a relayout for)
-  this.visible = visibleLines(display, cm.doc, viewport);
-  this.editorIsHidden = !display.wrapper.offsetWidth;
-  this.wrapperHeight = display.wrapper.clientHeight;
-  this.wrapperWidth = display.wrapper.clientWidth;
-  this.oldDisplayWidth = displayWidth(cm);
-  this.force = force;
-  this.dims = getDimensions(cm);
-  this.events = [];
-};
-
-DisplayUpdate.prototype.signal = function (emitter, type) {
-  if (hasHandler(emitter, type))
-    { this.events.push(arguments); }
-};
-DisplayUpdate.prototype.finish = function () {
-    var this$1 = this;
-
-  for (var i = 0; i < this.events.length; i++)
-    { signal.apply(null, this$1.events[i]); }
-};
-
-function maybeClipScrollbars(cm) {
-  var display = cm.display;
-  if (!display.scrollbarsClipped && display.scroller.offsetWidth) {
-    display.nativeBarWidth = display.scroller.offsetWidth - display.scroller.clientWidth;
-    display.heightForcer.style.height = scrollGap(cm) + "px";
-    display.sizer.style.marginBottom = -display.nativeBarWidth + "px";
-    display.sizer.style.borderRightWidth = scrollGap(cm) + "px";
-    display.scrollbarsClipped = true;
-  }
-}
-
-function selectionSnapshot(cm) {
-  if (cm.hasFocus()) { return null }
-  var active = activeElt();
-  if (!active || !contains(cm.display.lineDiv, active)) { return null }
-  var result = {activeElt: active};
-  if (window.getSelection) {
-    var sel = window.getSelection();
-    if (sel.anchorNode && sel.extend && contains(cm.display.lineDiv, sel.anchorNode)) {
-      result.anchorNode = sel.anchorNode;
-      result.anchorOffset = sel.anchorOffset;
-      result.focusNode = sel.focusNode;
-      result.focusOffset = sel.focusOffset;
-    }
-  }
-  return result
-}
-
-function restoreSelection(snapshot) {
-  if (!snapshot || !snapshot.activeElt || snapshot.activeElt == activeElt()) { return }
-  snapshot.activeElt.focus();
-  if (snapshot.anchorNode && contains(document.body, snapshot.anchorNode) && contains(document.body, snapshot.focusNode)) {
-    var sel = window.getSelection(), range$$1 = document.createRange();
-    range$$1.setEnd(snapshot.anchorNode, snapshot.anchorOffset);
-    range$$1.collapse(false);
-    sel.removeAllRanges();
-    sel.addRange(range$$1);
-    sel.extend(snapshot.focusNode, snapshot.focusOffset);
-  }
-}
-
-// Does the actual updating of the line display. Bails out
-// (returning false) when there is nothing to be done and forced is
-// false.
-function updateDisplayIfNeeded(cm, update) {
-  var display = cm.display, doc = cm.doc;
-
-  if (update.editorIsHidden) {
-    resetView(cm);
-    return false
-  }
-
-  // Bail out if the visible area is already rendered and nothing changed.
-  if (!update.force &&
-      update.visible.from >= display.viewFrom && update.visible.to <= display.viewTo &&
-      (display.updateLineNumbers == null || display.updateLineNumbers >= display.viewTo) &&
-      display.renderedView == display.view && countDirtyView(cm) == 0)
-    { return false }
-
-  if (maybeUpdateLineNumberWidth(cm)) {
-    resetView(cm);
-    update.dims = getDimensions(cm);
-  }
-
-  // Compute a suitable new viewport (from & to)
-  var end = doc.first + doc.size;
-  var from = Math.max(update.visible.from - cm.options.viewportMargin, doc.first);
-  var to = Math.min(end, update.visible.to + cm.options.viewportMargin);
-  if (display.viewFrom < from && from - display.viewFrom < 20) { from = Math.max(doc.first, display.viewFrom); }
-  if (display.viewTo > to && display.viewTo - to < 20) { to = Math.min(end, display.viewTo); }
-  if (sawCollapsedSpans) {
-    from = visualLineNo(cm.doc, from);
-    to = visualLineEndNo(cm.doc, to);
-  }
-
-  var different = from != display.viewFrom || to != display.viewTo ||
-    display.lastWrapHeight != update.wrapperHeight || display.lastWrapWidth != update.wrapperWidth;
-  adjustView(cm, from, to);
-
-  display.viewOffset = heightAtLine(getLine(cm.doc, display.viewFrom));
-  // Position the mover div to align with the current scroll position
-  cm.display.mover.style.top = display.viewOffset + "px";
-
-  var toUpdate = countDirtyView(cm);
-  if (!different && toUpdate == 0 && !update.force && display.renderedView == display.view &&
-      (display.updateLineNumbers == null || display.updateLineNumbers >= display.viewTo))
-    { return false }
-
-  // For big changes, we hide the enclosing element during the
-  // update, since that speeds up the operations on most browsers.
-  var selSnapshot = selectionSnapshot(cm);
-  if (toUpdate > 4) { display.lineDiv.style.display = "none"; }
-  patchDisplay(cm, display.updateLineNumbers, update.dims);
-  if (toUpdate > 4) { display.lineDiv.style.display = ""; }
-  display.renderedView = display.view;
-  // There might have been a widget with a focused element that got
-  // hidden or updated, if so re-focus it.
-  restoreSelection(selSnapshot);
-
-  // Prevent selection and cursors from interfering with the scroll
-  // width and height.
-  removeChildren(display.cursorDiv);
-  removeChildren(display.selectionDiv);
-  display.gutters.style.height = display.sizer.style.minHeight = 0;
-
-  if (different) {
-    display.lastWrapHeight = update.wrapperHeight;
-    display.lastWrapWidth = update.wrapperWidth;
-    startWorker(cm, 400);
-  }
-
-  display.updateLineNumbers = null;
-
-  return true
-}
-
-function postUpdateDisplay(cm, update) {
-  var viewport = update.viewport;
-
-  for (var first = true;; first = false) {
-    if (!first || !cm.options.lineWrapping || update.oldDisplayWidth == displayWidth(cm)) {
-      // Clip forced viewport to actual scrollable area.
-      if (viewport && viewport.top != null)
-        { viewport = {top: Math.min(cm.doc.height + paddingVert(cm.display) - displayHeight(cm), viewport.top)}; }
-      // Updated line heights might result in the drawn area not
-      // actually covering the viewport. Keep looping until it does.
-      update.visible = visibleLines(cm.display, cm.doc, viewport);
-      if (update.visible.from >= cm.display.viewFrom && update.visible.to <= cm.display.viewTo)
-        { break }
-    }
-    if (!updateDisplayIfNeeded(cm, update)) { break }
-    updateHeightsInViewport(cm);
-    var barMeasure = measureForScrollbars(cm);
-    updateSelection(cm);
-    updateScrollbars(cm, barMeasure);
-    setDocumentHeight(cm, barMeasure);
-    update.force = false;
-  }
-
-  update.signal(cm, "update", cm);
-  if (cm.display.viewFrom != cm.display.reportedViewFrom || cm.display.viewTo != cm.display.reportedViewTo) {
-    update.signal(cm, "viewportChange", cm, cm.display.viewFrom, cm.display.viewTo);
-    cm.display.reportedViewFrom = cm.display.viewFrom; cm.display.reportedViewTo = cm.display.viewTo;
-  }
-}
-
-function updateDisplaySimple(cm, viewport) {
-  var update = new DisplayUpdate(cm, viewport);
-  if (updateDisplayIfNeeded(cm, update)) {
-    updateHeightsInViewport(cm);
-    postUpdateDisplay(cm, update);
-    var barMeasure = measureForScrollbars(cm);
-    updateSelection(cm);
-    updateScrollbars(cm, barMeasure);
-    setDocumentHeight(cm, barMeasure);
-    update.finish();
-  }
-}
-
-// Sync the actual display DOM structure with display.view, removing
-// nodes for lines that are no longer in view, and creating the ones
-// that are not there yet, and updating the ones that are out of
-// date.
-function patchDisplay(cm, updateNumbersFrom, dims) {
-  var display = cm.display, lineNumbers = cm.options.lineNumbers;
-  var container = display.lineDiv, cur = container.firstChild;
-
-  function rm(node) {
-    var next = node.nextSibling;
-    // Works around a throw-scroll bug in OS X Webkit
-    if (webkit && mac && cm.display.currentWheelTarget == node)
-      { node.style.display = "none"; }
-    else
-      { node.parentNode.removeChild(node); }
-    return next
-  }
-
-  var view = display.view, lineN = display.viewFrom;
-  // Loop over the elements in the view, syncing cur (the DOM nodes
-  // in display.lineDiv) with the view as we go.
-  for (var i = 0; i < view.length; i++) {
-    var lineView = view[i];
-    if (lineView.hidden) {
-    } else if (!lineView.node || lineView.node.parentNode != container) { // Not drawn yet
-      var node = buildLineElement(cm, lineView, lineN, dims);
-      container.insertBefore(node, cur);
-    } else { // Already drawn
-      while (cur != lineView.node) { cur = rm(cur); }
-      var updateNumber = lineNumbers && updateNumbersFrom != null &&
-        updateNumbersFrom <= lineN && lineView.lineNumber;
-      if (lineView.changes) {
-        if (indexOf(lineView.changes, "gutter") > -1) { updateNumber = false; }
-        updateLineForChanges(cm, lineView, lineN, dims);
-      }
-      if (updateNumber) {
-        removeChildren(lineView.lineNumber);
-        lineView.lineNumber.appendChild(document.createTextNode(lineNumberFor(cm.options, lineN)));
-      }
-      cur = lineView.node.nextSibling;
-    }
-    lineN += lineView.size;
-  }
-  while (cur) { cur = rm(cur); }
-}
-
-function updateGutterSpace(cm) {
-  var width = cm.display.gutters.offsetWidth;
-  cm.display.sizer.style.marginLeft = width + "px";
-}
-
-function setDocumentHeight(cm, measure) {
-  cm.display.sizer.style.minHeight = measure.docHeight + "px";
-  cm.display.heightForcer.style.top = measure.docHeight + "px";
-  cm.display.gutters.style.height = (measure.docHeight + cm.display.barHeight + scrollGap(cm)) + "px";
-}
-
-// Rebuild the gutter elements, ensure the margin to the left of the
-// code matches their width.
-function updateGutters(cm) {
-  var gutters = cm.display.gutters, specs = cm.options.gutters;
-  removeChildren(gutters);
-  var i = 0;
-  for (; i < specs.length; ++i) {
-    var gutterClass = specs[i];
-    var gElt = gutters.appendChild(elt("div", null, "CodeMirror-gutter " + gutterClass));
-    if (gutterClass == "CodeMirror-linenumbers") {
-      cm.display.lineGutter = gElt;
-      gElt.style.width = (cm.display.lineNumWidth || 1) + "px";
-    }
-  }
-  gutters.style.display = i ? "" : "none";
-  updateGutterSpace(cm);
-}
-
-// Make sure the gutters options contains the element
-// "CodeMirror-linenumbers" when the lineNumbers option is true.
-function setGuttersForLineNumbers(options) {
-  var found = indexOf(options.gutters, "CodeMirror-linenumbers");
-  if (found == -1 && options.lineNumbers) {
-    options.gutters = options.gutters.concat(["CodeMirror-linenumbers"]);
-  } else if (found > -1 && !options.lineNumbers) {
-    options.gutters = options.gutters.slice(0);
-    options.gutters.splice(found, 1);
-  }
-}
-
-// Since the delta values reported on mouse wheel events are
-// unstandardized between browsers and even browser versions, and
-// generally horribly unpredictable, this code starts by measuring
-// the scroll effect that the first few mouse wheel events have,
-// and, from that, detects the way it can convert deltas to pixel
-// offsets afterwards.
-//
-// The reason we want to know the amount a wheel event will scroll
-// is that it gives us a chance to update the display before the
-// actual scrolling happens, reducing flickering.
-
-var wheelSamples = 0;
-var wheelPixelsPerUnit = null;
-// Fill in a browser-detected starting value on browsers where we
-// know one. These don't have to be accurate -- the result of them
-// being wrong would just be a slight flicker on the first wheel
-// scroll (if it is large enough).
-if (ie) { wheelPixelsPerUnit = -.53; }
-else if (gecko) { wheelPixelsPerUnit = 15; }
-else if (chrome) { wheelPixelsPerUnit = -.7; }
-else if (safari) { wheelPixelsPerUnit = -1/3; }
-
-function wheelEventDelta(e) {
-  var dx = e.wheelDeltaX, dy = e.wheelDeltaY;
-  if (dx == null && e.detail && e.axis == e.HORIZONTAL_AXIS) { dx = e.detail; }
-  if (dy == null && e.detail && e.axis == e.VERTICAL_AXIS) { dy = e.detail; }
-  else if (dy == null) { dy = e.wheelDelta; }
-  return {x: dx, y: dy}
-}
-function wheelEventPixels(e) {
-  var delta = wheelEventDelta(e);
-  delta.x *= wheelPixelsPerUnit;
-  delta.y *= wheelPixelsPerUnit;
-  return delta
-}
-
-function onScrollWheel(cm, e) {
-  var delta = wheelEventDelta(e), dx = delta.x, dy = delta.y;
-
-  var display = cm.display, scroll = display.scroller;
-  // Quit if there's nothing to scroll here
-  var canScrollX = scroll.scrollWidth > scroll.clientWidth;
-  var canScrollY = scroll.scrollHeight > scroll.clientHeight;
-  if (!(dx && canScrollX || dy && canScrollY)) { return }
-
-  // Webkit browsers on OS X abort momentum scrolls when the target
-  // of the scroll event is removed from the scrollable element.
-  // This hack (see related code in patchDisplay) makes sure the
-  // element is kept around.
-  if (dy && mac && webkit) {
-    outer: for (var cur = e.target, view = display.view; cur != scroll; cur = cur.parentNode) {
-      for (var i = 0; i < view.length; i++) {
-        if (view[i].node == cur) {
-          cm.display.currentWheelTarget = cur;
-          break outer
-        }
-      }
-    }
-  }
-
-  // On some browsers, horizontal scrolling will cause redraws to
-  // happen before the gutter has been realigned, causing it to
-  // wriggle around in a most unseemly way. When we have an
-  // estimated pixels/delta value, we just handle horizontal
-  // scrolling entirely here. It'll be slightly off from native, but
-  // better than glitching out.
-  if (dx && !gecko && !presto && wheelPixelsPerUnit != null) {
-    if (dy && canScrollY)
-      { updateScrollTop(cm, Math.max(0, scroll.scrollTop + dy * wheelPixelsPerUnit)); }
-    setScrollLeft(cm, Math.max(0, scroll.scrollLeft + dx * wheelPixelsPerUnit));
-    // Only prevent default scrolling if vertical scrolling is
-    // actually possible. Otherwise, it causes vertical scroll
-    // jitter on OSX trackpads when deltaX is small and deltaY
-    // is large (issue #3579)
-    if (!dy || (dy && canScrollY))
-      { e_preventDefault(e); }
-    display.wheelStartX = null; // Abort measurement, if in progress
-    return
-  }
-
-  // 'Project' the visible viewport to cover the area that is being
-  // scrolled into view (if we know enough to estimate it).
-  if (dy && wheelPixelsPerUnit != null) {
-    var pixels = dy * wheelPixelsPerUnit;
-    var top = cm.doc.scrollTop, bot = top + display.wrapper.clientHeight;
-    if (pixels < 0) { top = Math.max(0, top + pixels - 50); }
-    else { bot = Math.min(cm.doc.height, bot + pixels + 50); }
-    updateDisplaySimple(cm, {top: top, bottom: bot});
-  }
-
-  if (wheelSamples < 20) {
-    if (display.wheelStartX == null) {
-      display.wheelStartX = scroll.scrollLeft; display.wheelStartY = scroll.scrollTop;
-      display.wheelDX = dx; display.wheelDY = dy;
-      setTimeout(function () {
-        if (display.wheelStartX == null) { return }
-        var movedX = scroll.scrollLeft - display.wheelStartX;
-        var movedY = scroll.scrollTop - display.wheelStartY;
-        var sample = (movedY && display.wheelDY && movedY / display.wheelDY) ||
-          (movedX && display.wheelDX && movedX / display.wheelDX);
-        display.wheelStartX = display.wheelStartY = null;
-        if (!sample) { return }
-        wheelPixelsPerUnit = (wheelPixelsPerUnit * wheelSamples + sample) / (wheelSamples + 1);
-        ++wheelSamples;
-      }, 200);
-    } else {
-      display.wheelDX += dx; display.wheelDY += dy;
-    }
-  }
-}
-
-// Selection objects are immutable. A new one is created every time
-// the selection changes. A selection is one or more non-overlapping
-// (and non-touching) ranges, sorted, and an integer that indicates
-// which one is the primary selection (the one that's scrolled into
-// view, that getCursor returns, etc).
-var Selection = function(ranges, primIndex) {
-  this.ranges = ranges;
-  this.primIndex = primIndex;
-};
-
-Selection.prototype.primary = function () { return this.ranges[this.primIndex] };
-
-Selection.prototype.equals = function (other) {
-    var this$1 = this;
-
-  if (other == this) { return true }
-  if (other.primIndex != this.primIndex || other.ranges.length != this.ranges.length) { return false }
-  for (var i = 0; i < this.ranges.length; i++) {
-    var here = this$1.ranges[i], there = other.ranges[i];
-    if (!equalCursorPos(here.anchor, there.anchor) || !equalCursorPos(here.head, there.head)) { return false }
-  }
-  return true
-};
-
-Selection.prototype.deepCopy = function () {
-    var this$1 = this;
-
-  var out = [];
-  for (var i = 0; i < this.ranges.length; i++)
-    { out[i] = new Range(copyPos(this$1.ranges[i].anchor), copyPos(this$1.ranges[i].head)); }
-  return new Selection(out, this.primIndex)
-};
-
-Selection.prototype.somethingSelected = function () {
-    var this$1 = this;
-
-  for (var i = 0; i < this.ranges.length; i++)
-    { if (!this$1.ranges[i].empty()) { return true } }
-  return false
-};
-
-Selection.prototype.contains = function (pos, end) {
-    var this$1 = this;
-
-  if (!end) { end = pos; }
-  for (var i = 0; i < this.ranges.length; i++) {
-    var range = this$1.ranges[i];
-    if (cmp(end, range.from()) >= 0 && cmp(pos, range.to()) <= 0)
-      { return i }
-  }
-  return -1
-};
-
-var Range = function(anchor, head) {
-  this.anchor = anchor; this.head = head;
-};
-
-Range.prototype.from = function () { return minPos(this.anchor, this.head) };
-Range.prototype.to = function () { return maxPos(this.anchor, this.head) };
-Range.prototype.empty = function () { return this.head.line == this.anchor.line && this.head.ch == this.anchor.ch };
-
-// Take an unsorted, potentially overlapping set of ranges, and
-// build a selection out of it. 'Consumes' ranges array (modifying
-// it).
-function normalizeSelection(ranges, primIndex) {
-  var prim = ranges[primIndex];
-  ranges.sort(function (a, b) { return cmp(a.from(), b.from()); });
-  primIndex = indexOf(ranges, prim);
-  for (var i = 1; i < ranges.length; i++) {
-    var cur = ranges[i], prev = ranges[i - 1];
-    if (cmp(prev.to(), cur.from()) >= 0) {
-      var from = minPos(prev.from(), cur.from()), to = maxPos(prev.to(), cur.to());
-      var inv = prev.empty() ? cur.from() == cur.head : prev.from() == prev.head;
-      if (i <= primIndex) { --primIndex; }
-      ranges.splice(--i, 2, new Range(inv ? to : from, inv ? from : to));
-    }
-  }
-  return new Selection(ranges, primIndex)
-}
-
-function simpleSelection(anchor, head) {
-  return new Selection([new Range(anchor, head || anchor)], 0)
-}
-
-// Compute the position of the end of a change (its 'to' property
-// refers to the pre-change end).
-function changeEnd(change) {
-  if (!change.text) { return change.to }
-  return Pos(change.from.line + change.text.length - 1,
-             lst(change.text).length + (change.text.length == 1 ? change.from.ch : 0))
-}
-
-// Adjust a position to refer to the post-change position of the
-// same text, or the end of the change if the change covers it.
-function adjustForChange(pos, change) {
-  if (cmp(pos, change.from) < 0) { return pos }
-  if (cmp(pos, change.to) <= 0) { return changeEnd(change) }
-
-  var line = pos.line + change.text.length - (change.to.line - change.from.line) - 1, ch = pos.ch;
-  if (pos.line == change.to.line) { ch += changeEnd(change).ch - change.to.ch; }
-  return Pos(line, ch)
-}
-
-function computeSelAfterChange(doc, change) {
-  var out = [];
-  for (var i = 0; i < doc.sel.ranges.length; i++) {
-    var range = doc.sel.ranges[i];
-    out.push(new Range(adjustForChange(range.anchor, change),
-                       adjustForChange(range.head, change)));
-  }
-  return normalizeSelection(out, doc.sel.primIndex)
-}
-
-function offsetPos(pos, old, nw) {
-  if (pos.line == old.line)
-    { return Pos(nw.line, pos.ch - old.ch + nw.ch) }
-  else
-    { return Pos(nw.line + (pos.line - old.line), pos.ch) }
-}
-
-// Used by replaceSelections to allow moving the selection to the
-// start or around the replaced test. Hint may be "start" or "around".
-function computeReplacedSel(doc, changes, hint) {
-  var out = [];
-  var oldPrev = Pos(doc.first, 0), newPrev = oldPrev;
-  for (var i = 0; i < changes.length; i++) {
-    var change = changes[i];
-    var from = offsetPos(change.from, oldPrev, newPrev);
-    var to = offsetPos(changeEnd(change), oldPrev, newPrev);
-    oldPrev = change.to;
-    newPrev = to;
-    if (hint == "around") {
-      var range = doc.sel.ranges[i], inv = cmp(range.head, range.anchor) < 0;
-      out[i] = new Range(inv ? to : from, inv ? from : to);
-    } else {
-      out[i] = new Range(from, from);
-    }
-  }
-  return new Selection(out, doc.sel.primIndex)
-}
-
-// Used to get the editor into a consistent state again when options change.
-
-function loadMode(cm) {
-  cm.doc.mode = getMode(cm.options, cm.doc.modeOption);
-  resetModeState(cm);
-}
-
-function resetModeState(cm) {
-  cm.doc.iter(function (line) {
-    if (line.stateAfter) { line.stateAfter = null; }
-    if (line.styles) { line.styles = null; }
-  });
-  cm.doc.modeFrontier = cm.doc.highlightFrontier = cm.doc.first;
-  startWorker(cm, 100);
-  cm.state.modeGen++;
-  if (cm.curOp) { regChange(cm); }
-}
-
-// DOCUMENT DATA STRUCTURE
-
-// By default, updates that start and end at the beginning of a line
-// are treated specially, in order to make the association of line
-// widgets and marker elements with the text behave more intuitive.
-function isWholeLineUpdate(doc, change) {
-  return change.from.ch == 0 && change.to.ch == 0 && lst(change.text) == "" &&
-    (!doc.cm || doc.cm.options.wholeLineUpdateBefore)
-}
-
-// Perform a change on the document data structure.
-function updateDoc(doc, change, markedSpans, estimateHeight$$1) {
-  function spansFor(n) {return markedSpans ? markedSpans[n] : null}
-  function update(line, text, spans) {
-    updateLine(line, text, spans, estimateHeight$$1);
-    signalLater(line, "change", line, change);
-  }
-  function linesFor(start, end) {
-    var result = [];
-    for (var i = start; i < end; ++i)
-      { result.push(new Line(text[i], spansFor(i), estimateHeight$$1)); }
-    return result
-  }
-
-  var from = change.from, to = change.to, text = change.text;
-  var firstLine = getLine(doc, from.line), lastLine = getLine(doc, to.line);
-  var lastText = lst(text), lastSpans = spansFor(text.length - 1), nlines = to.line - from.line;
-
-  // Adjust the line structure
-  if (change.full) {
-    doc.insert(0, linesFor(0, text.length));
-    doc.remove(text.length, doc.size - text.length);
-  } else if (isWholeLineUpdate(doc, change)) {
-    // This is a whole-line replace. Treated specially to make
-    // sure line objects move the way they are supposed to.
-    var added = linesFor(0, text.length - 1);
-    update(lastLine, lastLine.text, lastSpans);
-    if (nlines) { doc.remove(from.line, nlines); }
-    if (added.length) { doc.insert(from.line, added); }
-  } else if (firstLine == lastLine) {
-    if (text.length == 1) {
-      update(firstLine, firstLine.text.slice(0, from.ch) + lastText + firstLine.text.slice(to.ch), lastSpans);
-    } else {
-      var added$1 = linesFor(1, text.length - 1);
-      added$1.push(new Line(lastText + firstLine.text.slice(to.ch), lastSpans, estimateHeight$$1));
-      update(firstLine, firstLine.text.slice(0, from.ch) + text[0], spansFor(0));
-      doc.insert(from.line + 1, added$1);
-    }
-  } else if (text.length == 1) {
-    update(firstLine, firstLine.text.slice(0, from.ch) + text[0] + lastLine.text.slice(to.ch), spansFor(0));
-    doc.remove(from.line + 1, nlines);
-  } else {
-    update(firstLine, firstLine.text.slice(0, from.ch) + text[0], spansFor(0));
-    update(lastLine, lastText + lastLine.text.slice(to.ch), lastSpans);
-    var added$2 = linesFor(1, text.length - 1);
-    if (nlines > 1) { doc.remove(from.line + 1, nlines - 1); }
-    doc.insert(from.line + 1, added$2);
-  }
-
-  signalLater(doc, "change", doc, change);
-}
-
-// Call f for all linked documents.
-function linkedDocs(doc, f, sharedHistOnly) {
-  function propagate(doc, skip, sharedHist) {
-    if (doc.linked) { for (var i = 0; i < doc.linked.length; ++i) {
-      var rel = doc.linked[i];
-      if (rel.doc == skip) { continue }
-      var shared = sharedHist && rel.sharedHist;
-      if (sharedHistOnly && !shared) { continue }
-      f(rel.doc, shared);
-      propagate(rel.doc, doc, shared);
-    } }
-  }
-  propagate(doc, null, true);
-}
-
-// Attach a document to an editor.
-function attachDoc(cm, doc) {
-  if (doc.cm) { throw new Error("This document is already in use.") }
-  cm.doc = doc;
-  doc.cm = cm;
-  estimateLineHeights(cm);
-  loadMode(cm);
-  setDirectionClass(cm);
-  if (!cm.options.lineWrapping) { findMaxLine(cm); }
-  cm.options.mode = doc.modeOption;
-  regChange(cm);
-}
-
-function setDirectionClass(cm) {
-  (cm.doc.direction == "rtl" ? addClass : rmClass)(cm.display.lineDiv, "CodeMirror-rtl");
-}
-
-function directionChanged(cm) {
-  runInOp(cm, function () {
-    setDirectionClass(cm);
-    regChange(cm);
-  });
-}
-
-function History(startGen) {
-  // Arrays of change events and selections. Doing something adds an
-  // event to done and clears undo. Undoing moves events from done
-  // to undone, redoing moves them in the other direction.
-  this.done = []; this.undone = [];
-  this.undoDepth = Infinity;
-  // Used to track when changes can be merged into a single undo
-  // event
-  this.lastModTime = this.lastSelTime = 0;
-  this.lastOp = this.lastSelOp = null;
-  this.lastOrigin = this.lastSelOrigin = null;
-  // Used by the isClean() method
-  this.generation = this.maxGeneration = startGen || 1;
-}
-
-// Create a history change event from an updateDoc-style change
-// object.
-function historyChangeFromChange(doc, change) {
-  var histChange = {from: copyPos(change.from), to: changeEnd(change), text: getBetween(doc, change.from, change.to)};
-  attachLocalSpans(doc, histChange, change.from.line, change.to.line + 1);
-  linkedDocs(doc, function (doc) { return attachLocalSpans(doc, histChange, change.from.line, change.to.line + 1); }, true);
-  return histChange
-}
-
-// Pop all selection events off the end of a history array. Stop at
-// a change event.
-function clearSelectionEvents(array) {
-  while (array.length) {
-    var last = lst(array);
-    if (last.ranges) { array.pop(); }
-    else { break }
-  }
-}
-
-// Find the top change event in the history. Pop off selection
-// events that are in the way.
-function lastChangeEvent(hist, force) {
-  if (force) {
-    clearSelectionEvents(hist.done);
-    return lst(hist.done)
-  } else if (hist.done.length && !lst(hist.done).ranges) {
-    return lst(hist.done)
-  } else if (hist.done.length > 1 && !hist.done[hist.done.length - 2].ranges) {
-    hist.done.pop();
-    return lst(hist.done)
-  }
-}
-
-// Register a change in the history. Merges changes that are within
-// a single operation, or are close together with an origin that
-// allows merging (starting with "+") into a single event.
-function addChangeToHistory(doc, change, selAfter, opId) {
-  var hist = doc.history;
-  hist.undone.length = 0;
-  var time = +new Date, cur;
-  var last;
-
-  if ((hist.lastOp == opId ||
-       hist.lastOrigin == change.origin && change.origin &&
-       ((change.origin.charAt(0) == "+" && doc.cm && hist.lastModTime > time - doc.cm.options.historyEventDelay) ||
-        change.origin.charAt(0) == "*")) &&
-      (cur = lastChangeEvent(hist, hist.lastOp == opId))) {
-    // Merge this change into the last event
-    last = lst(cur.changes);
-    if (cmp(change.from, change.to) == 0 && cmp(change.from, last.to) == 0) {
-      // Optimized case for simple insertion -- don't want to add
-      // new changesets for every character typed
-      last.to = changeEnd(change);
-    } else {
-      // Add new sub-event
-      cur.changes.push(historyChangeFromChange(doc, change));
-    }
-  } else {
-    // Can not be merged, start a new event.
-    var before = lst(hist.done);
-    if (!before || !before.ranges)
-      { pushSelectionToHistory(doc.sel, hist.done); }
-    cur = {changes: [historyChangeFromChange(doc, change)],
-           generation: hist.generation};
-    hist.done.push(cur);
-    while (hist.done.length > hist.undoDepth) {
-      hist.done.shift();
-      if (!hist.done[0].ranges) { hist.done.shift(); }
-    }
-  }
-  hist.done.push(selAfter);
-  hist.generation = ++hist.maxGeneration;
-  hist.lastModTime = hist.lastSelTime = time;
-  hist.lastOp = hist.lastSelOp = opId;
-  hist.lastOrigin = hist.lastSelOrigin = change.origin;
-
-  if (!last) { signal(doc, "historyAdded"); }
-}
-
-function selectionEventCanBeMerged(doc, origin, prev, sel) {
-  var ch = origin.charAt(0);
-  return ch == "*" ||
-    ch == "+" &&
-    prev.ranges.length == sel.ranges.length &&
-    prev.somethingSelected() == sel.somethingSelected() &&
-    new Date - doc.history.lastSelTime <= (doc.cm ? doc.cm.options.historyEventDelay : 500)
-}
-
-// Called whenever the selection changes, sets the new selection as
-// the pending selection in the history, and pushes the old pending
-// selection into the 'done' array when it was significantly
-// different (in number of selected ranges, emptiness, or time).
-function addSelectionToHistory(doc, sel, opId, options) {
-  var hist = doc.history, origin = options && options.origin;
-
-  // A new event is started when the previous origin does not match
-  // the current, or the origins don't allow matching. Origins
-  // starting with * are always merged, those starting with + are
-  // merged when similar and close together in time.
-  if (opId == hist.lastSelOp ||
-      (origin && hist.lastSelOrigin == origin &&
-       (hist.lastModTime == hist.lastSelTime && hist.lastOrigin == origin ||
-        selectionEventCanBeMerged(doc, origin, lst(hist.done), sel))))
-    { hist.done[hist.done.length - 1] = sel; }
-  else
-    { pushSelectionToHistory(sel, hist.done); }
-
-  hist.lastSelTime = +new Date;
-  hist.lastSelOrigin = origin;
-  hist.lastSelOp = opId;
-  if (options && options.clearRedo !== false)
-    { clearSelectionEvents(hist.undone); }
-}
-
-function pushSelectionToHistory(sel, dest) {
-  var top = lst(dest);
-  if (!(top && top.ranges && top.equals(sel)))
-    { dest.push(sel); }
-}
-
-// Used to store marked span information in the history.
-function attachLocalSpans(doc, change, from, to) {
-  var existing = change["spans_" + doc.id], n = 0;
-  doc.iter(Math.max(doc.first, from), Math.min(doc.first + doc.size, to), function (line) {
-    if (line.markedSpans)
-      { (existing || (existing = change["spans_" + doc.id] = {}))[n] = line.markedSpans; }
-    ++n;
-  });
-}
-
-// When un/re-doing restores text containing marked spans, those
-// that have been explicitly cleared should not be restored.
-function removeClearedSpans(spans) {
-  if (!spans) { return null }
-  var out;
-  for (var i = 0; i < spans.length; ++i) {
-    if (spans[i].marker.explicitlyCleared) { if (!out) { out = spans.slice(0, i); } }
-    else if (out) { out.push(spans[i]); }
-  }
-  return !out ? spans : out.length ? out : null
-}
-
-// Retrieve and filter the old marked spans stored in a change event.
-function getOldSpans(doc, change) {
-  var found = change["spans_" + doc.id];
-  if (!found) { return null }
-  var nw = [];
-  for (var i = 0; i < change.text.length; ++i)
-    { nw.push(removeClearedSpans(found[i])); }
-  return nw
-}
-
-// Used for un/re-doing changes from the history. Combines the
-// result of computing the existing spans with the set of spans that
-// existed in the history (so that deleting around a span and then
-// undoing brings back the span).
-function mergeOldSpans(doc, change) {
-  var old = getOldSpans(doc, change);
-  var stretched = stretchSpansOverChange(doc, change);
-  if (!old) { return stretched }
-  if (!stretched) { return old }
-
-  for (var i = 0; i < old.length; ++i) {
-    var oldCur = old[i], stretchCur = stretched[i];
-    if (oldCur && stretchCur) {
-      spans: for (var j = 0; j < stretchCur.length; ++j) {
-        var span = stretchCur[j];
-        for (var k = 0; k < oldCur.length; ++k)
-          { if (oldCur[k].marker == span.marker) { continue spans } }
-        oldCur.push(span);
-      }
-    } else if (stretchCur) {
-      old[i] = stretchCur;
-    }
-  }
-  return old
-}
-
-// Used both to provide a JSON-safe object in .getHistory, and, when
-// detaching a document, to split the history in two
-function copyHistoryArray(events, newGroup, instantiateSel) {
-  var copy = [];
-  for (var i = 0; i < events.length; ++i) {
-    var event = events[i];
-    if (event.ranges) {
-      copy.push(instantiateSel ? Selection.prototype.deepCopy.call(event) : event);
-      continue
-    }
-    var changes = event.changes, newChanges = [];
-    copy.push({changes: newChanges});
-    for (var j = 0; j < changes.length; ++j) {
-      var change = changes[j], m = (void 0);
-      newChanges.push({from: change.from, to: change.to, text: change.text});
-      if (newGroup) { for (var prop in change) { if (m = prop.match(/^spans_(\d+)$/)) {
-        if (indexOf(newGroup, Number(m[1])) > -1) {
-          lst(newChanges)[prop] = change[prop];
-          delete change[prop];
-        }
-      } } }
-    }
-  }
-  return copy
-}
-
-// The 'scroll' parameter given to many of these indicated whether
-// the new cursor position should be scrolled into view after
-// modifying the selection.
-
-// If shift is held or the extend flag is set, extends a range to
-// include a given position (and optionally a second position).
-// Otherwise, simply returns the range between the given positions.
-// Used for cursor motion and such.
-function extendRange(range, head, other, extend) {
-  if (extend) {
-    var anchor = range.anchor;
-    if (other) {
-      var posBefore = cmp(head, anchor) < 0;
-      if (posBefore != (cmp(other, anchor) < 0)) {
-        anchor = head;
-        head = other;
-      } else if (posBefore != (cmp(head, other) < 0)) {
-        head = other;
-      }
-    }
-    return new Range(anchor, head)
-  } else {
-    return new Range(other || head, head)
-  }
-}
-
-// Extend the primary selection range, discard the rest.
-function extendSelection(doc, head, other, options, extend) {
-  if (extend == null) { extend = doc.cm && (doc.cm.display.shift || doc.extend); }
-  setSelection(doc, new Selection([extendRange(doc.sel.primary(), head, other, extend)], 0), options);
-}
-
-// Extend all selections (pos is an array of selections with length
-// equal the number of selections)
-function extendSelections(doc, heads, options) {
-  var out = [];
-  var extend = doc.cm && (doc.cm.display.shift || doc.extend);
-  for (var i = 0; i < doc.sel.ranges.length; i++)
-    { out[i] = extendRange(doc.sel.ranges[i], heads[i], null, extend); }
-  var newSel = normalizeSelection(out, doc.sel.primIndex);
-  setSelection(doc, newSel, options);
-}
-
-// Updates a single range in the selection.
-function replaceOneSelection(doc, i, range, options) {
-  var ranges = doc.sel.ranges.slice(0);
-  ranges[i] = range;
-  setSelection(doc, normalizeSelection(ranges, doc.sel.primIndex), options);
-}
-
-// Reset the selection to a single range.
-function setSimpleSelection(doc, anchor, head, options) {
-  setSelection(doc, simpleSelection(anchor, head), options);
-}
-
-// Give beforeSelectionChange handlers a change to influence a
-// selection update.
-function filterSelectionChange(doc, sel, options) {
-  var obj = {
-    ranges: sel.ranges,
-    update: function(ranges) {
-      var this$1 = this;
-
-      this.ranges = [];
-      for (var i = 0; i < ranges.length; i++)
-        { this$1.ranges[i] = new Range(clipPos(doc, ranges[i].anchor),
-                                   clipPos(doc, ranges[i].head)); }
-    },
-    origin: options && options.origin
-  };
-  signal(doc, "beforeSelectionChange", doc, obj);
-  if (doc.cm) { signal(doc.cm, "beforeSelectionChange", doc.cm, obj); }
-  if (obj.ranges != sel.ranges) { return normalizeSelection(obj.ranges, obj.ranges.length - 1) }
-  else { return sel }
-}
-
-function setSelectionReplaceHistory(doc, sel, options) {
-  var done = doc.history.done, last = lst(done);
-  if (last && last.ranges) {
-    done[done.length - 1] = sel;
-    setSelectionNoUndo(doc, sel, options);
-  } else {
-    setSelection(doc, sel, options);
-  }
-}
-
-// Set a new selection.
-function setSelection(doc, sel, options) {
-  setSelectionNoUndo(doc, sel, options);
-  addSelectionToHistory(doc, doc.sel, doc.cm ? doc.cm.curOp.id : NaN, options);
-}
-
-function setSelectionNoUndo(doc, sel, options) {
-  if (hasHandler(doc, "beforeSelectionChange") || doc.cm && hasHandler(doc.cm, "beforeSelectionChange"))
-    { sel = filterSelectionChange(doc, sel, options); }
-
-  var bias = options && options.bias ||
-    (cmp(sel.primary().head, doc.sel.primary().head) < 0 ? -1 : 1);
-  setSelectionInner(doc, skipAtomicInSelection(doc, sel, bias, true));
-
-  if (!(options && options.scroll === false) && doc.cm)
-    { ensureCursorVisible(doc.cm); }
-}
-
-function setSelectionInner(doc, sel) {
-  if (sel.equals(doc.sel)) { return }
-
-  doc.sel = sel;
-
-  if (doc.cm) {
-    doc.cm.curOp.updateInput = doc.cm.curOp.selectionChanged = true;
-    signalCursorActivity(doc.cm);
-  }
-  signalLater(doc, "cursorActivity", doc);
-}
-
-// Verify that the selection does not partially select any atomic
-// marked ranges.
-function reCheckSelection(doc) {
-  setSelectionInner(doc, skipAtomicInSelection(doc, doc.sel, null, false));
-}
-
-// Return a selection that does not partially select any atomic
-// ranges.
-function skipAtomicInSelection(doc, sel, bias, mayClear) {
-  var out;
-  for (var i = 0; i < sel.ranges.length; i++) {
-    var range = sel.ranges[i];
-    var old = sel.ranges.length == doc.sel.ranges.length && doc.sel.ranges[i];
-    var newAnchor = skipAtomic(doc, range.anchor, old && old.anchor, bias, mayClear);
-    var newHead = skipAtomic(doc, range.head, old && old.head, bias, mayClear);
-    if (out || newAnchor != range.anchor || newHead != range.head) {
-      if (!out) { out = sel.ranges.slice(0, i); }
-      out[i] = new Range(newAnchor, newHead);
-    }
-  }
-  return out ? normalizeSelection(out, sel.primIndex) : sel
-}
-
-function skipAtomicInner(doc, pos, oldPos, dir, mayClear) {
-  var line = getLine(doc, pos.line);
-  if (line.markedSpans) { for (var i = 0; i < line.markedSpans.length; ++i) {
-    var sp = line.markedSpans[i], m = sp.marker;
-    if ((sp.from == null || (m.inclusiveLeft ? sp.from <= pos.ch : sp.from < pos.ch)) &&
-        (sp.to == null || (m.inclusiveRight ? sp.to >= pos.ch : sp.to > pos.ch))) {
-      if (mayClear) {
-        signal(m, "beforeCursorEnter");
-        if (m.explicitlyCleared) {
-          if (!line.markedSpans) { break }
-          else {--i; continue}
-        }
-      }
-      if (!m.atomic) { continue }
-
-      if (oldPos) {
-        var near = m.find(dir < 0 ? 1 : -1), diff = (void 0);
-        if (dir < 0 ? m.inclusiveRight : m.inclusiveLeft)
-          { near = movePos(doc, near, -dir, near && near.line == pos.line ? line : null); }
-        if (near && near.line == pos.line && (diff = cmp(near, oldPos)) && (dir < 0 ? diff < 0 : diff > 0))
-          { return skipAtomicInner(doc, near, pos, dir, mayClear) }
-      }
-
-      var far = m.find(dir < 0 ? -1 : 1);
-      if (dir < 0 ? m.inclusiveLeft : m.inclusiveRight)
-        { far = movePos(doc, far, dir, far.line == pos.line ? line : null); }
-      return far ? skipAtomicInner(doc, far, pos, dir, mayClear) : null
-    }
-  } }
-  return pos
-}
-
-// Ensure a given position is not inside an atomic range.
-function skipAtomic(doc, pos, oldPos, bias, mayClear) {
-  var dir = bias || 1;
-  var found = skipAtomicInner(doc, pos, oldPos, dir, mayClear) ||
-      (!mayClear && skipAtomicInner(doc, pos, oldPos, dir, true)) ||
-      skipAtomicInner(doc, pos, oldPos, -dir, mayClear) ||
-      (!mayClear && skipAtomicInner(doc, pos, oldPos, -dir, true));
-  if (!found) {
-    doc.cantEdit = true;
-    return Pos(doc.first, 0)
-  }
-  return found
-}
-
-function movePos(doc, pos, dir, line) {
-  if (dir < 0 && pos.ch == 0) {
-    if (pos.line > doc.first) { return clipPos(doc, Pos(pos.line - 1)) }
-    else { return null }
-  } else if (dir > 0 && pos.ch == (line || getLine(doc, pos.line)).text.length) {
-    if (pos.line < doc.first + doc.size - 1) { return Pos(pos.line + 1, 0) }
-    else { return null }
-  } else {
-    return new Pos(pos.line, pos.ch + dir)
-  }
-}
-
-function selectAll(cm) {
-  cm.setSelection(Pos(cm.firstLine(), 0), Pos(cm.lastLine()), sel_dontScroll);
-}
-
-// UPDATING
-
-// Allow "beforeChange" event handlers to influence a change
-function filterChange(doc, change, update) {
-  var obj = {
-    canceled: false,
-    from: change.from,
-    to: change.to,
-    text: change.text,
-    origin: change.origin,
-    cancel: function () { return obj.canceled = true; }
-  };
-  if (update) { obj.update = function (from, to, text, origin) {
-    if (from) { obj.from = clipPos(doc, from); }
-    if (to) { obj.to = clipPos(doc, to); }
-    if (text) { obj.text = text; }
-    if (origin !== undefined) { obj.origin = origin; }
-  }; }
-  signal(doc, "beforeChange", doc, obj);
-  if (doc.cm) { signal(doc.cm, "beforeChange", doc.cm, obj); }
-
-  if (obj.canceled) { return null }
-  return {from: obj.from, to: obj.to, text: obj.text, origin: obj.origin}
-}
-
-// Apply a change to a document, and add it to the document's
-// history, and propagating it to all linked documents.
-function makeChange(doc, change, ignoreReadOnly) {
-  if (doc.cm) {
-    if (!doc.cm.curOp) { return operation(doc.cm, makeChange)(doc, change, ignoreReadOnly) }
-    if (doc.cm.state.suppressEdits) { return }
-  }
-
-  if (hasHandler(doc, "beforeChange") || doc.cm && hasHandler(doc.cm, "beforeChange")) {
-    change = filterChange(doc, change, true);
-    if (!change) { return }
-  }
-
-  // Possibly split or suppress the update based on the presence
-  // of read-only spans in its range.
-  var split = sawReadOnlySpans && !ignoreReadOnly && removeReadOnlyRanges(doc, change.from, change.to);
-  if (split) {
-    for (var i = split.length - 1; i >= 0; --i)
-      { makeChangeInner(doc, {from: split[i].from, to: split[i].to, text: i ? [""] : change.text, origin: change.origin}); }
-  } else {
-    makeChangeInner(doc, change);
-  }
-}
-
-function makeChangeInner(doc, change) {
-  if (change.text.length == 1 && change.text[0] == "" && cmp(change.from, change.to) == 0) { return }
-  var selAfter = computeSelAfterChange(doc, change);
-  addChangeToHistory(doc, change, selAfter, doc.cm ? doc.cm.curOp.id : NaN);
-
-  makeChangeSingleDoc(doc, change, selAfter, stretchSpansOverChange(doc, change));
-  var rebased = [];
-
-  linkedDocs(doc, function (doc, sharedHist) {
-    if (!sharedHist && indexOf(rebased, doc.history) == -1) {
-      rebaseHist(doc.history, change);
-      rebased.push(doc.history);
-    }
-    makeChangeSingleDoc(doc, change, null, stretchSpansOverChange(doc, change));
-  });
-}
-
-// Revert a change stored in a document's history.
-function makeChangeFromHistory(doc, type, allowSelectionOnly) {
-  if (doc.cm && doc.cm.state.suppressEdits && !allowSelectionOnly) { return }
-
-  var hist = doc.history, event, selAfter = doc.sel;
-  var source = type == "undo" ? hist.done : hist.undone, dest = type == "undo" ? hist.undone : hist.done;
-
-  // Verify that there is a useable event (so that ctrl-z won't
-  // needlessly clear selection events)
-  var i = 0;
-  for (; i < source.length; i++) {
-    event = source[i];
-    if (allowSelectionOnly ? event.ranges && !event.equals(doc.sel) : !event.ranges)
-      { break }
-  }
-  if (i == source.length) { return }
-  hist.lastOrigin = hist.lastSelOrigin = null;
-
-  for (;;) {
-    event = source.pop();
-    if (event.ranges) {
-      pushSelectionToHistory(event, dest);
-      if (allowSelectionOnly && !event.equals(doc.sel)) {
-        setSelection(doc, event, {clearRedo: false});
-        return
-      }
-      selAfter = event;
-    }
-    else { break }
-  }
-
-  // Build up a reverse change object to add to the opposite history
-  // stack (redo when undoing, and vice versa).
-  var antiChanges = [];
-  pushSelectionToHistory(selAfter, dest);
-  dest.push({changes: antiChanges, generation: hist.generation});
-  hist.generation = event.generation || ++hist.maxGeneration;
-
-  var filter = hasHandler(doc, "beforeChange") || doc.cm && hasHandler(doc.cm, "beforeChange");
-
-  var loop = function ( i ) {
-    var change = event.changes[i];
-    change.origin = type;
-    if (filter && !filterChange(doc, change, false)) {
-      source.length = 0;
-      return {}
-    }
-
-    antiChanges.push(historyChangeFromChange(doc, change));
-
-    var after = i ? computeSelAfterChange(doc, change) : lst(source);
-    makeChangeSingleDoc(doc, change, after, mergeOldSpans(doc, change));
-    if (!i && doc.cm) { doc.cm.scrollIntoView({from: change.from, to: changeEnd(change)}); }
-    var rebased = [];
-
-    // Propagate to the linked documents
-    linkedDocs(doc, function (doc, sharedHist) {
-      if (!sharedHist && indexOf(rebased, doc.history) == -1) {
-        rebaseHist(doc.history, change);
-        rebased.push(doc.history);
-      }
-      makeChangeSingleDoc(doc, change, null, mergeOldSpans(doc, change));
-    });
-  };
-
-  for (var i$1 = event.changes.length - 1; i$1 >= 0; --i$1) {
-    var returned = loop( i$1 );
-
-    if ( returned ) return returned.v;
-  }
-}
-
-// Sub-views need their line numbers shifted when text is added
-// above or below them in the parent document.
-function shiftDoc(doc, distance) {
-  if (distance == 0) { return }
-  doc.first += distance;
-  doc.sel = new Selection(map(doc.sel.ranges, function (range) { return new Range(
-    Pos(range.anchor.line + distance, range.anchor.ch),
-    Pos(range.head.line + distance, range.head.ch)
-  ); }), doc.sel.primIndex);
-  if (doc.cm) {
-    regChange(doc.cm, doc.first, doc.first - distance, distance);
-    for (var d = doc.cm.display, l = d.viewFrom; l < d.viewTo; l++)
-      { regLineChange(doc.cm, l, "gutter"); }
-  }
-}
-
-// More lower-level change function, handling only a single document
-// (not linked ones).
-function makeChangeSingleDoc(doc, change, selAfter, spans) {
-  if (doc.cm && !doc.cm.curOp)
-    { return operation(doc.cm, makeChangeSingleDoc)(doc, change, selAfter, spans) }
-
-  if (change.to.line < doc.first) {
-    shiftDoc(doc, change.text.length - 1 - (change.to.line - change.from.line));
-    return
-  }
-  if (change.from.line > doc.lastLine()) { return }
-
-  // Clip the change to the size of this doc
-  if (change.from.line < doc.first) {
-    var shift = change.text.length - 1 - (doc.first - change.from.line);
-    shiftDoc(doc, shift);
-    change = {from: Pos(doc.first, 0), to: Pos(change.to.line + shift, change.to.ch),
-              text: [lst(change.text)], origin: change.origin};
-  }
-  var last = doc.lastLine();
-  if (change.to.line > last) {
-    change = {from: change.from, to: Pos(last, getLine(doc, last).text.length),
-              text: [change.text[0]], origin: change.origin};
-  }
-
-  change.removed = getBetween(doc, change.from, change.to);
-
-  if (!selAfter) { selAfter = computeSelAfterChange(doc, change); }
-  if (doc.cm) { makeChangeSingleDocInEditor(doc.cm, change, spans); }
-  else { updateDoc(doc, change, spans); }
-  setSelectionNoUndo(doc, selAfter, sel_dontScroll);
-}
-
-// Handle the interaction of a change to a document with the editor
-// that this document is part of.
-function makeChangeSingleDocInEditor(cm, change, spans) {
-  var doc = cm.doc, display = cm.display, from = change.from, to = change.to;
-
-  var recomputeMaxLength = false, checkWidthStart = from.line;
-  if (!cm.options.lineWrapping) {
-    checkWidthStart = lineNo(visualLine(getLine(doc, from.line)));
-    doc.iter(checkWidthStart, to.line + 1, function (line) {
-      if (line == display.maxLine) {
-        recomputeMaxLength = true;
-        return true
-      }
-    });
-  }
-
-  if (doc.sel.contains(change.from, change.to) > -1)
-    { signalCursorActivity(cm); }
-
-  updateDoc(doc, change, spans, estimateHeight(cm));
-
-  if (!cm.options.lineWrapping) {
-    doc.iter(checkWidthStart, from.line + change.text.length, function (line) {
-      var len = lineLength(line);
-      if (len > display.maxLineLength) {
-        display.maxLine = line;
-        display.maxLineLength = len;
-        display.maxLineChanged = true;
-        recomputeMaxLength = false;
-      }
-    });
-    if (recomputeMaxLength) { cm.curOp.updateMaxLine = true; }
-  }
-
-  retreatFrontier(doc, from.line);
-  startWorker(cm, 400);
-
-  var lendiff = change.text.length - (to.line - from.line) - 1;
-  // Remember that these lines changed, for updating the display
-  if (change.full)
-    { regChange(cm); }
-  else if (from.line == to.line && change.text.length == 1 && !isWholeLineUpdate(cm.doc, change))
-    { regLineChange(cm, from.line, "text"); }
-  else
-    { regChange(cm, from.line, to.line + 1, lendiff); }
-
-  var changesHandler = hasHandler(cm, "changes"), changeHandler = hasHandler(cm, "change");
-  if (changeHandler || changesHandler) {
-    var obj = {
-      from: from, to: to,
-      text: change.text,
-      removed: change.removed,
-      origin: change.origin
-    };
-    if (changeHandler) { signalLater(cm, "change", cm, obj); }
-    if (changesHandler) { (cm.curOp.changeObjs || (cm.curOp.changeObjs = [])).push(obj); }
-  }
-  cm.display.selForContextMenu = null;
-}
-
-function replaceRange(doc, code, from, to, origin) {
-  if (!to) { to = from; }
-  if (cmp(to, from) < 0) { var assign;
-    (assign = [to, from], from = assign[0], to = assign[1], assign); }
-  if (typeof code == "string") { code = doc.splitLines(code); }
-  makeChange(doc, {from: from, to: to, text: code, origin: origin});
-}
-
-// Rebasing/resetting history to deal with externally-sourced changes
-
-function rebaseHistSelSingle(pos, from, to, diff) {
-  if (to < pos.line) {
-    pos.line += diff;
-  } else if (from < pos.line) {
-    pos.line = from;
-    pos.ch = 0;
-  }
-}
-
-// Tries to rebase an array of history events given a change in the
-// document. If the change touches the same lines as the event, the
-// event, and everything 'behind' it, is discarded. If the change is
-// before the event, the event's positions are updated. Uses a
-// copy-on-write scheme for the positions, to avoid having to
-// reallocate them all on every rebase, but also avoid problems with
-// shared position objects being unsafely updated.
-function rebaseHistArray(array, from, to, diff) {
-  for (var i = 0; i < array.length; ++i) {
-    var sub = array[i], ok = true;
-    if (sub.ranges) {
-      if (!sub.copied) { sub = array[i] = sub.deepCopy(); sub.copied = true; }
-      for (var j = 0; j < sub.ranges.length; j++) {
-        rebaseHistSelSingle(sub.ranges[j].anchor, from, to, diff);
-        rebaseHistSelSingle(sub.ranges[j].head, from, to, diff);
-      }
-      continue
-    }
-    for (var j$1 = 0; j$1 < sub.changes.length; ++j$1) {
-      var cur = sub.changes[j$1];
-      if (to < cur.from.line) {
-        cur.from = Pos(cur.from.line + diff, cur.from.ch);
-        cur.to = Pos(cur.to.line + diff, cur.to.ch);
-      } else if (from <= cur.to.line) {
-        ok = false;
-        break
-      }
-    }
-    if (!ok) {
-      array.splice(0, i + 1);
-      i = 0;
-    }
-  }
-}
-
-function rebaseHist(hist, change) {
-  var from = change.from.line, to = change.to.line, diff = change.text.length - (to - from) - 1;
-  rebaseHistArray(hist.done, from, to, diff);
-  rebaseHistArray(hist.undone, from, to, diff);
-}
-
-// Utility for applying a change to a line by handle or number,
-// returning the number and optionally registering the line as
-// changed.
-function changeLine(doc, handle, changeType, op) {
-  var no = handle, line = handle;
-  if (typeof handle == "number") { line = getLine(doc, clipLine(doc, handle)); }
-  else { no = lineNo(handle); }
-  if (no == null) { return null }
-  if (op(line, no) && doc.cm) { regLineChange(doc.cm, no, changeType); }
-  return line
-}
-
-// The document is represented as a BTree consisting of leaves, with
-// chunk of lines in them, and branches, with up to ten leaves or
-// other branch nodes below them. The top node is always a branch
-// node, and is the document object itself (meaning it has
-// additional methods and properties).
-//
-// All nodes have parent links. The tree is used both to go from
-// line numbers to line objects, and to go from objects to numbers.
-// It also indexes by height, and is used to convert between height
-// and line object, and to find the total height of the document.
-//
-// See also http://marijnhaverbeke.nl/blog/codemirror-line-tree.html
-
-function LeafChunk(lines) {
-  var this$1 = this;
-
-  this.lines = lines;
-  this.parent = null;
-  var height = 0;
-  for (var i = 0; i < lines.length; ++i) {
-    lines[i].parent = this$1;
-    height += lines[i].height;
-  }
-  this.height = height;
-}
-
-LeafChunk.prototype = {
-  chunkSize: function chunkSize() { return this.lines.length },
-
-  // Remove the n lines at offset 'at'.
-  removeInner: function removeInner(at, n) {
-    var this$1 = this;
-
-    for (var i = at, e = at + n; i < e; ++i) {
-      var line = this$1.lines[i];
-      this$1.height -= line.height;
-      cleanUpLine(line);
-      signalLater(line, "delete");
-    }
-    this.lines.splice(at, n);
-  },
-
-  // Helper used to collapse a small branch into a single leaf.
-  collapse: function collapse(lines) {
-    lines.push.apply(lines, this.lines);
-  },
-
-  // Insert the given array of lines at offset 'at', count them as
-  // having the given height.
-  insertInner: function insertInner(at, lines, height) {
-    var this$1 = this;
-
-    this.height += height;
-    this.lines = this.lines.slice(0, at).concat(lines).concat(this.lines.slice(at));
-    for (var i = 0; i < lines.length; ++i) { lines[i].parent = this$1; }
-  },
-
-  // Used to iterate over a part of the tree.
-  iterN: function iterN(at, n, op) {
-    var this$1 = this;
-
-    for (var e = at + n; at < e; ++at)
-      { if (op(this$1.lines[at])) { return true } }
-  }
-};
-
-function BranchChunk(children) {
-  var this$1 = this;
-
-  this.children = children;
-  var size = 0, height = 0;
-  for (var i = 0; i < children.length; ++i) {
-    var ch = children[i];
-    size += ch.chunkSize(); height += ch.height;
-    ch.parent = this$1;
-  }
-  this.size = size;
-  this.height = height;
-  this.parent = null;
-}
-
-BranchChunk.prototype = {
-  chunkSize: function chunkSize() { return this.size },
-
-  removeInner: function removeInner(at, n) {
-    var this$1 = this;
-
-    this.size -= n;
-    for (var i = 0; i < this.children.length; ++i) {
-      var child = this$1.children[i], sz = child.chunkSize();
-      if (at < sz) {
-        var rm = Math.min(n, sz - at), oldHeight = child.height;
-        child.removeInner(at, rm);
-        this$1.height -= oldHeight - child.height;
-        if (sz == rm) { this$1.children.splice(i--, 1); child.parent = null; }
-        if ((n -= rm) == 0) { break }
-        at = 0;
-      } else { at -= sz; }
-    }
-    // If the result is smaller than 25 lines, ensure that it is a
-    // single leaf node.
-    if (this.size - n < 25 &&
-        (this.children.length > 1 || !(this.children[0] instanceof LeafChunk))) {
-      var lines = [];
-      this.collapse(lines);
-      this.children = [new LeafChunk(lines)];
-      this.children[0].parent = this;
-    }
-  },
-
-  collapse: function collapse(lines) {
-    var this$1 = this;
-
-    for (var i = 0; i < this.children.length; ++i) { this$1.children[i].collapse(lines); }
-  },
-
-  insertInner: function insertInner(at, lines, height) {
-    var this$1 = this;
-
-    this.size += lines.length;
-    this.height += height;
-    for (var i = 0; i < this.children.length; ++i) {
-      var child = this$1.children[i], sz = child.chunkSize();
-      if (at <= sz) {
-        child.insertInner(at, lines, height);
-        if (child.lines && child.lines.length > 50) {
-          // To avoid memory thrashing when child.lines is huge (e.g. first view of a large file), it's never spliced.
-          // Instead, small slices are taken. They're taken in order because sequential memory accesses are fastest.
-          var remaining = child.lines.length % 25 + 25;
-          for (var pos = remaining; pos < child.lines.length;) {
-            var leaf = new LeafChunk(child.lines.slice(pos, pos += 25));
-            child.height -= leaf.height;
-            this$1.children.splice(++i, 0, leaf);
-            leaf.parent = this$1;
-          }
-          child.lines = child.lines.slice(0, remaining);
-          this$1.maybeSpill();
-        }
-        break
-      }
-      at -= sz;
-    }
-  },
-
-  // When a node has grown, check whether it should be split.
-  maybeSpill: function maybeSpill() {
-    if (this.children.length <= 10) { return }
-    var me = this;
-    do {
-      var spilled = me.children.splice(me.children.length - 5, 5);
-      var sibling = new BranchChunk(spilled);
-      if (!me.parent) { // Become the parent node
-        var copy = new BranchChunk(me.children);
-        copy.parent = me;
-        me.children = [copy, sibling];
-        me = copy;
-     } else {
-        me.size -= sibling.size;
-        me.height -= sibling.height;
-        var myIndex = indexOf(me.parent.children, me);
-        me.parent.children.splice(myIndex + 1, 0, sibling);
-      }
-      sibling.parent = me.parent;
-    } while (me.children.length > 10)
-    me.parent.maybeSpill();
-  },
-
-  iterN: function iterN(at, n, op) {
-    var this$1 = this;
-
-    for (var i = 0; i < this.children.length; ++i) {
-      var child = this$1.children[i], sz = child.chunkSize();
-      if (at < sz) {
-        var used = Math.min(n, sz - at);
-        if (child.iterN(at, used, op)) { return true }
-        if ((n -= used) == 0) { break }
-        at = 0;
-      } else { at -= sz; }
-    }
-  }
-};
-
-// Line widgets are block elements displayed above or below a line.
-
-var LineWidget = function(doc, node, options) {
-  var this$1 = this;
-
-  if (options) { for (var opt in options) { if (options.hasOwnProperty(opt))
-    { this$1[opt] = options[opt]; } } }
-  this.doc = doc;
-  this.node = node;
-};
-
-LineWidget.prototype.clear = function () {
-    var this$1 = this;
-
-  var cm = this.doc.cm, ws = this.line.widgets, line = this.line, no = lineNo(line);
-  if (no == null || !ws) { return }
-  for (var i = 0; i < ws.length; ++i) { if (ws[i] == this$1) { ws.splice(i--, 1); } }
-  if (!ws.length) { line.widgets = null; }
-  var height = widgetHeight(this);
-  updateLineHeight(line, Math.max(0, line.height - height));
-  if (cm) {
-    runInOp(cm, function () {
-      adjustScrollWhenAboveVisible(cm, line, -height);
-      regLineChange(cm, no, "widget");
-    });
-    signalLater(cm, "lineWidgetCleared", cm, this, no);
-  }
-};
-
-LineWidget.prototype.changed = function () {
-    var this$1 = this;
-
-  var oldH = this.height, cm = this.doc.cm, line = this.line;
-  this.height = null;
-  var diff = widgetHeight(this) - oldH;
-  if (!diff) { return }
-  updateLineHeight(line, line.height + diff);
-  if (cm) {
-    runInOp(cm, function () {
-      cm.curOp.forceUpdate = true;
-      adjustScrollWhenAboveVisible(cm, line, diff);
-      signalLater(cm, "lineWidgetChanged", cm, this$1, lineNo(line));
-    });
-  }
-};
-eventMixin(LineWidget);
-
-function adjustScrollWhenAboveVisible(cm, line, diff) {
-  if (heightAtLine(line) < ((cm.curOp && cm.curOp.scrollTop) || cm.doc.scrollTop))
-    { addToScrollTop(cm, diff); }
-}
-
-function addLineWidget(doc, handle, node, options) {
-  var widget = new LineWidget(doc, node, options);
-  var cm = doc.cm;
-  if (cm && widget.noHScroll) { cm.display.alignWidgets = true; }
-  changeLine(doc, handle, "widget", function (line) {
-    var widgets = line.widgets || (line.widgets = []);
-    if (widget.insertAt == null) { widgets.push(widget); }
-    else { widgets.splice(Math.min(widgets.length - 1, Math.max(0, widget.insertAt)), 0, widget); }
-    widget.line = line;
-    if (cm && !lineIsHidden(doc, line)) {
-      var aboveVisible = heightAtLine(line) < doc.scrollTop;
-      updateLineHeight(line, line.height + widgetHeight(widget));
-      if (aboveVisible) { addToScrollTop(cm, widget.height); }
-      cm.curOp.forceUpdate = true;
-    }
-    return true
-  });
-  signalLater(cm, "lineWidgetAdded", cm, widget, typeof handle == "number" ? handle : lineNo(handle));
-  return widget
-}
-
-// TEXTMARKERS
-
-// Created with markText and setBookmark methods. A TextMarker is a
-// handle that can be used to clear or find a marked position in the
-// document. Line objects hold arrays (markedSpans) containing
-// {from, to, marker} object pointing to such marker objects, and
-// indicating that such a marker is present on that line. Multiple
-// lines may point to the same marker when it spans across lines.
-// The spans will have null for their from/to properties when the
-// marker continues beyond the start/end of the line. Markers have
-// links back to the lines they currently touch.
-
-// Collapsed markers have unique ids, in order to be able to order
-// them, which is needed for uniquely determining an outer marker
-// when they overlap (they may nest, but not partially overlap).
-var nextMarkerId = 0;
-
-var TextMarker = function(doc, type) {
-  this.lines = [];
-  this.type = type;
-  this.doc = doc;
-  this.id = ++nextMarkerId;
-};
-
-// Clear the marker.
-TextMarker.prototype.clear = function () {
-    var this$1 = this;
-
-  if (this.explicitlyCleared) { return }
-  var cm = this.doc.cm, withOp = cm && !cm.curOp;
-  if (withOp) { startOperation(cm); }
-  if (hasHandler(this, "clear")) {
-    var found = this.find();
-    if (found) { signalLater(this, "clear", found.from, found.to); }
-  }
-  var min = null, max = null;
-  for (var i = 0; i < this.lines.length; ++i) {
-    var line = this$1.lines[i];
-    var span = getMarkedSpanFor(line.markedSpans, this$1);
-    if (cm && !this$1.collapsed) { regLineChange(cm, lineNo(line), "text"); }
-    else if (cm) {
-      if (span.to != null) { max = lineNo(line); }
-      if (span.from != null) { min = lineNo(line); }
-    }
-    line.markedSpans = removeMarkedSpan(line.markedSpans, span);
-    if (span.from == null && this$1.collapsed && !lineIsHidden(this$1.doc, line) && cm)
-      { updateLineHeight(line, textHeight(cm.display)); }
-  }
-  if (cm && this.collapsed && !cm.options.lineWrapping) { for (var i$1 = 0; i$1 < this.lines.length; ++i$1) {
-    var visual = visualLine(this$1.lines[i$1]), len = lineLength(visual);
-    if (len > cm.display.maxLineLength) {
-      cm.display.maxLine = visual;
-      cm.display.maxLineLength = len;
-      cm.display.maxLineChanged = true;
-    }
-  } }
-
-  if (min != null && cm && this.collapsed) { regChange(cm, min, max + 1); }
-  this.lines.length = 0;
-  this.explicitlyCleared = true;
-  if (this.atomic && this.doc.cantEdit) {
-    this.doc.cantEdit = false;
-    if (cm) { reCheckSelection(cm.doc); }
-  }
-  if (cm) { signalLater(cm, "markerCleared", cm, this, min, max); }
-  if (withOp) { endOperation(cm); }
-  if (this.parent) { this.parent.clear(); }
-};
-
-// Find the position of the marker in the document. Returns a {from,
-// to} object by default. Side can be passed to get a specific side
-// -- 0 (both), -1 (left), or 1 (right). When lineObj is true, the
-// Pos objects returned contain a line object, rather than a line
-// number (used to prevent looking up the same line twice).
-TextMarker.prototype.find = function (side, lineObj) {
-    var this$1 = this;
-
-  if (side == null && this.type == "bookmark") { side = 1; }
-  var from, to;
-  for (var i = 0; i < this.lines.length; ++i) {
-    var line = this$1.lines[i];
-    var span = getMarkedSpanFor(line.markedSpans, this$1);
-    if (span.from != null) {
-      from = Pos(lineObj ? line : lineNo(line), span.from);
-      if (side == -1) { return from }
-    }
-    if (span.to != null) {
-      to = Pos(lineObj ? line : lineNo(line), span.to);
-      if (side == 1) { return to }
-    }
-  }
-  return from && {from: from, to: to}
-};
-
-// Signals that the marker's widget changed, and surrounding layout
-// should be recomputed.
-TextMarker.prototype.changed = function () {
-    var this$1 = this;
-
-  var pos = this.find(-1, true), widget = this, cm = this.doc.cm;
-  if (!pos || !cm) { return }
-  runInOp(cm, function () {
-    var line = pos.line, lineN = lineNo(pos.line);
-    var view = findViewForLine(cm, lineN);
-    if (view) {
-      clearLineMeasurementCacheFor(view);
-      cm.curOp.selectionChanged = cm.curOp.forceUpdate = true;
-    }
-    cm.curOp.updateMaxLine = true;
-    if (!lineIsHidden(widget.doc, line) && widget.height != null) {
-      var oldHeight = widget.height;
-      widget.height = null;
-      var dHeight = widgetHeight(widget) - oldHeight;
-      if (dHeight)
-        { updateLineHeight(line, line.height + dHeight); }
-    }
-    signalLater(cm, "markerChanged", cm, this$1);
-  });
-};
-
-TextMarker.prototype.attachLine = function (line) {
-  if (!this.lines.length && this.doc.cm) {
-    var op = this.doc.cm.curOp;
-    if (!op.maybeHiddenMarkers || indexOf(op.maybeHiddenMarkers, this) == -1)
-      { (op.maybeUnhiddenMarkers || (op.maybeUnhiddenMarkers = [])).push(this); }
-  }
-  this.lines.push(line);
-};
-
-TextMarker.prototype.detachLine = function (line) {
-  this.lines.splice(indexOf(this.lines, line), 1);
-  if (!this.lines.length && this.doc.cm) {
-    var op = this.doc.cm.curOp;(op.maybeHiddenMarkers || (op.maybeHiddenMarkers = [])).push(this);
-  }
-};
-eventMixin(TextMarker);
-
-// Create a marker, wire it up to the right lines, and
-function markText(doc, from, to, options, type) {
-  // Shared markers (across linked documents) are handled separately
-  // (markTextShared will call out to this again, once per
-  // document).
-  if (options && options.shared) { return markTextShared(doc, from, to, options, type) }
-  // Ensure we are in an operation.
-  if (doc.cm && !doc.cm.curOp) { return operation(doc.cm, markText)(doc, from, to, options, type) }
-
-  var marker = new TextMarker(doc, type), diff = cmp(from, to);
-  if (options) { copyObj(options, marker, false); }
-  // Don't connect empty markers unless clearWhenEmpty is false
-  if (diff > 0 || diff == 0 && marker.clearWhenEmpty !== false)
-    { return marker }
-  if (marker.replacedWith) {
-    // Showing up as a widget implies collapsed (widget replaces text)
-    marker.collapsed = true;
-    marker.widgetNode = eltP("span", [marker.replacedWith], "CodeMirror-widget");
-    if (!options.handleMouseEvents) { marker.widgetNode.setAttribute("cm-ignore-events", "true"); }
-    if (options.insertLeft) { marker.widgetNode.insertLeft = true; }
-  }
-  if (marker.collapsed) {
-    if (conflictingCollapsedRange(doc, from.line, from, to, marker) ||
-        from.line != to.line && conflictingCollapsedRange(doc, to.line, from, to, marker))
-      { throw new Error("Inserting collapsed marker partially overlapping an existing one") }
-    seeCollapsedSpans();
-  }
-
-  if (marker.addToHistory)
-    { addChangeToHistory(doc, {from: from, to: to, origin: "markText"}, doc.sel, NaN); }
-
-  var curLine = from.line, cm = doc.cm, updateMaxLine;
-  doc.iter(curLine, to.line + 1, function (line) {
-    if (cm && marker.collapsed && !cm.options.lineWrapping && visualLine(line) == cm.display.maxLine)
-      { updateMaxLine = true; }
-    if (marker.collapsed && curLine != from.line) { updateLineHeight(line, 0); }
-    addMarkedSpan(line, new MarkedSpan(marker,
-                                       curLine == from.line ? from.ch : null,
-                                       curLine == to.line ? to.ch : null));
-    ++curLine;
-  });
-  // lineIsHidden depends on the presence of the spans, so needs a second pass
-  if (marker.collapsed) { doc.iter(from.line, to.line + 1, function (line) {
-    if (lineIsHidden(doc, line)) { updateLineHeight(line, 0); }
-  }); }
-
-  if (marker.clearOnEnter) { on(marker, "beforeCursorEnter", function () { return marker.clear(); }); }
-
-  if (marker.readOnly) {
-    seeReadOnlySpans();
-    if (doc.history.done.length || doc.history.undone.length)
-      { doc.clearHistory(); }
-  }
-  if (marker.collapsed) {
-    marker.id = ++nextMarkerId;
-    marker.atomic = true;
-  }
-  if (cm) {
-    // Sync editor state
-    if (updateMaxLine) { cm.curOp.updateMaxLine = true; }
-    if (marker.collapsed)
-      { regChange(cm, from.line, to.line + 1); }
-    else if (marker.className || marker.title || marker.startStyle || marker.endStyle || marker.css)
-      { for (var i = from.line; i <= to.line; i++) { regLineChange(cm, i, "text"); } }
-    if (marker.atomic) { reCheckSelection(cm.doc); }
-    signalLater(cm, "markerAdded", cm, marker);
-  }
-  return marker
-}
-
-// SHARED TEXTMARKERS
-
-// A shared marker spans multiple linked documents. It is
-// implemented as a meta-marker-object controlling multiple normal
-// markers.
-var SharedTextMarker = function(markers, primary) {
-  var this$1 = this;
-
-  this.markers = markers;
-  this.primary = primary;
-  for (var i = 0; i < markers.length; ++i)
-    { markers[i].parent = this$1; }
-};
-
-SharedTextMarker.prototype.clear = function () {
-    var this$1 = this;
-
-  if (this.explicitlyCleared) { return }
-  this.explicitlyCleared = true;
-  for (var i = 0; i < this.markers.length; ++i)
-    { this$1.markers[i].clear(); }
-  signalLater(this, "clear");
-};
-
-SharedTextMarker.prototype.find = function (side, lineObj) {
-  return this.primary.find(side, lineObj)
-};
-eventMixin(SharedTextMarker);
-
-function markTextShared(doc, from, to, options, type) {
-  options = copyObj(options);
-  options.shared = false;
-  var markers = [markText(doc, from, to, options, type)], primary = markers[0];
-  var widget = options.widgetNode;
-  linkedDocs(doc, function (doc) {
-    if (widget) { options.widgetNode = widget.cloneNode(true); }
-    markers.push(markText(doc, clipPos(doc, from), clipPos(doc, to), options, type));
-    for (var i = 0; i < doc.linked.length; ++i)
-      { if (doc.linked[i].isParent) { return } }
-    primary = lst(markers);
-  });
-  return new SharedTextMarker(markers, primary)
-}
-
-function findSharedMarkers(doc) {
-  return doc.findMarks(Pos(doc.first, 0), doc.clipPos(Pos(doc.lastLine())), function (m) { return m.parent; })
-}
-
-function copySharedMarkers(doc, markers) {
-  for (var i = 0; i < markers.length; i++) {
-    var marker = markers[i], pos = marker.find();
-    var mFrom = doc.clipPos(pos.from), mTo = doc.clipPos(pos.to);
-    if (cmp(mFrom, mTo)) {
-      var subMark = markText(doc, mFrom, mTo, marker.primary, marker.primary.type);
-      marker.markers.push(subMark);
-      subMark.parent = marker;
-    }
-  }
-}
-
-function detachSharedMarkers(markers) {
-  var loop = function ( i ) {
-    var marker = markers[i], linked = [marker.primary.doc];
-    linkedDocs(marker.primary.doc, function (d) { return linked.push(d); });
-    for (var j = 0; j < marker.markers.length; j++) {
-      var subMarker = marker.markers[j];
-      if (indexOf(linked, subMarker.doc) == -1) {
-        subMarker.parent = null;
-        marker.markers.splice(j--, 1);
-      }
-    }
-  };
-
-  for (var i = 0; i < markers.length; i++) loop( i );
-}
-
-var nextDocId = 0;
-var Doc = function(text, mode, firstLine, lineSep, direction) {
-  if (!(this instanceof Doc)) { return new Doc(text, mode, firstLine, lineSep, direction) }
-  if (firstLine == null) { firstLine = 0; }
-
-  BranchChunk.call(this, [new LeafChunk([new Line("", null)])]);
-  this.first = firstLine;
-  this.scrollTop = this.scrollLeft = 0;
-  this.cantEdit = false;
-  this.cleanGeneration = 1;
-  this.modeFrontier = this.highlightFrontier = firstLine;
-  var start = Pos(firstLine, 0);
-  this.sel = simpleSelection(start);
-  this.history = new History(null);
-  this.id = ++nextDocId;
-  this.modeOption = mode;
-  this.lineSep = lineSep;
-  this.direction = (direction == "rtl") ? "rtl" : "ltr";
-  this.extend = false;
-
-  if (typeof text == "string") { text = this.splitLines(text); }
-  updateDoc(this, {from: start, to: start, text: text});
-  setSelection(this, simpleSelection(start), sel_dontScroll);
-};
-
-Doc.prototype = createObj(BranchChunk.prototype, {
-  constructor: Doc,
-  // Iterate over the document. Supports two forms -- with only one
-  // argument, it calls that for each line in the document. With
-  // three, it iterates over the range given by the first two (with
-  // the second being non-inclusive).
-  iter: function(from, to, op) {
-    if (op) { this.iterN(from - this.first, to - from, op); }
-    else { this.iterN(this.first, this.first + this.size, from); }
-  },
-
-  // Non-public interface for adding and removing lines.
-  insert: function(at, lines) {
-    var height = 0;
-    for (var i = 0; i < lines.length; ++i) { height += lines[i].height; }
-    this.insertInner(at - this.first, lines, height);
-  },
-  remove: function(at, n) { this.removeInner(at - this.first, n); },
-
-  // From here, the methods are part of the public interface. Most
-  // are also available from CodeMirror (editor) instances.
-
-  getValue: function(lineSep) {
-    var lines = getLines(this, this.first, this.first + this.size);
-    if (lineSep === false) { return lines }
-    return lines.join(lineSep || this.lineSeparator())
-  },
-  setValue: docMethodOp(function(code) {
-    var top = Pos(this.first, 0), last = this.first + this.size - 1;
-    makeChange(this, {from: top, to: Pos(last, getLine(this, last).text.length),
-                      text: this.splitLines(code), origin: "setValue", full: true}, true);
-    if (this.cm) { scrollToCoords(this.cm, 0, 0); }
-    setSelection(this, simpleSelection(top), sel_dontScroll);
-  }),
-  replaceRange: function(code, from, to, origin) {
-    from = clipPos(this, from);
-    to = to ? clipPos(this, to) : from;
-    replaceRange(this, code, from, to, origin);
-  },
-  getRange: function(from, to, lineSep) {
-    var lines = getBetween(this, clipPos(this, from), clipPos(this, to));
-    if (lineSep === false) { return lines }
-    return lines.join(lineSep || this.lineSeparator())
-  },
-
-  getLine: function(line) {var l = this.getLineHandle(line); return l && l.text},
-
-  getLineHandle: function(line) {if (isLine(this, line)) { return getLine(this, line) }},
-  getLineNumber: function(line) {return lineNo(line)},
-
-  getLineHandleVisualStart: function(line) {
-    if (typeof line == "number") { line = getLine(this, line); }
-    return visualLine(line)
-  },
-
-  lineCount: function() {return this.size},
-  firstLine: function() {return this.first},
-  lastLine: function() {return this.first + this.size - 1},
-
-  clipPos: function(pos) {return clipPos(this, pos)},
-
-  getCursor: function(start) {
-    var range$$1 = this.sel.primary(), pos;
-    if (start == null || start == "head") { pos = range$$1.head; }
-    else if (start == "anchor") { pos = range$$1.anchor; }
-    else if (start == "end" || start == "to" || start === false) { pos = range$$1.to(); }
-    else { pos = range$$1.from(); }
-    return pos
-  },
-  listSelections: function() { return this.sel.ranges },
-  somethingSelected: function() {return this.sel.somethingSelected()},
-
-  setCursor: docMethodOp(function(line, ch, options) {
-    setSimpleSelection(this, clipPos(this, typeof line == "number" ? Pos(line, ch || 0) : line), null, options);
-  }),
-  setSelection: docMethodOp(function(anchor, head, options) {
-    setSimpleSelection(this, clipPos(this, anchor), clipPos(this, head || anchor), options);
-  }),
-  extendSelection: docMethodOp(function(head, other, options) {
-    extendSelection(this, clipPos(this, head), other && clipPos(this, other), options);
-  }),
-  extendSelections: docMethodOp(function(heads, options) {
-    extendSelections(this, clipPosArray(this, heads), options);
-  }),
-  extendSelectionsBy: docMethodOp(function(f, options) {
-    var heads = map(this.sel.ranges, f);
-    extendSelections(this, clipPosArray(this, heads), options);
-  }),
-  setSelections: docMethodOp(function(ranges, primary, options) {
-    var this$1 = this;
-
-    if (!ranges.length) { return }
-    var out = [];
-    for (var i = 0; i < ranges.length; i++)
-      { out[i] = new Range(clipPos(this$1, ranges[i].anchor),
-                         clipPos(this$1, ranges[i].head)); }
-    if (primary == null) { primary = Math.min(ranges.length - 1, this.sel.primIndex); }
-    setSelection(this, normalizeSelection(out, primary), options);
-  }),
-  addSelection: docMethodOp(function(anchor, head, options) {
-    var ranges = this.sel.ranges.slice(0);
-    ranges.push(new Range(clipPos(this, anchor), clipPos(this, head || anchor)));
-    setSelection(this, normalizeSelection(ranges, ranges.length - 1), options);
-  }),
-
-  getSelection: function(lineSep) {
-    var this$1 = this;
-
-    var ranges = this.sel.ranges, lines;
-    for (var i = 0; i < ranges.length; i++) {
-      var sel = getBetween(this$1, ranges[i].from(), ranges[i].to());
-      lines = lines ? lines.concat(sel) : sel;
-    }
-    if (lineSep === false) { return lines }
-    else { return lines.join(lineSep || this.lineSeparator()) }
-  },
-  getSelections: function(lineSep) {
-    var this$1 = this;
-
-    var parts = [], ranges = this.sel.ranges;
-    for (var i = 0; i < ranges.length; i++) {
-      var sel = getBetween(this$1, ranges[i].from(), ranges[i].to());
-      if (lineSep !== false) { sel = sel.join(lineSep || this$1.lineSeparator()); }
-      parts[i] = sel;
-    }
-    return parts
-  },
-  replaceSelection: function(code, collapse, origin) {
-    var dup = [];
-    for (var i = 0; i < this.sel.ranges.length; i++)
-      { dup[i] = code; }
-    this.replaceSelections(dup, collapse, origin || "+input");
-  },
-  replaceSelections: docMethodOp(function(code, collapse, origin) {
-    var this$1 = this;
-
-    var changes = [], sel = this.sel;
-    for (var i = 0; i < sel.ranges.length; i++) {
-      var range$$1 = sel.ranges[i];
-      changes[i] = {from: range$$1.from(), to: range$$1.to(), text: this$1.splitLines(code[i]), origin: origin};
-    }
-    var newSel = collapse && collapse != "end" && computeReplacedSel(this, changes, collapse);
-    for (var i$1 = changes.length - 1; i$1 >= 0; i$1--)
-      { makeChange(this$1, changes[i$1]); }
-    if (newSel) { setSelectionReplaceHistory(this, newSel); }
-    else if (this.cm) { ensureCursorVisible(this.cm); }
-  }),
-  undo: docMethodOp(function() {makeChangeFromHistory(this, "undo");}),
-  redo: docMethodOp(function() {makeChangeFromHistory(this, "redo");}),
-  undoSelection: docMethodOp(function() {makeChangeFromHistory(this, "undo", true);}),
-  redoSelection: docMethodOp(function() {makeChangeFromHistory(this, "redo", true);}),
-
-  setExtending: function(val) {this.extend = val;},
-  getExtending: function() {return this.extend},
-
-  historySize: function() {
-    var hist = this.history, done = 0, undone = 0;
-    for (var i = 0; i < hist.done.length; i++) { if (!hist.done[i].ranges) { ++done; } }
-    for (var i$1 = 0; i$1 < hist.undone.length; i$1++) { if (!hist.undone[i$1].ranges) { ++undone; } }
-    return {undo: done, redo: undone}
-  },
-  clearHistory: function() {this.history = new History(this.history.maxGeneration);},
-
-  markClean: function() {
-    this.cleanGeneration = this.changeGeneration(true);
-  },
-  changeGeneration: function(forceSplit) {
-    if (forceSplit)
-      { this.history.lastOp = this.history.lastSelOp = this.history.lastOrigin = null; }
-    return this.history.generation
-  },
-  isClean: function (gen) {
-    return this.history.generation == (gen || this.cleanGeneration)
-  },
-
-  getHistory: function() {
-    return {done: copyHistoryArray(this.history.done),
-            undone: copyHistoryArray(this.history.undone)}
-  },
-  setHistory: function(histData) {
-    var hist = this.history = new History(this.history.maxGeneration);
-    hist.done = copyHistoryArray(histData.done.slice(0), null, true);
-    hist.undone = copyHistoryArray(histData.undone.slice(0), null, true);
-  },
-
-  setGutterMarker: docMethodOp(function(line, gutterID, value) {
-    return changeLine(this, line, "gutter", function (line) {
-      var markers = line.gutterMarkers || (line.gutterMarkers = {});
-      markers[gutterID] = value;
-      if (!value && isEmpty(markers)) { line.gutterMarkers = null; }
-      return true
-    })
-  }),
-
-  clearGutter: docMethodOp(function(gutterID) {
-    var this$1 = this;
-
-    this.iter(function (line) {
-      if (line.gutterMarkers && line.gutterMarkers[gutterID]) {
-        changeLine(this$1, line, "gutter", function () {
-          line.gutterMarkers[gutterID] = null;
-          if (isEmpty(line.gutterMarkers)) { line.gutterMarkers = null; }
-          return true
-        });
-      }
-    });
-  }),
-
-  lineInfo: function(line) {
-    var n;
-    if (typeof line == "number") {
-      if (!isLine(this, line)) { return null }
-      n = line;
-      line = getLine(this, line);
-      if (!line) { return null }
-    } else {
-      n = lineNo(line);
-      if (n == null) { return null }
-    }
-    return {line: n, handle: line, text: line.text, gutterMarkers: line.gutterMarkers,
-            textClass: line.textClass, bgClass: line.bgClass, wrapClass: line.wrapClass,
-            widgets: line.widgets}
-  },
-
-  addLineClass: docMethodOp(function(handle, where, cls) {
-    return changeLine(this, handle, where == "gutter" ? "gutter" : "class", function (line) {
-      var prop = where == "text" ? "textClass"
-               : where == "background" ? "bgClass"
-               : where == "gutter" ? "gutterClass" : "wrapClass";
-      if (!line[prop]) { line[prop] = cls; }
-      else if (classTest(cls).test(line[prop])) { return false }
-      else { line[prop] += " " + cls; }
-      return true
-    })
-  }),
-  removeLineClass: docMethodOp(function(handle, where, cls) {
-    return changeLine(this, handle, where == "gutter" ? "gutter" : "class", function (line) {
-      var prop = where == "text" ? "textClass"
-               : where == "background" ? "bgClass"
-               : where == "gutter" ? "gutterClass" : "wrapClass";
-      var cur = line[prop];
-      if (!cur) { return false }
-      else if (cls == null) { line[prop] = null; }
-      else {
-        var found = cur.match(classTest(cls));
-        if (!found) { return false }
-        var end = found.index + found[0].length;
-        line[prop] = cur.slice(0, found.index) + (!found.index || end == cur.length ? "" : " ") + cur.slice(end) || null;
-      }
-      return true
-    })
-  }),
-
-  addLineWidget: docMethodOp(function(handle, node, options) {
-    return addLineWidget(this, handle, node, options)
-  }),
-  removeLineWidget: function(widget) { widget.clear(); },
-
-  markText: function(from, to, options) {
-    return markText(this, clipPos(this, from), clipPos(this, to), options, options && options.type || "range")
-  },
-  setBookmark: function(pos, options) {
-    var realOpts = {replacedWith: options && (options.nodeType == null ? options.widget : options),
-                    insertLeft: options && options.insertLeft,
-                    clearWhenEmpty: false, shared: options && options.shared,
-                    handleMouseEvents: options && options.handleMouseEvents};
-    pos = clipPos(this, pos);
-    return markText(this, pos, pos, realOpts, "bookmark")
-  },
-  findMarksAt: function(pos) {
-    pos = clipPos(this, pos);
-    var markers = [], spans = getLine(this, pos.line).markedSpans;
-    if (spans) { for (var i = 0; i < spans.length; ++i) {
-      var span = spans[i];
-      if ((span.from == null || span.from <= pos.ch) &&
-          (span.to == null || span.to >= pos.ch))
-        { markers.push(span.marker.parent || span.marker); }
-    } }
-    return markers
-  },
-  findMarks: function(from, to, filter) {
-    from = clipPos(this, from); to = clipPos(this, to);
-    var found = [], lineNo$$1 = from.line;
-    this.iter(from.line, to.line + 1, function (line) {
-      var spans = line.markedSpans;
-      if (spans) { for (var i = 0; i < spans.length; i++) {
-        var span = spans[i];
-        if (!(span.to != null && lineNo$$1 == from.line && from.ch >= span.to ||
-              span.from == null && lineNo$$1 != from.line ||
-              span.from != null && lineNo$$1 == to.line && span.from >= to.ch) &&
-            (!filter || filter(span.marker)))
-          { found.push(span.marker.parent || span.marker); }
-      } }
-      ++lineNo$$1;
-    });
-    return found
-  },
-  getAllMarks: function() {
-    var markers = [];
-    this.iter(function (line) {
-      var sps = line.markedSpans;
-      if (sps) { for (var i = 0; i < sps.length; ++i)
-        { if (sps[i].from != null) { markers.push(sps[i].marker); } } }
-    });
-    return markers
-  },
-
-  posFromIndex: function(off) {
-    var ch, lineNo$$1 = this.first, sepSize = this.lineSeparator().length;
-    this.iter(function (line) {
-      var sz = line.text.length + sepSize;
-      if (sz > off) { ch = off; return true }
-      off -= sz;
-      ++lineNo$$1;
-    });
-    return clipPos(this, Pos(lineNo$$1, ch))
-  },
-  indexFromPos: function (coords) {
-    coords = clipPos(this, coords);
-    var index = coords.ch;
-    if (coords.line < this.first || coords.ch < 0) { return 0 }
-    var sepSize = this.lineSeparator().length;
-    this.iter(this.first, coords.line, function (line) { // iter aborts when callback returns a truthy value
-      index += line.text.length + sepSize;
-    });
-    return index
-  },
-
-  copy: function(copyHistory) {
-    var doc = new Doc(getLines(this, this.first, this.first + this.size),
-                      this.modeOption, this.first, this.lineSep, this.direction);
-    doc.scrollTop = this.scrollTop; doc.scrollLeft = this.scrollLeft;
-    doc.sel = this.sel;
-    doc.extend = false;
-    if (copyHistory) {
-      doc.history.undoDepth = this.history.undoDepth;
-      doc.setHistory(this.getHistory());
-    }
-    return doc
-  },
-
-  linkedDoc: function(options) {
-    if (!options) { options = {}; }
-    var from = this.first, to = this.first + this.size;
-    if (options.from != null && options.from > from) { from = options.from; }
-    if (options.to != null && options.to < to) { to = options.to; }
-    var copy = new Doc(getLines(this, from, to), options.mode || this.modeOption, from, this.lineSep, this.direction);
-    if (options.sharedHist) { copy.history = this.history
-    ; }(this.linked || (this.linked = [])).push({doc: copy, sharedHist: options.sharedHist});
-    copy.linked = [{doc: this, isParent: true, sharedHist: options.sharedHist}];
-    copySharedMarkers(copy, findSharedMarkers(this));
-    return copy
-  },
-  unlinkDoc: function(other) {
-    var this$1 = this;
-
-    if (other instanceof CodeMirror$1) { other = other.doc; }
-    if (this.linked) { for (var i = 0; i < this.linked.length; ++i) {
-      var link = this$1.linked[i];
-      if (link.doc != other) { continue }
-      this$1.linked.splice(i, 1);
-      other.unlinkDoc(this$1);
-      detachSharedMarkers(findSharedMarkers(this$1));
-      break
-    } }
-    // If the histories were shared, split them again
-    if (other.history == this.history) {
-      var splitIds = [other.id];
-      linkedDocs(other, function (doc) { return splitIds.push(doc.id); }, true);
-      other.history = new History(null);
-      other.history.done = copyHistoryArray(this.history.done, splitIds);
-      other.history.undone = copyHistoryArray(this.history.undone, splitIds);
-    }
-  },
-  iterLinkedDocs: function(f) {linkedDocs(this, f);},
-
-  getMode: function() {return this.mode},
-  getEditor: function() {return this.cm},
-
-  splitLines: function(str) {
-    if (this.lineSep) { return str.split(this.lineSep) }
-    return splitLinesAuto(str)
-  },
-  lineSeparator: function() { return this.lineSep || "\n" },
-
-  setDirection: docMethodOp(function (dir) {
-    if (dir != "rtl") { dir = "ltr"; }
-    if (dir == this.direction) { return }
-    this.direction = dir;
-    this.iter(function (line) { return line.order = null; });
-    if (this.cm) { directionChanged(this.cm); }
-  })
-});
-
-// Public alias.
-Doc.prototype.eachLine = Doc.prototype.iter;
-
-// Kludge to work around strange IE behavior where it'll sometimes
-// re-fire a series of drag-related events right after the drop (#1551)
-var lastDrop = 0;
-
-function onDrop(e) {
-  var cm = this;
-  clearDragCursor(cm);
-  if (signalDOMEvent(cm, e) || eventInWidget(cm.display, e))
-    { return }
-  e_preventDefault(e);
-  if (ie) { lastDrop = +new Date; }
-  var pos = posFromMouse(cm, e, true), files = e.dataTransfer.files;
-  if (!pos || cm.isReadOnly()) { return }
-  // Might be a file drop, in which case we simply extract the text
-  // and insert it.
-  if (files && files.length && window.FileReader && window.File) {
-    var n = files.length, text = Array(n), read = 0;
-    var loadFile = function (file, i) {
-      if (cm.options.allowDropFileTypes &&
-          indexOf(cm.options.allowDropFileTypes, file.type) == -1)
-        { return }
-
-      var reader = new FileReader;
-      reader.onload = operation(cm, function () {
-        var content = reader.result;
-        if (/[\x00-\x08\x0e-\x1f]{2}/.test(content)) { content = ""; }
-        text[i] = content;
-        if (++read == n) {
-          pos = clipPos(cm.doc, pos);
-          var change = {from: pos, to: pos,
-                        text: cm.doc.splitLines(text.join(cm.doc.lineSeparator())),
-                        origin: "paste"};
-          makeChange(cm.doc, change);
-          setSelectionReplaceHistory(cm.doc, simpleSelection(pos, changeEnd(change)));
-        }
-      });
-      reader.readAsText(file);
-    };
-    for (var i = 0; i < n; ++i) { loadFile(files[i], i); }
-  } else { // Normal drop
-    // Don't do a replace if the drop happened inside of the selected text.
-    if (cm.state.draggingText && cm.doc.sel.contains(pos) > -1) {
-      cm.state.draggingText(e);
-      // Ensure the editor is re-focused
-      setTimeout(function () { return cm.display.input.focus(); }, 20);
-      return
-    }
-    try {
-      var text$1 = e.dataTransfer.getData("Text");
-      if (text$1) {
-        var selected;
-        if (cm.state.draggingText && !cm.state.draggingText.copy)
-          { selected = cm.listSelections(); }
-        setSelectionNoUndo(cm.doc, simpleSelection(pos, pos));
-        if (selected) { for (var i$1 = 0; i$1 < selected.length; ++i$1)
-          { replaceRange(cm.doc, "", selected[i$1].anchor, selected[i$1].head, "drag"); } }
-        cm.replaceSelection(text$1, "around", "paste");
-        cm.display.input.focus();
-      }
-    }
-    catch(e){}
-  }
-}
-
-function onDragStart(cm, e) {
-  if (ie && (!cm.state.draggingText || +new Date - lastDrop < 100)) { e_stop(e); return }
-  if (signalDOMEvent(cm, e) || eventInWidget(cm.display, e)) { return }
-
-  e.dataTransfer.setData("Text", cm.getSelection());
-  e.dataTransfer.effectAllowed = "copyMove";
-
-  // Use dummy image instead of default browsers image.
-  // Recent Safari (~6.0.2) have a tendency to segfault when this happens, so we don't do it there.
-  if (e.dataTransfer.setDragImage && !safari) {
-    var img = elt("img", null, null, "position: fixed; left: 0; top: 0;");
-    img.src = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==";
-    if (presto) {
-      img.width = img.height = 1;
-      cm.display.wrapper.appendChild(img);
-      // Force a relayout, or Opera won't use our image for some obscure reason
-      img._top = img.offsetTop;
-    }
-    e.dataTransfer.setDragImage(img, 0, 0);
-    if (presto) { img.parentNode.removeChild(img); }
-  }
-}
-
-function onDragOver(cm, e) {
-  var pos = posFromMouse(cm, e);
-  if (!pos) { return }
-  var frag = document.createDocumentFragment();
-  drawSelectionCursor(cm, pos, frag);
-  if (!cm.display.dragCursor) {
-    cm.display.dragCursor = elt("div", null, "CodeMirror-cursors CodeMirror-dragcursors");
-    cm.display.lineSpace.insertBefore(cm.display.dragCursor, cm.display.cursorDiv);
-  }
-  removeChildrenAndAdd(cm.display.dragCursor, frag);
-}
-
-function clearDragCursor(cm) {
-  if (cm.display.dragCursor) {
-    cm.display.lineSpace.removeChild(cm.display.dragCursor);
-    cm.display.dragCursor = null;
-  }
-}
-
-// These must be handled carefully, because naively registering a
-// handler for each editor will cause the editors to never be
-// garbage collected.
-
-function forEachCodeMirror(f) {
-  if (!document.getElementsByClassName) { return }
-  var byClass = document.getElementsByClassName("CodeMirror");
-  for (var i = 0; i < byClass.length; i++) {
-    var cm = byClass[i].CodeMirror;
-    if (cm) { f(cm); }
-  }
-}
-
-var globalsRegistered = false;
-function ensureGlobalHandlers() {
-  if (globalsRegistered) { return }
-  registerGlobalHandlers();
-  globalsRegistered = true;
-}
-function registerGlobalHandlers() {
-  // When the window resizes, we need to refresh active editors.
-  var resizeTimer;
-  on(window, "resize", function () {
-    if (resizeTimer == null) { resizeTimer = setTimeout(function () {
-      resizeTimer = null;
-      forEachCodeMirror(onResize);
-    }, 100); }
-  });
-  // When the window loses focus, we want to show the editor as blurred
-  on(window, "blur", function () { return forEachCodeMirror(onBlur); });
-}
-// Called when the window resizes
-function onResize(cm) {
-  var d = cm.display;
-  if (d.lastWrapHeight == d.wrapper.clientHeight && d.lastWrapWidth == d.wrapper.clientWidth)
-    { return }
-  // Might be a text scaling operation, clear size caches.
-  d.cachedCharWidth = d.cachedTextHeight = d.cachedPaddingH = null;
-  d.scrollbarsClipped = false;
-  cm.setSize();
-}
-
-var keyNames = {
-  3: "Enter", 8: "Backspace", 9: "Tab", 13: "Enter", 16: "Shift", 17: "Ctrl", 18: "Alt",
-  19: "Pause", 20: "CapsLock", 27: "Esc", 32: "Space", 33: "PageUp", 34: "PageDown", 35: "End",
-  36: "Home", 37: "Left", 38: "Up", 39: "Right", 40: "Down", 44: "PrintScrn", 45: "Insert",
-  46: "Delete", 59: ";", 61: "=", 91: "Mod", 92: "Mod", 93: "Mod",
-  106: "*", 107: "=", 109: "-", 110: ".", 111: "/", 127: "Delete",
-  173: "-", 186: ";", 187: "=", 188: ",", 189: "-", 190: ".", 191: "/", 192: "`", 219: "[", 220: "\\",
-  221: "]", 222: "'", 63232: "Up", 63233: "Down", 63234: "Left", 63235: "Right", 63272: "Delete",
-  63273: "Home", 63275: "End", 63276: "PageUp", 63277: "PageDown", 63302: "Insert"
-};
-
-// Number keys
-for (var i = 0; i < 10; i++) { keyNames[i + 48] = keyNames[i + 96] = String(i); }
-// Alphabetic keys
-for (var i$1 = 65; i$1 <= 90; i$1++) { keyNames[i$1] = String.fromCharCode(i$1); }
-// Function keys
-for (var i$2 = 1; i$2 <= 12; i$2++) { keyNames[i$2 + 111] = keyNames[i$2 + 63235] = "F" + i$2; }
-
-var keyMap = {};
-
-keyMap.basic = {
-  "Left": "goCharLeft", "Right": "goCharRight", "Up": "goLineUp", "Down": "goLineDown",
-  "End": "goLineEnd", "Home": "goLineStartSmart", "PageUp": "goPageUp", "PageDown": "goPageDown",
-  "Delete": "delCharAfter", "Backspace": "delCharBefore", "Shift-Backspace": "delCharBefore",
-  "Tab": "defaultTab", "Shift-Tab": "indentAuto",
-  "Enter": "newlineAndIndent", "Insert": "toggleOverwrite",
-  "Esc": "singleSelection"
-};
-// Note that the save and find-related commands aren't defined by
-// default. User code or addons can define them. Unknown commands
-// are simply ignored.
-keyMap.pcDefault = {
-  "Ctrl-A": "selectAll", "Ctrl-D": "deleteLine", "Ctrl-Z": "undo", "Shift-Ctrl-Z": "redo", "Ctrl-Y": "redo",
-  "Ctrl-Home": "goDocStart", "Ctrl-End": "goDocEnd", "Ctrl-Up": "goLineUp", "Ctrl-Down": "goLineDown",
-  "Ctrl-Left": "goGroupLeft", "Ctrl-Right": "goGroupRight", "Alt-Left": "goLineStart", "Alt-Right": "goLineEnd",
-  "Ctrl-Backspace": "delGroupBefore", "Ctrl-Delete": "delGroupAfter", "Ctrl-S": "save", "Ctrl-F": "find",
-  "Ctrl-G": "findNext", "Shift-Ctrl-G": "findPrev", "Shift-Ctrl-F": "replace", "Shift-Ctrl-R": "replaceAll",
-  "Ctrl-[": "indentLess", "Ctrl-]": "indentMore",
-  "Ctrl-U": "undoSelection", "Shift-Ctrl-U": "redoSelection", "Alt-U": "redoSelection",
-  fallthrough: "basic"
-};
-// Very basic readline/emacs-style bindings, which are standard on Mac.
-keyMap.emacsy = {
-  "Ctrl-F": "goCharRight", "Ctrl-B": "goCharLeft", "Ctrl-P": "goLineUp", "Ctrl-N": "goLineDown",
-  "Alt-F": "goWordRight", "Alt-B": "goWordLeft", "Ctrl-A": "goLineStart", "Ctrl-E": "goLineEnd",
-  "Ctrl-V": "goPageDown", "Shift-Ctrl-V": "goPageUp", "Ctrl-D": "delCharAfter", "Ctrl-H": "delCharBefore",
-  "Alt-D": "delWordAfter", "Alt-Backspace": "delWordBefore", "Ctrl-K": "killLine", "Ctrl-T": "transposeChars",
-  "Ctrl-O": "openLine"
-};
-keyMap.macDefault = {
-  "Cmd-A": "selectAll", "Cmd-D": "deleteLine", "Cmd-Z": "undo", "Shift-Cmd-Z": "redo", "Cmd-Y": "redo",
-  "Cmd-Home": "goDocStart", "Cmd-Up": "goDocStart", "Cmd-End": "goDocEnd", "Cmd-Down": "goDocEnd", "Alt-Left": "goGroupLeft",
-  "Alt-Right": "goGroupRight", "Cmd-Left": "goLineLeft", "Cmd-Right": "goLineRight", "Alt-Backspace": "delGroupBefore",
-  "Ctrl-Alt-Backspace": "delGroupAfter", "Alt-Delete": "delGroupAfter", "Cmd-S": "save", "Cmd-F": "find",
-  "Cmd-G": "findNext", "Shift-Cmd-G": "findPrev", "Cmd-Alt-F": "replace", "Shift-Cmd-Alt-F": "replaceAll",
-  "Cmd-[": "indentLess", "Cmd-]": "indentMore", "Cmd-Backspace": "delWrappedLineLeft", "Cmd-Delete": "delWrappedLineRight",
-  "Cmd-U": "undoSelection", "Shift-Cmd-U": "redoSelection", "Ctrl-Up": "goDocStart", "Ctrl-Down": "goDocEnd",
-  fallthrough: ["basic", "emacsy"]
-};
-keyMap["default"] = mac ? keyMap.macDefault : keyMap.pcDefault;
-
-// KEYMAP DISPATCH
-
-function normalizeKeyName(name) {
-  var parts = name.split(/-(?!$)/);
-  name = parts[parts.length - 1];
-  var alt, ctrl, shift, cmd;
-  for (var i = 0; i < parts.length - 1; i++) {
-    var mod = parts[i];
-    if (/^(cmd|meta|m)$/i.test(mod)) { cmd = true; }
-    else if (/^a(lt)?$/i.test(mod)) { alt = true; }
-    else if (/^(c|ctrl|control)$/i.test(mod)) { ctrl = true; }
-    else if (/^s(hift)?$/i.test(mod)) { shift = true; }
-    else { throw new Error("Unrecognized modifier name: " + mod) }
-  }
-  if (alt) { name = "Alt-" + name; }
-  if (ctrl) { name = "Ctrl-" + name; }
-  if (cmd) { name = "Cmd-" + name; }
-  if (shift) { name = "Shift-" + name; }
-  return name
-}
-
-// This is a kludge to keep keymaps mostly working as raw objects
-// (backwards compatibility) while at the same time support features
-// like normalization and multi-stroke key bindings. It compiles a
-// new normalized keymap, and then updates the old object to reflect
-// this.
-function normalizeKeyMap(keymap) {
-  var copy = {};
-  for (var keyname in keymap) { if (keymap.hasOwnProperty(keyname)) {
-    var value = keymap[keyname];
-    if (/^(name|fallthrough|(de|at)tach)$/.test(keyname)) { continue }
-    if (value == "...") { delete keymap[keyname]; continue }
-
-    var keys = map(keyname.split(" "), normalizeKeyName);
-    for (var i = 0; i < keys.length; i++) {
-      var val = (void 0), name = (void 0);
-      if (i == keys.length - 1) {
-        name = keys.join(" ");
-        val = value;
-      } else {
-        name = keys.slice(0, i + 1).join(" ");
-        val = "...";
-      }
-      var prev = copy[name];
-      if (!prev) { copy[name] = val; }
-      else if (prev != val) { throw new Error("Inconsistent bindings for " + name) }
-    }
-    delete keymap[keyname];
-  } }
-  for (var prop in copy) { keymap[prop] = copy[prop]; }
-  return keymap
-}
-
-function lookupKey(key, map$$1, handle, context) {
-  map$$1 = getKeyMap(map$$1);
-  var found = map$$1.call ? map$$1.call(key, context) : map$$1[key];
-  if (found === false) { return "nothing" }
-  if (found === "...") { return "multi" }
-  if (found != null && handle(found)) { return "handled" }
-
-  if (map$$1.fallthrough) {
-    if (Object.prototype.toString.call(map$$1.fallthrough) != "[object Array]")
-      { return lookupKey(key, map$$1.fallthrough, handle, context) }
-    for (var i = 0; i < map$$1.fallthrough.length; i++) {
-      var result = lookupKey(key, map$$1.fallthrough[i], handle, context);
-      if (result) { return result }
-    }
-  }
-}
-
-// Modifier key presses don't count as 'real' key presses for the
-// purpose of keymap fallthrough.
-function isModifierKey(value) {
-  var name = typeof value == "string" ? value : keyNames[value.keyCode];
-  return name == "Ctrl" || name == "Alt" || name == "Shift" || name == "Mod"
-}
-
-function addModifierNames(name, event, noShift) {
-  var base = name;
-  if (event.altKey && base != "Alt") { name = "Alt-" + name; }
-  if ((flipCtrlCmd ? event.metaKey : event.ctrlKey) && base != "Ctrl") { name = "Ctrl-" + name; }
-  if ((flipCtrlCmd ? event.ctrlKey : event.metaKey) && base != "Cmd") { name = "Cmd-" + name; }
-  if (!noShift && event.shiftKey && base != "Shift") { name = "Shift-" + name; }
-  return name
-}
-
-// Look up the name of a key as indicated by an event object.
-function keyName(event, noShift) {
-  if (presto && event.keyCode == 34 && event["char"]) { return false }
-  var name = keyNames[event.keyCode];
-  if (name == null || event.altGraphKey) { return false }
-  return addModifierNames(name, event, noShift)
-}
-
-function getKeyMap(val) {
-  return typeof val == "string" ? keyMap[val] : val
-}
-
-// Helper for deleting text near the selection(s), used to implement
-// backspace, delete, and similar functionality.
-function deleteNearSelection(cm, compute) {
-  var ranges = cm.doc.sel.ranges, kill = [];
-  // Build up a set of ranges to kill first, merging overlapping
-  // ranges.
-  for (var i = 0; i < ranges.length; i++) {
-    var toKill = compute(ranges[i]);
-    while (kill.length && cmp(toKill.from, lst(kill).to) <= 0) {
-      var replaced = kill.pop();
-      if (cmp(replaced.from, toKill.from) < 0) {
-        toKill.from = replaced.from;
-        break
-      }
-    }
-    kill.push(toKill);
-  }
-  // Next, remove those actual ranges.
-  runInOp(cm, function () {
-    for (var i = kill.length - 1; i >= 0; i--)
-      { replaceRange(cm.doc, "", kill[i].from, kill[i].to, "+delete"); }
-    ensureCursorVisible(cm);
-  });
-}
-
-function moveCharLogically(line, ch, dir) {
-  var target = skipExtendingChars(line.text, ch + dir, dir);
-  return target < 0 || target > line.text.length ? null : target
-}
-
-function moveLogically(line, start, dir) {
-  var ch = moveCharLogically(line, start.ch, dir);
-  return ch == null ? null : new Pos(start.line, ch, dir < 0 ? "after" : "before")
-}
-
-function endOfLine(visually, cm, lineObj, lineNo, dir) {
-  if (visually) {
-    var order = getOrder(lineObj, cm.doc.direction);
-    if (order) {
-      var part = dir < 0 ? lst(order) : order[0];
-      var moveInStorageOrder = (dir < 0) == (part.level == 1);
-      var sticky = moveInStorageOrder ? "after" : "before";
-      var ch;
-      // With a wrapped rtl chunk (possibly spanning multiple bidi parts),
-      // it could be that the last bidi part is not on the last visual line,
-      // since visual lines contain content order-consecutive chunks.
-      // Thus, in rtl, we are looking for the first (content-order) character
-      // in the rtl chunk that is on the last line (that is, the same line
-      // as the last (content-order) character).
-      if (part.level > 0 || cm.doc.direction == "rtl") {
-        var prep = prepareMeasureForLine(cm, lineObj);
-        ch = dir < 0 ? lineObj.text.length - 1 : 0;
-        var targetTop = measureCharPrepared(cm, prep, ch).top;
-        ch = findFirst(function (ch) { return measureCharPrepared(cm, prep, ch).top == targetTop; }, (dir < 0) == (part.level == 1) ? part.from : part.to - 1, ch);
-        if (sticky == "before") { ch = moveCharLogically(lineObj, ch, 1); }
-      } else { ch = dir < 0 ? part.to : part.from; }
-      return new Pos(lineNo, ch, sticky)
-    }
-  }
-  return new Pos(lineNo, dir < 0 ? lineObj.text.length : 0, dir < 0 ? "before" : "after")
-}
-
-function moveVisually(cm, line, start, dir) {
-  var bidi = getOrder(line, cm.doc.direction);
-  if (!bidi) { return moveLogically(line, start, dir) }
-  if (start.ch >= line.text.length) {
-    start.ch = line.text.length;
-    start.sticky = "before";
-  } else if (start.ch <= 0) {
-    start.ch = 0;
-    start.sticky = "after";
-  }
-  var partPos = getBidiPartAt(bidi, start.ch, start.sticky), part = bidi[partPos];
-  if (cm.doc.direction == "ltr" && part.level % 2 == 0 && (dir > 0 ? part.to > start.ch : part.from < start.ch)) {
-    // Case 1: We move within an ltr part in an ltr editor. Even with wrapped lines,
-    // nothing interesting happens.
-    return moveLogically(line, start, dir)
-  }
-
-  var mv = function (pos, dir) { return moveCharLogically(line, pos instanceof Pos ? pos.ch : pos, dir); };
-  var prep;
-  var getWrappedLineExtent = function (ch) {
-    if (!cm.options.lineWrapping) { return {begin: 0, end: line.text.length} }
-    prep = prep || prepareMeasureForLine(cm, line);
-    return wrappedLineExtentChar(cm, line, prep, ch)
-  };
-  var wrappedLineExtent = getWrappedLineExtent(start.sticky == "before" ? mv(start, -1) : start.ch);
-
-  if (cm.doc.direction == "rtl" || part.level == 1) {
-    var moveInStorageOrder = (part.level == 1) == (dir < 0);
-    var ch = mv(start, moveInStorageOrder ? 1 : -1);
-    if (ch != null && (!moveInStorageOrder ? ch >= part.from && ch >= wrappedLineExtent.begin : ch <= part.to && ch <= wrappedLineExtent.end)) {
-      // Case 2: We move within an rtl part or in an rtl editor on the same visual line
-      var sticky = moveInStorageOrder ? "before" : "after";
-      return new Pos(start.line, ch, sticky)
-    }
-  }
-
-  // Case 3: Could not move within this bidi part in this visual line, so leave
-  // the current bidi part
-
-  var searchInVisualLine = function (partPos, dir, wrappedLineExtent) {
-    var getRes = function (ch, moveInStorageOrder) { return moveInStorageOrder
-      ? new Pos(start.line, mv(ch, 1), "before")
-      : new Pos(start.line, ch, "after"); };
-
-    for (; partPos >= 0 && partPos < bidi.length; partPos += dir) {
-      var part = bidi[partPos];
-      var moveInStorageOrder = (dir > 0) == (part.level != 1);
-      var ch = moveInStorageOrder ? wrappedLineExtent.begin : mv(wrappedLineExtent.end, -1);
-      if (part.from <= ch && ch < part.to) { return getRes(ch, moveInStorageOrder) }
-      ch = moveInStorageOrder ? part.from : mv(part.to, -1);
-      if (wrappedLineExtent.begin <= ch && ch < wrappedLineExtent.end) { return getRes(ch, moveInStorageOrder) }
-    }
-  };
-
-  // Case 3a: Look for other bidi parts on the same visual line
-  var res = searchInVisualLine(partPos + dir, dir, wrappedLineExtent);
-  if (res) { return res }
-
-  // Case 3b: Look for other bidi parts on the next visual line
-  var nextCh = dir > 0 ? wrappedLineExtent.end : mv(wrappedLineExtent.begin, -1);
-  if (nextCh != null && !(dir > 0 && nextCh == line.text.length)) {
-    res = searchInVisualLine(dir > 0 ? 0 : bidi.length - 1, dir, getWrappedLineExtent(nextCh));
-    if (res) { return res }
-  }
-
-  // Case 4: Nowhere to move
-  return null
-}
-
-// Commands are parameter-less actions that can be performed on an
-// editor, mostly used for keybindings.
-var commands = {
-  selectAll: selectAll,
-  singleSelection: function (cm) { return cm.setSelection(cm.getCursor("anchor"), cm.getCursor("head"), sel_dontScroll); },
-  killLine: function (cm) { return deleteNearSelection(cm, function (range) {
-    if (range.empty()) {
-      var len = getLine(cm.doc, range.head.line).text.length;
-      if (range.head.ch == len && range.head.line < cm.lastLine())
-        { return {from: range.head, to: Pos(range.head.line + 1, 0)} }
-      else
-        { return {from: range.head, to: Pos(range.head.line, len)} }
-    } else {
-      return {from: range.from(), to: range.to()}
-    }
-  }); },
-  deleteLine: function (cm) { return deleteNearSelection(cm, function (range) { return ({
-    from: Pos(range.from().line, 0),
-    to: clipPos(cm.doc, Pos(range.to().line + 1, 0))
-  }); }); },
-  delLineLeft: function (cm) { return deleteNearSelection(cm, function (range) { return ({
-    from: Pos(range.from().line, 0), to: range.from()
-  }); }); },
-  delWrappedLineLeft: function (cm) { return deleteNearSelection(cm, function (range) {
-    var top = cm.charCoords(range.head, "div").top + 5;
-    var leftPos = cm.coordsChar({left: 0, top: top}, "div");
-    return {from: leftPos, to: range.from()}
-  }); },
-  delWrappedLineRight: function (cm) { return deleteNearSelection(cm, function (range) {
-    var top = cm.charCoords(range.head, "div").top + 5;
-    var rightPos = cm.coordsChar({left: cm.display.lineDiv.offsetWidth + 100, top: top}, "div");
-    return {from: range.from(), to: rightPos }
-  }); },
-  undo: function (cm) { return cm.undo(); },
-  redo: function (cm) { return cm.redo(); },
-  undoSelection: function (cm) { return cm.undoSelection(); },
-  redoSelection: function (cm) { return cm.redoSelection(); },
-  goDocStart: function (cm) { return cm.extendSelection(Pos(cm.firstLine(), 0)); },
-  goDocEnd: function (cm) { return cm.extendSelection(Pos(cm.lastLine())); },
-  goLineStart: function (cm) { return cm.extendSelectionsBy(function (range) { return lineStart(cm, range.head.line); },
-    {origin: "+move", bias: 1}
-  ); },
-  goLineStartSmart: function (cm) { return cm.extendSelectionsBy(function (range) { return lineStartSmart(cm, range.head); },
-    {origin: "+move", bias: 1}
-  ); },
-  goLineEnd: function (cm) { return cm.extendSelectionsBy(function (range) { return lineEnd(cm, range.head.line); },
-    {origin: "+move", bias: -1}
-  ); },
-  goLineRight: function (cm) { return cm.extendSelectionsBy(function (range) {
-    var top = cm.cursorCoords(range.head, "div").top + 5;
-    return cm.coordsChar({left: cm.display.lineDiv.offsetWidth + 100, top: top}, "div")
-  }, sel_move); },
-  goLineLeft: function (cm) { return cm.extendSelectionsBy(function (range) {
-    var top = cm.cursorCoords(range.head, "div").top + 5;
-    return cm.coordsChar({left: 0, top: top}, "div")
-  }, sel_move); },
-  goLineLeftSmart: function (cm) { return cm.extendSelectionsBy(function (range) {
-    var top = cm.cursorCoords(range.head, "div").top + 5;
-    var pos = cm.coordsChar({left: 0, top: top}, "div");
-    if (pos.ch < cm.getLine(pos.line).search(/\S/)) { return lineStartSmart(cm, range.head) }
-    return pos
-  }, sel_move); },
-  goLineUp: function (cm) { return cm.moveV(-1, "line"); },
-  goLineDown: function (cm) { return cm.moveV(1, "line"); },
-  goPageUp: function (cm) { return cm.moveV(-1, "page"); },
-  goPageDown: function (cm) { return cm.moveV(1, "page"); },
-  goCharLeft: function (cm) { return cm.moveH(-1, "char"); },
-  goCharRight: function (cm) { return cm.moveH(1, "char"); },
-  goColumnLeft: function (cm) { return cm.moveH(-1, "column"); },
-  goColumnRight: function (cm) { return cm.moveH(1, "column"); },
-  goWordLeft: function (cm) { return cm.moveH(-1, "word"); },
-  goGroupRight: function (cm) { return cm.moveH(1, "group"); },
-  goGroupLeft: function (cm) { return cm.moveH(-1, "group"); },
-  goWordRight: function (cm) { return cm.moveH(1, "word"); },
-  delCharBefore: function (cm) { return cm.deleteH(-1, "char"); },
-  delCharAfter: function (cm) { return cm.deleteH(1, "char"); },
-  delWordBefore: function (cm) { return cm.deleteH(-1, "word"); },
-  delWordAfter: function (cm) { return cm.deleteH(1, "word"); },
-  delGroupBefore: function (cm) { return cm.deleteH(-1, "group"); },
-  delGroupAfter: function (cm) { return cm.deleteH(1, "group"); },
-  indentAuto: function (cm) { return cm.indentSelection("smart"); },
-  indentMore: function (cm) { return cm.indentSelection("add"); },
-  indentLess: function (cm) { return cm.indentSelection("subtract"); },
-  insertTab: function (cm) { return cm.replaceSelection("\t"); },
-  insertSoftTab: function (cm) {
-    var spaces = [], ranges = cm.listSelections(), tabSize = cm.options.tabSize;
-    for (var i = 0; i < ranges.length; i++) {
-      var pos = ranges[i].from();
-      var col = countColumn(cm.getLine(pos.line), pos.ch, tabSize);
-      spaces.push(spaceStr(tabSize - col % tabSize));
-    }
-    cm.replaceSelections(spaces);
-  },
-  defaultTab: function (cm) {
-    if (cm.somethingSelected()) { cm.indentSelection("add"); }
-    else { cm.execCommand("insertTab"); }
-  },
-  // Swap the two chars left and right of each selection's head.
-  // Move cursor behind the two swapped characters afterwards.
-  //
-  // Doesn't consider line feeds a character.
-  // Doesn't scan more than one line above to find a character.
-  // Doesn't do anything on an empty line.
-  // Doesn't do anything with non-empty selections.
-  transposeChars: function (cm) { return runInOp(cm, function () {
-    var ranges = cm.listSelections(), newSel = [];
-    for (var i = 0; i < ranges.length; i++) {
-      if (!ranges[i].empty()) { continue }
-      var cur = ranges[i].head, line = getLine(cm.doc, cur.line).text;
-      if (line) {
-        if (cur.ch == line.length) { cur = new Pos(cur.line, cur.ch - 1); }
-        if (cur.ch > 0) {
-          cur = new Pos(cur.line, cur.ch + 1);
-          cm.replaceRange(line.charAt(cur.ch - 1) + line.charAt(cur.ch - 2),
-                          Pos(cur.line, cur.ch - 2), cur, "+transpose");
-        } else if (cur.line > cm.doc.first) {
-          var prev = getLine(cm.doc, cur.line - 1).text;
-          if (prev) {
-            cur = new Pos(cur.line, 1);
-            cm.replaceRange(line.charAt(0) + cm.doc.lineSeparator() +
-                            prev.charAt(prev.length - 1),
-                            Pos(cur.line - 1, prev.length - 1), cur, "+transpose");
-          }
-        }
-      }
-      newSel.push(new Range(cur, cur));
-    }
-    cm.setSelections(newSel);
-  }); },
-  newlineAndIndent: function (cm) { return runInOp(cm, function () {
-    var sels = cm.listSelections();
-    for (var i = sels.length - 1; i >= 0; i--)
-      { cm.replaceRange(cm.doc.lineSeparator(), sels[i].anchor, sels[i].head, "+input"); }
-    sels = cm.listSelections();
-    for (var i$1 = 0; i$1 < sels.length; i$1++)
-      { cm.indentLine(sels[i$1].from().line, null, true); }
-    ensureCursorVisible(cm);
-  }); },
-  openLine: function (cm) { return cm.replaceSelection("\n", "start"); },
-  toggleOverwrite: function (cm) { return cm.toggleOverwrite(); }
-};
-
-
-function lineStart(cm, lineN) {
-  var line = getLine(cm.doc, lineN);
-  var visual = visualLine(line);
-  if (visual != line) { lineN = lineNo(visual); }
-  return endOfLine(true, cm, visual, lineN, 1)
-}
-function lineEnd(cm, lineN) {
-  var line = getLine(cm.doc, lineN);
-  var visual = visualLineEnd(line);
-  if (visual != line) { lineN = lineNo(visual); }
-  return endOfLine(true, cm, line, lineN, -1)
-}
-function lineStartSmart(cm, pos) {
-  var start = lineStart(cm, pos.line);
-  var line = getLine(cm.doc, start.line);
-  var order = getOrder(line, cm.doc.direction);
-  if (!order || order[0].level == 0) {
-    var firstNonWS = Math.max(0, line.text.search(/\S/));
-    var inWS = pos.line == start.line && pos.ch <= firstNonWS && pos.ch;
-    return Pos(start.line, inWS ? 0 : firstNonWS, start.sticky)
-  }
-  return start
-}
-
-// Run a handler that was bound to a key.
-function doHandleBinding(cm, bound, dropShift) {
-  if (typeof bound == "string") {
-    bound = commands[bound];
-    if (!bound) { return false }
-  }
-  // Ensure previous input has been read, so that the handler sees a
-  // consistent view of the document
-  cm.display.input.ensurePolled();
-  var prevShift = cm.display.shift, done = false;
-  try {
-    if (cm.isReadOnly()) { cm.state.suppressEdits = true; }
-    if (dropShift) { cm.display.shift = false; }
-    done = bound(cm) != Pass;
-  } finally {
-    cm.display.shift = prevShift;
-    cm.state.suppressEdits = false;
-  }
-  return done
-}
-
-function lookupKeyForEditor(cm, name, handle) {
-  for (var i = 0; i < cm.state.keyMaps.length; i++) {
-    var result = lookupKey(name, cm.state.keyMaps[i], handle, cm);
-    if (result) { return result }
-  }
-  return (cm.options.extraKeys && lookupKey(name, cm.options.extraKeys, handle, cm))
-    || lookupKey(name, cm.options.keyMap, handle, cm)
-}
-
-// Note that, despite the name, this function is also used to check
-// for bound mouse clicks.
-
-var stopSeq = new Delayed;
-function dispatchKey(cm, name, e, handle) {
-  var seq = cm.state.keySeq;
-  if (seq) {
-    if (isModifierKey(name)) { return "handled" }
-    stopSeq.set(50, function () {
-      if (cm.state.keySeq == seq) {
-        cm.state.keySeq = null;
-        cm.display.input.reset();
-      }
-    });
-    name = seq + " " + name;
-  }
-  var result = lookupKeyForEditor(cm, name, handle);
-
-  if (result == "multi")
-    { cm.state.keySeq = name; }
-  if (result == "handled")
-    { signalLater(cm, "keyHandled", cm, name, e); }
-
-  if (result == "handled" || result == "multi") {
-    e_preventDefault(e);
-    restartBlink(cm);
-  }
-
-  if (seq && !result && /\'$/.test(name)) {
-    e_preventDefault(e);
-    return true
-  }
-  return !!result
-}
-
-// Handle a key from the keydown event.
-function handleKeyBinding(cm, e) {
-  var name = keyName(e, true);
-  if (!name) { return false }
-
-  if (e.shiftKey && !cm.state.keySeq) {
-    // First try to resolve full name (including 'Shift-'). Failing
-    // that, see if there is a cursor-motion command (starting with
-    // 'go') bound to the keyname without 'Shift-'.
-    return dispatchKey(cm, "Shift-" + name, e, function (b) { return doHandleBinding(cm, b, true); })
-        || dispatchKey(cm, name, e, function (b) {
-             if (typeof b == "string" ? /^go[A-Z]/.test(b) : b.motion)
-               { return doHandleBinding(cm, b) }
-           })
-  } else {
-    return dispatchKey(cm, name, e, function (b) { return doHandleBinding(cm, b); })
-  }
-}
-
-// Handle a key from the keypress event
-function handleCharBinding(cm, e, ch) {
-  return dispatchKey(cm, "'" + ch + "'", e, function (b) { return doHandleBinding(cm, b, true); })
-}
-
-var lastStoppedKey = null;
-function onKeyDown(e) {
-  var cm = this;
-  cm.curOp.focus = activeElt();
-  if (signalDOMEvent(cm, e)) { return }
-  // IE does strange things with escape.
-  if (ie && ie_version < 11 && e.keyCode == 27) { e.returnValue = false; }
-  var code = e.keyCode;
-  cm.display.shift = code == 16 || e.shiftKey;
-  var handled = handleKeyBinding(cm, e);
-  if (presto) {
-    lastStoppedKey = handled ? code : null;
-    // Opera has no cut event... we try to at least catch the key combo
-    if (!handled && code == 88 && !hasCopyEvent && (mac ? e.metaKey : e.ctrlKey))
-      { cm.replaceSelection("", null, "cut"); }
-  }
-
-  // Turn mouse into crosshair when Alt is held on Mac.
-  if (code == 18 && !/\bCodeMirror-crosshair\b/.test(cm.display.lineDiv.className))
-    { showCrossHair(cm); }
-}
-
-function showCrossHair(cm) {
-  var lineDiv = cm.display.lineDiv;
-  addClass(lineDiv, "CodeMirror-crosshair");
-
-  function up(e) {
-    if (e.keyCode == 18 || !e.altKey) {
-      rmClass(lineDiv, "CodeMirror-crosshair");
-      off(document, "keyup", up);
-      off(document, "mouseover", up);
-    }
-  }
-  on(document, "keyup", up);
-  on(document, "mouseover", up);
-}
-
-function onKeyUp(e) {
-  if (e.keyCode == 16) { this.doc.sel.shift = false; }
-  signalDOMEvent(this, e);
-}
-
-function onKeyPress(e) {
-  var cm = this;
-  if (eventInWidget(cm.display, e) || signalDOMEvent(cm, e) || e.ctrlKey && !e.altKey || mac && e.metaKey) { return }
-  var keyCode = e.keyCode, charCode = e.charCode;
-  if (presto && keyCode == lastStoppedKey) {lastStoppedKey = null; e_preventDefault(e); return}
-  if ((presto && (!e.which || e.which < 10)) && handleKeyBinding(cm, e)) { return }
-  var ch = String.fromCharCode(charCode == null ? keyCode : charCode);
-  // Some browsers fire keypress events for backspace
-  if (ch == "\x08") { return }
-  if (handleCharBinding(cm, e, ch)) { return }
-  cm.display.input.onKeyPress(e);
-}
-
-var DOUBLECLICK_DELAY = 400;
-
-var PastClick = function(time, pos, button) {
-  this.time = time;
-  this.pos = pos;
-  this.button = button;
-};
-
-PastClick.prototype.compare = function (time, pos, button) {
-  return this.time + DOUBLECLICK_DELAY > time &&
-    cmp(pos, this.pos) == 0 && button == this.button
-};
-
-var lastClick;
-var lastDoubleClick;
-function clickRepeat(pos, button) {
-  var now = +new Date;
-  if (lastDoubleClick && lastDoubleClick.compare(now, pos, button)) {
-    lastClick = lastDoubleClick = null;
-    return "triple"
-  } else if (lastClick && lastClick.compare(now, pos, button)) {
-    lastDoubleClick = new PastClick(now, pos, button);
-    lastClick = null;
-    return "double"
-  } else {
-    lastClick = new PastClick(now, pos, button);
-    lastDoubleClick = null;
-    return "single"
-  }
-}
-
-// A mouse down can be a single click, double click, triple click,
-// start of selection drag, start of text drag, new cursor
-// (ctrl-click), rectangle drag (alt-drag), or xwin
-// middle-click-paste. Or it might be a click on something we should
-// not interfere with, such as a scrollbar or widget.
-function onMouseDown(e) {
-  var cm = this, display = cm.display;
-  if (signalDOMEvent(cm, e) || display.activeTouch && display.input.supportsTouch()) { return }
-  display.input.ensurePolled();
-  display.shift = e.shiftKey;
-
-  if (eventInWidget(display, e)) {
-    if (!webkit) {
-      // Briefly turn off draggability, to allow widgets to do
-      // normal dragging things.
-      display.scroller.draggable = false;
-      setTimeout(function () { return display.scroller.draggable = true; }, 100);
-    }
-    return
-  }
-  if (clickInGutter(cm, e)) { return }
-  var pos = posFromMouse(cm, e), button = e_button(e), repeat = pos ? clickRepeat(pos, button) : "single";
-  window.focus();
-
-  // #3261: make sure, that we're not starting a second selection
-  if (button == 1 && cm.state.selectingText)
-    { cm.state.selectingText(e); }
-
-  if (pos && handleMappedButton(cm, button, pos, repeat, e)) { return }
-
-  if (button == 1) {
-    if (pos) { leftButtonDown(cm, pos, repeat, e); }
-    else if (e_target(e) == display.scroller) { e_preventDefault(e); }
-  } else if (button == 2) {
-    if (pos) { extendSelection(cm.doc, pos); }
-    setTimeout(function () { return display.input.focus(); }, 20);
-  } else if (button == 3) {
-    if (captureRightClick) { onContextMenu(cm, e); }
-    else { delayBlurEvent(cm); }
-  }
-}
-
-function handleMappedButton(cm, button, pos, repeat, event) {
-  var name = "Click";
-  if (repeat == "double") { name = "Double" + name; }
-  else if (repeat == "triple") { name = "Triple" + name; }
-  name = (button == 1 ? "Left" : button == 2 ? "Middle" : "Right") + name;
-
-  return dispatchKey(cm,  addModifierNames(name, event), event, function (bound) {
-    if (typeof bound == "string") { bound = commands[bound]; }
-    if (!bound) { return false }
-    var done = false;
-    try {
-      if (cm.isReadOnly()) { cm.state.suppressEdits = true; }
-      done = bound(cm, pos) != Pass;
-    } finally {
-      cm.state.suppressEdits = false;
-    }
-    return done
-  })
-}
-
-function configureMouse(cm, repeat, event) {
-  var option = cm.getOption("configureMouse");
-  var value = option ? option(cm, repeat, event) : {};
-  if (value.unit == null) {
-    var rect = chromeOS ? event.shiftKey && event.metaKey : event.altKey;
-    value.unit = rect ? "rectangle" : repeat == "single" ? "char" : repeat == "double" ? "word" : "line";
-  }
-  if (value.extend == null || cm.doc.extend) { value.extend = cm.doc.extend || event.shiftKey; }
-  if (value.addNew == null) { value.addNew = mac ? event.metaKey : event.ctrlKey; }
-  if (value.moveOnDrag == null) { value.moveOnDrag = !(mac ? event.altKey : event.ctrlKey); }
-  return value
-}
-
-function leftButtonDown(cm, pos, repeat, event) {
-  if (ie) { setTimeout(bind(ensureFocus, cm), 0); }
-  else { cm.curOp.focus = activeElt(); }
-
-  var behavior = configureMouse(cm, repeat, event);
-
-  var sel = cm.doc.sel, contained;
-  if (cm.options.dragDrop && dragAndDrop && !cm.isReadOnly() &&
-      repeat == "single" && (contained = sel.contains(pos)) > -1 &&
-      (cmp((contained = sel.ranges[contained]).from(), pos) < 0 || pos.xRel > 0) &&
-      (cmp(contained.to(), pos) > 0 || pos.xRel < 0))
-    { leftButtonStartDrag(cm, event, pos, behavior); }
-  else
-    { leftButtonSelect(cm, event, pos, behavior); }
-}
-
-// Start a text drag. When it ends, see if any dragging actually
-// happen, and treat as a click if it didn't.
-function leftButtonStartDrag(cm, event, pos, behavior) {
-  var display = cm.display, moved = false;
-  var dragEnd = operation(cm, function (e) {
-    if (webkit) { display.scroller.draggable = false; }
-    cm.state.draggingText = false;
-    off(document, "mouseup", dragEnd);
-    off(document, "mousemove", mouseMove);
-    off(display.scroller, "dragstart", dragStart);
-    off(display.scroller, "drop", dragEnd);
-    if (!moved) {
-      e_preventDefault(e);
-      if (!behavior.addNew)
-        { extendSelection(cm.doc, pos, null, null, behavior.extend); }
-      // Work around unexplainable focus problem in IE9 (#2127) and Chrome (#3081)
-      if (webkit || ie && ie_version == 9)
-        { setTimeout(function () {document.body.focus(); display.input.focus();}, 20); }
-      else
-        { display.input.focus(); }
-    }
-  });
-  var mouseMove = function(e2) {
-    moved = moved || Math.abs(event.clientX - e2.clientX) + Math.abs(event.clientY - e2.clientY) >= 10;
-  };
-  var dragStart = function () { return moved = true; };
-  // Let the drag handler handle this.
-  if (webkit) { display.scroller.draggable = true; }
-  cm.state.draggingText = dragEnd;
-  dragEnd.copy = !behavior.moveOnDrag;
-  // IE's approach to draggable
-  if (display.scroller.dragDrop) { display.scroller.dragDrop(); }
-  on(document, "mouseup", dragEnd);
-  on(document, "mousemove", mouseMove);
-  on(display.scroller, "dragstart", dragStart);
-  on(display.scroller, "drop", dragEnd);
-
-  delayBlurEvent(cm);
-  setTimeout(function () { return display.input.focus(); }, 20);
-}
-
-function rangeForUnit(cm, pos, unit) {
-  if (unit == "char") { return new Range(pos, pos) }
-  if (unit == "word") { return cm.findWordAt(pos) }
-  if (unit == "line") { return new Range(Pos(pos.line, 0), clipPos(cm.doc, Pos(pos.line + 1, 0))) }
-  var result = unit(cm, pos);
-  return new Range(result.from, result.to)
-}
-
-// Normal selection, as opposed to text dragging.
-function leftButtonSelect(cm, event, start, behavior) {
-  var display = cm.display, doc = cm.doc;
-  e_preventDefault(event);
-
-  var ourRange, ourIndex, startSel = doc.sel, ranges = startSel.ranges;
-  if (behavior.addNew && !behavior.extend) {
-    ourIndex = doc.sel.contains(start);
-    if (ourIndex > -1)
-      { ourRange = ranges[ourIndex]; }
-    else
-      { ourRange = new Range(start, start); }
-  } else {
-    ourRange = doc.sel.primary();
-    ourIndex = doc.sel.primIndex;
-  }
-
-  if (behavior.unit == "rectangle") {
-    if (!behavior.addNew) { ourRange = new Range(start, start); }
-    start = posFromMouse(cm, event, true, true);
-    ourIndex = -1;
-  } else {
-    var range$$1 = rangeForUnit(cm, start, behavior.unit);
-    if (behavior.extend)
-      { ourRange = extendRange(ourRange, range$$1.anchor, range$$1.head, behavior.extend); }
-    else
-      { ourRange = range$$1; }
-  }
-
-  if (!behavior.addNew) {
-    ourIndex = 0;
-    setSelection(doc, new Selection([ourRange], 0), sel_mouse);
-    startSel = doc.sel;
-  } else if (ourIndex == -1) {
-    ourIndex = ranges.length;
-    setSelection(doc, normalizeSelection(ranges.concat([ourRange]), ourIndex),
-                 {scroll: false, origin: "*mouse"});
-  } else if (ranges.length > 1 && ranges[ourIndex].empty() && behavior.unit == "char" && !behavior.extend) {
-    setSelection(doc, normalizeSelection(ranges.slice(0, ourIndex).concat(ranges.slice(ourIndex + 1)), 0),
-                 {scroll: false, origin: "*mouse"});
-    startSel = doc.sel;
-  } else {
-    replaceOneSelection(doc, ourIndex, ourRange, sel_mouse);
-  }
-
-  var lastPos = start;
-  function extendTo(pos) {
-    if (cmp(lastPos, pos) == 0) { return }
-    lastPos = pos;
-
-    if (behavior.unit == "rectangle") {
-      var ranges = [], tabSize = cm.options.tabSize;
-      var startCol = countColumn(getLine(doc, start.line).text, start.ch, tabSize);
-      var posCol = countColumn(getLine(doc, pos.line).text, pos.ch, tabSize);
-      var left = Math.min(startCol, posCol), right = Math.max(startCol, posCol);
-      for (var line = Math.min(start.line, pos.line), end = Math.min(cm.lastLine(), Math.max(start.line, pos.line));
-           line <= end; line++) {
-        var text = getLine(doc, line).text, leftPos = findColumn(text, left, tabSize);
-        if (left == right)
-          { ranges.push(new Range(Pos(line, leftPos), Pos(line, leftPos))); }
-        else if (text.length > leftPos)
-          { ranges.push(new Range(Pos(line, leftPos), Pos(line, findColumn(text, right, tabSize)))); }
-      }
-      if (!ranges.length) { ranges.push(new Range(start, start)); }
-      setSelection(doc, normalizeSelection(startSel.ranges.slice(0, ourIndex).concat(ranges), ourIndex),
-                   {origin: "*mouse", scroll: false});
-      cm.scrollIntoView(pos);
-    } else {
-      var oldRange = ourRange;
-      var range$$1 = rangeForUnit(cm, pos, behavior.unit);
-      var anchor = oldRange.anchor, head;
-      if (cmp(range$$1.anchor, anchor) > 0) {
-        head = range$$1.head;
-        anchor = minPos(oldRange.from(), range$$1.anchor);
-      } else {
-        head = range$$1.anchor;
-        anchor = maxPos(oldRange.to(), range$$1.head);
-      }
-      var ranges$1 = startSel.ranges.slice(0);
-      ranges$1[ourIndex] = bidiSimplify(cm, new Range(clipPos(doc, anchor), head));
-      setSelection(doc, normalizeSelection(ranges$1, ourIndex), sel_mouse);
-    }
-  }
-
-  var editorSize = display.wrapper.getBoundingClientRect();
-  // Used to ensure timeout re-tries don't fire when another extend
-  // happened in the meantime (clearTimeout isn't reliable -- at
-  // least on Chrome, the timeouts still happen even when cleared,
-  // if the clear happens after their scheduled firing time).
-  var counter = 0;
-
-  function extend(e) {
-    var curCount = ++counter;
-    var cur = posFromMouse(cm, e, true, behavior.unit == "rectangle");
-    if (!cur) { return }
-    if (cmp(cur, lastPos) != 0) {
-      cm.curOp.focus = activeElt();
-      extendTo(cur);
-      var visible = visibleLines(display, doc);
-      if (cur.line >= visible.to || cur.line < visible.from)
-        { setTimeout(operation(cm, function () {if (counter == curCount) { extend(e); }}), 150); }
-    } else {
-      var outside = e.clientY < editorSize.top ? -20 : e.clientY > editorSize.bottom ? 20 : 0;
-      if (outside) { setTimeout(operation(cm, function () {
-        if (counter != curCount) { return }
-        display.scroller.scrollTop += outside;
-        extend(e);
-      }), 50); }
-    }
-  }
-
-  function done(e) {
-    cm.state.selectingText = false;
-    counter = Infinity;
-    e_preventDefault(e);
-    display.input.focus();
-    off(document, "mousemove", move);
-    off(document, "mouseup", up);
-    doc.history.lastSelOrigin = null;
-  }
-
-  var move = operation(cm, function (e) {
-    if (!e_button(e)) { done(e); }
-    else { extend(e); }
-  });
-  var up = operation(cm, done);
-  cm.state.selectingText = up;
-  on(document, "mousemove", move);
-  on(document, "mouseup", up);
-}
-
-// Used when mouse-selecting to adjust the anchor to the proper side
-// of a bidi jump depending on the visual position of the head.
-function bidiSimplify(cm, range$$1) {
-  var anchor = range$$1.anchor;
-  var head = range$$1.head;
-  var anchorLine = getLine(cm.doc, anchor.line);
-  if (cmp(anchor, head) == 0 && anchor.sticky == head.sticky) { return range$$1 }
-  var order = getOrder(anchorLine);
-  if (!order) { return range$$1 }
-  var index = getBidiPartAt(order, anchor.ch, anchor.sticky), part = order[index];
-  if (part.from != anchor.ch && part.to != anchor.ch) { return range$$1 }
-  var boundary = index + ((part.from == anchor.ch) == (part.level != 1) ? 0 : 1);
-  if (boundary == 0 || boundary == order.length) { return range$$1 }
-
-  // Compute the relative visual position of the head compared to the
-  // anchor (<0 is to the left, >0 to the right)
-  var leftSide;
-  if (head.line != anchor.line) {
-    leftSide = (head.line - anchor.line) * (cm.doc.direction == "ltr" ? 1 : -1) > 0;
-  } else {
-    var headIndex = getBidiPartAt(order, head.ch, head.sticky);
-    var dir = headIndex - index || (head.ch - anchor.ch) * (part.level == 1 ? -1 : 1);
-    if (headIndex == boundary - 1 || headIndex == boundary)
-      { leftSide = dir < 0; }
-    else
-      { leftSide = dir > 0; }
-  }
-
-  var usePart = order[boundary + (leftSide ? -1 : 0)];
-  var from = leftSide == (usePart.level == 1);
-  var ch = from ? usePart.from : usePart.to, sticky = from ? "after" : "before";
-  return anchor.ch == ch && anchor.sticky == sticky ? range$$1 : new Range(new Pos(anchor.line, ch, sticky), head)
-}
-
-
-// Determines whether an event happened in the gutter, and fires the
-// handlers for the corresponding event.
-function gutterEvent(cm, e, type, prevent) {
-  var mX, mY;
-  if (e.touches) {
-    mX = e.touches[0].clientX;
-    mY = e.touches[0].clientY;
-  } else {
-    try { mX = e.clientX; mY = e.clientY; }
-    catch(e) { return false }
-  }
-  if (mX >= Math.floor(cm.display.gutters.getBoundingClientRect().right)) { return false }
-  if (prevent) { e_preventDefault(e); }
-
-  var display = cm.display;
-  var lineBox = display.lineDiv.getBoundingClientRect();
-
-  if (mY > lineBox.bottom || !hasHandler(cm, type)) { return e_defaultPrevented(e) }
-  mY -= lineBox.top - display.viewOffset;
-
-  for (var i = 0; i < cm.options.gutters.length; ++i) {
-    var g = display.gutters.childNodes[i];
-    if (g && g.getBoundingClientRect().right >= mX) {
-      var line = lineAtHeight(cm.doc, mY);
-      var gutter = cm.options.gutters[i];
-      signal(cm, type, cm, line, gutter, e);
-      return e_defaultPrevented(e)
-    }
-  }
-}
-
-function clickInGutter(cm, e) {
-  return gutterEvent(cm, e, "gutterClick", true)
-}
-
-// CONTEXT MENU HANDLING
-
-// To make the context menu work, we need to briefly unhide the
-// textarea (making it as unobtrusive as possible) to let the
-// right-click take effect on it.
-function onContextMenu(cm, e) {
-  if (eventInWidget(cm.display, e) || contextMenuInGutter(cm, e)) { return }
-  if (signalDOMEvent(cm, e, "contextmenu")) { return }
-  cm.display.input.onContextMenu(e);
-}
-
-function contextMenuInGutter(cm, e) {
-  if (!hasHandler(cm, "gutterContextMenu")) { return false }
-  return gutterEvent(cm, e, "gutterContextMenu", false)
-}
-
-function themeChanged(cm) {
-  cm.display.wrapper.className = cm.display.wrapper.className.replace(/\s*cm-s-\S+/g, "") +
-    cm.options.theme.replace(/(^|\s)\s*/g, " cm-s-");
-  clearCaches(cm);
-}
-
-var Init = {toString: function(){return "CodeMirror.Init"}};
-
-var defaults = {};
-var optionHandlers = {};
-
-function defineOptions(CodeMirror) {
-  var optionHandlers = CodeMirror.optionHandlers;
-
-  function option(name, deflt, handle, notOnInit) {
-    CodeMirror.defaults[name] = deflt;
-    if (handle) { optionHandlers[name] =
-      notOnInit ? function (cm, val, old) {if (old != Init) { handle(cm, val, old); }} : handle; }
-  }
-
-  CodeMirror.defineOption = option;
-
-  // Passed to option handlers when there is no old value.
-  CodeMirror.Init = Init;
-
-  // These two are, on init, called from the constructor because they
-  // have to be initialized before the editor can start at all.
-  option("value", "", function (cm, val) { return cm.setValue(val); }, true);
-  option("mode", null, function (cm, val) {
-    cm.doc.modeOption = val;
-    loadMode(cm);
-  }, true);
-
-  option("indentUnit", 2, loadMode, true);
-  option("indentWithTabs", false);
-  option("smartIndent", true);
-  option("tabSize", 4, function (cm) {
-    resetModeState(cm);
-    clearCaches(cm);
-    regChange(cm);
-  }, true);
-  option("lineSeparator", null, function (cm, val) {
-    cm.doc.lineSep = val;
-    if (!val) { return }
-    var newBreaks = [], lineNo = cm.doc.first;
-    cm.doc.iter(function (line) {
-      for (var pos = 0;;) {
-        var found = line.text.indexOf(val, pos);
-        if (found == -1) { break }
-        pos = found + val.length;
-        newBreaks.push(Pos(lineNo, found));
-      }
-      lineNo++;
-    });
-    for (var i = newBreaks.length - 1; i >= 0; i--)
-      { replaceRange(cm.doc, val, newBreaks[i], Pos(newBreaks[i].line, newBreaks[i].ch + val.length)); }
-  });
-  option("specialChars", /[\u0000-\u001f\u007f-\u009f\u00ad\u061c\u200b-\u200f\u2028\u2029\ufeff]/g, function (cm, val, old) {
-    cm.state.specialChars = new RegExp(val.source + (val.test("\t") ? "" : "|\t"), "g");
-    if (old != Init) { cm.refresh(); }
-  });
-  option("specialCharPlaceholder", defaultSpecialCharPlaceholder, function (cm) { return cm.refresh(); }, true);
-  option("electricChars", true);
-  option("inputStyle", mobile ? "contenteditable" : "textarea", function () {
-    throw new Error("inputStyle can not (yet) be changed in a running editor") // FIXME
-  }, true);
-  option("spellcheck", false, function (cm, val) { return cm.getInputField().spellcheck = val; }, true);
-  option("rtlMoveVisually", !windows);
-  option("wholeLineUpdateBefore", true);
-
-  option("theme", "default", function (cm) {
-    themeChanged(cm);
-    guttersChanged(cm);
-  }, true);
-  option("keyMap", "default", function (cm, val, old) {
-    var next = getKeyMap(val);
-    var prev = old != Init && getKeyMap(old);
-    if (prev && prev.detach) { prev.detach(cm, next); }
-    if (next.attach) { next.attach(cm, prev || null); }
-  });
-  option("extraKeys", null);
-  option("configureMouse", null);
-
-  option("lineWrapping", false, wrappingChanged, true);
-  option("gutters", [], function (cm) {
-    setGuttersForLineNumbers(cm.options);
-    guttersChanged(cm);
-  }, true);
-  option("fixedGutter", true, function (cm, val) {
-    cm.display.gutters.style.left = val ? compensateForHScroll(cm.display) + "px" : "0";
-    cm.refresh();
-  }, true);
-  option("coverGutterNextToScrollbar", false, function (cm) { return updateScrollbars(cm); }, true);
-  option("scrollbarStyle", "native", function (cm) {
-    initScrollbars(cm);
-    updateScrollbars(cm);
-    cm.display.scrollbars.setScrollTop(cm.doc.scrollTop);
-    cm.display.scrollbars.setScrollLeft(cm.doc.scrollLeft);
-  }, true);
-  option("lineNumbers", false, function (cm) {
-    setGuttersForLineNumbers(cm.options);
-    guttersChanged(cm);
-  }, true);
-  option("firstLineNumber", 1, guttersChanged, true);
-  option("lineNumberFormatter", function (integer) { return integer; }, guttersChanged, true);
-  option("showCursorWhenSelecting", false, updateSelection, true);
-
-  option("resetSelectionOnContextMenu", true);
-  option("lineWiseCopyCut", true);
-  option("pasteLinesPerSelection", true);
-
-  option("readOnly", false, function (cm, val) {
-    if (val == "nocursor") {
-      onBlur(cm);
-      cm.display.input.blur();
-    }
-    cm.display.input.readOnlyChanged(val);
-  });
-  option("disableInput", false, function (cm, val) {if (!val) { cm.display.input.reset(); }}, true);
-  option("dragDrop", true, dragDropChanged);
-  option("allowDropFileTypes", null);
-
-  option("cursorBlinkRate", 530);
-  option("cursorScrollMargin", 0);
-  option("cursorHeight", 1, updateSelection, true);
-  option("singleCursorHeightPerLine", true, updateSelection, true);
-  option("workTime", 100);
-  option("workDelay", 100);
-  option("flattenSpans", true, resetModeState, true);
-  option("addModeClass", false, resetModeState, true);
-  option("pollInterval", 100);
-  option("undoDepth", 200, function (cm, val) { return cm.doc.history.undoDepth = val; });
-  option("historyEventDelay", 1250);
-  option("viewportMargin", 10, function (cm) { return cm.refresh(); }, true);
-  option("maxHighlightLength", 10000, resetModeState, true);
-  option("moveInputWithCursor", true, function (cm, val) {
-    if (!val) { cm.display.input.resetPosition(); }
-  });
-
-  option("tabindex", null, function (cm, val) { return cm.display.input.getField().tabIndex = val || ""; });
-  option("autofocus", null);
-  option("direction", "ltr", function (cm, val) { return cm.doc.setDirection(val); }, true);
-}
-
-function guttersChanged(cm) {
-  updateGutters(cm);
-  regChange(cm);
-  alignHorizontally(cm);
-}
-
-function dragDropChanged(cm, value, old) {
-  var wasOn = old && old != Init;
-  if (!value != !wasOn) {
-    var funcs = cm.display.dragFunctions;
-    var toggle = value ? on : off;
-    toggle(cm.display.scroller, "dragstart", funcs.start);
-    toggle(cm.display.scroller, "dragenter", funcs.enter);
-    toggle(cm.display.scroller, "dragover", funcs.over);
-    toggle(cm.display.scroller, "dragleave", funcs.leave);
-    toggle(cm.display.scroller, "drop", funcs.drop);
-  }
-}
-
-function wrappingChanged(cm) {
-  if (cm.options.lineWrapping) {
-    addClass(cm.display.wrapper, "CodeMirror-wrap");
-    cm.display.sizer.style.minWidth = "";
-    cm.display.sizerWidth = null;
-  } else {
-    rmClass(cm.display.wrapper, "CodeMirror-wrap");
-    findMaxLine(cm);
-  }
-  estimateLineHeights(cm);
-  regChange(cm);
-  clearCaches(cm);
-  setTimeout(function () { return updateScrollbars(cm); }, 100);
-}
-
-// A CodeMirror instance represents an editor. This is the object
-// that user code is usually dealing with.
-
-function CodeMirror$1(place, options) {
-  var this$1 = this;
-
-  if (!(this instanceof CodeMirror$1)) { return new CodeMirror$1(place, options) }
-
-  this.options = options = options ? copyObj(options) : {};
-  // Determine effective options based on given values and defaults.
-  copyObj(defaults, options, false);
-  setGuttersForLineNumbers(options);
-
-  var doc = options.value;
-  if (typeof doc == "string") { doc = new Doc(doc, options.mode, null, options.lineSeparator, options.direction); }
-  this.doc = doc;
-
-  var input = new CodeMirror$1.inputStyles[options.inputStyle](this);
-  var display = this.display = new Display(place, doc, input);
-  display.wrapper.CodeMirror = this;
-  updateGutters(this);
-  themeChanged(this);
-  if (options.lineWrapping)
-    { this.display.wrapper.className += " CodeMirror-wrap"; }
-  initScrollbars(this);
-
-  this.state = {
-    keyMaps: [],  // stores maps added by addKeyMap
-    overlays: [], // highlighting overlays, as added by addOverlay
-    modeGen: 0,   // bumped when mode/overlay changes, used to invalidate highlighting info
-    overwrite: false,
-    delayingBlurEvent: false,
-    focused: false,
-    suppressEdits: false, // used to disable editing during key handlers when in readOnly mode
-    pasteIncoming: false, cutIncoming: false, // help recognize paste/cut edits in input.poll
-    selectingText: false,
-    draggingText: false,
-    highlight: new Delayed(), // stores highlight worker timeout
-    keySeq: null,  // Unfinished key sequence
-    specialChars: null
-  };
-
-  if (options.autofocus && !mobile) { display.input.focus(); }
-
-  // Override magic textarea content restore that IE sometimes does
-  // on our hidden textarea on reload
-  if (ie && ie_version < 11) { setTimeout(function () { return this$1.display.input.reset(true); }, 20); }
-
-  registerEventHandlers(this);
-  ensureGlobalHandlers();
-
-  startOperation(this);
-  this.curOp.forceUpdate = true;
-  attachDoc(this, doc);
-
-  if ((options.autofocus && !mobile) || this.hasFocus())
-    { setTimeout(bind(onFocus, this), 20); }
-  else
-    { onBlur(this); }
-
-  for (var opt in optionHandlers) { if (optionHandlers.hasOwnProperty(opt))
-    { optionHandlers[opt](this$1, options[opt], Init); } }
-  maybeUpdateLineNumberWidth(this);
-  if (options.finishInit) { options.finishInit(this); }
-  for (var i = 0; i < initHooks.length; ++i) { initHooks[i](this$1); }
-  endOperation(this);
-  // Suppress optimizelegibility in Webkit, since it breaks text
-  // measuring on line wrapping boundaries.
-  if (webkit && options.lineWrapping &&
-      getComputedStyle(display.lineDiv).textRendering == "optimizelegibility")
-    { display.lineDiv.style.textRendering = "auto"; }
-}
-
-// The default configuration options.
-CodeMirror$1.defaults = defaults;
-// Functions to run when options are changed.
-CodeMirror$1.optionHandlers = optionHandlers;
-
-// Attach the necessary event handlers when initializing the editor
-function registerEventHandlers(cm) {
-  var d = cm.display;
-  on(d.scroller, "mousedown", operation(cm, onMouseDown));
-  // Older IE's will not fire a second mousedown for a double click
-  if (ie && ie_version < 11)
-    { on(d.scroller, "dblclick", operation(cm, function (e) {
-      if (signalDOMEvent(cm, e)) { return }
-      var pos = posFromMouse(cm, e);
-      if (!pos || clickInGutter(cm, e) || eventInWidget(cm.display, e)) { return }
-      e_preventDefault(e);
-      var word = cm.findWordAt(pos);
-      extendSelection(cm.doc, word.anchor, word.head);
-    })); }
-  else
-    { on(d.scroller, "dblclick", function (e) { return signalDOMEvent(cm, e) || e_preventDefault(e); }); }
-  // Some browsers fire contextmenu *after* opening the menu, at
-  // which point we can't mess with it anymore. Context menu is
-  // handled in onMouseDown for these browsers.
-  if (!captureRightClick) { on(d.scroller, "contextmenu", function (e) { return onContextMenu(cm, e); }); }
-
-  // Used to suppress mouse event handling when a touch happens
-  var touchFinished, prevTouch = {end: 0};
-  function finishTouch() {
-    if (d.activeTouch) {
-      touchFinished = setTimeout(function () { return d.activeTouch = null; }, 1000);
-      prevTouch = d.activeTouch;
-      prevTouch.end = +new Date;
-    }
-  }
-  function isMouseLikeTouchEvent(e) {
-    if (e.touches.length != 1) { return false }
-    var touch = e.touches[0];
-    return touch.radiusX <= 1 && touch.radiusY <= 1
-  }
-  function farAway(touch, other) {
-    if (other.left == null) { return true }
-    var dx = other.left - touch.left, dy = other.top - touch.top;
-    return dx * dx + dy * dy > 20 * 20
-  }
-  on(d.scroller, "touchstart", function (e) {
-    if (!signalDOMEvent(cm, e) && !isMouseLikeTouchEvent(e) && !clickInGutter(cm, e)) {
-      d.input.ensurePolled();
-      clearTimeout(touchFinished);
-      var now = +new Date;
-      d.activeTouch = {start: now, moved: false,
-                       prev: now - prevTouch.end <= 300 ? prevTouch : null};
-      if (e.touches.length == 1) {
-        d.activeTouch.left = e.touches[0].pageX;
-        d.activeTouch.top = e.touches[0].pageY;
-      }
-    }
-  });
-  on(d.scroller, "touchmove", function () {
-    if (d.activeTouch) { d.activeTouch.moved = true; }
-  });
-  on(d.scroller, "touchend", function (e) {
-    var touch = d.activeTouch;
-    if (touch && !eventInWidget(d, e) && touch.left != null &&
-        !touch.moved && new Date - touch.start < 300) {
-      var pos = cm.coordsChar(d.activeTouch, "page"), range;
-      if (!touch.prev || farAway(touch, touch.prev)) // Single tap
-        { range = new Range(pos, pos); }
-      else if (!touch.prev.prev || farAway(touch, touch.prev.prev)) // Double tap
-        { range = cm.findWordAt(pos); }
-      else // Triple tap
-        { range = new Range(Pos(pos.line, 0), clipPos(cm.doc, Pos(pos.line + 1, 0))); }
-      cm.setSelection(range.anchor, range.head);
-      cm.focus();
-      e_preventDefault(e);
-    }
-    finishTouch();
-  });
-  on(d.scroller, "touchcancel", finishTouch);
-
-  // Sync scrolling between fake scrollbars and real scrollable
-  // area, ensure viewport is updated when scrolling.
-  on(d.scroller, "scroll", function () {
-    if (d.scroller.clientHeight) {
-      updateScrollTop(cm, d.scroller.scrollTop);
-      setScrollLeft(cm, d.scroller.scrollLeft, true);
-      signal(cm, "scroll", cm);
-    }
-  });
-
-  // Listen to wheel events in order to try and update the viewport on time.
-  on(d.scroller, "mousewheel", function (e) { return onScrollWheel(cm, e); });
-  on(d.scroller, "DOMMouseScroll", function (e) { return onScrollWheel(cm, e); });
-
-  // Prevent wrapper from ever scrolling
-  on(d.wrapper, "scroll", function () { return d.wrapper.scrollTop = d.wrapper.scrollLeft = 0; });
-
-  d.dragFunctions = {
-    enter: function (e) {if (!signalDOMEvent(cm, e)) { e_stop(e); }},
-    over: function (e) {if (!signalDOMEvent(cm, e)) { onDragOver(cm, e); e_stop(e); }},
-    start: function (e) { return onDragStart(cm, e); },
-    drop: operation(cm, onDrop),
-    leave: function (e) {if (!signalDOMEvent(cm, e)) { clearDragCursor(cm); }}
-  };
-
-  var inp = d.input.getField();
-  on(inp, "keyup", function (e) { return onKeyUp.call(cm, e); });
-  on(inp, "keydown", operation(cm, onKeyDown));
-  on(inp, "keypress", operation(cm, onKeyPress));
-  on(inp, "focus", function (e) { return onFocus(cm, e); });
-  on(inp, "blur", function (e) { return onBlur(cm, e); });
-}
-
-var initHooks = [];
-CodeMirror$1.defineInitHook = function (f) { return initHooks.push(f); };
-
-// Indent the given line. The how parameter can be "smart",
-// "add"/null, "subtract", or "prev". When aggressive is false
-// (typically set to true for forced single-line indents), empty
-// lines are not indented, and places where the mode returns Pass
-// are left alone.
-function indentLine(cm, n, how, aggressive) {
-  var doc = cm.doc, state;
-  if (how == null) { how = "add"; }
-  if (how == "smart") {
-    // Fall back to "prev" when the mode doesn't have an indentation
-    // method.
-    if (!doc.mode.indent) { how = "prev"; }
-    else { state = getContextBefore(cm, n).state; }
-  }
-
-  var tabSize = cm.options.tabSize;
-  var line = getLine(doc, n), curSpace = countColumn(line.text, null, tabSize);
-  if (line.stateAfter) { line.stateAfter = null; }
-  var curSpaceString = line.text.match(/^\s*/)[0], indentation;
-  if (!aggressive && !/\S/.test(line.text)) {
-    indentation = 0;
-    how = "not";
-  } else if (how == "smart") {
-    indentation = doc.mode.indent(state, line.text.slice(curSpaceString.length), line.text);
-    if (indentation == Pass || indentation > 150) {
-      if (!aggressive) { return }
-      how = "prev";
-    }
-  }
-  if (how == "prev") {
-    if (n > doc.first) { indentation = countColumn(getLine(doc, n-1).text, null, tabSize); }
-    else { indentation = 0; }
-  } else if (how == "add") {
-    indentation = curSpace + cm.options.indentUnit;
-  } else if (how == "subtract") {
-    indentation = curSpace - cm.options.indentUnit;
-  } else if (typeof how == "number") {
-    indentation = curSpace + how;
-  }
-  indentation = Math.max(0, indentation);
-
-  var indentString = "", pos = 0;
-  if (cm.options.indentWithTabs)
-    { for (var i = Math.floor(indentation / tabSize); i; --i) {pos += tabSize; indentString += "\t";} }
-  if (pos < indentation) { indentString += spaceStr(indentation - pos); }
-
-  if (indentString != curSpaceString) {
-    replaceRange(doc, indentString, Pos(n, 0), Pos(n, curSpaceString.length), "+input");
-    line.stateAfter = null;
-    return true
-  } else {
-    // Ensure that, if the cursor was in the whitespace at the start
-    // of the line, it is moved to the end of that space.
-    for (var i$1 = 0; i$1 < doc.sel.ranges.length; i$1++) {
-      var range = doc.sel.ranges[i$1];
-      if (range.head.line == n && range.head.ch < curSpaceString.length) {
-        var pos$1 = Pos(n, curSpaceString.length);
-        replaceOneSelection(doc, i$1, new Range(pos$1, pos$1));
-        break
-      }
-    }
-  }
-}
-
-// This will be set to a {lineWise: bool, text: [string]} object, so
-// that, when pasting, we know what kind of selections the copied
-// text was made out of.
-var lastCopied = null;
-
-function setLastCopied(newLastCopied) {
-  lastCopied = newLastCopied;
-}
-
-function applyTextInput(cm, inserted, deleted, sel, origin) {
-  var doc = cm.doc;
-  cm.display.shift = false;
-  if (!sel) { sel = doc.sel; }
-
-  var paste = cm.state.pasteIncoming || origin == "paste";
-  var textLines = splitLinesAuto(inserted), multiPaste = null;
-  // When pasing N lines into N selections, insert one line per selection
-  if (paste && sel.ranges.length > 1) {
-    if (lastCopied && lastCopied.text.join("\n") == inserted) {
-      if (sel.ranges.length % lastCopied.text.length == 0) {
-        multiPaste = [];
-        for (var i = 0; i < lastCopied.text.length; i++)
-          { multiPaste.push(doc.splitLines(lastCopied.text[i])); }
-      }
-    } else if (textLines.length == sel.ranges.length && cm.options.pasteLinesPerSelection) {
-      multiPaste = map(textLines, function (l) { return [l]; });
-    }
-  }
-
-  var updateInput;
-  // Normal behavior is to insert the new text into every selection
-  for (var i$1 = sel.ranges.length - 1; i$1 >= 0; i$1--) {
-    var range$$1 = sel.ranges[i$1];
-    var from = range$$1.from(), to = range$$1.to();
-    if (range$$1.empty()) {
-      if (deleted && deleted > 0) // Handle deletion
-        { from = Pos(from.line, from.ch - deleted); }
-      else if (cm.state.overwrite && !paste) // Handle overwrite
-        { to = Pos(to.line, Math.min(getLine(doc, to.line).text.length, to.ch + lst(textLines).length)); }
-      else if (lastCopied && lastCopied.lineWise && lastCopied.text.join("\n") == inserted)
-        { from = to = Pos(from.line, 0); }
-    }
-    updateInput = cm.curOp.updateInput;
-    var changeEvent = {from: from, to: to, text: multiPaste ? multiPaste[i$1 % multiPaste.length] : textLines,
-                       origin: origin || (paste ? "paste" : cm.state.cutIncoming ? "cut" : "+input")};
-    makeChange(cm.doc, changeEvent);
-    signalLater(cm, "inputRead", cm, changeEvent);
-  }
-  if (inserted && !paste)
-    { triggerElectric(cm, inserted); }
-
-  ensureCursorVisible(cm);
-  cm.curOp.updateInput = updateInput;
-  cm.curOp.typing = true;
-  cm.state.pasteIncoming = cm.state.cutIncoming = false;
-}
-
-function handlePaste(e, cm) {
-  var pasted = e.clipboardData && e.clipboardData.getData("Text");
-  if (pasted) {
-    e.preventDefault();
-    if (!cm.isReadOnly() && !cm.options.disableInput)
-      { runInOp(cm, function () { return applyTextInput(cm, pasted, 0, null, "paste"); }); }
-    return true
-  }
-}
-
-function triggerElectric(cm, inserted) {
-  // When an 'electric' character is inserted, immediately trigger a reindent
-  if (!cm.options.electricChars || !cm.options.smartIndent) { return }
-  var sel = cm.doc.sel;
-
-  for (var i = sel.ranges.length - 1; i >= 0; i--) {
-    var range$$1 = sel.ranges[i];
-    if (range$$1.head.ch > 100 || (i && sel.ranges[i - 1].head.line == range$$1.head.line)) { continue }
-    var mode = cm.getModeAt(range$$1.head);
-    var indented = false;
-    if (mode.electricChars) {
-      for (var j = 0; j < mode.electricChars.length; j++)
-        { if (inserted.indexOf(mode.electricChars.charAt(j)) > -1) {
-          indented = indentLine(cm, range$$1.head.line, "smart");
-          break
-        } }
-    } else if (mode.electricInput) {
-      if (mode.electricInput.test(getLine(cm.doc, range$$1.head.line).text.slice(0, range$$1.head.ch)))
-        { indented = indentLine(cm, range$$1.head.line, "smart"); }
-    }
-    if (indented) { signalLater(cm, "electricInput", cm, range$$1.head.line); }
-  }
-}
-
-function copyableRanges(cm) {
-  var text = [], ranges = [];
-  for (var i = 0; i < cm.doc.sel.ranges.length; i++) {
-    var line = cm.doc.sel.ranges[i].head.line;
-    var lineRange = {anchor: Pos(line, 0), head: Pos(line + 1, 0)};
-    ranges.push(lineRange);
-    text.push(cm.getRange(lineRange.anchor, lineRange.head));
-  }
-  return {text: text, ranges: ranges}
-}
-
-function disableBrowserMagic(field, spellcheck) {
-  field.setAttribute("autocorrect", "off");
-  field.setAttribute("autocapitalize", "off");
-  field.setAttribute("spellcheck", !!spellcheck);
-}
-
-function hiddenTextarea() {
-  var te = elt("textarea", null, null, "position: absolute; bottom: -1em; padding: 0; width: 1px; height: 1em; outline: none");
-  var div = elt("div", [te], null, "overflow: hidden; position: relative; width: 3px; height: 0px;");
-  // The textarea is kept positioned near the cursor to prevent the
-  // fact that it'll be scrolled into view on input from scrolling
-  // our fake cursor out of view. On webkit, when wrap=off, paste is
-  // very slow. So make the area wide instead.
-  if (webkit) { te.style.width = "1000px"; }
-  else { te.setAttribute("wrap", "off"); }
-  // If border: 0; -- iOS fails to open keyboard (issue #1287)
-  if (ios) { te.style.border = "1px solid black"; }
-  disableBrowserMagic(te);
-  return div
-}
-
-// The publicly visible API. Note that methodOp(f) means
-// 'wrap f in an operation, performed on its `this` parameter'.
-
-// This is not the complete set of editor methods. Most of the
-// methods defined on the Doc type are also injected into
-// CodeMirror.prototype, for backwards compatibility and
-// convenience.
-
-var addEditorMethods = function(CodeMirror) {
-  var optionHandlers = CodeMirror.optionHandlers;
-
-  var helpers = CodeMirror.helpers = {};
-
-  CodeMirror.prototype = {
-    constructor: CodeMirror,
-    focus: function(){window.focus(); this.display.input.focus();},
-
-    setOption: function(option, value) {
-      var options = this.options, old = options[option];
-      if (options[option] == value && option != "mode") { return }
-      options[option] = value;
-      if (optionHandlers.hasOwnProperty(option))
-        { operation(this, optionHandlers[option])(this, value, old); }
-      signal(this, "optionChange", this, option);
-    },
-
-    getOption: function(option) {return this.options[option]},
-    getDoc: function() {return this.doc},
-
-    addKeyMap: function(map$$1, bottom) {
-      this.state.keyMaps[bottom ? "push" : "unshift"](getKeyMap(map$$1));
-    },
-    removeKeyMap: function(map$$1) {
-      var maps = this.state.keyMaps;
-      for (var i = 0; i < maps.length; ++i)
-        { if (maps[i] == map$$1 || maps[i].name == map$$1) {
-          maps.splice(i, 1);
-          return true
-        } }
-    },
-
-    addOverlay: methodOp(function(spec, options) {
-      var mode = spec.token ? spec : CodeMirror.getMode(this.options, spec);
-      if (mode.startState) { throw new Error("Overlays may not be stateful.") }
-      insertSorted(this.state.overlays,
-                   {mode: mode, modeSpec: spec, opaque: options && options.opaque,
-                    priority: (options && options.priority) || 0},
-                   function (overlay) { return overlay.priority; });
-      this.state.modeGen++;
-      regChange(this);
-    }),
-    removeOverlay: methodOp(function(spec) {
-      var this$1 = this;
-
-      var overlays = this.state.overlays;
-      for (var i = 0; i < overlays.length; ++i) {
-        var cur = overlays[i].modeSpec;
-        if (cur == spec || typeof spec == "string" && cur.name == spec) {
-          overlays.splice(i, 1);
-          this$1.state.modeGen++;
-          regChange(this$1);
-          return
-        }
-      }
-    }),
-
-    indentLine: methodOp(function(n, dir, aggressive) {
-      if (typeof dir != "string" && typeof dir != "number") {
-        if (dir == null) { dir = this.options.smartIndent ? "smart" : "prev"; }
-        else { dir = dir ? "add" : "subtract"; }
-      }
-      if (isLine(this.doc, n)) { indentLine(this, n, dir, aggressive); }
-    }),
-    indentSelection: methodOp(function(how) {
-      var this$1 = this;
-
-      var ranges = this.doc.sel.ranges, end = -1;
-      for (var i = 0; i < ranges.length; i++) {
-        var range$$1 = ranges[i];
-        if (!range$$1.empty()) {
-          var from = range$$1.from(), to = range$$1.to();
-          var start = Math.max(end, from.line);
-          end = Math.min(this$1.lastLine(), to.line - (to.ch ? 0 : 1)) + 1;
-          for (var j = start; j < end; ++j)
-            { indentLine(this$1, j, how); }
-          var newRanges = this$1.doc.sel.ranges;
-          if (from.ch == 0 && ranges.length == newRanges.length && newRanges[i].from().ch > 0)
-            { replaceOneSelection(this$1.doc, i, new Range(from, newRanges[i].to()), sel_dontScroll); }
-        } else if (range$$1.head.line > end) {
-          indentLine(this$1, range$$1.head.line, how, true);
-          end = range$$1.head.line;
-          if (i == this$1.doc.sel.primIndex) { ensureCursorVisible(this$1); }
-        }
-      }
-    }),
-
-    // Fetch the parser token for a given character. Useful for hacks
-    // that want to inspect the mode state (say, for completion).
-    getTokenAt: function(pos, precise) {
-      return takeToken(this, pos, precise)
-    },
-
-    getLineTokens: function(line, precise) {
-      return takeToken(this, Pos(line), precise, true)
-    },
-
-    getTokenTypeAt: function(pos) {
-      pos = clipPos(this.doc, pos);
-      var styles = getLineStyles(this, getLine(this.doc, pos.line));
-      var before = 0, after = (styles.length - 1) / 2, ch = pos.ch;
-      var type;
-      if (ch == 0) { type = styles[2]; }
-      else { for (;;) {
-        var mid = (before + after) >> 1;
-        if ((mid ? styles[mid * 2 - 1] : 0) >= ch) { after = mid; }
-        else if (styles[mid * 2 + 1] < ch) { before = mid + 1; }
-        else { type = styles[mid * 2 + 2]; break }
-      } }
-      var cut = type ? type.indexOf("overlay ") : -1;
-      return cut < 0 ? type : cut == 0 ? null : type.slice(0, cut - 1)
-    },
-
-    getModeAt: function(pos) {
-      var mode = this.doc.mode;
-      if (!mode.innerMode) { return mode }
-      return CodeMirror.innerMode(mode, this.getTokenAt(pos).state).mode
-    },
-
-    getHelper: function(pos, type) {
-      return this.getHelpers(pos, type)[0]
-    },
-
-    getHelpers: function(pos, type) {
-      var this$1 = this;
-
-      var found = [];
-      if (!helpers.hasOwnProperty(type)) { return found }
-      var help = helpers[type], mode = this.getModeAt(pos);
-      if (typeof mode[type] == "string") {
-        if (help[mode[type]]) { found.push(help[mode[type]]); }
-      } else if (mode[type]) {
-        for (var i = 0; i < mode[type].length; i++) {
-          var val = help[mode[type][i]];
-          if (val) { found.push(val); }
-        }
-      } else if (mode.helperType && help[mode.helperType]) {
-        found.push(help[mode.helperType]);
-      } else if (help[mode.name]) {
-        found.push(help[mode.name]);
-      }
-      for (var i$1 = 0; i$1 < help._global.length; i$1++) {
-        var cur = help._global[i$1];
-        if (cur.pred(mode, this$1) && indexOf(found, cur.val) == -1)
-          { found.push(cur.val); }
-      }
-      return found
-    },
-
-    getStateAfter: function(line, precise) {
-      var doc = this.doc;
-      line = clipLine(doc, line == null ? doc.first + doc.size - 1: line);
-      return getContextBefore(this, line + 1, precise).state
-    },
-
-    cursorCoords: function(start, mode) {
-      var pos, range$$1 = this.doc.sel.primary();
-      if (start == null) { pos = range$$1.head; }
-      else if (typeof start == "object") { pos = clipPos(this.doc, start); }
-      else { pos = start ? range$$1.from() : range$$1.to(); }
-      return cursorCoords(this, pos, mode || "page")
-    },
-
-    charCoords: function(pos, mode) {
-      return charCoords(this, clipPos(this.doc, pos), mode || "page")
-    },
-
-    coordsChar: function(coords, mode) {
-      coords = fromCoordSystem(this, coords, mode || "page");
-      return coordsChar(this, coords.left, coords.top)
-    },
-
-    lineAtHeight: function(height, mode) {
-      height = fromCoordSystem(this, {top: height, left: 0}, mode || "page").top;
-      return lineAtHeight(this.doc, height + this.display.viewOffset)
-    },
-    heightAtLine: function(line, mode, includeWidgets) {
-      var end = false, lineObj;
-      if (typeof line == "number") {
-        var last = this.doc.first + this.doc.size - 1;
-        if (line < this.doc.first) { line = this.doc.first; }
-        else if (line > last) { line = last; end = true; }
-        lineObj = getLine(this.doc, line);
-      } else {
-        lineObj = line;
-      }
-      return intoCoordSystem(this, lineObj, {top: 0, left: 0}, mode || "page", includeWidgets || end).top +
-        (end ? this.doc.height - heightAtLine(lineObj) : 0)
-    },
-
-    defaultTextHeight: function() { return textHeight(this.display) },
-    defaultCharWidth: function() { return charWidth(this.display) },
-
-    getViewport: function() { return {from: this.display.viewFrom, to: this.display.viewTo}},
-
-    addWidget: function(pos, node, scroll, vert, horiz) {
-      var display = this.display;
-      pos = cursorCoords(this, clipPos(this.doc, pos));
-      var top = pos.bottom, left = pos.left;
-      node.style.position = "absolute";
-      node.setAttribute("cm-ignore-events", "true");
-      this.display.input.setUneditable(node);
-      display.sizer.appendChild(node);
-      if (vert == "over") {
-        top = pos.top;
-      } else if (vert == "above" || vert == "near") {
-        var vspace = Math.max(display.wrapper.clientHeight, this.doc.height),
-        hspace = Math.max(display.sizer.clientWidth, display.lineSpace.clientWidth);
-        // Default to positioning above (if specified and possible); otherwise default to positioning below
-        if ((vert == 'above' || pos.bottom + node.offsetHeight > vspace) && pos.top > node.offsetHeight)
-          { top = pos.top - node.offsetHeight; }
-        else if (pos.bottom + node.offsetHeight <= vspace)
-          { top = pos.bottom; }
-        if (left + node.offsetWidth > hspace)
-          { left = hspace - node.offsetWidth; }
-      }
-      node.style.top = top + "px";
-      node.style.left = node.style.right = "";
-      if (horiz == "right") {
-        left = display.sizer.clientWidth - node.offsetWidth;
-        node.style.right = "0px";
-      } else {
-        if (horiz == "left") { left = 0; }
-        else if (horiz == "middle") { left = (display.sizer.clientWidth - node.offsetWidth) / 2; }
-        node.style.left = left + "px";
-      }
-      if (scroll)
-        { scrollIntoView(this, {left: left, top: top, right: left + node.offsetWidth, bottom: top + node.offsetHeight}); }
-    },
-
-    triggerOnKeyDown: methodOp(onKeyDown),
-    triggerOnKeyPress: methodOp(onKeyPress),
-    triggerOnKeyUp: onKeyUp,
-    triggerOnMouseDown: methodOp(onMouseDown),
-
-    execCommand: function(cmd) {
-      if (commands.hasOwnProperty(cmd))
-        { return commands[cmd].call(null, this) }
-    },
-
-    triggerElectric: methodOp(function(text) { triggerElectric(this, text); }),
-
-    findPosH: function(from, amount, unit, visually) {
-      var this$1 = this;
-
-      var dir = 1;
-      if (amount < 0) { dir = -1; amount = -amount; }
-      var cur = clipPos(this.doc, from);
-      for (var i = 0; i < amount; ++i) {
-        cur = findPosH(this$1.doc, cur, dir, unit, visually);
-        if (cur.hitSide) { break }
-      }
-      return cur
-    },
-
-    moveH: methodOp(function(dir, unit) {
-      var this$1 = this;
-
-      this.extendSelectionsBy(function (range$$1) {
-        if (this$1.display.shift || this$1.doc.extend || range$$1.empty())
-          { return findPosH(this$1.doc, range$$1.head, dir, unit, this$1.options.rtlMoveVisually) }
-        else
-          { return dir < 0 ? range$$1.from() : range$$1.to() }
-      }, sel_move);
-    }),
-
-    deleteH: methodOp(function(dir, unit) {
-      var sel = this.doc.sel, doc = this.doc;
-      if (sel.somethingSelected())
-        { doc.replaceSelection("", null, "+delete"); }
-      else
-        { deleteNearSelection(this, function (range$$1) {
-          var other = findPosH(doc, range$$1.head, dir, unit, false);
-          return dir < 0 ? {from: other, to: range$$1.head} : {from: range$$1.head, to: other}
-        }); }
-    }),
-
-    findPosV: function(from, amount, unit, goalColumn) {
-      var this$1 = this;
-
-      var dir = 1, x = goalColumn;
-      if (amount < 0) { dir = -1; amount = -amount; }
-      var cur = clipPos(this.doc, from);
-      for (var i = 0; i < amount; ++i) {
-        var coords = cursorCoords(this$1, cur, "div");
-        if (x == null) { x = coords.left; }
-        else { coords.left = x; }
-        cur = findPosV(this$1, coords, dir, unit);
-        if (cur.hitSide) { break }
-      }
-      return cur
-    },
-
-    moveV: methodOp(function(dir, unit) {
-      var this$1 = this;
-
-      var doc = this.doc, goals = [];
-      var collapse = !this.display.shift && !doc.extend && doc.sel.somethingSelected();
-      doc.extendSelectionsBy(function (range$$1) {
-        if (collapse)
-          { return dir < 0 ? range$$1.from() : range$$1.to() }
-        var headPos = cursorCoords(this$1, range$$1.head, "div");
-        if (range$$1.goalColumn != null) { headPos.left = range$$1.goalColumn; }
-        goals.push(headPos.left);
-        var pos = findPosV(this$1, headPos, dir, unit);
-        if (unit == "page" && range$$1 == doc.sel.primary())
-          { addToScrollTop(this$1, charCoords(this$1, pos, "div").top - headPos.top); }
-        return pos
-      }, sel_move);
-      if (goals.length) { for (var i = 0; i < doc.sel.ranges.length; i++)
-        { doc.sel.ranges[i].goalColumn = goals[i]; } }
-    }),
-
-    // Find the word at the given position (as returned by coordsChar).
-    findWordAt: function(pos) {
-      var doc = this.doc, line = getLine(doc, pos.line).text;
-      var start = pos.ch, end = pos.ch;
-      if (line) {
-        var helper = this.getHelper(pos, "wordChars");
-        if ((pos.sticky == "before" || end == line.length) && start) { --start; } else { ++end; }
-        var startChar = line.charAt(start);
-        var check = isWordChar(startChar, helper)
-          ? function (ch) { return isWordChar(ch, helper); }
-          : /\s/.test(startChar) ? function (ch) { return /\s/.test(ch); }
-          : function (ch) { return (!/\s/.test(ch) && !isWordChar(ch)); };
-        while (start > 0 && check(line.charAt(start - 1))) { --start; }
-        while (end < line.length && check(line.charAt(end))) { ++end; }
-      }
-      return new Range(Pos(pos.line, start), Pos(pos.line, end))
-    },
-
-    toggleOverwrite: function(value) {
-      if (value != null && value == this.state.overwrite) { return }
-      if (this.state.overwrite = !this.state.overwrite)
-        { addClass(this.display.cursorDiv, "CodeMirror-overwrite"); }
-      else
-        { rmClass(this.display.cursorDiv, "CodeMirror-overwrite"); }
-
-      signal(this, "overwriteToggle", this, this.state.overwrite);
-    },
-    hasFocus: function() { return this.display.input.getField() == activeElt() },
-    isReadOnly: function() { return !!(this.options.readOnly || this.doc.cantEdit) },
-
-    scrollTo: methodOp(function (x, y) { scrollToCoords(this, x, y); }),
-    getScrollInfo: function() {
-      var scroller = this.display.scroller;
-      return {left: scroller.scrollLeft, top: scroller.scrollTop,
-              height: scroller.scrollHeight - scrollGap(this) - this.display.barHeight,
-              width: scroller.scrollWidth - scrollGap(this) - this.display.barWidth,
-              clientHeight: displayHeight(this), clientWidth: displayWidth(this)}
-    },
-
-    scrollIntoView: methodOp(function(range$$1, margin) {
-      if (range$$1 == null) {
-        range$$1 = {from: this.doc.sel.primary().head, to: null};
-        if (margin == null) { margin = this.options.cursorScrollMargin; }
-      } else if (typeof range$$1 == "number") {
-        range$$1 = {from: Pos(range$$1, 0), to: null};
-      } else if (range$$1.from == null) {
-        range$$1 = {from: range$$1, to: null};
-      }
-      if (!range$$1.to) { range$$1.to = range$$1.from; }
-      range$$1.margin = margin || 0;
-
-      if (range$$1.from.line != null) {
-        scrollToRange(this, range$$1);
-      } else {
-        scrollToCoordsRange(this, range$$1.from, range$$1.to, range$$1.margin);
-      }
-    }),
-
-    setSize: methodOp(function(width, height) {
-      var this$1 = this;
-
-      var interpret = function (val) { return typeof val == "number" || /^\d+$/.test(String(val)) ? val + "px" : val; };
-      if (width != null) { this.display.wrapper.style.width = interpret(width); }
-      if (height != null) { this.display.wrapper.style.height = interpret(height); }
-      if (this.options.lineWrapping) { clearLineMeasurementCache(this); }
-      var lineNo$$1 = this.display.viewFrom;
-      this.doc.iter(lineNo$$1, this.display.viewTo, function (line) {
-        if (line.widgets) { for (var i = 0; i < line.widgets.length; i++)
-          { if (line.widgets[i].noHScroll) { regLineChange(this$1, lineNo$$1, "widget"); break } } }
-        ++lineNo$$1;
-      });
-      this.curOp.forceUpdate = true;
-      signal(this, "refresh", this);
-    }),
-
-    operation: function(f){return runInOp(this, f)},
-    startOperation: function(){return startOperation(this)},
-    endOperation: function(){return endOperation(this)},
-
-    refresh: methodOp(function() {
-      var oldHeight = this.display.cachedTextHeight;
-      regChange(this);
-      this.curOp.forceUpdate = true;
-      clearCaches(this);
-      scrollToCoords(this, this.doc.scrollLeft, this.doc.scrollTop);
-      updateGutterSpace(this);
-      if (oldHeight == null || Math.abs(oldHeight - textHeight(this.display)) > .5)
-        { estimateLineHeights(this); }
-      signal(this, "refresh", this);
-    }),
-
-    swapDoc: methodOp(function(doc) {
-      var old = this.doc;
-      old.cm = null;
-      attachDoc(this, doc);
-      clearCaches(this);
-      this.display.input.reset();
-      scrollToCoords(this, doc.scrollLeft, doc.scrollTop);
-      this.curOp.forceScroll = true;
-      signalLater(this, "swapDoc", this, old);
-      return old
-    }),
-
-    getInputField: function(){return this.display.input.getField()},
-    getWrapperElement: function(){return this.display.wrapper},
-    getScrollerElement: function(){return this.display.scroller},
-    getGutterElement: function(){return this.display.gutters}
-  };
-  eventMixin(CodeMirror);
-
-  CodeMirror.registerHelper = function(type, name, value) {
-    if (!helpers.hasOwnProperty(type)) { helpers[type] = CodeMirror[type] = {_global: []}; }
-    helpers[type][name] = value;
-  };
-  CodeMirror.registerGlobalHelper = function(type, name, predicate, value) {
-    CodeMirror.registerHelper(type, name, value);
-    helpers[type]._global.push({pred: predicate, val: value});
-  };
-};
-
-// Used for horizontal relative motion. Dir is -1 or 1 (left or
-// right), unit can be "char", "column" (like char, but doesn't
-// cross line boundaries), "word" (across next word), or "group" (to
-// the start of next group of word or non-word-non-whitespace
-// chars). The visually param controls whether, in right-to-left
-// text, direction 1 means to move towards the next index in the
-// string, or towards the character to the right of the current
-// position. The resulting position will have a hitSide=true
-// property if it reached the end of the document.
-function findPosH(doc, pos, dir, unit, visually) {
-  var oldPos = pos;
-  var origDir = dir;
-  var lineObj = getLine(doc, pos.line);
-  function findNextLine() {
-    var l = pos.line + dir;
-    if (l < doc.first || l >= doc.first + doc.size) { return false }
-    pos = new Pos(l, pos.ch, pos.sticky);
-    return lineObj = getLine(doc, l)
-  }
-  function moveOnce(boundToLine) {
-    var next;
-    if (visually) {
-      next = moveVisually(doc.cm, lineObj, pos, dir);
-    } else {
-      next = moveLogically(lineObj, pos, dir);
-    }
-    if (next == null) {
-      if (!boundToLine && findNextLine())
-        { pos = endOfLine(visually, doc.cm, lineObj, pos.line, dir); }
-      else
-        { return false }
-    } else {
-      pos = next;
-    }
-    return true
-  }
-
-  if (unit == "char") {
-    moveOnce();
-  } else if (unit == "column") {
-    moveOnce(true);
-  } else if (unit == "word" || unit == "group") {
-    var sawType = null, group = unit == "group";
-    var helper = doc.cm && doc.cm.getHelper(pos, "wordChars");
-    for (var first = true;; first = false) {
-      if (dir < 0 && !moveOnce(!first)) { break }
-      var cur = lineObj.text.charAt(pos.ch) || "\n";
-      var type = isWordChar(cur, helper) ? "w"
-        : group && cur == "\n" ? "n"
-        : !group || /\s/.test(cur) ? null
-        : "p";
-      if (group && !first && !type) { type = "s"; }
-      if (sawType && sawType != type) {
-        if (dir < 0) {dir = 1; moveOnce(); pos.sticky = "after";}
-        break
-      }
-
-      if (type) { sawType = type; }
-      if (dir > 0 && !moveOnce(!first)) { break }
-    }
-  }
-  var result = skipAtomic(doc, pos, oldPos, origDir, true);
-  if (equalCursorPos(oldPos, result)) { result.hitSide = true; }
-  return result
-}
-
-// For relative vertical movement. Dir may be -1 or 1. Unit can be
-// "page" or "line". The resulting position will have a hitSide=true
-// property if it reached the end of the document.
-function findPosV(cm, pos, dir, unit) {
-  var doc = cm.doc, x = pos.left, y;
-  if (unit == "page") {
-    var pageSize = Math.min(cm.display.wrapper.clientHeight, window.innerHeight || document.documentElement.clientHeight);
-    var moveAmount = Math.max(pageSize - .5 * textHeight(cm.display), 3);
-    y = (dir > 0 ? pos.bottom : pos.top) + dir * moveAmount;
-
-  } else if (unit == "line") {
-    y = dir > 0 ? pos.bottom + 3 : pos.top - 3;
-  }
-  var target;
-  for (;;) {
-    target = coordsChar(cm, x, y);
-    if (!target.outside) { break }
-    if (dir < 0 ? y <= 0 : y >= doc.height) { target.hitSide = true; break }
-    y += dir * 5;
-  }
-  return target
-}
-
-// CONTENTEDITABLE INPUT STYLE
-
-var ContentEditableInput = function(cm) {
-  this.cm = cm;
-  this.lastAnchorNode = this.lastAnchorOffset = this.lastFocusNode = this.lastFocusOffset = null;
-  this.polling = new Delayed();
-  this.composing = null;
-  this.gracePeriod = false;
-  this.readDOMTimeout = null;
-};
-
-ContentEditableInput.prototype.init = function (display) {
-    var this$1 = this;
-
-  var input = this, cm = input.cm;
-  var div = input.div = display.lineDiv;
-  disableBrowserMagic(div, cm.options.spellcheck);
-
-  on(div, "paste", function (e) {
-    if (signalDOMEvent(cm, e) || handlePaste(e, cm)) { return }
-    // IE doesn't fire input events, so we schedule a read for the pasted content in this way
-    if (ie_version <= 11) { setTimeout(operation(cm, function () { return this$1.updateFromDOM(); }), 20); }
-  });
-
-  on(div, "compositionstart", function (e) {
-    this$1.composing = {data: e.data, done: false};
-  });
-  on(div, "compositionupdate", function (e) {
-    if (!this$1.composing) { this$1.composing = {data: e.data, done: false}; }
-  });
-  on(div, "compositionend", function (e) {
-    if (this$1.composing) {
-      if (e.data != this$1.composing.data) { this$1.readFromDOMSoon(); }
-      this$1.composing.done = true;
-    }
-  });
-
-  on(div, "touchstart", function () { return input.forceCompositionEnd(); });
-
-  on(div, "input", function () {
-    if (!this$1.composing) { this$1.readFromDOMSoon(); }
-  });
-
-  function onCopyCut(e) {
-    if (signalDOMEvent(cm, e)) { return }
-    if (cm.somethingSelected()) {
-      setLastCopied({lineWise: false, text: cm.getSelections()});
-      if (e.type == "cut") { cm.replaceSelection("", null, "cut"); }
-    } else if (!cm.options.lineWiseCopyCut) {
-      return
-    } else {
-      var ranges = copyableRanges(cm);
-      setLastCopied({lineWise: true, text: ranges.text});
-      if (e.type == "cut") {
-        cm.operation(function () {
-          cm.setSelections(ranges.ranges, 0, sel_dontScroll);
-          cm.replaceSelection("", null, "cut");
-        });
-      }
-    }
-    if (e.clipboardData) {
-      e.clipboardData.clearData();
-      var content = lastCopied.text.join("\n");
-      // iOS exposes the clipboard API, but seems to discard content inserted into it
-      e.clipboardData.setData("Text", content);
-      if (e.clipboardData.getData("Text") == content) {
-        e.preventDefault();
-        return
-      }
-    }
-    // Old-fashioned briefly-focus-a-textarea hack
-    var kludge = hiddenTextarea(), te = kludge.firstChild;
-    cm.display.lineSpace.insertBefore(kludge, cm.display.lineSpace.firstChild);
-    te.value = lastCopied.text.join("\n");
-    var hadFocus = document.activeElement;
-    selectInput(te);
-    setTimeout(function () {
-      cm.display.lineSpace.removeChild(kludge);
-      hadFocus.focus();
-      if (hadFocus == div) { input.showPrimarySelection(); }
-    }, 50);
-  }
-  on(div, "copy", onCopyCut);
-  on(div, "cut", onCopyCut);
-};
-
-ContentEditableInput.prototype.prepareSelection = function () {
-  var result = prepareSelection(this.cm, false);
-  result.focus = this.cm.state.focused;
-  return result
-};
-
-ContentEditableInput.prototype.showSelection = function (info, takeFocus) {
-  if (!info || !this.cm.display.view.length) { return }
-  if (info.focus || takeFocus) { this.showPrimarySelection(); }
-  this.showMultipleSelections(info);
-};
-
-ContentEditableInput.prototype.showPrimarySelection = function () {
-  var sel = window.getSelection(), cm = this.cm, prim = cm.doc.sel.primary();
-  var from = prim.from(), to = prim.to();
-
-  if (cm.display.viewTo == cm.display.viewFrom || from.line >= cm.display.viewTo || to.line < cm.display.viewFrom) {
-    sel.removeAllRanges();
-    return
-  }
-
-  var curAnchor = domToPos(cm, sel.anchorNode, sel.anchorOffset);
-  var curFocus = domToPos(cm, sel.focusNode, sel.focusOffset);
-  if (curAnchor && !curAnchor.bad && curFocus && !curFocus.bad &&
-      cmp(minPos(curAnchor, curFocus), from) == 0 &&
-      cmp(maxPos(curAnchor, curFocus), to) == 0)
-    { return }
-
-  var view = cm.display.view;
-  var start = (from.line >= cm.display.viewFrom && posToDOM(cm, from)) ||
-      {node: view[0].measure.map[2], offset: 0};
-  var end = to.line < cm.display.viewTo && posToDOM(cm, to);
-  if (!end) {
-    var measure = view[view.length - 1].measure;
-    var map$$1 = measure.maps ? measure.maps[measure.maps.length - 1] : measure.map;
-    end = {node: map$$1[map$$1.length - 1], offset: map$$1[map$$1.length - 2] - map$$1[map$$1.length - 3]};
-  }
-
-  if (!start || !end) {
-    sel.removeAllRanges();
-    return
-  }
-
-  var old = sel.rangeCount && sel.getRangeAt(0), rng;
-  try { rng = range(start.node, start.offset, end.offset, end.node); }
-  catch(e) {} // Our model of the DOM might be outdated, in which case the range we try to set can be impossible
-  if (rng) {
-    if (!gecko && cm.state.focused) {
-      sel.collapse(start.node, start.offset);
-      if (!rng.collapsed) {
-        sel.removeAllRanges();
-        sel.addRange(rng);
-      }
-    } else {
-      sel.removeAllRanges();
-      sel.addRange(rng);
-    }
-    if (old && sel.anchorNode == null) { sel.addRange(old); }
-    else if (gecko) { this.startGracePeriod(); }
-  }
-  this.rememberSelection();
-};
-
-ContentEditableInput.prototype.startGracePeriod = function () {
-    var this$1 = this;
-
-  clearTimeout(this.gracePeriod);
-  this.gracePeriod = setTimeout(function () {
-    this$1.gracePeriod = false;
-    if (this$1.selectionChanged())
-      { this$1.cm.operation(function () { return this$1.cm.curOp.selectionChanged = true; }); }
-  }, 20);
-};
-
-ContentEditableInput.prototype.showMultipleSelections = function (info) {
-  removeChildrenAndAdd(this.cm.display.cursorDiv, info.cursors);
-  removeChildrenAndAdd(this.cm.display.selectionDiv, info.selection);
-};
-
-ContentEditableInput.prototype.rememberSelection = function () {
-  var sel = window.getSelection();
-  this.lastAnchorNode = sel.anchorNode; this.lastAnchorOffset = sel.anchorOffset;
-  this.lastFocusNode = sel.focusNode; this.lastFocusOffset = sel.focusOffset;
-};
-
-ContentEditableInput.prototype.selectionInEditor = function () {
-  var sel = window.getSelection();
-  if (!sel.rangeCount) { return false }
-  var node = sel.getRangeAt(0).commonAncestorContainer;
-  return contains(this.div, node)
-};
-
-ContentEditableInput.prototype.focus = function () {
-  if (this.cm.options.readOnly != "nocursor") {
-    if (!this.selectionInEditor())
-      { this.showSelection(this.prepareSelection(), true); }
-    this.div.focus();
-  }
-};
-ContentEditableInput.prototype.blur = function () { this.div.blur(); };
-ContentEditableInput.prototype.getField = function () { return this.div };
-
-ContentEditableInput.prototype.supportsTouch = function () { return true };
-
-ContentEditableInput.prototype.receivedFocus = function () {
-  var input = this;
-  if (this.selectionInEditor())
-    { this.pollSelection(); }
-  else
-    { runInOp(this.cm, function () { return input.cm.curOp.selectionChanged = true; }); }
-
-  function poll() {
-    if (input.cm.state.focused) {
-      input.pollSelection();
-      input.polling.set(input.cm.options.pollInterval, poll);
-    }
-  }
-  this.polling.set(this.cm.options.pollInterval, poll);
-};
-
-ContentEditableInput.prototype.selectionChanged = function () {
-  var sel = window.getSelection();
-  return sel.anchorNode != this.lastAnchorNode || sel.anchorOffset != this.lastAnchorOffset ||
-    sel.focusNode != this.lastFocusNode || sel.focusOffset != this.lastFocusOffset
-};
-
-ContentEditableInput.prototype.pollSelection = function () {
-  if (this.readDOMTimeout != null || this.gracePeriod || !this.selectionChanged()) { return }
-  var sel = window.getSelection(), cm = this.cm;
-  // On Android Chrome (version 56, at least), backspacing into an
-  // uneditable block element will put the cursor in that element,
-  // and then, because it's not editable, hide the virtual keyboard.
-  // Because Android doesn't allow us to actually detect backspace
-  // presses in a sane way, this code checks for when that happens
-  // and simulates a backspace press in this case.
-  if (android && chrome && this.cm.options.gutters.length && isInGutter(sel.anchorNode)) {
-    this.cm.triggerOnKeyDown({type: "keydown", keyCode: 8, preventDefault: Math.abs});
-    this.blur();
-    this.focus();
-    return
-  }
-  if (this.composing) { return }
-  this.rememberSelection();
-  var anchor = domToPos(cm, sel.anchorNode, sel.anchorOffset);
-  var head = domToPos(cm, sel.focusNode, sel.focusOffset);
-  if (anchor && head) { runInOp(cm, function () {
-    setSelection(cm.doc, simpleSelection(anchor, head), sel_dontScroll);
-    if (anchor.bad || head.bad) { cm.curOp.selectionChanged = true; }
-  }); }
-};
-
-ContentEditableInput.prototype.pollContent = function () {
-  if (this.readDOMTimeout != null) {
-    clearTimeout(this.readDOMTimeout);
-    this.readDOMTimeout = null;
-  }
-
-  var cm = this.cm, display = cm.display, sel = cm.doc.sel.primary();
-  var from = sel.from(), to = sel.to();
-  if (from.ch == 0 && from.line > cm.firstLine())
-    { from = Pos(from.line - 1, getLine(cm.doc, from.line - 1).length); }
-  if (to.ch == getLine(cm.doc, to.line).text.length && to.line < cm.lastLine())
-    { to = Pos(to.line + 1, 0); }
-  if (from.line < display.viewFrom || to.line > display.viewTo - 1) { return false }
-
-  var fromIndex, fromLine, fromNode;
-  if (from.line == display.viewFrom || (fromIndex = findViewIndex(cm, from.line)) == 0) {
-    fromLine = lineNo(display.view[0].line);
-    fromNode = display.view[0].node;
-  } else {
-    fromLine = lineNo(display.view[fromIndex].line);
-    fromNode = display.view[fromIndex - 1].node.nextSibling;
-  }
-  var toIndex = findViewIndex(cm, to.line);
-  var toLine, toNode;
-  if (toIndex == display.view.length - 1) {
-    toLine = display.viewTo - 1;
-    toNode = display.lineDiv.lastChild;
-  } else {
-    toLine = lineNo(display.view[toIndex + 1].line) - 1;
-    toNode = display.view[toIndex + 1].node.previousSibling;
-  }
-
-  if (!fromNode) { return false }
-  var newText = cm.doc.splitLines(domTextBetween(cm, fromNode, toNode, fromLine, toLine));
-  var oldText = getBetween(cm.doc, Pos(fromLine, 0), Pos(toLine, getLine(cm.doc, toLine).text.length));
-  while (newText.length > 1 && oldText.length > 1) {
-    if (lst(newText) == lst(oldText)) { newText.pop(); oldText.pop(); toLine--; }
-    else if (newText[0] == oldText[0]) { newText.shift(); oldText.shift(); fromLine++; }
-    else { break }
-  }
-
-  var cutFront = 0, cutEnd = 0;
-  var newTop = newText[0], oldTop = oldText[0], maxCutFront = Math.min(newTop.length, oldTop.length);
-  while (cutFront < maxCutFront && newTop.charCodeAt(cutFront) == oldTop.charCodeAt(cutFront))
-    { ++cutFront; }
-  var newBot = lst(newText), oldBot = lst(oldText);
-  var maxCutEnd = Math.min(newBot.length - (newText.length == 1 ? cutFront : 0),
-                           oldBot.length - (oldText.length == 1 ? cutFront : 0));
-  while (cutEnd < maxCutEnd &&
-         newBot.charCodeAt(newBot.length - cutEnd - 1) == oldBot.charCodeAt(oldBot.length - cutEnd - 1))
-    { ++cutEnd; }
-  // Try to move start of change to start of selection if ambiguous
-  if (newText.length == 1 && oldText.length == 1 && fromLine == from.line) {
-    while (cutFront && cutFront > from.ch &&
-           newBot.charCodeAt(newBot.length - cutEnd - 1) == oldBot.charCodeAt(oldBot.length - cutEnd - 1)) {
-      cutFront--;
-      cutEnd++;
-    }
-  }
-
-  newText[newText.length - 1] = newBot.slice(0, newBot.length - cutEnd).replace(/^\u200b+/, "");
-  newText[0] = newText[0].slice(cutFront).replace(/\u200b+$/, "");
-
-  var chFrom = Pos(fromLine, cutFront);
-  var chTo = Pos(toLine, oldText.length ? lst(oldText).length - cutEnd : 0);
-  if (newText.length > 1 || newText[0] || cmp(chFrom, chTo)) {
-    replaceRange(cm.doc, newText, chFrom, chTo, "+input");
-    return true
-  }
-};
-
-ContentEditableInput.prototype.ensurePolled = function () {
-  this.forceCompositionEnd();
-};
-ContentEditableInput.prototype.reset = function () {
-  this.forceCompositionEnd();
-};
-ContentEditableInput.prototype.forceCompositionEnd = function () {
-  if (!this.composing) { return }
-  clearTimeout(this.readDOMTimeout);
-  this.composing = null;
-  this.updateFromDOM();
-  this.div.blur();
-  this.div.focus();
-};
-ContentEditableInput.prototype.readFromDOMSoon = function () {
-    var this$1 = this;
-
-  if (this.readDOMTimeout != null) { return }
-  this.readDOMTimeout = setTimeout(function () {
-    this$1.readDOMTimeout = null;
-    if (this$1.composing) {
-      if (this$1.composing.done) { this$1.composing = null; }
-      else { return }
-    }
-    this$1.updateFromDOM();
-  }, 80);
-};
-
-ContentEditableInput.prototype.updateFromDOM = function () {
-    var this$1 = this;
-
-  if (this.cm.isReadOnly() || !this.pollContent())
-    { runInOp(this.cm, function () { return regChange(this$1.cm); }); }
-};
-
-ContentEditableInput.prototype.setUneditable = function (node) {
-  node.contentEditable = "false";
-};
-
-ContentEditableInput.prototype.onKeyPress = function (e) {
-  if (e.charCode == 0) { return }
-  e.preventDefault();
-  if (!this.cm.isReadOnly())
-    { operation(this.cm, applyTextInput)(this.cm, String.fromCharCode(e.charCode == null ? e.keyCode : e.charCode), 0); }
-};
-
-ContentEditableInput.prototype.readOnlyChanged = function (val) {
-  this.div.contentEditable = String(val != "nocursor");
-};
-
-ContentEditableInput.prototype.onContextMenu = function () {};
-ContentEditableInput.prototype.resetPosition = function () {};
-
-ContentEditableInput.prototype.needsContentAttribute = true;
-
-function posToDOM(cm, pos) {
-  var view = findViewForLine(cm, pos.line);
-  if (!view || view.hidden) { return null }
-  var line = getLine(cm.doc, pos.line);
-  var info = mapFromLineView(view, line, pos.line);
-
-  var order = getOrder(line, cm.doc.direction), side = "left";
-  if (order) {
-    var partPos = getBidiPartAt(order, pos.ch);
-    side = partPos % 2 ? "right" : "left";
-  }
-  var result = nodeAndOffsetInLineMap(info.map, pos.ch, side);
-  result.offset = result.collapse == "right" ? result.end : result.start;
-  return result
-}
-
-function isInGutter(node) {
-  for (var scan = node; scan; scan = scan.parentNode)
-    { if (/CodeMirror-gutter-wrapper/.test(scan.className)) { return true } }
-  return false
-}
-
-function badPos(pos, bad) { if (bad) { pos.bad = true; } return pos }
-
-function domTextBetween(cm, from, to, fromLine, toLine) {
-  var text = "", closing = false, lineSep = cm.doc.lineSeparator();
-  function recognizeMarker(id) { return function (marker) { return marker.id == id; } }
-  function close() {
-    if (closing) {
-      text += lineSep;
-      closing = false;
-    }
-  }
-  function addText(str) {
-    if (str) {
-      close();
-      text += str;
-    }
-  }
-  function walk(node) {
-    if (node.nodeType == 1) {
-      var cmText = node.getAttribute("cm-text");
-      if (cmText != null) {
-        addText(cmText || node.textContent.replace(/\u200b/g, ""));
-        return
-      }
-      var markerID = node.getAttribute("cm-marker"), range$$1;
-      if (markerID) {
-        var found = cm.findMarks(Pos(fromLine, 0), Pos(toLine + 1, 0), recognizeMarker(+markerID));
-        if (found.length && (range$$1 = found[0].find(0)))
-          { addText(getBetween(cm.doc, range$$1.from, range$$1.to).join(lineSep)); }
-        return
-      }
-      if (node.getAttribute("contenteditable") == "false") { return }
-      var isBlock = /^(pre|div|p)$/i.test(node.nodeName);
-      if (isBlock) { close(); }
-      for (var i = 0; i < node.childNodes.length; i++)
-        { walk(node.childNodes[i]); }
-      if (isBlock) { closing = true; }
-    } else if (node.nodeType == 3) {
-      addText(node.nodeValue);
-    }
-  }
-  for (;;) {
-    walk(from);
-    if (from == to) { break }
-    from = from.nextSibling;
-  }
-  return text
-}
-
-function domToPos(cm, node, offset) {
-  var lineNode;
-  if (node == cm.display.lineDiv) {
-    lineNode = cm.display.lineDiv.childNodes[offset];
-    if (!lineNode) { return badPos(cm.clipPos(Pos(cm.display.viewTo - 1)), true) }
-    node = null; offset = 0;
-  } else {
-    for (lineNode = node;; lineNode = lineNode.parentNode) {
-      if (!lineNode || lineNode == cm.display.lineDiv) { return null }
-      if (lineNode.parentNode && lineNode.parentNode == cm.display.lineDiv) { break }
-    }
-  }
-  for (var i = 0; i < cm.display.view.length; i++) {
-    var lineView = cm.display.view[i];
-    if (lineView.node == lineNode)
-      { return locateNodeInLineView(lineView, node, offset) }
-  }
-}
-
-function locateNodeInLineView(lineView, node, offset) {
-  var wrapper = lineView.text.firstChild, bad = false;
-  if (!node || !contains(wrapper, node)) { return badPos(Pos(lineNo(lineView.line), 0), true) }
-  if (node == wrapper) {
-    bad = true;
-    node = wrapper.childNodes[offset];
-    offset = 0;
-    if (!node) {
-      var line = lineView.rest ? lst(lineView.rest) : lineView.line;
-      return badPos(Pos(lineNo(line), line.text.length), bad)
-    }
-  }
-
-  var textNode = node.nodeType == 3 ? node : null, topNode = node;
-  if (!textNode && node.childNodes.length == 1 && node.firstChild.nodeType == 3) {
-    textNode = node.firstChild;
-    if (offset) { offset = textNode.nodeValue.length; }
-  }
-  while (topNode.parentNode != wrapper) { topNode = topNode.parentNode; }
-  var measure = lineView.measure, maps = measure.maps;
-
-  function find(textNode, topNode, offset) {
-    for (var i = -1; i < (maps ? maps.length : 0); i++) {
-      var map$$1 = i < 0 ? measure.map : maps[i];
-      for (var j = 0; j < map$$1.length; j += 3) {
-        var curNode = map$$1[j + 2];
-        if (curNode == textNode || curNode == topNode) {
-          var line = lineNo(i < 0 ? lineView.line : lineView.rest[i]);
-          var ch = map$$1[j] + offset;
-          if (offset < 0 || curNode != textNode) { ch = map$$1[j + (offset ? 1 : 0)]; }
-          return Pos(line, ch)
-        }
-      }
-    }
-  }
-  var found = find(textNode, topNode, offset);
-  if (found) { return badPos(found, bad) }
-
-  // FIXME this is all really shaky. might handle the few cases it needs to handle, but likely to cause problems
-  for (var after = topNode.nextSibling, dist = textNode ? textNode.nodeValue.length - offset : 0; after; after = after.nextSibling) {
-    found = find(after, after.firstChild, 0);
-    if (found)
-      { return badPos(Pos(found.line, found.ch - dist), bad) }
-    else
-      { dist += after.textContent.length; }
-  }
-  for (var before = topNode.previousSibling, dist$1 = offset; before; before = before.previousSibling) {
-    found = find(before, before.firstChild, -1);
-    if (found)
-      { return badPos(Pos(found.line, found.ch + dist$1), bad) }
-    else
-      { dist$1 += before.textContent.length; }
-  }
-}
-
-// TEXTAREA INPUT STYLE
-
-var TextareaInput = function(cm) {
-  this.cm = cm;
-  // See input.poll and input.reset
-  this.prevInput = "";
-
-  // Flag that indicates whether we expect input to appear real soon
-  // now (after some event like 'keypress' or 'input') and are
-  // polling intensively.
-  this.pollingFast = false;
-  // Self-resetting timeout for the poller
-  this.polling = new Delayed();
-  // Used to work around IE issue with selection being forgotten when focus moves away from textarea
-  this.hasSelection = false;
-  this.composing = null;
-};
-
-TextareaInput.prototype.init = function (display) {
-    var this$1 = this;
-
-  var input = this, cm = this.cm;
-
-  // Wraps and hides input textarea
-  var div = this.wrapper = hiddenTextarea();
-  // The semihidden textarea that is focused when the editor is
-  // focused, and receives input.
-  var te = this.textarea = div.firstChild;
-  display.wrapper.insertBefore(div, display.wrapper.firstChild);
-
-  // Needed to hide big blue blinking cursor on Mobile Safari (doesn't seem to work in iOS 8 anymore)
-  if (ios) { te.style.width = "0px"; }
-
-  on(te, "input", function () {
-    if (ie && ie_version >= 9 && this$1.hasSelection) { this$1.hasSelection = null; }
-    input.poll();
-  });
-
-  on(te, "paste", function (e) {
-    if (signalDOMEvent(cm, e) || handlePaste(e, cm)) { return }
-
-    cm.state.pasteIncoming = true;
-    input.fastPoll();
-  });
-
-  function prepareCopyCut(e) {
-    if (signalDOMEvent(cm, e)) { return }
-    if (cm.somethingSelected()) {
-      setLastCopied({lineWise: false, text: cm.getSelections()});
-    } else if (!cm.options.lineWiseCopyCut) {
-      return
-    } else {
-      var ranges = copyableRanges(cm);
-      setLastCopied({lineWise: true, text: ranges.text});
-      if (e.type == "cut") {
-        cm.setSelections(ranges.ranges, null, sel_dontScroll);
-      } else {
-        input.prevInput = "";
-        te.value = ranges.text.join("\n");
-        selectInput(te);
-      }
-    }
-    if (e.type == "cut") { cm.state.cutIncoming = true; }
-  }
-  on(te, "cut", prepareCopyCut);
-  on(te, "copy", prepareCopyCut);
-
-  on(display.scroller, "paste", function (e) {
-    if (eventInWidget(display, e) || signalDOMEvent(cm, e)) { return }
-    cm.state.pasteIncoming = true;
-    input.focus();
-  });
-
-  // Prevent normal selection in the editor (we handle our own)
-  on(display.lineSpace, "selectstart", function (e) {
-    if (!eventInWidget(display, e)) { e_preventDefault(e); }
-  });
-
-  on(te, "compositionstart", function () {
-    var start = cm.getCursor("from");
-    if (input.composing) { input.composing.range.clear(); }
-    input.composing = {
-      start: start,
-      range: cm.markText(start, cm.getCursor("to"), {className: "CodeMirror-composing"})
-    };
-  });
-  on(te, "compositionend", function () {
-    if (input.composing) {
-      input.poll();
-      input.composing.range.clear();
-      input.composing = null;
-    }
-  });
-};
-
-TextareaInput.prototype.prepareSelection = function () {
-  // Redraw the selection and/or cursor
-  var cm = this.cm, display = cm.display, doc = cm.doc;
-  var result = prepareSelection(cm);
-
-  // Move the hidden textarea near the cursor to prevent scrolling artifacts
-  if (cm.options.moveInputWithCursor) {
-    var headPos = cursorCoords(cm, doc.sel.primary().head, "div");
-    var wrapOff = display.wrapper.getBoundingClientRect(), lineOff = display.lineDiv.getBoundingClientRect();
-    result.teTop = Math.max(0, Math.min(display.wrapper.clientHeight - 10,
-                                        headPos.top + lineOff.top - wrapOff.top));
-    result.teLeft = Math.max(0, Math.min(display.wrapper.clientWidth - 10,
-                                         headPos.left + lineOff.left - wrapOff.left));
-  }
-
-  return result
-};
-
-TextareaInput.prototype.showSelection = function (drawn) {
-  var cm = this.cm, display = cm.display;
-  removeChildrenAndAdd(display.cursorDiv, drawn.cursors);
-  removeChildrenAndAdd(display.selectionDiv, drawn.selection);
-  if (drawn.teTop != null) {
-    this.wrapper.style.top = drawn.teTop + "px";
-    this.wrapper.style.left = drawn.teLeft + "px";
-  }
-};
-
-// Reset the input to correspond to the selection (or to be empty,
-// when not typing and nothing is selected)
-TextareaInput.prototype.reset = function (typing) {
-  if (this.contextMenuPending || this.composing) { return }
-  var cm = this.cm;
-  if (cm.somethingSelected()) {
-    this.prevInput = "";
-    var content = cm.getSelection();
-    this.textarea.value = content;
-    if (cm.state.focused) { selectInput(this.textarea); }
-    if (ie && ie_version >= 9) { this.hasSelection = content; }
-  } else if (!typing) {
-    this.prevInput = this.textarea.value = "";
-    if (ie && ie_version >= 9) { this.hasSelection = null; }
-  }
-};
-
-TextareaInput.prototype.getField = function () { return this.textarea };
-
-TextareaInput.prototype.supportsTouch = function () { return false };
-
-TextareaInput.prototype.focus = function () {
-  if (this.cm.options.readOnly != "nocursor" && (!mobile || activeElt() != this.textarea)) {
-    try { this.textarea.focus(); }
-    catch (e) {} // IE8 will throw if the textarea is display: none or not in DOM
-  }
-};
-
-TextareaInput.prototype.blur = function () { this.textarea.blur(); };
-
-TextareaInput.prototype.resetPosition = function () {
-  this.wrapper.style.top = this.wrapper.style.left = 0;
-};
-
-TextareaInput.prototype.receivedFocus = function () { this.slowPoll(); };
-
-// Poll for input changes, using the normal rate of polling. This
-// runs as long as the editor is focused.
-TextareaInput.prototype.slowPoll = function () {
-    var this$1 = this;
-
-  if (this.pollingFast) { return }
-  this.polling.set(this.cm.options.pollInterval, function () {
-    this$1.poll();
-    if (this$1.cm.state.focused) { this$1.slowPoll(); }
-  });
-};
-
-// When an event has just come in that is likely to add or change
-// something in the input textarea, we poll faster, to ensure that
-// the change appears on the screen quickly.
-TextareaInput.prototype.fastPoll = function () {
-  var missed = false, input = this;
-  input.pollingFast = true;
-  function p() {
-    var changed = input.poll();
-    if (!changed && !missed) {missed = true; input.polling.set(60, p);}
-    else {input.pollingFast = false; input.slowPoll();}
-  }
-  input.polling.set(20, p);
-};
-
-// Read input from the textarea, and update the document to match.
-// When something is selected, it is present in the textarea, and
-// selected (unless it is huge, in which case a placeholder is
-// used). When nothing is selected, the cursor sits after previously
-// seen text (can be empty), which is stored in prevInput (we must
-// not reset the textarea when typing, because that breaks IME).
-TextareaInput.prototype.poll = function () {
-    var this$1 = this;
-
-  var cm = this.cm, input = this.textarea, prevInput = this.prevInput;
-  // Since this is called a *lot*, try to bail out as cheaply as
-  // possible when it is clear that nothing happened. hasSelection
-  // will be the case when there is a lot of text in the textarea,
-  // in which case reading its value would be expensive.
-  if (this.contextMenuPending || !cm.state.focused ||
-      (hasSelection(input) && !prevInput && !this.composing) ||
-      cm.isReadOnly() || cm.options.disableInput || cm.state.keySeq)
-    { return false }
-
-  var text = input.value;
-  // If nothing changed, bail.
-  if (text == prevInput && !cm.somethingSelected()) { return false }
-  // Work around nonsensical selection resetting in IE9/10, and
-  // inexplicable appearance of private area unicode characters on
-  // some key combos in Mac (#2689).
-  if (ie && ie_version >= 9 && this.hasSelection === text ||
-      mac && /[\uf700-\uf7ff]/.test(text)) {
-    cm.display.input.reset();
-    return false
-  }
-
-  if (cm.doc.sel == cm.display.selForContextMenu) {
-    var first = text.charCodeAt(0);
-    if (first == 0x200b && !prevInput) { prevInput = "\u200b"; }
-    if (first == 0x21da) { this.reset(); return this.cm.execCommand("undo") }
-  }
-  // Find the part of the input that is actually new
-  var same = 0, l = Math.min(prevInput.length, text.length);
-  while (same < l && prevInput.charCodeAt(same) == text.charCodeAt(same)) { ++same; }
-
-  runInOp(cm, function () {
-    applyTextInput(cm, text.slice(same), prevInput.length - same,
-                   null, this$1.composing ? "*compose" : null);
-
-    // Don't leave long text in the textarea, since it makes further polling slow
-    if (text.length > 1000 || text.indexOf("\n") > -1) { input.value = this$1.prevInput = ""; }
-    else { this$1.prevInput = text; }
-
-    if (this$1.composing) {
-      this$1.composing.range.clear();
-      this$1.composing.range = cm.markText(this$1.composing.start, cm.getCursor("to"),
-                                         {className: "CodeMirror-composing"});
-    }
-  });
-  return true
-};
-
-TextareaInput.prototype.ensurePolled = function () {
-  if (this.pollingFast && this.poll()) { this.pollingFast = false; }
-};
-
-TextareaInput.prototype.onKeyPress = function () {
-  if (ie && ie_version >= 9) { this.hasSelection = null; }
-  this.fastPoll();
-};
-
-TextareaInput.prototype.onContextMenu = function (e) {
-  var input = this, cm = input.cm, display = cm.display, te = input.textarea;
-  var pos = posFromMouse(cm, e), scrollPos = display.scroller.scrollTop;
-  if (!pos || presto) { return } // Opera is difficult.
-
-  // Reset the current text selection only if the click is done outside of the selection
-  // and 'resetSelectionOnContextMenu' option is true.
-  var reset = cm.options.resetSelectionOnContextMenu;
-  if (reset && cm.doc.sel.contains(pos) == -1)
-    { operation(cm, setSelection)(cm.doc, simpleSelection(pos), sel_dontScroll); }
-
-  var oldCSS = te.style.cssText, oldWrapperCSS = input.wrapper.style.cssText;
-  input.wrapper.style.cssText = "position: absolute";
-  var wrapperBox = input.wrapper.getBoundingClientRect();
-  te.style.cssText = "position: absolute; width: 30px; height: 30px;\n      top: " + (e.clientY - wrapperBox.top - 5) + "px; left: " + (e.clientX - wrapperBox.left - 5) + "px;\n      z-index: 1000; background: " + (ie ? "rgba(255, 255, 255, .05)" : "transparent") + ";\n      outline: none; border-width: 0; outline: none; overflow: hidden; opacity: .05; filter: alpha(opacity=5);";
-  var oldScrollY;
-  if (webkit) { oldScrollY = window.scrollY; } // Work around Chrome issue (#2712)
-  display.input.focus();
-  if (webkit) { window.scrollTo(null, oldScrollY); }
-  display.input.reset();
-  // Adds "Select all" to context menu in FF
-  if (!cm.somethingSelected()) { te.value = input.prevInput = " "; }
-  input.contextMenuPending = true;
-  display.selForContextMenu = cm.doc.sel;
-  clearTimeout(display.detectingSelectAll);
-
-  // Select-all will be greyed out if there's nothing to select, so
-  // this adds a zero-width space so that we can later check whether
-  // it got selected.
-  function prepareSelectAllHack() {
-    if (te.selectionStart != null) {
-      var selected = cm.somethingSelected();
-      var extval = "\u200b" + (selected ? te.value : "");
-      te.value = "\u21da"; // Used to catch context-menu undo
-      te.value = extval;
-      input.prevInput = selected ? "" : "\u200b";
-      te.selectionStart = 1; te.selectionEnd = extval.length;
-      // Re-set this, in case some other handler touched the
-      // selection in the meantime.
-      display.selForContextMenu = cm.doc.sel;
-    }
-  }
-  function rehide() {
-    input.contextMenuPending = false;
-    input.wrapper.style.cssText = oldWrapperCSS;
-    te.style.cssText = oldCSS;
-    if (ie && ie_version < 9) { display.scrollbars.setScrollTop(display.scroller.scrollTop = scrollPos); }
-
-    // Try to detect the user choosing select-all
-    if (te.selectionStart != null) {
-      if (!ie || (ie && ie_version < 9)) { prepareSelectAllHack(); }
-      var i = 0, poll = function () {
-        if (display.selForContextMenu == cm.doc.sel && te.selectionStart == 0 &&
-            te.selectionEnd > 0 && input.prevInput == "\u200b") {
-          operation(cm, selectAll)(cm);
-        } else if (i++ < 10) {
-          display.detectingSelectAll = setTimeout(poll, 500);
-        } else {
-          display.selForContextMenu = null;
-          display.input.reset();
-        }
-      };
-      display.detectingSelectAll = setTimeout(poll, 200);
-    }
-  }
-
-  if (ie && ie_version >= 9) { prepareSelectAllHack(); }
-  if (captureRightClick) {
-    e_stop(e);
-    var mouseup = function () {
-      off(window, "mouseup", mouseup);
-      setTimeout(rehide, 20);
-    };
-    on(window, "mouseup", mouseup);
-  } else {
-    setTimeout(rehide, 50);
-  }
-};
-
-TextareaInput.prototype.readOnlyChanged = function (val) {
-  if (!val) { this.reset(); }
-  this.textarea.disabled = val == "nocursor";
-};
-
-TextareaInput.prototype.setUneditable = function () {};
-
-TextareaInput.prototype.needsContentAttribute = false;
-
-function fromTextArea(textarea, options) {
-  options = options ? copyObj(options) : {};
-  options.value = textarea.value;
-  if (!options.tabindex && textarea.tabIndex)
-    { options.tabindex = textarea.tabIndex; }
-  if (!options.placeholder && textarea.placeholder)
-    { options.placeholder = textarea.placeholder; }
-  // Set autofocus to true if this textarea is focused, or if it has
-  // autofocus and no other element is focused.
-  if (options.autofocus == null) {
-    var hasFocus = activeElt();
-    options.autofocus = hasFocus == textarea ||
-      textarea.getAttribute("autofocus") != null && hasFocus == document.body;
-  }
-
-  function save() {textarea.value = cm.getValue();}
-
-  var realSubmit;
-  if (textarea.form) {
-    on(textarea.form, "submit", save);
-    // Deplorable hack to make the submit method do the right thing.
-    if (!options.leaveSubmitMethodAlone) {
-      var form = textarea.form;
-      realSubmit = form.submit;
-      try {
-        var wrappedSubmit = form.submit = function () {
-          save();
-          form.submit = realSubmit;
-          form.submit();
-          form.submit = wrappedSubmit;
-        };
-      } catch(e) {}
-    }
-  }
-
-  options.finishInit = function (cm) {
-    cm.save = save;
-    cm.getTextArea = function () { return textarea; };
-    cm.toTextArea = function () {
-      cm.toTextArea = isNaN; // Prevent this from being ran twice
-      save();
-      textarea.parentNode.removeChild(cm.getWrapperElement());
-      textarea.style.display = "";
-      if (textarea.form) {
-        off(textarea.form, "submit", save);
-        if (typeof textarea.form.submit == "function")
-          { textarea.form.submit = realSubmit; }
-      }
-    };
-  };
-
-  textarea.style.display = "none";
-  var cm = CodeMirror$1(function (node) { return textarea.parentNode.insertBefore(node, textarea.nextSibling); },
-    options);
-  return cm
-}
-
-function addLegacyProps(CodeMirror) {
-  CodeMirror.off = off;
-  CodeMirror.on = on;
-  CodeMirror.wheelEventPixels = wheelEventPixels;
-  CodeMirror.Doc = Doc;
-  CodeMirror.splitLines = splitLinesAuto;
-  CodeMirror.countColumn = countColumn;
-  CodeMirror.findColumn = findColumn;
-  CodeMirror.isWordChar = isWordCharBasic;
-  CodeMirror.Pass = Pass;
-  CodeMirror.signal = signal;
-  CodeMirror.Line = Line;
-  CodeMirror.changeEnd = changeEnd;
-  CodeMirror.scrollbarModel = scrollbarModel;
-  CodeMirror.Pos = Pos;
-  CodeMirror.cmpPos = cmp;
-  CodeMirror.modes = modes;
-  CodeMirror.mimeModes = mimeModes;
-  CodeMirror.resolveMode = resolveMode;
-  CodeMirror.getMode = getMode;
-  CodeMirror.modeExtensions = modeExtensions;
-  CodeMirror.extendMode = extendMode;
-  CodeMirror.copyState = copyState;
-  CodeMirror.startState = startState;
-  CodeMirror.innerMode = innerMode;
-  CodeMirror.commands = commands;
-  CodeMirror.keyMap = keyMap;
-  CodeMirror.keyName = keyName;
-  CodeMirror.isModifierKey = isModifierKey;
-  CodeMirror.lookupKey = lookupKey;
-  CodeMirror.normalizeKeyMap = normalizeKeyMap;
-  CodeMirror.StringStream = StringStream;
-  CodeMirror.SharedTextMarker = SharedTextMarker;
-  CodeMirror.TextMarker = TextMarker;
-  CodeMirror.LineWidget = LineWidget;
-  CodeMirror.e_preventDefault = e_preventDefault;
-  CodeMirror.e_stopPropagation = e_stopPropagation;
-  CodeMirror.e_stop = e_stop;
-  CodeMirror.addClass = addClass;
-  CodeMirror.contains = contains;
-  CodeMirror.rmClass = rmClass;
-  CodeMirror.keyNames = keyNames;
-}
-
-// EDITOR CONSTRUCTOR
-
-defineOptions(CodeMirror$1);
-
-addEditorMethods(CodeMirror$1);
-
-// Set up methods on CodeMirror's prototype to redirect to the editor's document.
-var dontDelegate = "iter insert remove copy getEditor constructor".split(" ");
-for (var prop in Doc.prototype) { if (Doc.prototype.hasOwnProperty(prop) && indexOf(dontDelegate, prop) < 0)
-  { CodeMirror$1.prototype[prop] = (function(method) {
-    return function() {return method.apply(this.doc, arguments)}
-  })(Doc.prototype[prop]); } }
-
-eventMixin(Doc);
-
-// INPUT HANDLING
-
-CodeMirror$1.inputStyles = {"textarea": TextareaInput, "contenteditable": ContentEditableInput};
-
-// MODE DEFINITION AND QUERYING
-
-// Extra arguments are stored as the mode's dependencies, which is
-// used by (legacy) mechanisms like loadmode.js to automatically
-// load a mode. (Preferred mechanism is the require/define calls.)
-CodeMirror$1.defineMode = function(name/*, mode, …*/) {
-  if (!CodeMirror$1.defaults.mode && name != "null") { CodeMirror$1.defaults.mode = name; }
-  defineMode.apply(this, arguments);
-};
-
-CodeMirror$1.defineMIME = defineMIME;
-
-// Minimal default mode.
-CodeMirror$1.defineMode("null", function () { return ({token: function (stream) { return stream.skipToEnd(); }}); });
-CodeMirror$1.defineMIME("text/plain", "null");
-
-// EXTENSIONS
-
-CodeMirror$1.defineExtension = function (name, func) {
-  CodeMirror$1.prototype[name] = func;
-};
-CodeMirror$1.defineDocExtension = function (name, func) {
-  Doc.prototype[name] = func;
-};
-
-CodeMirror$1.fromTextArea = fromTextArea;
-
-addLegacyProps(CodeMirror$1);
-
-CodeMirror$1.version = "5.31.0";
-
-return CodeMirror$1;
-
-})));
diff --git a/datasette/static/codemirror-5.57.0-sql.min.js b/datasette/static/codemirror-5.57.0-sql.min.js
new file mode 100644
index 00000000..13f667c6
--- /dev/null
+++ b/datasette/static/codemirror-5.57.0-sql.min.js
@@ -0,0 +1,5 @@
+/*
+  CodeMirror, copyright (c) by Marijn Haverbeke and others
+  Distributed under an MIT license: https://codemirror.net/LICENSE
+*/
+(function(mod){if(typeof exports=="object"&&typeof module=="object")mod(require("../../lib/codemirror"));else if(typeof define=="function"&&define.amd)define(["../../lib/codemirror"],mod);else mod(CodeMirror)})(function(CodeMirror){"use strict";CodeMirror.defineMode("sql",function(config,parserConfig){var client=parserConfig.client||{},atoms=parserConfig.atoms||{false:true,true:true,null:true},builtin=parserConfig.builtin||set(defaultBuiltin),keywords=parserConfig.keywords||set(sqlKeywords),operatorChars=parserConfig.operatorChars||/^[*+\-%<>!=&|~^\/]/,support=parserConfig.support||{},hooks=parserConfig.hooks||{},dateSQL=parserConfig.dateSQL||{date:true,time:true,timestamp:true},backslashStringEscapes=parserConfig.backslashStringEscapes!==false,brackets=parserConfig.brackets||/^[\{}\(\)\[\]]/,punctuation=parserConfig.punctuation||/^[;.,:]/;function tokenBase(stream,state){var ch=stream.next();if(hooks[ch]){var result=hooks[ch](stream,state);if(result!==false)return result}if(support.hexNumber&&(ch=="0"&&stream.match(/^[xX][0-9a-fA-F]+/)||(ch=="x"||ch=="X")&&stream.match(/^'[0-9a-fA-F]+'/))){return"number"}else if(support.binaryNumber&&((ch=="b"||ch=="B")&&stream.match(/^'[01]+'/)||ch=="0"&&stream.match(/^b[01]+/))){return"number"}else if(ch.charCodeAt(0)>47&&ch.charCodeAt(0)<58){stream.match(/^[0-9]*(\.[0-9]+)?([eE][-+]?[0-9]+)?/);support.decimallessFloat&&stream.match(/^\.(?!\.)/);return"number"}else if(ch=="?"&&(stream.eatSpace()||stream.eol()||stream.eat(";"))){return"variable-3"}else if(ch=="'"||ch=='"'&&support.doubleQuote){state.tokenize=tokenLiteral(ch);return state.tokenize(stream,state)}else if((support.nCharCast&&(ch=="n"||ch=="N")||support.charsetCast&&ch=="_"&&stream.match(/[a-z][a-z0-9]*/i))&&(stream.peek()=="'"||stream.peek()=='"')){return"keyword"}else if(support.escapeConstant&&(ch=="e"||ch=="E")&&(stream.peek()=="'"||stream.peek()=='"'&&support.doubleQuote)){state.tokenize=function(stream,state){return(state.tokenize=tokenLiteral(stream.next(),true))(stream,state)};return"keyword"}else if(support.commentSlashSlash&&ch=="/"&&stream.eat("/")){stream.skipToEnd();return"comment"}else if(support.commentHash&&ch=="#"||ch=="-"&&stream.eat("-")&&(!support.commentSpaceRequired||stream.eat(" "))){stream.skipToEnd();return"comment"}else if(ch=="/"&&stream.eat("*")){state.tokenize=tokenComment(1);return state.tokenize(stream,state)}else if(ch=="."){if(support.zerolessFloat&&stream.match(/^(?:\d+(?:e[+-]?\d+)?)/i))return"number";if(stream.match(/^\.+/))return null;if(support.ODBCdotTable&&stream.match(/^[\w\d_$#]+/))return"variable-2"}else if(operatorChars.test(ch)){stream.eatWhile(operatorChars);return"operator"}else if(brackets.test(ch)){return"bracket"}else if(punctuation.test(ch)){stream.eatWhile(punctuation);return"punctuation"}else if(ch=="{"&&(stream.match(/^( )*(d|D|t|T|ts|TS)( )*'[^']*'( )*}/)||stream.match(/^( )*(d|D|t|T|ts|TS)( )*"[^"]*"( )*}/))){return"number"}else{stream.eatWhile(/^[_\w\d]/);var word=stream.current().toLowerCase();if(dateSQL.hasOwnProperty(word)&&(stream.match(/^( )+'[^']*'/)||stream.match(/^( )+"[^"]*"/)))return"number";if(atoms.hasOwnProperty(word))return"atom";if(builtin.hasOwnProperty(word))return"builtin";if(keywords.hasOwnProperty(word))return"keyword";if(client.hasOwnProperty(word))return"string-2";return null}}function tokenLiteral(quote,backslashEscapes){return function(stream,state){var escaped=false,ch;while((ch=stream.next())!=null){if(ch==quote&&!escaped){state.tokenize=tokenBase;break}escaped=(backslashStringEscapes||backslashEscapes)&&!escaped&&ch=="\\"}return"string"}}function tokenComment(depth){return function(stream,state){var m=stream.match(/^.*?(\/\*|\*\/)/);if(!m)stream.skipToEnd();else if(m[1]=="/*")state.tokenize=tokenComment(depth+1);else if(depth>1)state.tokenize=tokenComment(depth-1);else state.tokenize=tokenBase;return"comment"}}function pushContext(stream,state,type){state.context={prev:state.context,indent:stream.indentation(),col:stream.column(),type:type}}function popContext(state){state.indent=state.context.indent;state.context=state.context.prev}return{startState:function(){return{tokenize:tokenBase,context:null}},token:function(stream,state){if(stream.sol()){if(state.context&&state.context.align==null)state.context.align=false}if(state.tokenize==tokenBase&&stream.eatSpace())return null;var style=state.tokenize(stream,state);if(style=="comment")return style;if(state.context&&state.context.align==null)state.context.align=true;var tok=stream.current();if(tok=="(")pushContext(stream,state,")");else if(tok=="[")pushContext(stream,state,"]");else if(state.context&&state.context.type==tok)popContext(state);return style},indent:function(state,textAfter){var cx=state.context;if(!cx)return CodeMirror.Pass;var closing=textAfter.charAt(0)==cx.type;if(cx.align)return cx.col+(closing?0:1);else return cx.indent+(closing?0:config.indentUnit)},blockCommentStart:"/*",blockCommentEnd:"*/",lineComment:support.commentSlashSlash?"//":support.commentHash?"#":"--",closeBrackets:"()[]{}''\"\"``"}});function hookIdentifier(stream){var ch;while((ch=stream.next())!=null){if(ch=="`"&&!stream.eat("`"))return"variable-2"}stream.backUp(stream.current().length-1);return stream.eatWhile(/\w/)?"variable-2":null}function hookIdentifierDoublequote(stream){var ch;while((ch=stream.next())!=null){if(ch=='"'&&!stream.eat('"'))return"variable-2"}stream.backUp(stream.current().length-1);return stream.eatWhile(/\w/)?"variable-2":null}function hookVar(stream){if(stream.eat("@")){stream.match(/^session\./);stream.match(/^local\./);stream.match(/^global\./)}if(stream.eat("'")){stream.match(/^.*'/);return"variable-2"}else if(stream.eat('"')){stream.match(/^.*"/);return"variable-2"}else if(stream.eat("`")){stream.match(/^.*`/);return"variable-2"}else if(stream.match(/^[0-9a-zA-Z$\.\_]+/)){return"variable-2"}return null}function hookClient(stream){if(stream.eat("N")){return"atom"}return stream.match(/^[a-zA-Z.#!?]/)?"variable-2":null}var sqlKeywords="alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit ";function set(str){var obj={},words=str.split(" ");for(var i=0;i!=^\&|\/]/,brackets:/^[\{}\(\)]/,punctuation:/^[;.,:/]/,backslashStringEscapes:false,dateSQL:set("date datetimeoffset datetime2 smalldatetime datetime time"),hooks:{"@":hookVar}});CodeMirror.defineMIME("text/x-mysql",{name:"sql",client:set("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),keywords:set(sqlKeywords+"accessible action add after algorithm all analyze asensitive at authors auto_increment autocommit avg avg_row_length before binary binlog both btree cache call cascade cascaded case catalog_name chain change changed character check checkpoint checksum class_origin client_statistics close coalesce code collate collation collations column columns comment commit committed completion concurrent condition connection consistent constraint contains continue contributors convert cross current current_date current_time current_timestamp current_user cursor data database databases day_hour day_microsecond day_minute day_second deallocate dec declare default delay_key_write delayed delimiter des_key_file describe deterministic dev_pop dev_samp deviance diagnostics directory disable discard distinctrow div dual dumpfile each elseif enable enclosed end ends engine engines enum errors escape escaped even event events every execute exists exit explain extended fast fetch field fields first flush for force foreign found_rows full fulltext function general get global grant grants group group_concat handler hash help high_priority hosts hour_microsecond hour_minute hour_second if ignore ignore_server_ids import index index_statistics infile inner innodb inout insensitive insert_method install interval invoker isolation iterate key keys kill language last leading leave left level limit linear lines list load local localtime localtimestamp lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters match max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modifies modify mutex mysql_errno natural next no no_write_to_binlog offline offset one online open optimize option optionally out outer outfile pack_keys parser partition partitions password phase plugin plugins prepare preserve prev primary privileges procedure processlist profile profiles purge query quick range read read_write reads real rebuild recover references regexp relaylog release remove rename reorganize repair repeatable replace require resignal restrict resume return returns revoke right rlike rollback rollup row row_format rtree savepoint schedule schema schema_name schemas second_microsecond security sensitive separator serializable server session share show signal slave slow smallint snapshot soname spatial specific sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result sqlexception sqlstate sqlwarning ssl start starting starts status std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace temporary terminated to trailing transaction trigger triggers truncate uncommitted undo uninstall unique unlock upgrade usage use use_frm user user_resources user_statistics using utc_date utc_time utc_timestamp value variables varying view views warnings when while with work write xa xor year_month zerofill begin do then else loop repeat"),builtin:set("bool boolean bit blob decimal double float long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision date datetime year unsigned signed numeric"),atoms:set("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^]/,dateSQL:set("date time timestamp"),support:set("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber doubleQuote nCharCast charsetCast commentHash commentSpaceRequired"),hooks:{"@":hookVar,"`":hookIdentifier,"\\":hookClient}});CodeMirror.defineMIME("text/x-mariadb",{name:"sql",client:set("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),keywords:set(sqlKeywords+"accessible action add after algorithm all always analyze asensitive at authors auto_increment autocommit avg avg_row_length before binary binlog both btree cache call cascade cascaded case catalog_name chain change changed character check checkpoint checksum class_origin client_statistics close coalesce code collate collation collations column columns comment commit committed completion concurrent condition connection consistent constraint contains continue contributors convert cross current current_date current_time current_timestamp current_user cursor data database databases day_hour day_microsecond day_minute day_second deallocate dec declare default delay_key_write delayed delimiter des_key_file describe deterministic dev_pop dev_samp deviance diagnostics directory disable discard distinctrow div dual dumpfile each elseif enable enclosed end ends engine engines enum errors escape escaped even event events every execute exists exit explain extended fast fetch field fields first flush for force foreign found_rows full fulltext function general generated get global grant grants group groupby_concat handler hard hash help high_priority hosts hour_microsecond hour_minute hour_second if ignore ignore_server_ids import index index_statistics infile inner innodb inout insensitive insert_method install interval invoker isolation iterate key keys kill language last leading leave left level limit linear lines list load local localtime localtimestamp lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters match max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modifies modify mutex mysql_errno natural next no no_write_to_binlog offline offset one online open optimize option optionally out outer outfile pack_keys parser partition partitions password persistent phase plugin plugins prepare preserve prev primary privileges procedure processlist profile profiles purge query quick range read read_write reads real rebuild recover references regexp relaylog release remove rename reorganize repair repeatable replace require resignal restrict resume return returns revoke right rlike rollback rollup row row_format rtree savepoint schedule schema schema_name schemas second_microsecond security sensitive separator serializable server session share show shutdown signal slave slow smallint snapshot soft soname spatial specific sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result sqlexception sqlstate sqlwarning ssl start starting starts status std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace temporary terminated to trailing transaction trigger triggers truncate uncommitted undo uninstall unique unlock upgrade usage use use_frm user user_resources user_statistics using utc_date utc_time utc_timestamp value variables varying view views virtual warnings when while with work write xa xor year_month zerofill begin do then else loop repeat"),builtin:set("bool boolean bit blob decimal double float long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision date datetime year unsigned signed numeric"),atoms:set("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^]/,dateSQL:set("date time timestamp"),support:set("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber doubleQuote nCharCast charsetCast commentHash commentSpaceRequired"),hooks:{"@":hookVar,"`":hookIdentifier,"\\":hookClient}});CodeMirror.defineMIME("text/x-sqlite",{name:"sql",client:set("auth backup bail binary changes check clone databases dbinfo dump echo eqp exit explain fullschema headers help import imposter indexes iotrace limit lint load log mode nullvalue once open output print prompt quit read restore save scanstats schema separator session shell show stats system tables testcase timeout timer trace vfsinfo vfslist vfsname width"),keywords:set(sqlKeywords+"abort action add after all analyze attach autoincrement before begin cascade case cast check collate column commit conflict constraint cross current_date current_time current_timestamp database default deferrable deferred detach each else end escape except exclusive exists explain fail for foreign full glob if ignore immediate index indexed initially inner instead intersect isnull key left limit match natural no notnull null of offset outer plan pragma primary query raise recursive references regexp reindex release rename replace restrict right rollback row savepoint temp temporary then to transaction trigger unique using vacuum view virtual when with without"),builtin:set("bool boolean bit blob decimal double float long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text clob bigint int int2 int8 integer float double char varchar date datetime year unsigned signed numeric real"),atoms:set("null current_date current_time current_timestamp"),operatorChars:/^[*+\-%<>!=&|/~]/,dateSQL:set("date time timestamp datetime"),support:set("decimallessFloat zerolessFloat"),identifierQuote:'"',hooks:{"@":hookVar,":":hookVar,"?":hookVar,$:hookVar,'"':hookIdentifierDoublequote,"`":hookIdentifier}});CodeMirror.defineMIME("text/x-cassandra",{name:"sql",client:{},keywords:set("add all allow alter and any apply as asc authorize batch begin by clustering columnfamily compact consistency count create custom delete desc distinct drop each_quorum exists filtering from grant if in index insert into key keyspace keyspaces level limit local_one local_quorum modify nan norecursive nosuperuser not of on one order password permission permissions primary quorum rename revoke schema select set storage superuser table three to token truncate ttl two type unlogged update use user users using values where with writetime"),builtin:set("ascii bigint blob boolean counter decimal double float frozen inet int list map static text timestamp timeuuid tuple uuid varchar varint"),atoms:set("false true infinity NaN"),operatorChars:/^[<>=]/,dateSQL:{},support:set("commentSlashSlash decimallessFloat"),hooks:{}});CodeMirror.defineMIME("text/x-plsql",{name:"sql",client:set("appinfo arraysize autocommit autoprint autorecovery autotrace blockterminator break btitle cmdsep colsep compatibility compute concat copycommit copytypecheck define describe echo editfile embedded escape exec execute feedback flagger flush heading headsep instance linesize lno loboffset logsource long longchunksize markup native newpage numformat numwidth pagesize pause pno recsep recsepchar release repfooter repheader serveroutput shiftinout show showmode size spool sqlblanklines sqlcase sqlcode sqlcontinue sqlnumber sqlpluscompatibility sqlprefix sqlprompt sqlterminator suffix tab term termout time timing trimout trimspool ttitle underline verify version wrap"),keywords:set("abort accept access add all alter and any array arraylen as asc assert assign at attributes audit authorization avg base_table begin between binary_integer body boolean by case cast char char_base check close cluster clusters colauth column comment commit compress connect connected constant constraint crash create current currval cursor data_base database date dba deallocate debugoff debugon decimal declare default definition delay delete desc digits dispose distinct do drop else elseif elsif enable end entry escape exception exception_init exchange exclusive exists exit external fast fetch file for force form from function generic goto grant group having identified if immediate in increment index indexes indicator initial initrans insert interface intersect into is key level library like limited local lock log logging long loop master maxextents maxtrans member minextents minus mislabel mode modify multiset new next no noaudit nocompress nologging noparallel not nowait number_base object of off offline on online only open option or order out package parallel partition pctfree pctincrease pctused pls_integer positive positiven pragma primary prior private privileges procedure public raise range raw read rebuild record ref references refresh release rename replace resource restrict return returning returns reverse revoke rollback row rowid rowlabel rownum rows run savepoint schema segment select separate session set share snapshot some space split sql start statement storage subtype successful synonym tabauth table tables tablespace task terminate then to trigger truncate type union unique unlimited unrecoverable unusable update use using validate value values variable view views when whenever where while with work"),builtin:set("abs acos add_months ascii asin atan atan2 average bfile bfilename bigserial bit blob ceil character chartorowid chr clob concat convert cos cosh count dec decode deref dual dump dup_val_on_index empty error exp false float floor found glb greatest hextoraw initcap instr instrb int integer isopen last_day least length lengthb ln lower lpad ltrim lub make_ref max min mlslabel mod months_between natural naturaln nchar nclob new_time next_day nextval nls_charset_decl_len nls_charset_id nls_charset_name nls_initcap nls_lower nls_sort nls_upper nlssort no_data_found notfound null number numeric nvarchar2 nvl others power rawtohex real reftohex round rowcount rowidtochar rowtype rpad rtrim serial sign signtype sin sinh smallint soundex sqlcode sqlerrm sqrt stddev string substr substrb sum sysdate tan tanh to_char text to_date to_label to_multi_byte to_number to_single_byte translate true trunc uid unlogged upper user userenv varchar varchar2 variance varying vsize xml"),operatorChars:/^[*\/+\-%<>!=~]/,dateSQL:set("date time timestamp"),support:set("doubleQuote nCharCast zerolessFloat binaryNumber hexNumber")});CodeMirror.defineMIME("text/x-hive",{name:"sql",keywords:set("select alter $elem$ $key$ $value$ add after all analyze and archive as asc before between binary both bucket buckets by cascade case cast change cluster clustered clusterstatus collection column columns comment compute concatenate continue create cross cursor data database databases dbproperties deferred delete delimited desc describe directory disable distinct distribute drop else enable end escaped exclusive exists explain export extended external fetch fields fileformat first format formatted from full function functions grant group having hold_ddltime idxproperties if import in index indexes inpath inputdriver inputformat insert intersect into is items join keys lateral left like limit lines load local location lock locks mapjoin materialized minus msck no_drop nocompress not of offline on option or order out outer outputdriver outputformat overwrite partition partitioned partitions percent plus preserve procedure purge range rcfile read readonly reads rebuild recordreader recordwriter recover reduce regexp rename repair replace restrict revoke right rlike row schema schemas semi sequencefile serde serdeproperties set shared show show_database sort sorted ssl statistics stored streamtable table tables tablesample tblproperties temporary terminated textfile then tmp to touch transform trigger unarchive undo union uniquejoin unlock update use using utc utc_tmestamp view when where while with admin authorization char compact compactions conf cube current current_date current_timestamp day decimal defined dependency directories elem_type exchange file following for grouping hour ignore inner interval jar less logical macro minute month more none noscan over owner partialscan preceding pretty principals protection reload rewrite role roles rollup rows second server sets skewed transactions truncate unbounded unset uri user values window year"),builtin:set("bool boolean long timestamp tinyint smallint bigint int float double date datetime unsigned string array struct map uniontype key_type utctimestamp value_type varchar"),atoms:set("false true null unknown"),operatorChars:/^[*+\-%<>!=]/,dateSQL:set("date timestamp"),support:set("ODBCdotTable doubleQuote binaryNumber hexNumber")});CodeMirror.defineMIME("text/x-pgsql",{name:"sql",client:set("source"),keywords:set(sqlKeywords+"a abort abs absent absolute access according action ada add admin after aggregate alias all allocate also alter always analyse analyze and any are array array_agg array_max_cardinality as asc asensitive assert assertion assignment asymmetric at atomic attach attribute attributes authorization avg backward base64 before begin begin_frame begin_partition bernoulli between bigint binary bit bit_length blob blocked bom boolean both breadth by c cache call called cardinality cascade cascaded case cast catalog catalog_name ceil ceiling chain char char_length character character_length character_set_catalog character_set_name character_set_schema characteristics characters check checkpoint class class_origin clob close cluster coalesce cobol collate collation collation_catalog collation_name collation_schema collect column column_name columns command_function command_function_code comment comments commit committed concurrently condition condition_number configuration conflict connect connection connection_name constant constraint constraint_catalog constraint_name constraint_schema constraints constructor contains content continue control conversion convert copy corr corresponding cost count covar_pop covar_samp create cross csv cube cume_dist current current_catalog current_date current_default_transform_group current_path current_role current_row current_schema current_time current_timestamp current_transform_group_for_type current_user cursor cursor_name cycle data database datalink datatype date datetime_interval_code datetime_interval_precision day db deallocate debug dec decimal declare default defaults deferrable deferred defined definer degree delete delimiter delimiters dense_rank depends depth deref derived desc describe descriptor detach detail deterministic diagnostics dictionary disable discard disconnect dispatch distinct dlnewcopy dlpreviouscopy dlurlcomplete dlurlcompleteonly dlurlcompletewrite dlurlpath dlurlpathonly dlurlpathwrite dlurlscheme dlurlserver dlvalue do document domain double drop dump dynamic dynamic_function dynamic_function_code each element else elseif elsif empty enable encoding encrypted end end_frame end_partition endexec enforced enum equals errcode error escape event every except exception exclude excluding exclusive exec execute exists exit exp explain expression extension external extract false family fetch file filter final first first_value flag float floor following for force foreach foreign fortran forward found frame_row free freeze from fs full function functions fusion g general generated get global go goto grant granted greatest group grouping groups handler having header hex hierarchy hint hold hour id identity if ignore ilike immediate immediately immutable implementation implicit import in include including increment indent index indexes indicator info inherit inherits initially inline inner inout input insensitive insert instance instantiable instead int integer integrity intersect intersection interval into invoker is isnull isolation join k key key_member key_type label lag language large last last_value lateral lead leading leakproof least left length level library like like_regex limit link listen ln load local localtime localtimestamp location locator lock locked log logged loop lower m map mapping match matched materialized max max_cardinality maxvalue member merge message message_length message_octet_length message_text method min minute minvalue mod mode modifies module month more move multiset mumps name names namespace national natural nchar nclob nesting new next nfc nfd nfkc nfkd nil no none normalize normalized not nothing notice notify notnull nowait nth_value ntile null nullable nullif nulls number numeric object occurrences_regex octet_length octets of off offset oids old on only open operator option options or order ordering ordinality others out outer output over overlaps overlay overriding owned owner p pad parallel parameter parameter_mode parameter_name parameter_ordinal_position parameter_specific_catalog parameter_specific_name parameter_specific_schema parser partial partition pascal passing passthrough password path percent percent_rank percentile_cont percentile_disc perform period permission pg_context pg_datatype_name pg_exception_context pg_exception_detail pg_exception_hint placing plans pli policy portion position position_regex power precedes preceding precision prepare prepared preserve primary print_strict_params prior privileges procedural procedure procedures program public publication query quote raise range rank read reads real reassign recheck recovery recursive ref references referencing refresh regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy regr_syy reindex relative release rename repeatable replace replica requiring reset respect restart restore restrict result result_oid return returned_cardinality returned_length returned_octet_length returned_sqlstate returning returns reverse revoke right role rollback rollup routine routine_catalog routine_name routine_schema routines row row_count row_number rows rowtype rule savepoint scale schema schema_name schemas scope scope_catalog scope_name scope_schema scroll search second section security select selective self sensitive sequence sequences serializable server server_name session session_user set setof sets share show similar simple size skip slice smallint snapshot some source space specific specific_name specifictype sql sqlcode sqlerror sqlexception sqlstate sqlwarning sqrt stable stacked standalone start state statement static statistics stddev_pop stddev_samp stdin stdout storage strict strip structure style subclass_origin submultiset subscription substring substring_regex succeeds sum symmetric sysid system system_time system_user t table table_name tables tablesample tablespace temp template temporary text then ties time timestamp timezone_hour timezone_minute to token top_level_count trailing transaction transaction_active transactions_committed transactions_rolled_back transform transforms translate translate_regex translation treat trigger trigger_catalog trigger_name trigger_schema trim trim_array true truncate trusted type types uescape unbounded uncommitted under unencrypted union unique unknown unlink unlisten unlogged unnamed unnest until untyped update upper uri usage use_column use_variable user user_defined_type_catalog user_defined_type_code user_defined_type_name user_defined_type_schema using vacuum valid validate validator value value_of values var_pop var_samp varbinary varchar variable_conflict variadic varying verbose version versioning view views volatile warning when whenever where while whitespace width_bucket window with within without work wrapper write xml xmlagg xmlattributes xmlbinary xmlcast xmlcomment xmlconcat xmldeclaration xmldocument xmlelement xmlexists xmlforest xmliterate xmlnamespaces xmlparse xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltext xmlvalidate year yes zone"),builtin:set("bigint int8 bigserial serial8 bit varying varbit boolean bool box bytea character char varchar cidr circle date double precision float8 inet integer int int4 interval json jsonb line lseg macaddr macaddr8 money numeric decimal path pg_lsn point polygon real float4 smallint int2 smallserial serial2 serial serial4 text time without zone with timetz timestamp timestamptz tsquery tsvector txid_snapshot uuid xml"),atoms:set("false true null unknown"),operatorChars:/^[*\/+\-%<>!=&|^\/#@?~]/,backslashStringEscapes:false,dateSQL:set("date time timestamp"),support:set("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber nCharCast charsetCast escapeConstant")});CodeMirror.defineMIME("text/x-gql",{name:"sql",keywords:set("ancestor and asc by contains desc descendant distinct from group has in is limit offset on order select superset where"),atoms:set("false true"),builtin:set("blob datetime first key __key__ string integer double boolean null"),operatorChars:/^[*+\-%<>!=]/});CodeMirror.defineMIME("text/x-gpsql",{name:"sql",client:set("source"),keywords:set("abort absolute access action active add admin after aggregate all also alter always analyse analyze and any array as asc assertion assignment asymmetric at authorization backward before begin between bigint binary bit boolean both by cache called cascade cascaded case cast chain char character characteristics check checkpoint class close cluster coalesce codegen collate column comment commit committed concurrency concurrently configuration connection constraint constraints contains content continue conversion copy cost cpu_rate_limit create createdb createexttable createrole createuser cross csv cube current current_catalog current_date current_role current_schema current_time current_timestamp current_user cursor cycle data database day deallocate dec decimal declare decode default defaults deferrable deferred definer delete delimiter delimiters deny desc dictionary disable discard distinct distributed do document domain double drop dxl each else enable encoding encrypted end enum errors escape every except exchange exclude excluding exclusive execute exists explain extension external extract false family fetch fields filespace fill filter first float following for force foreign format forward freeze from full function global grant granted greatest group group_id grouping handler hash having header hold host hour identity if ignore ilike immediate immutable implicit in including inclusive increment index indexes inherit inherits initially inline inner inout input insensitive insert instead int integer intersect interval into invoker is isnull isolation join key language large last leading least left level like limit list listen load local localtime localtimestamp location lock log login mapping master match maxvalue median merge minute minvalue missing mode modifies modify month move name names national natural nchar new newline next no nocreatedb nocreateexttable nocreaterole nocreateuser noinherit nologin none noovercommit nosuperuser not nothing notify notnull nowait null nullif nulls numeric object of off offset oids old on only operator option options or order ordered others out outer over overcommit overlaps overlay owned owner parser partial partition partitions passing password percent percentile_cont percentile_disc placing plans position preceding precision prepare prepared preserve primary prior privileges procedural procedure protocol queue quote randomly range read readable reads real reassign recheck recursive ref references reindex reject relative release rename repeatable replace replica reset resource restart restrict returning returns revoke right role rollback rollup rootpartition row rows rule savepoint scatter schema scroll search second security segment select sequence serializable session session_user set setof sets share show similar simple smallint some split sql stable standalone start statement statistics stdin stdout storage strict strip subpartition subpartitions substring superuser symmetric sysid system table tablespace temp template temporary text then threshold ties time timestamp to trailing transaction treat trigger trim true truncate trusted type unbounded uncommitted unencrypted union unique unknown unlisten until update user using vacuum valid validation validator value values varchar variadic varying verbose version view volatile web when where whitespace window with within without work writable write xml xmlattributes xmlconcat xmlelement xmlexists xmlforest xmlparse xmlpi xmlroot xmlserialize year yes zone"),builtin:set("bigint int8 bigserial serial8 bit varying varbit boolean bool box bytea character char varchar cidr circle date double precision float float8 inet integer int int4 interval json jsonb line lseg macaddr macaddr8 money numeric decimal path pg_lsn point polygon real float4 smallint int2 smallserial serial2 serial serial4 text time without zone with timetz timestamp timestamptz tsquery tsvector txid_snapshot uuid xml"),atoms:set("false true null unknown"),operatorChars:/^[*+\-%<>!=&|^\/#@?~]/,dateSQL:set("date time timestamp"),support:set("ODBCdotTable decimallessFloat zerolessFloat binaryNumber hexNumber nCharCast charsetCast")});CodeMirror.defineMIME("text/x-sparksql",{name:"sql",keywords:set("add after all alter analyze and anti archive array as asc at between bucket buckets by cache cascade case cast change clear cluster clustered codegen collection column columns comment commit compact compactions compute concatenate cost create cross cube current current_date current_timestamp database databases datata dbproperties defined delete delimited deny desc describe dfs directories distinct distribute drop else end escaped except exchange exists explain export extended external false fields fileformat first following for format formatted from full function functions global grant group grouping having if ignore import in index indexes inner inpath inputformat insert intersect interval into is items join keys last lateral lazy left like limit lines list load local location lock locks logical macro map minus msck natural no not null nulls of on optimize option options or order out outer outputformat over overwrite partition partitioned partitions percent preceding principals purge range recordreader recordwriter recover reduce refresh regexp rename repair replace reset restrict revoke right rlike role roles rollback rollup row rows schema schemas select semi separated serde serdeproperties set sets show skewed sort sorted start statistics stored stratify struct table tables tablesample tblproperties temp temporary terminated then to touch transaction transactions transform true truncate unarchive unbounded uncache union unlock unset use using values view when where window with"),builtin:set("tinyint smallint int bigint boolean float double string binary timestamp decimal array map struct uniontype delimited serde sequencefile textfile rcfile inputformat outputformat"),atoms:set("false true null"),operatorChars:/^[*\/+\-%<>!=~&|^]/,dateSQL:set("date time timestamp"),support:set("ODBCdotTable doubleQuote zerolessFloat")});CodeMirror.defineMIME("text/x-esper",{name:"sql",client:set("source"),keywords:set("alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit after all and as at asc avedev avg between by case cast coalesce count create current_timestamp day days delete define desc distinct else end escape events every exists false first from full group having hour hours in inner insert instanceof into irstream is istream join last lastweekday left limit like max match_recognize matches median measures metadatasql min minute minutes msec millisecond milliseconds not null offset on or order outer output partition pattern prev prior regexp retain-union retain-intersection right rstream sec second seconds select set some snapshot sql stddev sum then true unidirectional until update variable weekday when where window"),builtin:{},atoms:set("false true null"),operatorChars:/^[*+\-%<>!=&|^\/#@?~]/,dateSQL:set("time"),support:set("decimallessFloat zerolessFloat binaryNumber hexNumber")})});
\ No newline at end of file
diff --git a/datasette/static/codemirror-5.57.0.min.css b/datasette/static/codemirror-5.57.0.min.css
new file mode 100644
index 00000000..0adf786f
--- /dev/null
+++ b/datasette/static/codemirror-5.57.0.min.css
@@ -0,0 +1 @@
+.CodeMirror{font-family:monospace;height:300px;color:#000;direction:ltr}.CodeMirror-lines{padding:4px 0}.CodeMirror pre.CodeMirror-line,.CodeMirror pre.CodeMirror-line-like{padding:0 4px}.CodeMirror-gutter-filler,.CodeMirror-scrollbar-filler{background-color:#fff}.CodeMirror-gutters{border-right:1px solid #ddd;background-color:#f7f7f7;white-space:nowrap}.CodeMirror-linenumber{padding:0 3px 0 5px;min-width:20px;text-align:right;color:#999;white-space:nowrap}.CodeMirror-guttermarker{color:#000}.CodeMirror-guttermarker-subtle{color:#999}.CodeMirror-cursor{border-left:1px solid #000;border-right:none;width:0}.CodeMirror div.CodeMirror-secondarycursor{border-left:1px solid silver}.cm-fat-cursor .CodeMirror-cursor{width:auto;border:0!important;background:#7e7}.cm-fat-cursor div.CodeMirror-cursors{z-index:1}.cm-fat-cursor-mark{background-color:rgba(20,255,20,.5);-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite}.cm-animate-fat-cursor{width:auto;border:0;-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite;background-color:#7e7}@-moz-keyframes blink{50%{background-color:transparent}}@-webkit-keyframes blink{50%{background-color:transparent}}@keyframes blink{50%{background-color:transparent}}.cm-tab{display:inline-block;text-decoration:inherit}.CodeMirror-rulers{position:absolute;left:0;right:0;top:-50px;bottom:0;overflow:hidden}.CodeMirror-ruler{border-left:1px solid #ccc;top:0;bottom:0;position:absolute}.cm-s-default .cm-header{color:#00f}.cm-s-default .cm-quote{color:#090}.cm-negative{color:#d44}.cm-positive{color:#292}.cm-header,.cm-strong{font-weight:700}.cm-em{font-style:italic}.cm-link{text-decoration:underline}.cm-strikethrough{text-decoration:line-through}.cm-s-default .cm-keyword{color:#708}.cm-s-default .cm-atom{color:#219}.cm-s-default .cm-number{color:#164}.cm-s-default .cm-def{color:#00f}.cm-s-default .cm-variable-2{color:#05a}.cm-s-default .cm-type,.cm-s-default .cm-variable-3{color:#085}.cm-s-default .cm-comment{color:#a50}.cm-s-default .cm-string{color:#a11}.cm-s-default .cm-string-2{color:#f50}.cm-s-default .cm-meta{color:#555}.cm-s-default .cm-qualifier{color:#555}.cm-s-default .cm-builtin{color:#30a}.cm-s-default .cm-bracket{color:#997}.cm-s-default .cm-tag{color:#170}.cm-s-default .cm-attribute{color:#00c}.cm-s-default .cm-hr{color:#999}.cm-s-default .cm-link{color:#00c}.cm-s-default .cm-error{color:red}.cm-invalidchar{color:red}.CodeMirror-composing{border-bottom:2px solid}div.CodeMirror span.CodeMirror-matchingbracket{color:#0b0}div.CodeMirror span.CodeMirror-nonmatchingbracket{color:#a22}.CodeMirror-matchingtag{background:rgba(255,150,0,.3)}.CodeMirror-activeline-background{background:#e8f2ff}.CodeMirror{position:relative;overflow:hidden;background:#fff}.CodeMirror-scroll{overflow:scroll!important;margin-bottom:-50px;margin-right:-50px;padding-bottom:50px;height:100%;outline:0;position:relative}.CodeMirror-sizer{position:relative;border-right:50px solid transparent}.CodeMirror-gutter-filler,.CodeMirror-hscrollbar,.CodeMirror-scrollbar-filler,.CodeMirror-vscrollbar{position:absolute;z-index:6;display:none}.CodeMirror-vscrollbar{right:0;top:0;overflow-x:hidden;overflow-y:scroll}.CodeMirror-hscrollbar{bottom:0;left:0;overflow-y:hidden;overflow-x:scroll}.CodeMirror-scrollbar-filler{right:0;bottom:0}.CodeMirror-gutter-filler{left:0;bottom:0}.CodeMirror-gutters{position:absolute;left:0;top:0;min-height:100%;z-index:3}.CodeMirror-gutter{white-space:normal;height:100%;display:inline-block;vertical-align:top;margin-bottom:-50px}.CodeMirror-gutter-wrapper{position:absolute;z-index:4;background:0 0!important;border:none!important}.CodeMirror-gutter-background{position:absolute;top:0;bottom:0;z-index:4}.CodeMirror-gutter-elt{position:absolute;cursor:default;z-index:4}.CodeMirror-gutter-wrapper ::selection{background-color:transparent}.CodeMirror-gutter-wrapper ::-moz-selection{background-color:transparent}.CodeMirror-lines{cursor:text;min-height:1px}.CodeMirror pre.CodeMirror-line,.CodeMirror pre.CodeMirror-line-like{-moz-border-radius:0;-webkit-border-radius:0;border-radius:0;border-width:0;background:0 0;font-family:inherit;font-size:inherit;margin:0;white-space:pre;word-wrap:normal;line-height:inherit;color:inherit;z-index:2;position:relative;overflow:visible;-webkit-tap-highlight-color:transparent;-webkit-font-variant-ligatures:contextual;font-variant-ligatures:contextual}.CodeMirror-wrap pre.CodeMirror-line,.CodeMirror-wrap pre.CodeMirror-line-like{word-wrap:break-word;white-space:pre-wrap;word-break:normal}.CodeMirror-linebackground{position:absolute;left:0;right:0;top:0;bottom:0;z-index:0}.CodeMirror-linewidget{position:relative;z-index:2;padding:.1px}.CodeMirror-rtl pre{direction:rtl}.CodeMirror-code{outline:0}.CodeMirror-gutter,.CodeMirror-gutters,.CodeMirror-linenumber,.CodeMirror-scroll,.CodeMirror-sizer{-moz-box-sizing:content-box;box-sizing:content-box}.CodeMirror-measure{position:absolute;width:100%;height:0;overflow:hidden;visibility:hidden}.CodeMirror-cursor{position:absolute;pointer-events:none}.CodeMirror-measure pre{position:static}div.CodeMirror-cursors{visibility:hidden;position:relative;z-index:3}div.CodeMirror-dragcursors{visibility:visible}.CodeMirror-focused div.CodeMirror-cursors{visibility:visible}.CodeMirror-selected{background:#d9d9d9}.CodeMirror-focused .CodeMirror-selected{background:#d7d4f0}.CodeMirror-crosshair{cursor:crosshair}.CodeMirror-line::selection,.CodeMirror-line>span::selection,.CodeMirror-line>span>span::selection{background:#d7d4f0}.CodeMirror-line::-moz-selection,.CodeMirror-line>span::-moz-selection,.CodeMirror-line>span>span::-moz-selection{background:#d7d4f0}.cm-searching{background-color:#ffa;background-color:rgba(255,255,0,.4)}.cm-force-border{padding-right:.1px}@media print{.CodeMirror div.CodeMirror-cursors{visibility:hidden}}.cm-tab-wrap-hack:after{content:''}span.CodeMirror-selectedtext{background:0 0}
\ No newline at end of file
diff --git a/datasette/static/codemirror-5.57.0.min.js b/datasette/static/codemirror-5.57.0.min.js
new file mode 100644
index 00000000..a8ef1854
--- /dev/null
+++ b/datasette/static/codemirror-5.57.0.min.js
@@ -0,0 +1,11 @@
+/*
+  CodeMirror, copyright (c) by Marijn Haverbeke and others
+  Distributed under an MIT license: https://codemirror.net/LICENSE
+
+  This is CodeMirror (https://codemirror.net), a code editor
+  implemented in JavaScript on top of the browser's DOM.
+
+  You can find some technical background for some of the code below
+  at http://marijnhaverbeke.nl/blog/#cm-internals .
+*/
+(function(global,factory){typeof exports==="object"&&typeof module!=="undefined"?module.exports=factory():typeof define==="function"&&define.amd?define(factory):(global=global||self,global.CodeMirror=factory())})(this,function(){"use strict";var userAgent=navigator.userAgent;var platform=navigator.platform;var gecko=/gecko\/\d/i.test(userAgent);var ie_upto10=/MSIE \d/.test(userAgent);var ie_11up=/Trident\/(?:[7-9]|\d{2,})\..*rv:(\d+)/.exec(userAgent);var edge=/Edge\/(\d+)/.exec(userAgent);var ie=ie_upto10||ie_11up||edge;var ie_version=ie&&(ie_upto10?document.documentMode||6:+(edge||ie_11up)[1]);var webkit=!edge&&/WebKit\//.test(userAgent);var qtwebkit=webkit&&/Qt\/\d+\.\d+/.test(userAgent);var chrome=!edge&&/Chrome\//.test(userAgent);var presto=/Opera\//.test(userAgent);var safari=/Apple Computer/.test(navigator.vendor);var mac_geMountainLion=/Mac OS X 1\d\D([8-9]|\d\d)\D/.test(userAgent);var phantom=/PhantomJS/.test(userAgent);var ios=!edge&&/AppleWebKit/.test(userAgent)&&/Mobile\/\w+/.test(userAgent);var android=/Android/.test(userAgent);var mobile=ios||android||/webOS|BlackBerry|Opera Mini|Opera Mobi|IEMobile/i.test(userAgent);var mac=ios||/Mac/.test(platform);var chromeOS=/\bCrOS\b/.test(userAgent);var windows=/win/i.test(platform);var presto_version=presto&&userAgent.match(/Version\/(\d*\.\d*)/);if(presto_version){presto_version=Number(presto_version[1])}if(presto_version&&presto_version>=15){presto=false;webkit=true}var flipCtrlCmd=mac&&(qtwebkit||presto&&(presto_version==null||presto_version<12.11));var captureRightClick=gecko||ie&&ie_version>=9;function classTest(cls){return new RegExp("(^|\\s)"+cls+"(?:$|\\s)\\s*")}var rmClass=function(node,cls){var current=node.className;var match=classTest(cls).exec(current);if(match){var after=current.slice(match.index+match[0].length);node.className=current.slice(0,match.index)+(after?match[1]+after:"")}};function removeChildren(e){for(var count=e.childNodes.length;count>0;--count){e.removeChild(e.firstChild)}return e}function removeChildrenAndAdd(parent,e){return removeChildren(parent).appendChild(e)}function elt(tag,content,className,style){var e=document.createElement(tag);if(className){e.className=className}if(style){e.style.cssText=style}if(typeof content=="string"){e.appendChild(document.createTextNode(content))}else if(content){for(var i=0;i=end){return n+(end-i)}n+=nextTab-i;n+=tabSize-n%tabSize;i=nextTab+1}}var Delayed=function(){this.id=null;this.f=null;this.time=0;this.handler=bind(this.onTimeout,this)};Delayed.prototype.onTimeout=function(self){self.id=0;if(self.time<=+new Date){self.f()}else{setTimeout(self.handler,self.time-+new Date)}};Delayed.prototype.set=function(ms,f){this.f=f;var time=+new Date+ms;if(!this.id||time=goal){return pos+Math.min(skipped,goal-col)}col+=nextTab-pos;col+=tabSize-col%tabSize;pos=nextTab+1;if(col>=goal){return pos}}}var spaceStrs=[""];function spaceStr(n){while(spaceStrs.length<=n){spaceStrs.push(lst(spaceStrs)+" ")}return spaceStrs[n]}function lst(arr){return arr[arr.length-1]}function map(array,f){var out=[];for(var i=0;i"€"&&(ch.toUpperCase()!=ch.toLowerCase()||nonASCIISingleCaseWordChar.test(ch))}function isWordChar(ch,helper){if(!helper){return isWordCharBasic(ch)}if(helper.source.indexOf("\\w")>-1&&isWordCharBasic(ch)){return true}return helper.test(ch)}function isEmpty(obj){for(var n in obj){if(obj.hasOwnProperty(n)&&obj[n]){return false}}return true}var extendingChars=/[\u0300-\u036f\u0483-\u0489\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u064b-\u065e\u0670\u06d6-\u06dc\u06de-\u06e4\u06e7\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0900-\u0902\u093c\u0941-\u0948\u094d\u0951-\u0955\u0962\u0963\u0981\u09bc\u09be\u09c1-\u09c4\u09cd\u09d7\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a70\u0a71\u0a75\u0a81\u0a82\u0abc\u0ac1-\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3e\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b57\u0b62\u0b63\u0b82\u0bbe\u0bc0\u0bcd\u0bd7\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62\u0c63\u0cbc\u0cbf\u0cc2\u0cc6\u0ccc\u0ccd\u0cd5\u0cd6\u0ce2\u0ce3\u0d3e\u0d41-\u0d44\u0d4d\u0d57\u0d62\u0d63\u0dca\u0dcf\u0dd2-\u0dd4\u0dd6\u0ddf\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb\u0ebc\u0ec8-\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86\u0f87\u0f90-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039\u103a\u103d\u103e\u1058\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085\u1086\u108d\u109d\u135f\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927\u1928\u1932\u1939-\u193b\u1a17\u1a18\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80\u1b81\u1ba2-\u1ba5\u1ba8\u1ba9\u1c2c-\u1c33\u1c36\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1dc0-\u1de6\u1dfd-\u1dff\u200c\u200d\u20d0-\u20f0\u2cef-\u2cf1\u2de0-\u2dff\u302a-\u302f\u3099\u309a\ua66f-\ua672\ua67c\ua67d\ua6f0\ua6f1\ua802\ua806\ua80b\ua825\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31\uaa32\uaa35\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7\uaab8\uaabe\uaabf\uaac1\uabe5\uabe8\uabed\udc00-\udfff\ufb1e\ufe00-\ufe0f\ufe20-\ufe26\uff9e\uff9f]/;function isExtendingChar(ch){return ch.charCodeAt(0)>=768&&extendingChars.test(ch)}function skipExtendingChars(str,pos,dir){while((dir<0?pos>0:posto?-1:1;for(;;){if(from==to){return from}var midF=(from+to)/2,mid=dir<0?Math.ceil(midF):Math.floor(midF);if(mid==from){return pred(mid)?from:to}if(pred(mid)){to=mid}else{from=mid+dir}}}function iterateBidiSections(order,from,to,f){if(!order){return f(from,to,"ltr",0)}var found=false;for(var i=0;ifrom||from==to&&part.to==from){f(Math.max(part.from,from),Math.min(part.to,to),part.level==1?"rtl":"ltr",i);found=true}}if(!found){f(from,to,"ltr")}}var bidiOther=null;function getBidiPartAt(order,ch,sticky){var found;bidiOther=null;for(var i=0;ich){return i}if(cur.to==ch){if(cur.from!=cur.to&&sticky=="before"){found=i}else{bidiOther=i}}if(cur.from==ch){if(cur.from!=cur.to&&sticky!="before"){found=i}else{bidiOther=i}}}return found!=null?found:bidiOther}var bidiOrdering=function(){var lowTypes="bbbbbbbbbtstwsbbbbbbbbbbbbbbssstwNN%%%NNNNNN,N,N1111111111NNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNbbbbbbsbbbbbbbbbbbbbbbbbbbbbbbbbb,N%%%%NNNNLNNNNN%%11NLNNN1LNNNNNLLLLLLLLLLLLLLLLLLLLLLLNLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLN";var arabicTypes="nnnnnnNNr%%r,rNNmmmmmmmmmmmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmmmmmmmmmmmmmmmnnnnnnnnnn%nnrrrmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmnNmmmmmmrrmmNmmmmrr1111111111";function charType(code){if(code<=247){return lowTypes.charAt(code)}else if(1424<=code&&code<=1524){return"R"}else if(1536<=code&&code<=1785){return arabicTypes.charAt(code-1536)}else if(1774<=code&&code<=2220){return"r"}else if(8192<=code&&code<=8203){return"w"}else if(code==8204){return"b"}else{return"L"}}var bidiRE=/[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac]/;var isNeutral=/[stwN]/,isStrong=/[LRr]/,countsAsLeft=/[Lb1n]/,countsAsNum=/[1n]/;function BidiSpan(level,from,to){this.level=level;this.from=from;this.to=to}return function(str,direction){var outerType=direction=="ltr"?"L":"R";if(str.length==0||direction=="ltr"&&!bidiRE.test(str)){return false}var len=str.length,types=[];for(var i=0;i-1){map[type]=arr.slice(0,index).concat(arr.slice(index+1))}}}}function signal(emitter,type){var handlers=getHandlers(emitter,type);if(!handlers.length){return}var args=Array.prototype.slice.call(arguments,2);for(var i=0;i0}function eventMixin(ctor){ctor.prototype.on=function(type,f){on(this,type,f)};ctor.prototype.off=function(type,f){off(this,type,f)}}function e_preventDefault(e){if(e.preventDefault){e.preventDefault()}else{e.returnValue=false}}function e_stopPropagation(e){if(e.stopPropagation){e.stopPropagation()}else{e.cancelBubble=true}}function e_defaultPrevented(e){return e.defaultPrevented!=null?e.defaultPrevented:e.returnValue==false}function e_stop(e){e_preventDefault(e);e_stopPropagation(e)}function e_target(e){return e.target||e.srcElement}function e_button(e){var b=e.which;if(b==null){if(e.button&1){b=1}else if(e.button&2){b=3}else if(e.button&4){b=2}}if(mac&&e.ctrlKey&&b==1){b=3}return b}var dragAndDrop=function(){if(ie&&ie_version<9){return false}var div=elt("div");return"draggable"in div||"dragDrop"in div}();var zwspSupported;function zeroWidthElement(measure){if(zwspSupported==null){var test=elt("span","​");removeChildrenAndAdd(measure,elt("span",[test,document.createTextNode("x")]));if(measure.firstChild.offsetHeight!=0){zwspSupported=test.offsetWidth<=1&&test.offsetHeight>2&&!(ie&&ie_version<8)}}var node=zwspSupported?elt("span","​"):elt("span"," ",null,"display: inline-block; width: 1px; margin-right: -1px");node.setAttribute("cm-text","");return node}var badBidiRects;function hasBadBidiRects(measure){if(badBidiRects!=null){return badBidiRects}var txt=removeChildrenAndAdd(measure,document.createTextNode("AخA"));var r0=range(txt,0,1).getBoundingClientRect();var r1=range(txt,1,2).getBoundingClientRect();removeChildren(measure);if(!r0||r0.left==r0.right){return false}return badBidiRects=r1.right-r0.right<3}var splitLinesAuto="\n\nb".split(/\n/).length!=3?function(string){var pos=0,result=[],l=string.length;while(pos<=l){var nl=string.indexOf("\n",pos);if(nl==-1){nl=string.length}var line=string.slice(pos,string.charAt(nl-1)=="\r"?nl-1:nl);var rt=line.indexOf("\r");if(rt!=-1){result.push(line.slice(0,rt));pos+=rt+1}else{result.push(line);pos=nl+1}}return result}:function(string){return string.split(/\r\n?|\n/)};var hasSelection=window.getSelection?function(te){try{return te.selectionStart!=te.selectionEnd}catch(e){return false}}:function(te){var range;try{range=te.ownerDocument.selection.createRange()}catch(e){}if(!range||range.parentElement()!=te){return false}return range.compareEndPoints("StartToEnd",range)!=0};var hasCopyEvent=function(){var e=elt("div");if("oncopy"in e){return true}e.setAttribute("oncopy","return;");return typeof e.oncopy=="function"}();var badZoomedRects=null;function hasBadZoomedRects(measure){if(badZoomedRects!=null){return badZoomedRects}var node=removeChildrenAndAdd(measure,elt("span","x"));var normal=node.getBoundingClientRect();var fromRange=range(node,0,1).getBoundingClientRect();return badZoomedRects=Math.abs(normal.left-fromRange.left)>1}var modes={},mimeModes={};function defineMode(name,mode){if(arguments.length>2){mode.dependencies=Array.prototype.slice.call(arguments,2)}modes[name]=mode}function defineMIME(mime,spec){mimeModes[mime]=spec}function resolveMode(spec){if(typeof spec=="string"&&mimeModes.hasOwnProperty(spec)){spec=mimeModes[spec]}else if(spec&&typeof spec.name=="string"&&mimeModes.hasOwnProperty(spec.name)){var found=mimeModes[spec.name];if(typeof found=="string"){found={name:found}}spec=createObj(found,spec);spec.name=found.name}else if(typeof spec=="string"&&/^[\w\-]+\/[\w\-]+\+xml$/.test(spec)){return resolveMode("application/xml")}else if(typeof spec=="string"&&/^[\w\-]+\/[\w\-]+\+json$/.test(spec)){return resolveMode("application/json")}if(typeof spec=="string"){return{name:spec}}else{return spec||{name:"null"}}}function getMode(options,spec){spec=resolveMode(spec);var mfactory=modes[spec.name];if(!mfactory){return getMode(options,"text/plain")}var modeObj=mfactory(options,spec);if(modeExtensions.hasOwnProperty(spec.name)){var exts=modeExtensions[spec.name];for(var prop in exts){if(!exts.hasOwnProperty(prop)){continue}if(modeObj.hasOwnProperty(prop)){modeObj["_"+prop]=modeObj[prop]}modeObj[prop]=exts[prop]}}modeObj.name=spec.name;if(spec.helperType){modeObj.helperType=spec.helperType}if(spec.modeProps){for(var prop$1 in spec.modeProps){modeObj[prop$1]=spec.modeProps[prop$1]}}return modeObj}var modeExtensions={};function extendMode(mode,properties){var exts=modeExtensions.hasOwnProperty(mode)?modeExtensions[mode]:modeExtensions[mode]={};copyObj(properties,exts)}function copyState(mode,state){if(state===true){return state}if(mode.copyState){return mode.copyState(state)}var nstate={};for(var n in state){var val=state[n];if(val instanceof Array){val=val.concat([])}nstate[n]=val}return nstate}function innerMode(mode,state){var info;while(mode.innerMode){info=mode.innerMode(state);if(!info||info.mode==mode){break}state=info.state;mode=info.mode}return info||{mode:mode,state:state}}function startState(mode,a1,a2){return mode.startState?mode.startState(a1,a2):true}var StringStream=function(string,tabSize,lineOracle){this.pos=this.start=0;this.string=string;this.tabSize=tabSize||8;this.lastColumnPos=this.lastColumnValue=0;this.lineStart=0;this.lineOracle=lineOracle};StringStream.prototype.eol=function(){return this.pos>=this.string.length};StringStream.prototype.sol=function(){return this.pos==this.lineStart};StringStream.prototype.peek=function(){return this.string.charAt(this.pos)||undefined};StringStream.prototype.next=function(){if(this.posstart};StringStream.prototype.eatSpace=function(){var start=this.pos;while(/[\s\u00a0]/.test(this.string.charAt(this.pos))){++this.pos}return this.pos>start};StringStream.prototype.skipToEnd=function(){this.pos=this.string.length};StringStream.prototype.skipTo=function(ch){var found=this.string.indexOf(ch,this.pos);if(found>-1){this.pos=found;return true}};StringStream.prototype.backUp=function(n){this.pos-=n};StringStream.prototype.column=function(){if(this.lastColumnPos0){return null}if(match&&consume!==false){this.pos+=match[0].length}return match}};StringStream.prototype.current=function(){return this.string.slice(this.start,this.pos)};StringStream.prototype.hideFirstChars=function(n,inner){this.lineStart+=n;try{return inner()}finally{this.lineStart-=n}};StringStream.prototype.lookAhead=function(n){var oracle=this.lineOracle;return oracle&&oracle.lookAhead(n)};StringStream.prototype.baseToken=function(){var oracle=this.lineOracle;return oracle&&oracle.baseToken(this.pos)};function getLine(doc,n){n-=doc.first;if(n<0||n>=doc.size){throw new Error("There is no line "+(n+doc.first)+" in the document.")}var chunk=doc;while(!chunk.lines){for(var i=0;;++i){var child=chunk.children[i],sz=child.chunkSize();if(n=doc.first&&llast){return Pos(last,getLine(doc,last).text.length)}return clipToLen(pos,getLine(doc,pos.line).text.length)}function clipToLen(pos,linelen){var ch=pos.ch;if(ch==null||ch>linelen){return Pos(pos.line,linelen)}else if(ch<0){return Pos(pos.line,0)}else{return pos}}function clipPosArray(doc,array){var out=[];for(var i=0;ithis.maxLookAhead){this.maxLookAhead=n}return line};Context.prototype.baseToken=function(n){if(!this.baseTokens){return null}while(this.baseTokens[this.baseTokenPos]<=n){this.baseTokenPos+=2}var type=this.baseTokens[this.baseTokenPos+1];return{type:type&&type.replace(/( |^)overlay .*/,""),size:this.baseTokens[this.baseTokenPos]-n}};Context.prototype.nextLine=function(){this.line++;if(this.maxLookAhead>0){this.maxLookAhead--}};Context.fromSaved=function(doc,saved,line){if(saved instanceof SavedContext){return new Context(doc,copyState(doc.mode,saved.state),line,saved.lookAhead)}else{return new Context(doc,copyState(doc.mode,saved),line)}};Context.prototype.save=function(copy){var state=copy!==false?copyState(this.doc.mode,this.state):this.state;return this.maxLookAhead>0?new SavedContext(state,this.maxLookAhead):state};function highlightLine(cm,line,context,forceToEnd){var st=[cm.state.modeGen],lineClasses={};runMode(cm,line.text,cm.doc.mode,context,function(end,style){return st.push(end,style)},lineClasses,forceToEnd);var state=context.state;var loop=function(o){context.baseTokens=st;var overlay=cm.state.overlays[o],i=1,at=0;context.state=true;runMode(cm,line.text,overlay.mode,context,function(end,style){var start=i;while(atend){st.splice(i,1,end,st[i+1],i_end)}i+=2;at=Math.min(end,i_end)}if(!style){return}if(overlay.opaque){st.splice(start,i-start,end,"overlay "+style);i=start+2}else{for(;startcm.options.maxHighlightLength&©State(cm.doc.mode,context.state);var result=highlightLine(cm,line,context);if(resetState){context.state=resetState}line.stateAfter=context.save(!resetState);line.styles=result.styles;if(result.classes){line.styleClasses=result.classes}else if(line.styleClasses){line.styleClasses=null}if(updateFrontier===cm.doc.highlightFrontier){cm.doc.modeFrontier=Math.max(cm.doc.modeFrontier,++cm.doc.highlightFrontier)}}return line.styles}function getContextBefore(cm,n,precise){var doc=cm.doc,display=cm.display;if(!doc.mode.startState){return new Context(doc,true,n)}var start=findStartLine(cm,n,precise);var saved=start>doc.first&&getLine(doc,start-1).stateAfter;var context=saved?Context.fromSaved(doc,saved,start):new Context(doc,startState(doc.mode),start);doc.iter(start,n,function(line){processLine(cm,line.text,context);var pos=context.line;line.stateAfter=pos==n-1||pos%5==0||pos>=display.viewFrom&&posstream.start){return style}}throw new Error("Mode "+mode.name+" failed to advance stream.")}var Token=function(stream,type,state){this.start=stream.start;this.end=stream.pos;this.string=stream.current();this.type=type||null;this.state=state};function takeToken(cm,pos,precise,asArray){var doc=cm.doc,mode=doc.mode,style;pos=clipPos(doc,pos);var line=getLine(doc,pos.line),context=getContextBefore(cm,pos.line,precise);var stream=new StringStream(line.text,cm.options.tabSize,context),tokens;if(asArray){tokens=[]}while((asArray||stream.poscm.options.maxHighlightLength){flattenSpans=false;if(forceToEnd){processLine(cm,text,context,stream.pos)}stream.pos=text.length;style=null}else{style=extractLineClasses(readToken(mode,stream,context.state,inner),lineClasses)}if(inner){var mName=inner[0].name;if(mName){style="m-"+(style?mName+" "+style:mName)}}if(!flattenSpans||curStyle!=style){while(curStartlim;--search){if(search<=doc.first){return doc.first}var line=getLine(doc,search-1),after=line.stateAfter;if(after&&(!precise||search+(after instanceof SavedContext?after.lookAhead:0)<=doc.modeFrontier)){return search}var indented=countColumn(line.text,null,cm.options.tabSize);if(minline==null||minindent>indented){minline=search-1;minindent=indented}}return minline}function retreatFrontier(doc,n){doc.modeFrontier=Math.min(doc.modeFrontier,n);if(doc.highlightFrontierstart;line--){var saved=getLine(doc,line).stateAfter;if(saved&&(!(saved instanceof SavedContext)||line+saved.lookAhead=startCh:span.to>startCh);(nw||(nw=[])).push(new MarkedSpan(marker,span.from,endsAfter?null:span.to))}}}return nw}function markedSpansAfter(old,endCh,isInsert){var nw;if(old){for(var i=0;i=endCh:span.to>endCh);if(endsAfter||span.from==endCh&&marker.type=="bookmark"&&(!isInsert||span.marker.insertLeft)){var startsBefore=span.from==null||(marker.inclusiveLeft?span.from<=endCh:span.from0&&first){for(var i$2=0;i$20){continue}var newParts=[j,1],dfrom=cmp(p.from,m.from),dto=cmp(p.to,m.to);if(dfrom<0||!mk.inclusiveLeft&&!dfrom){newParts.push({from:p.from,to:m.from})}if(dto>0||!mk.inclusiveRight&&!dto){newParts.push({from:m.to,to:p.to})}parts.splice.apply(parts,newParts);j+=newParts.length-3}}return parts}function detachMarkedSpans(line){var spans=line.markedSpans;if(!spans){return}for(var i=0;ich)&&(!found||compareCollapsedMarkers(found,sp.marker)<0)){found=sp.marker}}}return found}function conflictingCollapsedRange(doc,lineNo,from,to,marker){var line=getLine(doc,lineNo);var sps=sawCollapsedSpans&&line.markedSpans;if(sps){for(var i=0;i=0&&toCmp<=0||fromCmp<=0&&toCmp>=0){continue}if(fromCmp<=0&&(sp.marker.inclusiveRight&&marker.inclusiveLeft?cmp(found.to,from)>=0:cmp(found.to,from)>0)||fromCmp>=0&&(sp.marker.inclusiveRight&&marker.inclusiveLeft?cmp(found.from,to)<=0:cmp(found.from,to)<0)){return true}}}}function visualLine(line){var merged;while(merged=collapsedSpanAtStart(line)){line=merged.find(-1,true).line}return line}function visualLineEnd(line){var merged;while(merged=collapsedSpanAtEnd(line)){line=merged.find(1,true).line}return line}function visualLineContinued(line){var merged,lines;while(merged=collapsedSpanAtEnd(line)){line=merged.find(1,true).line;(lines||(lines=[])).push(line)}return lines}function visualLineNo(doc,lineN){var line=getLine(doc,lineN),vis=visualLine(line);if(line==vis){return lineN}return lineNo(vis)}function visualLineEndNo(doc,lineN){if(lineN>doc.lastLine()){return lineN}var line=getLine(doc,lineN),merged;if(!lineIsHidden(doc,line)){return lineN}while(merged=collapsedSpanAtEnd(line)){line=merged.find(1,true).line}return lineNo(line)+1}function lineIsHidden(doc,line){var sps=sawCollapsedSpans&&line.markedSpans;if(sps){for(var sp=void 0,i=0;id.maxLineLength){d.maxLineLength=len;d.maxLine=line}})}var Line=function(text,markedSpans,estimateHeight){this.text=text;attachMarkedSpans(this,markedSpans);this.height=estimateHeight?estimateHeight(this):1};Line.prototype.lineNo=function(){return lineNo(this)};eventMixin(Line);function updateLine(line,text,markedSpans,estimateHeight){line.text=text;if(line.stateAfter){line.stateAfter=null}if(line.styles){line.styles=null}if(line.order!=null){line.order=null}detachMarkedSpans(line);attachMarkedSpans(line,markedSpans);var estHeight=estimateHeight?estimateHeight(line):1;if(estHeight!=line.height){updateLineHeight(line,estHeight)}}function cleanUpLine(line){line.parent=null;detachMarkedSpans(line)}var styleToClassCache={},styleToClassCacheWithMode={};function interpretTokenStyle(style,options){if(!style||/^\s*$/.test(style)){return null}var cache=options.addModeClass?styleToClassCacheWithMode:styleToClassCache;return cache[style]||(cache[style]=style.replace(/\S+/g,"cm-$&"))}function buildLineContent(cm,lineView){var content=eltP("span",null,null,webkit?"padding-right: .1px":null);var builder={pre:eltP("pre",[content],"CodeMirror-line"),content:content,col:0,pos:0,cm:cm,trailingSpace:false,splitSpaces:cm.getOption("lineWrapping")};lineView.measure={};for(var i=0;i<=(lineView.rest?lineView.rest.length:0);i++){var line=i?lineView.rest[i-1]:lineView.line,order=void 0;builder.pos=0;builder.addToken=buildToken;if(hasBadBidiRects(cm.display.measure)&&(order=getOrder(line,cm.doc.direction))){builder.addToken=buildTokenBadBidi(builder.addToken,order)}builder.map=[];var allowFrontierUpdate=lineView!=cm.display.externalMeasured&&lineNo(line);insertLineContent(line,builder,getLineStyles(cm,line,allowFrontierUpdate));if(line.styleClasses){if(line.styleClasses.bgClass){builder.bgClass=joinClasses(line.styleClasses.bgClass,builder.bgClass||"")}if(line.styleClasses.textClass){builder.textClass=joinClasses(line.styleClasses.textClass,builder.textClass||"")}}if(builder.map.length==0){builder.map.push(0,0,builder.content.appendChild(zeroWidthElement(cm.display.measure)))}if(i==0){lineView.measure.map=builder.map;lineView.measure.cache={}}else{(lineView.measure.maps||(lineView.measure.maps=[])).push(builder.map);(lineView.measure.caches||(lineView.measure.caches=[])).push({})}}if(webkit){var last=builder.content.lastChild;if(/\bcm-tab\b/.test(last.className)||last.querySelector&&last.querySelector(".cm-tab")){builder.content.className="cm-tab-wrap-hack"}}signal(cm,"renderLine",cm,lineView.line,builder.pre);if(builder.pre.className){builder.textClass=joinClasses(builder.pre.className,builder.textClass||"")}return builder}function defaultSpecialCharPlaceholder(ch){var token=elt("span","•","cm-invalidchar");token.title="\\u"+ch.charCodeAt(0).toString(16);token.setAttribute("aria-label",token.title);return token}function buildToken(builder,text,style,startStyle,endStyle,css,attributes){if(!text){return}var displayText=builder.splitSpaces?splitSpaces(text,builder.trailingSpace):text;var special=builder.cm.state.specialChars,mustWrap=false;var content;if(!special.test(text)){builder.col+=text.length;content=document.createTextNode(displayText);builder.map.push(builder.pos,builder.pos+text.length,content);if(ie&&ie_version<9){mustWrap=true}builder.pos+=text.length}else{content=document.createDocumentFragment();var pos=0;while(true){special.lastIndex=pos;var m=special.exec(text);var skipped=m?m.index-pos:text.length-pos;if(skipped){var txt=document.createTextNode(displayText.slice(pos,pos+skipped));if(ie&&ie_version<9){content.appendChild(elt("span",[txt]))}else{content.appendChild(txt)}builder.map.push(builder.pos,builder.pos+skipped,txt);builder.col+=skipped;builder.pos+=skipped}if(!m){break}pos+=skipped+1;var txt$1=void 0;if(m[0]=="\t"){var tabSize=builder.cm.options.tabSize,tabWidth=tabSize-builder.col%tabSize;txt$1=content.appendChild(elt("span",spaceStr(tabWidth),"cm-tab"));txt$1.setAttribute("role","presentation");txt$1.setAttribute("cm-text","\t");builder.col+=tabWidth}else if(m[0]=="\r"||m[0]=="\n"){txt$1=content.appendChild(elt("span",m[0]=="\r"?"␍":"␤","cm-invalidchar"));txt$1.setAttribute("cm-text",m[0]);builder.col+=1}else{txt$1=builder.cm.options.specialCharPlaceholder(m[0]);txt$1.setAttribute("cm-text",m[0]);if(ie&&ie_version<9){content.appendChild(elt("span",[txt$1]))}else{content.appendChild(txt$1)}builder.col+=1}builder.map.push(builder.pos,builder.pos+1,txt$1);builder.pos++}}builder.trailingSpace=displayText.charCodeAt(text.length-1)==32;if(style||startStyle||endStyle||mustWrap||css){var fullStyle=style||"";if(startStyle){fullStyle+=startStyle}if(endStyle){fullStyle+=endStyle}var token=elt("span",[content],fullStyle,css);if(attributes){for(var attr in attributes){if(attributes.hasOwnProperty(attr)&&attr!="style"&&attr!="class"){token.setAttribute(attr,attributes[attr])}}}return builder.content.appendChild(token)}builder.content.appendChild(content)}function splitSpaces(text,trailingBefore){if(text.length>1&&!/  /.test(text)){return text}var spaceBefore=trailingBefore,result="";for(var i=0;istart&&part.from<=start){break}}if(part.to>=end){return inner(builder,text,style,startStyle,endStyle,css,attributes)}inner(builder,text.slice(0,part.to-start),style,startStyle,null,css,attributes);startStyle=null;text=text.slice(part.to-start);start=part.to}}}function buildCollapsedSpan(builder,size,marker,ignoreWidget){var widget=!ignoreWidget&&marker.widgetNode;if(widget){builder.map.push(builder.pos,builder.pos+size,widget)}if(!ignoreWidget&&builder.cm.display.input.needsContentAttribute){if(!widget){widget=builder.content.appendChild(document.createElement("span"))}widget.setAttribute("cm-marker",marker.id)}if(widget){builder.cm.display.input.setUneditable(widget);builder.content.appendChild(widget)}builder.pos+=size;builder.trailingSpace=false}function insertLineContent(line,builder,styles){var spans=line.markedSpans,allText=line.text,at=0;if(!spans){for(var i$1=1;i$1pos||m.collapsed&&sp.to==pos&&sp.from==pos)){if(sp.to!=null&&sp.to!=pos&&nextChange>sp.to){nextChange=sp.to;spanEndStyle=""}if(m.className){spanStyle+=" "+m.className}if(m.css){css=(css?css+";":"")+m.css}if(m.startStyle&&sp.from==pos){spanStartStyle+=" "+m.startStyle}if(m.endStyle&&sp.to==nextChange){(endStyles||(endStyles=[])).push(m.endStyle,sp.to)}if(m.title){(attributes||(attributes={})).title=m.title}if(m.attributes){for(var attr in m.attributes){(attributes||(attributes={}))[attr]=m.attributes[attr]}}if(m.collapsed&&(!collapsed||compareCollapsedMarkers(collapsed.marker,m)<0)){collapsed=sp}}else if(sp.from>pos&&nextChange>sp.from){nextChange=sp.from}}if(endStyles){for(var j$1=0;j$1=len){break}var upto=Math.min(len,nextChange);while(true){if(text){var end=pos+text.length;if(!collapsed){var tokenText=end>upto?text.slice(0,upto-pos):text;builder.addToken(builder,tokenText,style?style+spanStyle:spanStyle,spanStartStyle,pos+tokenText.length==nextChange?spanEndStyle:"",css,attributes)}if(end>=upto){text=text.slice(upto-pos);pos=upto;break}pos=end;spanStartStyle=""}text=allText.slice(at,at=styles[i++]);style=interpretTokenStyle(styles[i++],builder.cm.options)}}}function LineView(doc,line,lineN){this.line=line;this.rest=visualLineContinued(line);this.size=this.rest?lineNo(lst(this.rest))-lineN+1:1;this.node=this.text=null;this.hidden=lineIsHidden(doc,line)}function buildViewArray(cm,from,to){var array=[],nextPos;for(var pos=from;pos2){heights.push((cur.bottom+next.top)/2-rect.top)}}}heights.push(rect.bottom-rect.top)}}function mapFromLineView(lineView,line,lineN){if(lineView.line==line){return{map:lineView.measure.map,cache:lineView.measure.cache}}for(var i=0;ilineN){return{map:lineView.measure.maps[i$1],cache:lineView.measure.caches[i$1],before:true}}}}function updateExternalMeasurement(cm,line){line=visualLine(line);var lineN=lineNo(line);var view=cm.display.externalMeasured=new LineView(cm.doc,line,lineN);view.lineN=lineN;var built=view.built=buildLineContent(cm,view);view.text=built.pre;removeChildrenAndAdd(cm.display.lineMeasure,built.pre);return view}function measureChar(cm,line,ch,bias){return measureCharPrepared(cm,prepareMeasureForLine(cm,line),ch,bias)}function findViewForLine(cm,lineN){if(lineN>=cm.display.viewFrom&&lineN=ext.lineN&&lineNch){end=mEnd-mStart;start=end-1;if(ch>=mEnd){collapse="right"}}if(start!=null){node=map[i+2];if(mStart==mEnd&&bias==(node.insertLeft?"left":"right")){collapse=bias}if(bias=="left"&&start==0){while(i&&map[i-2]==map[i-3]&&map[i-1].insertLeft){node=map[(i-=3)+2];collapse="left"}}if(bias=="right"&&start==mEnd-mStart){while(i=0;i$1--){if((rect=rects[i$1]).left!=rect.right){break}}}return rect}function measureCharInner(cm,prepared,ch,bias){var place=nodeAndOffsetInLineMap(prepared.map,ch,bias);var node=place.node,start=place.start,end=place.end,collapse=place.collapse;var rect;if(node.nodeType==3){for(var i$1=0;i$1<4;i$1++){while(start&&isExtendingChar(prepared.line.text.charAt(place.coverStart+start))){--start}while(place.coverStart+end0){collapse=bias="right"}var rects;if(cm.options.lineWrapping&&(rects=node.getClientRects()).length>1){rect=rects[bias=="right"?rects.length-1:0]}else{rect=node.getBoundingClientRect()}}if(ie&&ie_version<9&&!start&&(!rect||!rect.left&&!rect.right)){var rSpan=node.parentNode.getClientRects()[0];if(rSpan){rect={left:rSpan.left,right:rSpan.left+charWidth(cm.display),top:rSpan.top,bottom:rSpan.bottom}}else{rect=nullRect}}var rtop=rect.top-prepared.rect.top,rbot=rect.bottom-prepared.rect.top;var mid=(rtop+rbot)/2;var heights=prepared.view.measure.heights;var i=0;for(;i=lineObj.text.length){ch=lineObj.text.length;sticky="before"}else if(ch<=0){ch=0;sticky="after"}if(!order){return get(sticky=="before"?ch-1:ch,sticky=="before")}function getBidi(ch,partPos,invert){var part=order[partPos],right=part.level==1;return get(invert?ch-1:ch,right!=invert)}var partPos=getBidiPartAt(order,ch,sticky);var other=bidiOther;var val=getBidi(ch,partPos,sticky=="before");if(other!=null){val.other=getBidi(ch,other,sticky!="before")}return val}function estimateCoords(cm,pos){var left=0;pos=clipPos(cm.doc,pos);if(!cm.options.lineWrapping){left=charWidth(cm.display)*pos.ch}var lineObj=getLine(cm.doc,pos.line);var top=heightAtLine(lineObj)+paddingTop(cm.display);return{left:left,right:left,top:top,bottom:top+lineObj.height}}function PosWithInfo(line,ch,sticky,outside,xRel){var pos=Pos(line,ch,sticky);pos.xRel=xRel;if(outside){pos.outside=outside}return pos}function coordsChar(cm,x,y){var doc=cm.doc;y+=cm.display.viewOffset;if(y<0){return PosWithInfo(doc.first,0,null,-1,-1)}var lineN=lineAtHeight(doc,y),last=doc.first+doc.size-1;if(lineN>last){return PosWithInfo(doc.first+doc.size-1,getLine(doc,last).text.length,null,1,1)}if(x<0){x=0}var lineObj=getLine(doc,lineN);for(;;){var found=coordsCharInner(cm,lineObj,lineN,x,y);var collapsed=collapsedSpanAround(lineObj,found.ch+(found.xRel>0||found.outside>0?1:0));if(!collapsed){return found}var rangeEnd=collapsed.find(1);if(rangeEnd.line==lineN){return rangeEnd}lineObj=getLine(doc,lineN=rangeEnd.line)}}function wrappedLineExtent(cm,lineObj,preparedMeasure,y){y-=widgetTopHeight(lineObj);var end=lineObj.text.length;var begin=findFirst(function(ch){return measureCharPrepared(cm,preparedMeasure,ch-1).bottom<=y},end,0);end=findFirst(function(ch){return measureCharPrepared(cm,preparedMeasure,ch).top>y},begin,end);return{begin:begin,end:end}}function wrappedLineExtentChar(cm,lineObj,preparedMeasure,target){if(!preparedMeasure){preparedMeasure=prepareMeasureForLine(cm,lineObj)}var targetTop=intoCoordSystem(cm,lineObj,measureCharPrepared(cm,preparedMeasure,target),"line").top;return wrappedLineExtent(cm,lineObj,preparedMeasure,targetTop)}function boxIsAfter(box,x,y,left){return box.bottom<=y?false:box.top>y?true:(left?box.left:box.right)>x}function coordsCharInner(cm,lineObj,lineNo,x,y){y-=heightAtLine(lineObj);var preparedMeasure=prepareMeasureForLine(cm,lineObj);var widgetHeight=widgetTopHeight(lineObj);var begin=0,end=lineObj.text.length,ltr=true;var order=getOrder(lineObj,cm.doc.direction);if(order){var part=(cm.options.lineWrapping?coordsBidiPartWrapped:coordsBidiPart)(cm,lineObj,lineNo,preparedMeasure,order,x,y);ltr=part.level!=1;begin=ltr?part.from:part.to-1;end=ltr?part.to:part.from-1}var chAround=null,boxAround=null;var ch=findFirst(function(ch){var box=measureCharPrepared(cm,preparedMeasure,ch);box.top+=widgetHeight;box.bottom+=widgetHeight;if(!boxIsAfter(box,x,y,false)){return false}if(box.top<=y&&box.left<=x){chAround=ch;boxAround=box}return true},begin,end);var baseX,sticky,outside=false;if(boxAround){var atLeft=x-boxAround.left=coords.bottom?1:0}ch=skipExtendingChars(lineObj.text,ch,1);return PosWithInfo(lineNo,ch,sticky,outside,x-baseX)}function coordsBidiPart(cm,lineObj,lineNo,preparedMeasure,order,x,y){var index=findFirst(function(i){var part=order[i],ltr=part.level!=1;return boxIsAfter(cursorCoords(cm,Pos(lineNo,ltr?part.to:part.from,ltr?"before":"after"),"line",lineObj,preparedMeasure),x,y,true)},0,order.length-1);var part=order[index];if(index>0){var ltr=part.level!=1;var start=cursorCoords(cm,Pos(lineNo,ltr?part.from:part.to,ltr?"after":"before"),"line",lineObj,preparedMeasure);if(boxIsAfter(start,x,y,true)&&start.top>y){part=order[index-1]}}return part}function coordsBidiPartWrapped(cm,lineObj,_lineNo,preparedMeasure,order,x,y){var ref=wrappedLineExtent(cm,lineObj,preparedMeasure,y);var begin=ref.begin;var end=ref.end;if(/\s/.test(lineObj.text.charAt(end-1))){end--}var part=null,closestDist=null;for(var i=0;i=end||p.to<=begin){continue}var ltr=p.level!=1;var endX=measureCharPrepared(cm,preparedMeasure,ltr?Math.min(end,p.to)-1:Math.max(begin,p.from)).right;var dist=endXdist){part=p;closestDist=dist}}if(!part){part=order[order.length-1]}if(part.fromend){part={from:part.from,to:end,level:part.level}}return part}var measureText;function textHeight(display){if(display.cachedTextHeight!=null){return display.cachedTextHeight}if(measureText==null){measureText=elt("pre",null,"CodeMirror-line-like");for(var i=0;i<49;++i){measureText.appendChild(document.createTextNode("x"));measureText.appendChild(elt("br"))}measureText.appendChild(document.createTextNode("x"))}removeChildrenAndAdd(display.measure,measureText);var height=measureText.offsetHeight/50;if(height>3){display.cachedTextHeight=height}removeChildren(display.measure);return height||1}function charWidth(display){if(display.cachedCharWidth!=null){return display.cachedCharWidth}var anchor=elt("span","xxxxxxxxxx");var pre=elt("pre",[anchor],"CodeMirror-line-like");removeChildrenAndAdd(display.measure,pre);var rect=anchor.getBoundingClientRect(),width=(rect.right-rect.left)/10;if(width>2){display.cachedCharWidth=width}return width||10}function getDimensions(cm){var d=cm.display,left={},width={};var gutterLeft=d.gutters.clientLeft;for(var n=d.gutters.firstChild,i=0;n;n=n.nextSibling,++i){var id=cm.display.gutterSpecs[i].className;left[id]=n.offsetLeft+n.clientLeft+gutterLeft;width[id]=n.clientWidth}return{fixedPos:compensateForHScroll(d),gutterTotalWidth:d.gutters.offsetWidth,gutterLeft:left,gutterWidth:width,wrapperWidth:d.wrapper.clientWidth}}function compensateForHScroll(display){return display.scroller.getBoundingClientRect().left-display.sizer.getBoundingClientRect().left}function estimateHeight(cm){var th=textHeight(cm.display),wrapping=cm.options.lineWrapping;var perLine=wrapping&&Math.max(5,cm.display.scroller.clientWidth/charWidth(cm.display)-3);return function(line){if(lineIsHidden(cm.doc,line)){return 0}var widgetsHeight=0;if(line.widgets){for(var i=0;i0&&(line=getLine(cm.doc,coords.line).text).length==coords.ch){var colDiff=countColumn(line,line.length,cm.options.tabSize)-line.length;coords=Pos(coords.line,Math.max(0,Math.round((x-paddingH(cm.display).left)/charWidth(cm.display))-colDiff))}return coords}function findViewIndex(cm,n){if(n>=cm.display.viewTo){return null}n-=cm.display.viewFrom;if(n<0){return null}var view=cm.display.view;for(var i=0;ifrom)){display.updateLineNumbers=from}cm.curOp.viewChanged=true;if(from>=display.viewTo){if(sawCollapsedSpans&&visualLineNo(cm.doc,from)display.viewFrom){resetView(cm)}else{display.viewFrom+=lendiff;display.viewTo+=lendiff}}else if(from<=display.viewFrom&&to>=display.viewTo){resetView(cm)}else if(from<=display.viewFrom){var cut=viewCuttingPoint(cm,to,to+lendiff,1);if(cut){display.view=display.view.slice(cut.index);display.viewFrom=cut.lineN;display.viewTo+=lendiff}else{resetView(cm)}}else if(to>=display.viewTo){var cut$1=viewCuttingPoint(cm,from,from,-1);if(cut$1){display.view=display.view.slice(0,cut$1.index);display.viewTo=cut$1.lineN}else{resetView(cm)}}else{var cutTop=viewCuttingPoint(cm,from,from,-1);var cutBot=viewCuttingPoint(cm,to,to+lendiff,1);if(cutTop&&cutBot){display.view=display.view.slice(0,cutTop.index).concat(buildViewArray(cm,cutTop.lineN,cutBot.lineN)).concat(display.view.slice(cutBot.index));display.viewTo+=lendiff}else{resetView(cm)}}var ext=display.externalMeasured;if(ext){if(to=ext.lineN&&line=display.viewTo){return}var lineView=display.view[findViewIndex(cm,line)];if(lineView.node==null){return}var arr=lineView.changes||(lineView.changes=[]);if(indexOf(arr,type)==-1){arr.push(type)}}function resetView(cm){cm.display.viewFrom=cm.display.viewTo=cm.doc.first;cm.display.view=[];cm.display.viewOffset=0}function viewCuttingPoint(cm,oldN,newN,dir){var index=findViewIndex(cm,oldN),diff,view=cm.display.view;if(!sawCollapsedSpans||newN==cm.doc.first+cm.doc.size){return{index:index,lineN:newN}}var n=cm.display.viewFrom;for(var i=0;i0){if(index==view.length-1){return null}diff=n+view[index].size-oldN;index++}else{diff=n-oldN}oldN+=diff;newN+=diff}while(visualLineNo(cm.doc,newN)!=newN){if(index==(dir<0?0:view.length-1)){return null}newN+=dir*view[index-(dir<0?1:0)].size;index+=dir}return{index:index,lineN:newN}}function adjustView(cm,from,to){var display=cm.display,view=display.view;if(view.length==0||from>=display.viewTo||to<=display.viewFrom){display.view=buildViewArray(cm,from,to);display.viewFrom=from}else{if(display.viewFrom>from){display.view=buildViewArray(cm,from,display.viewFrom).concat(display.view)}else if(display.viewFromto){display.view=display.view.slice(0,findViewIndex(cm,to))}}display.viewTo=to}function countDirtyView(cm){var view=cm.display.view,dirty=0;for(var i=0;i=cm.display.viewTo||range.to().line0){display.blinker=setInterval(function(){return display.cursorDiv.style.visibility=(on=!on)?"":"hidden"},cm.options.cursorBlinkRate)}else if(cm.options.cursorBlinkRate<0){display.cursorDiv.style.visibility="hidden"}}function ensureFocus(cm){if(!cm.state.focused){cm.display.input.focus();onFocus(cm)}}function delayBlurEvent(cm){cm.state.delayingBlurEvent=true;setTimeout(function(){if(cm.state.delayingBlurEvent){cm.state.delayingBlurEvent=false;onBlur(cm)}},100)}function onFocus(cm,e){if(cm.state.delayingBlurEvent){cm.state.delayingBlurEvent=false}if(cm.options.readOnly=="nocursor"){return}if(!cm.state.focused){signal(cm,"focus",cm,e);cm.state.focused=true;addClass(cm.display.wrapper,"CodeMirror-focused");if(!cm.curOp&&cm.display.selForContextMenu!=cm.doc.sel){cm.display.input.reset();if(webkit){setTimeout(function(){return cm.display.input.reset(true)},20)}}cm.display.input.receivedFocus()}restartBlink(cm)}function onBlur(cm,e){if(cm.state.delayingBlurEvent){return}if(cm.state.focused){signal(cm,"blur",cm,e);cm.state.focused=false;rmClass(cm.display.wrapper,"CodeMirror-focused")}clearInterval(cm.display.blinker);setTimeout(function(){if(!cm.state.focused){cm.display.shift=false}},150)}function updateHeightsInViewport(cm){var display=cm.display;var prevBottom=display.lineDiv.offsetTop;for(var i=0;i.005||diff<-.005){updateLineHeight(cur.line,height);updateWidgetHeight(cur.line);if(cur.rest){for(var j=0;jcm.display.sizerWidth){var chWidth=Math.ceil(width/charWidth(cm.display));if(chWidth>cm.display.maxLineLength){cm.display.maxLineLength=chWidth;cm.display.maxLine=cur.line;cm.display.maxLineChanged=true}}}}function updateWidgetHeight(line){if(line.widgets){for(var i=0;i=to){from=lineAtHeight(doc,heightAtLine(getLine(doc,ensureTo))-display.wrapper.clientHeight);to=ensureTo}}return{from:from,to:Math.max(to,from+1)}}function maybeScrollWindow(cm,rect){if(signalDOMEvent(cm,"scrollCursorIntoView")){return}var display=cm.display,box=display.sizer.getBoundingClientRect(),doScroll=null;if(rect.top+box.top<0){doScroll=true}else if(rect.bottom+box.top>(window.innerHeight||document.documentElement.clientHeight)){doScroll=false}if(doScroll!=null&&!phantom){var scrollNode=elt("div","​",null,"position: absolute;\n                         top: "+(rect.top-display.viewOffset-paddingTop(cm.display))+"px;\n                         height: "+(rect.bottom-rect.top+scrollGap(cm)+display.barHeight)+"px;\n                         left: "+rect.left+"px; width: "+Math.max(2,rect.right-rect.left)+"px;");cm.display.lineSpace.appendChild(scrollNode);scrollNode.scrollIntoView(doScroll);cm.display.lineSpace.removeChild(scrollNode)}}function scrollPosIntoView(cm,pos,end,margin){if(margin==null){margin=0}var rect;if(!cm.options.lineWrapping&&pos==end){pos=pos.ch?Pos(pos.line,pos.sticky=="before"?pos.ch-1:pos.ch,"after"):pos;end=pos.sticky=="before"?Pos(pos.line,pos.ch+1,"before"):pos}for(var limit=0;limit<5;limit++){var changed=false;var coords=cursorCoords(cm,pos);var endCoords=!end||end==pos?coords:cursorCoords(cm,end);rect={left:Math.min(coords.left,endCoords.left),top:Math.min(coords.top,endCoords.top)-margin,right:Math.max(coords.left,endCoords.left),bottom:Math.max(coords.bottom,endCoords.bottom)+margin};var scrollPos=calculateScrollPos(cm,rect);var startTop=cm.doc.scrollTop,startLeft=cm.doc.scrollLeft;if(scrollPos.scrollTop!=null){updateScrollTop(cm,scrollPos.scrollTop);if(Math.abs(cm.doc.scrollTop-startTop)>1){changed=true}}if(scrollPos.scrollLeft!=null){setScrollLeft(cm,scrollPos.scrollLeft);if(Math.abs(cm.doc.scrollLeft-startLeft)>1){changed=true}}if(!changed){break}}return rect}function scrollIntoView(cm,rect){var scrollPos=calculateScrollPos(cm,rect);if(scrollPos.scrollTop!=null){updateScrollTop(cm,scrollPos.scrollTop)}if(scrollPos.scrollLeft!=null){setScrollLeft(cm,scrollPos.scrollLeft)}}function calculateScrollPos(cm,rect){var display=cm.display,snapMargin=textHeight(cm.display);if(rect.top<0){rect.top=0}var screentop=cm.curOp&&cm.curOp.scrollTop!=null?cm.curOp.scrollTop:display.scroller.scrollTop;var screen=displayHeight(cm),result={};if(rect.bottom-rect.top>screen){rect.bottom=rect.top+screen}var docBottom=cm.doc.height+paddingVert(display);var atTop=rect.topdocBottom-snapMargin;if(rect.topscreentop+screen){var newTop=Math.min(rect.top,(atBottom?docBottom:rect.bottom)-screen);if(newTop!=screentop){result.scrollTop=newTop}}var screenleft=cm.curOp&&cm.curOp.scrollLeft!=null?cm.curOp.scrollLeft:display.scroller.scrollLeft;var screenw=displayWidth(cm)-(cm.options.fixedGutter?display.gutters.offsetWidth:0);var tooWide=rect.right-rect.left>screenw;if(tooWide){rect.right=rect.left+screenw}if(rect.left<10){result.scrollLeft=0}else if(rect.leftscreenw+screenleft-3){result.scrollLeft=rect.right+(tooWide?0:10)-screenw}return result}function addToScrollTop(cm,top){if(top==null){return}resolveScrollToPos(cm);cm.curOp.scrollTop=(cm.curOp.scrollTop==null?cm.doc.scrollTop:cm.curOp.scrollTop)+top}function ensureCursorVisible(cm){resolveScrollToPos(cm);var cur=cm.getCursor();cm.curOp.scrollToPos={from:cur,to:cur,margin:cm.options.cursorScrollMargin}}function scrollToCoords(cm,x,y){if(x!=null||y!=null){resolveScrollToPos(cm)}if(x!=null){cm.curOp.scrollLeft=x}if(y!=null){cm.curOp.scrollTop=y}}function scrollToRange(cm,range){resolveScrollToPos(cm);cm.curOp.scrollToPos=range}function resolveScrollToPos(cm){var range=cm.curOp.scrollToPos;if(range){cm.curOp.scrollToPos=null;var from=estimateCoords(cm,range.from),to=estimateCoords(cm,range.to);scrollToCoordsRange(cm,from,to,range.margin)}}function scrollToCoordsRange(cm,from,to,margin){var sPos=calculateScrollPos(cm,{left:Math.min(from.left,to.left),top:Math.min(from.top,to.top)-margin,right:Math.max(from.right,to.right),bottom:Math.max(from.bottom,to.bottom)+margin});scrollToCoords(cm,sPos.scrollLeft,sPos.scrollTop)}function updateScrollTop(cm,val){if(Math.abs(cm.doc.scrollTop-val)<2){return}if(!gecko){updateDisplaySimple(cm,{top:val})}setScrollTop(cm,val,true);if(gecko){updateDisplaySimple(cm)}startWorker(cm,100)}function setScrollTop(cm,val,forceScroll){val=Math.max(0,Math.min(cm.display.scroller.scrollHeight-cm.display.scroller.clientHeight,val));if(cm.display.scroller.scrollTop==val&&!forceScroll){return}cm.doc.scrollTop=val;cm.display.scrollbars.setScrollTop(val);if(cm.display.scroller.scrollTop!=val){cm.display.scroller.scrollTop=val}}function setScrollLeft(cm,val,isScroller,forceScroll){val=Math.max(0,Math.min(val,cm.display.scroller.scrollWidth-cm.display.scroller.clientWidth));if((isScroller?val==cm.doc.scrollLeft:Math.abs(cm.doc.scrollLeft-val)<2)&&!forceScroll){return}cm.doc.scrollLeft=val;alignHorizontally(cm);if(cm.display.scroller.scrollLeft!=val){cm.display.scroller.scrollLeft=val}cm.display.scrollbars.setScrollLeft(val)}function measureForScrollbars(cm){var d=cm.display,gutterW=d.gutters.offsetWidth;var docH=Math.round(cm.doc.height+paddingVert(cm.display));return{clientHeight:d.scroller.clientHeight,viewHeight:d.wrapper.clientHeight,scrollWidth:d.scroller.scrollWidth,clientWidth:d.scroller.clientWidth,viewWidth:d.wrapper.clientWidth,barLeft:cm.options.fixedGutter?gutterW:0,docHeight:docH,scrollHeight:docH+scrollGap(cm)+d.barHeight,nativeBarWidth:d.nativeBarWidth,gutterWidth:gutterW}}var NativeScrollbars=function(place,scroll,cm){this.cm=cm;var vert=this.vert=elt("div",[elt("div",null,null,"min-width: 1px")],"CodeMirror-vscrollbar");var horiz=this.horiz=elt("div",[elt("div",null,null,"height: 100%; min-height: 1px")],"CodeMirror-hscrollbar");vert.tabIndex=horiz.tabIndex=-1;place(vert);place(horiz);on(vert,"scroll",function(){if(vert.clientHeight){scroll(vert.scrollTop,"vertical")}});on(horiz,"scroll",function(){if(horiz.clientWidth){scroll(horiz.scrollLeft,"horizontal")}});this.checkedZeroWidth=false;if(ie&&ie_version<8){this.horiz.style.minHeight=this.vert.style.minWidth="18px"}};NativeScrollbars.prototype.update=function(measure){var needsH=measure.scrollWidth>measure.clientWidth+1;var needsV=measure.scrollHeight>measure.clientHeight+1;var sWidth=measure.nativeBarWidth;if(needsV){this.vert.style.display="block";this.vert.style.bottom=needsH?sWidth+"px":"0";var totalHeight=measure.viewHeight-(needsH?sWidth:0);this.vert.firstChild.style.height=Math.max(0,measure.scrollHeight-measure.clientHeight+totalHeight)+"px"}else{this.vert.style.display="";this.vert.firstChild.style.height="0"}if(needsH){this.horiz.style.display="block";this.horiz.style.right=needsV?sWidth+"px":"0";this.horiz.style.left=measure.barLeft+"px";var totalWidth=measure.viewWidth-measure.barLeft-(needsV?sWidth:0);this.horiz.firstChild.style.width=Math.max(0,measure.scrollWidth-measure.clientWidth+totalWidth)+"px"}else{this.horiz.style.display="";this.horiz.firstChild.style.width="0"}if(!this.checkedZeroWidth&&measure.clientHeight>0){if(sWidth==0){this.zeroWidthHack()}this.checkedZeroWidth=true}return{right:needsV?sWidth:0,bottom:needsH?sWidth:0}};NativeScrollbars.prototype.setScrollLeft=function(pos){if(this.horiz.scrollLeft!=pos){this.horiz.scrollLeft=pos}if(this.disableHoriz){this.enableZeroWidthBar(this.horiz,this.disableHoriz,"horiz")}};NativeScrollbars.prototype.setScrollTop=function(pos){if(this.vert.scrollTop!=pos){this.vert.scrollTop=pos}if(this.disableVert){this.enableZeroWidthBar(this.vert,this.disableVert,"vert")}};NativeScrollbars.prototype.zeroWidthHack=function(){var w=mac&&!mac_geMountainLion?"12px":"18px";this.horiz.style.height=this.vert.style.width=w;this.horiz.style.pointerEvents=this.vert.style.pointerEvents="none";this.disableHoriz=new Delayed;this.disableVert=new Delayed};NativeScrollbars.prototype.enableZeroWidthBar=function(bar,delay,type){bar.style.pointerEvents="auto";function maybeDisable(){var box=bar.getBoundingClientRect();var elt=type=="vert"?document.elementFromPoint(box.right-1,(box.top+box.bottom)/2):document.elementFromPoint((box.right+box.left)/2,box.bottom-1);if(elt!=bar){bar.style.pointerEvents="none"}else{delay.set(1e3,maybeDisable)}}delay.set(1e3,maybeDisable)};NativeScrollbars.prototype.clear=function(){var parent=this.horiz.parentNode;parent.removeChild(this.horiz);parent.removeChild(this.vert)};var NullScrollbars=function(){};NullScrollbars.prototype.update=function(){return{bottom:0,right:0}};NullScrollbars.prototype.setScrollLeft=function(){};NullScrollbars.prototype.setScrollTop=function(){};NullScrollbars.prototype.clear=function(){};function updateScrollbars(cm,measure){if(!measure){measure=measureForScrollbars(cm)}var startWidth=cm.display.barWidth,startHeight=cm.display.barHeight;updateScrollbarsInner(cm,measure);for(var i=0;i<4&&startWidth!=cm.display.barWidth||startHeight!=cm.display.barHeight;i++){if(startWidth!=cm.display.barWidth&&cm.options.lineWrapping){updateHeightsInViewport(cm)}updateScrollbarsInner(cm,measureForScrollbars(cm));startWidth=cm.display.barWidth;startHeight=cm.display.barHeight}}function updateScrollbarsInner(cm,measure){var d=cm.display;var sizes=d.scrollbars.update(measure);d.sizer.style.paddingRight=(d.barWidth=sizes.right)+"px";d.sizer.style.paddingBottom=(d.barHeight=sizes.bottom)+"px";d.heightForcer.style.borderBottom=sizes.bottom+"px solid transparent";if(sizes.right&&sizes.bottom){d.scrollbarFiller.style.display="block";d.scrollbarFiller.style.height=sizes.bottom+"px";d.scrollbarFiller.style.width=sizes.right+"px"}else{d.scrollbarFiller.style.display=""}if(sizes.bottom&&cm.options.coverGutterNextToScrollbar&&cm.options.fixedGutter){d.gutterFiller.style.display="block";d.gutterFiller.style.height=sizes.bottom+"px";d.gutterFiller.style.width=measure.gutterWidth+"px"}else{d.gutterFiller.style.display=""}}var scrollbarModel={native:NativeScrollbars,null:NullScrollbars};function initScrollbars(cm){if(cm.display.scrollbars){cm.display.scrollbars.clear();if(cm.display.scrollbars.addClass){rmClass(cm.display.wrapper,cm.display.scrollbars.addClass)}}cm.display.scrollbars=new scrollbarModel[cm.options.scrollbarStyle](function(node){cm.display.wrapper.insertBefore(node,cm.display.scrollbarFiller);on(node,"mousedown",function(){if(cm.state.focused){setTimeout(function(){return cm.display.input.focus()},0)}});node.setAttribute("cm-not-content","true")},function(pos,axis){if(axis=="horizontal"){setScrollLeft(cm,pos)}else{updateScrollTop(cm,pos)}},cm);if(cm.display.scrollbars.addClass){addClass(cm.display.wrapper,cm.display.scrollbars.addClass)}}var nextOpId=0;function startOperation(cm){cm.curOp={cm:cm,viewChanged:false,startHeight:cm.doc.height,forceUpdate:false,updateInput:0,typing:false,changeObjs:null,cursorActivityHandlers:null,cursorActivityCalled:0,selectionChanged:false,updateMaxLine:false,scrollLeft:null,scrollTop:null,scrollToPos:null,focus:false,id:++nextOpId};pushOperation(cm.curOp)}function endOperation(cm){var op=cm.curOp;if(op){finishOperation(op,function(group){for(var i=0;i=display.viewTo)||display.maxLineChanged&&cm.options.lineWrapping;op.update=op.mustUpdate&&new DisplayUpdate(cm,op.mustUpdate&&{top:op.scrollTop,ensure:op.scrollToPos},op.forceUpdate)}function endOperation_W1(op){op.updatedDisplay=op.mustUpdate&&updateDisplayIfNeeded(op.cm,op.update)}function endOperation_R2(op){var cm=op.cm,display=cm.display;if(op.updatedDisplay){updateHeightsInViewport(cm)}op.barMeasure=measureForScrollbars(cm);if(display.maxLineChanged&&!cm.options.lineWrapping){op.adjustWidthTo=measureChar(cm,display.maxLine,display.maxLine.text.length).left+3;cm.display.sizerWidth=op.adjustWidthTo;op.barMeasure.scrollWidth=Math.max(display.scroller.clientWidth,display.sizer.offsetLeft+op.adjustWidthTo+scrollGap(cm)+cm.display.barWidth);op.maxScrollLeft=Math.max(0,display.sizer.offsetLeft+op.adjustWidthTo-displayWidth(cm))}if(op.updatedDisplay||op.selectionChanged){op.preparedSelection=display.input.prepareSelection()}}function endOperation_W2(op){var cm=op.cm;if(op.adjustWidthTo!=null){cm.display.sizer.style.minWidth=op.adjustWidthTo+"px";if(op.maxScrollLeft

    +

    `; var DROPDOWN_ICON_SVG = ` @@ -166,6 +167,14 @@ var DROPDOWN_ICON_SVG = `
    {% for column in display_columns %} - + {% if not column.sortable %} {{ column.name }} {% else %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 211352b5..466e8a47 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -51,6 +51,14 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} +{% if metadata.columns %} +
    + {% for column_name, column_description in metadata.columns.items() %} +
    {{ column_name }}
    {{ column_description }}
    + {% endfor %} +
    +{% endif %} + {% if filtered_table_rows_count or human_description_en %}

    {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %} diff --git a/datasette/views/table.py b/datasette/views/table.py index 456d8069..486a6131 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -125,6 +125,7 @@ class RowTableShared(DataView): """Returns columns, rows for specified table - including fancy foreign key treatment""" db = self.ds.databases[database] table_metadata = self.ds.table_metadata(database, table) + column_descriptions = table_metadata.get("columns") or {} column_details = {col.name: col for col in await db.table_column_details(table)} sortable_columns = await self.sortable_columns_for_table(database, table, True) pks = await db.primary_keys(table) @@ -147,6 +148,7 @@ class RowTableShared(DataView): "is_pk": r[0] in pks_for_display, "type": type_, "notnull": notnull, + "description": column_descriptions.get(r[0]), } ) diff --git a/docs/metadata.rst b/docs/metadata.rst index dad5adca..35b8aede 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -78,6 +78,34 @@ The three visible metadata fields you can apply to everything, specific database For each of these you can provide just the ``*_url`` field and Datasette will treat that as the default link label text and display the URL directly on the page. +.. _metadata_column_descriptions: + +Column descriptions +------------------- + +You can include descriptions for your columns by adding a ``"columns": {"name-of-column": "description-of-column"}`` block to your table metadata: + +.. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "columns": { + "column1": "Description of column 1", + "column2": "Description of column 2" + } + } + } + } + } + } + +These will be displayed at the top of the table page, and will also show in the cog menu for each column. + +You can see an example of how these look at `latest.datasette.io/fixtures/roadside_attractions `__. + Specifying units for a column ----------------------------- diff --git a/tests/fixtures.py b/tests/fixtures.py index 880e4347..4a420e4b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -336,6 +336,12 @@ METADATA = { "fts_table": "searchable_fts", "fts_pk": "pk", }, + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, "attraction_characteristic": {"sort_desc": "pk"}, "facet_cities": {"sort": "name"}, "paginated_view": {"size": 25}, diff --git a/tests/test_html.py b/tests/test_html.py index b1b6c1f3..f12f89cd 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1777,3 +1777,21 @@ def test_trace_correctly_escaped(app_client): response = app_client.get("/fixtures?sql=select+'

    Hello'&_trace=1") assert "select '

    Hello" not in response.text assert "select '<h1>Hello" in response.text + + +def test_column_metadata(app_client): + response = app_client.get("/fixtures/roadside_attractions") + soup = Soup(response.body, "html.parser") + dl = soup.find("dl") + assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [ + ("name", "The name of the attraction"), + ("address", "The street address for the attraction"), + ] + assert ( + soup.select("th[data-column=name]")[0]["data-column-description"] + == "The name of the attraction" + ) + assert ( + soup.select("th[data-column=address]")[0]["data-column-description"] + == "The street address for the attraction" + ) From 77f46297a88ac7e49dad2139410b01ee56d5f99c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 12 Aug 2021 18:01:57 -0700 Subject: [PATCH 1154/2303] Rename --help-config to --help-settings, closes #1431 --- datasette/cli.py | 12 ++++++------ docs/datasette-serve-help.txt | 2 +- tests/test_cli.py | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index e53f3d8e..d4e23c70 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -51,7 +51,7 @@ class Config(click.ParamType): name, value = config.split(":", 1) if name not in DEFAULT_SETTINGS: self.fail( - f"{name} is not a valid option (--help-config to see all)", + f"{name} is not a valid option (--help-settings to see all)", param, ctx, ) @@ -84,7 +84,7 @@ class Setting(CompositeParamType): name, value = config if name not in DEFAULT_SETTINGS: self.fail( - f"{name} is not a valid option (--help-config to see all)", + f"{name} is not a valid option (--help-settings to see all)", param, ctx, ) @@ -408,7 +408,7 @@ def uninstall(packages, yes): help="Run an HTTP GET request against this path, print results and exit", ) @click.option("--version-note", help="Additional note to show on /-/versions") -@click.option("--help-config", is_flag=True, help="Show available config options") +@click.option("--help-settings", is_flag=True, help="Show available settings") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") @click.option( "-o", @@ -456,7 +456,7 @@ def serve( root, get, version_note, - help_config, + help_settings, pdb, open_browser, create, @@ -466,9 +466,9 @@ def serve( return_instance=False, ): """Serve up specified SQLite database files with a web UI""" - if help_config: + if help_settings: formatter = formatting.HelpFormatter() - with formatter.section("Config options"): + with formatter.section("Settings"): formatter.write_dl( [ (option.name, f"{option.help} (default={option.default})") diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index ec3f41a0..2911977a 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -32,7 +32,7 @@ Options: --get TEXT Run an HTTP GET request against this path, print results and exit --version-note TEXT Additional note to show on /-/versions - --help-config Show available config options + --help-settings Show available settings --pdb Launch debugger on any errors -o, --open Open Datasette in your web browser --create Create database files if they do not exist diff --git a/tests/test_cli.py b/tests/test_cli.py index e31a305e..763fe2e7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,6 +5,7 @@ from .fixtures import ( EXPECTED_PLUGINS, ) import asyncio +from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ @@ -147,7 +148,7 @@ def test_metadata_yaml(): root=False, version_note=None, get=None, - help_config=False, + help_settings=False, pdb=False, crossdb=False, open_browser=False, @@ -291,3 +292,10 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename): cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))] ) assert result2.exit_code == 0, result2.output + + +def test_help_settings(): + runner = CliRunner() + result = runner.invoke(cli, ["--help-settings"]) + for setting in SETTINGS: + assert setting.name in result.output From ca4f83dc7b1d573b92a8921fca96d3ed490614c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 12 Aug 2021 18:10:36 -0700 Subject: [PATCH 1155/2303] Rename config= to settings=, refs #1432 --- datasette/app.py | 8 ++++---- datasette/cli.py | 8 ++++---- datasette/templates/table.html | 2 +- datasette/views/base.py | 2 +- datasette/views/database.py | 2 +- tests/fixtures.py | 20 ++++++++++---------- tests/test_api.py | 8 ++++---- tests/test_custom_pages.py | 2 +- tests/test_facets.py | 2 +- tests/test_html.py | 14 ++++++++------ 10 files changed, 35 insertions(+), 33 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f2f75884..8cbaaf9f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -200,7 +200,7 @@ class Datasette: plugins_dir=None, static_mounts=None, memory=False, - config=None, + settings=None, secret=None, version_note=None, config_dir=None, @@ -279,7 +279,7 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: config = json.loads((config_dir / "settings.json").read_text()) - self._settings = dict(DEFAULT_SETTINGS, **(config or {})) + self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( @@ -419,8 +419,8 @@ class Datasette: def setting(self, key): return self._settings.get(key, None) - def config_dict(self): - # Returns a fully resolved config dictionary, useful for templates + def settings_dict(self): + # Returns a fully resolved settings dictionary, useful for templates return {option.name: self.setting(option.name) for option in SETTINGS} def _metadata_recursive_update(self, orig, updated): diff --git a/datasette/cli.py b/datasette/cli.py index d4e23c70..ea6da748 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -495,14 +495,14 @@ def serve( if metadata: metadata_data = parse_metadata(metadata.read()) - combined_config = {} + combined_settings = {} if config: click.echo( "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead", err=True, ) - combined_config.update(config) - combined_config.update(settings) + combined_settings.update(config) + combined_settings.update(settings) kwargs = dict( immutables=immutable, @@ -514,7 +514,7 @@ def serve( template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - config=combined_config, + settings=combined_settings, memory=memory, secret=secret, version_note=version_note, diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 466e8a47..a28945ad 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -201,7 +201,7 @@ CSV options: {% if expandable_columns %}{% endif %} - {% if next_url and config.allow_csv_stream %}{% endif %} + {% if next_url and settings.allow_csv_stream %}{% endif %} {% for key, value in url_csv_hidden_args %} diff --git a/datasette/views/base.py b/datasette/views/base.py index 1cea1386..3333781c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -614,7 +614,7 @@ class DataView(BaseView): ] + [("_size", "max")], "datasette_version": __version__, - "config": self.ds.config_dict(), + "settings": self.ds.settings_dict(), }, } if "metadata" not in context: diff --git a/datasette/views/database.py b/datasette/views/database.py index 7c36034c..e3070ce6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -456,7 +456,7 @@ class QueryView(DataView): "canned_query": canned_query, "edit_sql_url": edit_sql_url, "metadata": metadata, - "config": self.ds.config_dict(), + "settings": self.ds.settings_dict(), "request": request, "show_hide_link": show_hide_link, "show_hide_text": show_hide_text, diff --git a/tests/fixtures.py b/tests/fixtures.py index 4a420e4b..dc22c609 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -99,7 +99,7 @@ def make_app_client( max_returned_rows=None, cors=False, memory=False, - config=None, + settings=None, filename="fixtures.db", is_immutable=False, extra_databases=None, @@ -129,7 +129,7 @@ def make_app_client( # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) - config = config or {} + settings = settings or {} for key, value in { "default_page_size": 50, "max_returned_rows": max_returned_rows or 100, @@ -138,8 +138,8 @@ def make_app_client( # errors when running the full test suite: "num_sql_threads": 1, }.items(): - if key not in config: - config[key] = value + if key not in settings: + settings[key] = value ds = Datasette( files, immutables=immutables, @@ -147,7 +147,7 @@ def make_app_client( cors=cors, metadata=metadata or METADATA, plugins_dir=PLUGINS_DIR, - config=config, + settings=settings, inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, @@ -171,7 +171,7 @@ def app_client_no_files(): @pytest.fixture(scope="session") def app_client_base_url_prefix(): - with make_app_client(config={"base_url": "/prefix/"}) as client: + with make_app_client(settings={"base_url": "/prefix/"}) as client: yield client @@ -210,13 +210,13 @@ def app_client_two_attached_databases_one_immutable(): @pytest.fixture(scope="session") def app_client_with_hash(): - with make_app_client(config={"hash_urls": True}, is_immutable=True) as client: + with make_app_client(settings={"hash_urls": True}, is_immutable=True) as client: yield client @pytest.fixture(scope="session") def app_client_with_trace(): - with make_app_client(config={"trace_debug": True}, is_immutable=True) as client: + with make_app_client(settings={"trace_debug": True}, is_immutable=True) as client: yield client @@ -234,13 +234,13 @@ def app_client_returned_rows_matches_page_size(): @pytest.fixture(scope="session") def app_client_larger_cache_size(): - with make_app_client(config={"cache_size_kb": 2500}) as client: + with make_app_client(settings={"cache_size_kb": 2500}) as client: yield client @pytest.fixture(scope="session") def app_client_csv_max_mb_one(): - with make_app_client(config={"max_csv_mb": 1}) as client: + with make_app_client(settings={"max_csv_mb": 1}) as client: yield client diff --git a/tests/test_api.py b/tests/test_api.py index 83cca521..1e93c62e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1711,14 +1711,14 @@ def test_suggested_facets(app_client): def test_allow_facet_off(): - with make_app_client(config={"allow_facet": False}) as client: + with make_app_client(settings={"allow_facet": False}) as client: assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status # Should not suggest any facets either: assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] def test_suggest_facets_off(): - with make_app_client(config={"suggest_facets": False}) as client: + with make_app_client(settings={"suggest_facets": False}) as client: # Now suggested_facets should be [] assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] @@ -1883,7 +1883,7 @@ def test_config_cache_size(app_client_larger_cache_size): def test_config_force_https_urls(): - with make_app_client(config={"force_https_urls": True}) as client: + with make_app_client(settings={"force_https_urls": True}) as client: response = client.get("/fixtures/facetable.json?_size=3&_facet=state") assert response.json["next_url"].startswith("https://") assert response.json["facet_results"]["state"]["results"][0][ @@ -1921,7 +1921,7 @@ def test_custom_query_with_unicode_characters(app_client): @pytest.mark.parametrize("trace_debug", (True, False)) def test_trace(trace_debug): - with make_app_client(config={"trace_debug": trace_debug}) as client: + with make_app_client(settings={"trace_debug": trace_debug}) as client: response = client.get("/fixtures/simple_primary_key.json?_trace=1") assert response.status == 200 diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 5a71f56d..76c67397 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -14,7 +14,7 @@ def custom_pages_client(): @pytest.fixture(scope="session") def custom_pages_client_with_base_url(): with make_app_client( - template_dir=TEST_TEMPLATE_DIRS, config={"base_url": "/prefix/"} + template_dir=TEST_TEMPLATE_DIRS, settings={"base_url": "/prefix/"} ) as client: yield client diff --git a/tests/test_facets.py b/tests/test_facets.py index 18fb8c3b..22927512 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -351,7 +351,7 @@ async def test_json_array_with_blanks_and_nulls(): @pytest.mark.asyncio async def test_facet_size(): - ds = Datasette([], memory=True, config={"max_returned_rows": 50}) + ds = Datasette([], memory=True, settings={"max_returned_rows": 50}) db = ds.add_database(Database(ds, memory_name="test_facet_size")) await db.execute_write( "create table neighbourhoods(city text, neighbourhood text)", block=True diff --git a/tests/test_html.py b/tests/test_html.py index f12f89cd..90fcdae7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -214,7 +214,7 @@ def test_definition_sql(path, expected_definition_sql, app_client): def test_table_cell_truncation(): - with make_app_client(config={"truncate_cells_html": 5}) as client: + with make_app_client(settings={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -239,7 +239,7 @@ def test_table_cell_truncation(): def test_row_page_does_not_truncate(): - with make_app_client(config={"truncate_cells_html": 5}) as client: + with make_app_client(settings={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable/1") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -1072,7 +1072,9 @@ def test_database_download_disallowed_for_memory(): def test_allow_download_off(): - with make_app_client(is_immutable=True, config={"allow_download": False}) as client: + with make_app_client( + is_immutable=True, settings={"allow_download": False} + ) as client: response = client.get("/fixtures") soup = Soup(response.body, "html.parser") assert not len(soup.findAll("a", {"href": re.compile(r"\.db$")})) @@ -1486,7 +1488,7 @@ def test_query_error(app_client): def test_config_template_debug_on(): - with make_app_client(config={"template_debug": True}) as client: + with make_app_client(settings={"template_debug": True}) as client: response = client.get("/fixtures/facetable?_context=1") assert response.status == 200 assert response.text.startswith("
    {")
@@ -1500,7 +1502,7 @@ def test_config_template_debug_off(app_client):
 
 def test_debug_context_includes_extra_template_vars():
     # https://github.com/simonw/datasette/issues/693
-    with make_app_client(config={"template_debug": True}) as client:
+    with make_app_client(settings={"template_debug": True}) as client:
         response = client.get("/fixtures/facetable?_context=1")
         # scope_path is added by PLUGIN1
         assert "scope_path" in response.text
@@ -1744,7 +1746,7 @@ def test_facet_more_links(
     expected_ellipses_url,
 ):
     with make_app_client(
-        config={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
+        settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
     ) as client:
         response = client.get(path)
         soup = Soup(response.body, "html.parser")

From bbc4756f9e8180c7a40c57f8a35e39dee7be7807 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 12 Aug 2021 20:54:25 -0700
Subject: [PATCH 1156/2303] Settings fix, refs #1433

---
 datasette/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index 8cbaaf9f..adc543ef 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -277,7 +277,7 @@ class Datasette:
         self.static_mounts = static_mounts or []
         if config_dir and (config_dir / "config.json").exists():
             raise StartupError("config.json should be renamed to settings.json")
-        if config_dir and (config_dir / "settings.json").exists() and not config:
+        if config_dir and (config_dir / "settings.json").exists() and not settings:
             config = json.loads((config_dir / "settings.json").read_text())
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions

From 2883098770fc66e50183b2b231edbde20848d4d6 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 12 Aug 2021 22:10:07 -0700
Subject: [PATCH 1157/2303] Fixed config_dir mode, refs #1432

---
 datasette/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index adc543ef..06db740e 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -278,7 +278,7 @@ class Datasette:
         if config_dir and (config_dir / "config.json").exists():
             raise StartupError("config.json should be renamed to settings.json")
         if config_dir and (config_dir / "settings.json").exists() and not settings:
-            config = json.loads((config_dir / "settings.json").read_text())
+            settings = json.loads((config_dir / "settings.json").read_text())
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note

From adb5b70de5cec3c3dd37184defe606a082c232cf Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 16 Aug 2021 11:56:32 -0700
Subject: [PATCH 1158/2303] Show count of facet values if ?_facet_size=max,
 closes #1423

---
 datasette/static/app.css       |  5 +++++
 datasette/templates/table.html |  4 +++-
 datasette/views/table.py       |  1 +
 tests/test_html.py             | 22 +++++++++++++++++++++-
 4 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/datasette/static/app.css b/datasette/static/app.css
index bf068fdf..af3e14d5 100644
--- a/datasette/static/app.css
+++ b/datasette/static/app.css
@@ -633,6 +633,11 @@ form button[type=button] {
     width: 250px;
     margin-right: 15px;
 }
+.facet-info-total {
+    font-size: 0.8em;
+    color: #666;
+    padding-right: 0.25em;
+}
 .facet-info li,
 .facet-info ul {
     margin: 0;
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index a28945ad..6ba301b5 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -156,7 +156,9 @@
         {% for facet_info in sorted_facet_results %}
             
    
                 
    
-                    {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
+                    {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
+                        {% if show_facet_counts %} {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}{% endif %}
+                    
                     {% if facet_info.hideable %}
                         ✖
                     {% endif %}
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 486a6131..83f7c7cb 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -928,6 +928,7 @@ class TableView(RowTableShared):
                     key=lambda f: (len(f["results"]), f["name"]),
                     reverse=True,
                 ),
+                "show_facet_counts": special_args.get("_facet_size") == "max",
                 "extra_wheres_for_ui": extra_wheres_for_ui,
                 "form_hidden_args": form_hidden_args,
                 "is_sortable": any(c["sortable"] for c in display_columns),
diff --git a/tests/test_html.py b/tests/test_html.py
index 90fcdae7..e73ccd2f 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -479,7 +479,7 @@ def test_facet_display(app_client):
     for div in divs:
         actual.append(
             {
-                "name": div.find("strong").text,
+                "name": div.find("strong").text.split()[0],
                 "items": [
                     {
                         "name": a.text,
@@ -1797,3 +1797,23 @@ def test_column_metadata(app_client):
         soup.select("th[data-column=address]")[0]["data-column-description"]
         == "The street address for the attraction"
     )
+
+
+@pytest.mark.parametrize("use_facet_size_max", (True, False))
+def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
+    # https://github.com/simonw/datasette/issues/1423
+    with make_app_client(settings={"max_returned_rows": 100}) as client:
+        path = "/fixtures/sortable?_facet=content&_facet=pk1"
+        if use_facet_size_max:
+            path += "&_facet_size=max"
+        response = client.get(path)
+        assert response.status == 200
+    fragments = (
+        '>100',
+        '8',
+    )
+    for fragment in fragments:
+        if use_facet_size_max:
+            assert fragment in response.text
+        else:
+            assert fragment not in response.text

From d84e574e59c51ddcd6cf60a6f9b3d45182daf824 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 19 Aug 2021 14:09:38 -0700
Subject: [PATCH 1159/2303] Ability to deploy demos of branches

* Ability to deploy additional branch demos, closes #1442
* Only run tests before deploy on main branch
* Documentation for continuous deployment
---
 .github/workflows/deploy-latest.yml |  8 +++++++-
 docs/contributing.rst               | 11 +++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index 849adb40..1a07503a 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -29,6 +29,7 @@ jobs:
         python -m pip install -e .[docs]
         python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
@@ -50,6 +51,8 @@ jobs:
       run: |-
         gcloud config set run/region us-central1
         gcloud config set project datasette-222320
+        export SUFFIX="-${GITHUB_REF#refs/heads/}"
+        export SUFFIX=${SUFFIX#-main}
         datasette publish cloudrun fixtures.db extra_database.db \
             -m fixtures.json \
             --plugins-dir=plugins \
@@ -57,7 +60,10 @@ jobs:
             --version-note=$GITHUB_SHA \
             --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \
             --install=pysqlite3-binary \
-            --service=datasette-latest
+            --service "datasette-latest$SUFFIX"
+    - name: Deploy to docs as well (only for main)
+      if: ${{ github.ref == 'refs/heads/main' }}
+      run: |-
         # Deploy docs.db to a different service
         datasette publish cloudrun docs.db \
             --branch=$GITHUB_SHA \
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 8a638e0b..07f2a0e4 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -202,6 +202,17 @@ For added productivity, you can use use `sphinx-autobuild `__ is re-deployed automatically to Google Cloud Run for every push to ``main`` that passes the test suite. This is implemented by the GitHub Actions workflow at `.github/workflows/deploy-latest.yml `__.
+
+Specific branches can also be set to automatically deploy by adding them to the ``on: push: branches`` block at the top of the workflow YAML file. Branches configured in this way will be deployed to a new Cloud Run service whether or not their tests pass.
+
+The Cloud Run URL for a branch demo can be found in the GitHub Actions logs.
+
 .. _contributing_release:
 
 Release process

From 4eb3ae40fb223a66ae574fb84fac99e96183b08d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 19 Aug 2021 14:17:44 -0700
Subject: [PATCH 1160/2303] Don't bother building docs if not on main

Refs ##1442
---
 .github/workflows/deploy-latest.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index 1a07503a..1ae96e89 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -36,6 +36,7 @@ jobs:
     - name: Build fixtures.db
       run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db
     - name: Build docs.db
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         sphinx-build -b xml . _build

From 7e15422aacfa9e9735cb9f9beaa32250edbf4905 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 19 Aug 2021 14:23:43 -0700
Subject: [PATCH 1161/2303] Documentation for datasette.databases property,
 closes #1443

---
 docs/internals.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index 058a8969..d5db7ffa 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -196,6 +196,17 @@ Datasette class
 
 This object is an instance of the ``Datasette`` class, passed to many plugin hooks as an argument called ``datasette``.
 
+.. _datasette_databases:
+
+.databases
+----------
+
+Property exposing an ordered dictionary of databases currently connected to Datasette.
+
+The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances.
+
+All databases are listed, irrespective of user permissions. This means that the ``_internal`` database will always be listed here.
+
 .. _datasette_plugin_config:
 
 .plugin_config(plugin_name, database=None, table=None)

From 92a99d969c01633dba14cceebeda65daaedaec17 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 24 Aug 2021 11:13:42 -0700
Subject: [PATCH 1162/2303] Added not-footer wrapper div, refs #1446

---
 datasette/templates/base.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/base.html b/datasette/templates/base.html
index e61edc4f..c9aa7e31 100644
--- a/datasette/templates/base.html
+++ b/datasette/templates/base.html
@@ -13,6 +13,7 @@
 {% block extra_head %}{% endblock %}
 
 
+
    
 
    
 
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    
 
 {% include "_close_open_menus.html" %}

From 93c3a7ffbfb3378f743ebce87d033cf1ce7689e0 Mon Sep 17 00:00:00 2001
From: Tim Sherratt 
Date: Wed, 25 Aug 2021 11:28:58 +1000
Subject: [PATCH 1163/2303] Remove underscore from search mode parameter name
 (#1447)

The text refers to the parameter as `searchmode` but the `metadata.json` example uses `search_mode`. The latter doesn't actually seem to work.
---
 docs/full_text_search.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst
index f549296f..90b2e8c1 100644
--- a/docs/full_text_search.rst
+++ b/docs/full_text_search.rst
@@ -70,7 +70,7 @@ Here is an example which enables full-text search (with SQLite advanced search o
                     "display_ads": {
                         "fts_table": "ads_fts",
                         "fts_pk": "id",
-                        "search_mode": "raw"
+                        "searchmode": "raw"
                     }
                 }
             }

From 5161422b7fa249c6b7d6dc47ec6f483d3fdbd170 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Aug 2021 18:29:26 -0700
Subject: [PATCH 1164/2303] Update trustme requirement from <0.9,>=0.7 to
 >=0.7,<0.10 (#1433)

Updates the requirements on [trustme](https://github.com/python-trio/trustme) to permit the latest version.
- [Release notes](https://github.com/python-trio/trustme/releases)
- [Commits](https://github.com/python-trio/trustme/compare/v0.7.0...v0.9.0)

---
updated-dependencies:
- dependency-name: trustme
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 65e99848..a3866515 100644
--- a/setup.py
+++ b/setup.py
@@ -73,7 +73,7 @@ setup(
             "beautifulsoup4>=4.8.1,<4.10.0",
             "black==21.6b0",
             "pytest-timeout>=1.4.2,<1.5",
-            "trustme>=0.7,<0.9",
+            "trustme>=0.7,<0.10",
         ],
         "rich": ["rich"],
     },

From a1a33bb5822214be1cebd98cd858b2058d91a4aa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Aug 2021 18:29:55 -0700
Subject: [PATCH 1165/2303] Bump black from 21.6b0 to 21.7b0 (#1400)

Bumps [black](https://github.com/psf/black) from 21.6b0 to 21.7b0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a3866515..84f32087 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.4",
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
-            "black==21.6b0",
+            "black==21.7b0",
             "pytest-timeout>=1.4.2,<1.5",
             "trustme>=0.7,<0.10",
         ],

From 3655bb49a464bcc8004e491cc4d4de292f1acd62 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 27 Aug 2021 17:48:54 -0700
Subject: [PATCH 1166/2303] Better default help text, closes #1450

---
 datasette/cli.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index ea6da748..65da5613 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -123,7 +123,11 @@ def sqlite_extensions(fn):
 @click.version_option(version=__version__)
 def cli():
     """
-    Datasette!
+    Datasette is an open source multi-tool for exploring and publishing data
+
+    \b
+    About Datasette: https://datasette.io/
+    Full documentation: https://docs.datasette.io/
     """
 
 

From 30c18576d603366dc3bd83ba50de1b7e70844430 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 27 Aug 2021 18:39:42 -0700
Subject: [PATCH 1167/2303] register_commands() plugin hook, closes #1449

---
 datasette/cli.py       |  3 +++
 datasette/hookspecs.py |  5 ++++
 docs/plugin_hooks.rst  | 45 +++++++++++++++++++++++++++++++++
 tests/test_plugins.py  | 57 +++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 109 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 65da5613..22e2338a 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -595,6 +595,9 @@ def serve(
     uvicorn.run(ds.app(), **uvicorn_kwargs)
 
 
+pm.hook.register_commands(cli=cli)
+
+
 async def check_databases(ds):
     # Run check_connection against every connected database
     # to confirm they are all usable
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 56c79d23..1d4e3b27 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -79,6 +79,11 @@ def register_routes(datasette):
     """Register URL routes: return a list of (regex, view_function) pairs"""
 
 
+@hookspec
+def register_commands(cli):
+    """Register additional CLI commands, e.g. 'datasette mycommand ...'"""
+
+
 @hookspec
 def actor_from_request(datasette, request):
     """Return an actor dictionary based on the incoming request"""
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 5cdb1623..a6fe1071 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -587,6 +587,51 @@ See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes u
 
 Examples: `datasette-auth-github `__, `datasette-psutil `__
 
+.. _plugin_register_commands:
+
+register_commands(cli)
+----------------------
+
+``cli`` - the root Datasette `Click command group `__
+    Use this to register additional CLI commands
+
+Register additional CLI commands that can be run using ``datsette yourcommand ...``. This provides a mechanism by which plugins can add new CLI commands to Datasette.
+
+This example registers a new ``datasette verify file1.db file2.db`` command that checks if the provided file paths are valid SQLite databases:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import click
+    import sqlite3
+
+    @hookimpl
+    def register_commands(cli):
+        @cli.command()
+        @click.argument("files", type=click.Path(exists=True), nargs=-1)
+        def verify(files):
+            "Verify that files can be opened by Datasette"
+            for file in files:
+                conn = sqlite3.connect(str(file))
+                try:
+                    conn.execute("select * from sqlite_master")
+                except sqlite3.DatabaseError:
+                    raise click.ClickException("Invalid database: {}".format(file))
+
+The new command can then be executed like so::
+
+    datasette verify fixtures.db
+
+Help text (from the docstring for the function plus any defined Click arguments or options) will become available using::
+
+    datasette verify --help
+
+Plugins can register multiple commands by making multiple calls to the ``@cli.command()`` decorator.Consult the `Click documentation `__ for full details on how to build a CLI command, including how to define arguments and options.
+
+Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-dir mechanism ` - they need to be installed into the same virtual environment as Datasette using ``pip install``. Provided it has a ``setup.py`` file (see :ref:`writing_plugins_packaging`) you can run ``pip install`` directly against the directory in which you are developing your plugin like so::
+
+    pip install -e path/to/my/datasette-plugin
+
 .. _plugin_register_facet_classes:
 
 register_facet_classes()
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index ec8ff0c5..a024c39b 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -6,13 +6,15 @@ from .fixtures import (
     TEMP_PLUGIN_SECRET_FILE,
     TestClient as _TestClient,
 )  # noqa
+from click.testing import CliRunner
 from datasette.app import Datasette
-from datasette import cli
+from datasette import cli, hookimpl
 from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
 from datasette.utils.sqlite import sqlite3
 from datasette.utils import CustomRow
 from jinja2.environment import Template
 import base64
+import importlib
 import json
 import os
 import pathlib
@@ -902,3 +904,56 @@ def test_hook_get_metadata(app_client):
     assert "Hello from local metadata" == meta["databases"]["from-local"]["title"]
     assert "Hello from the plugin hook" == meta["databases"]["from-hook"]["title"]
     pm.hook.get_metadata = og_pm_hook_get_metadata
+
+
+def _extract_commands(output):
+    lines = output.split("Commands:\n", 1)[1].split("\n")
+    return {line.split()[0].replace("*", "") for line in lines if line.strip()}
+
+
+def test_hook_register_commands():
+    # Without the plugin should have seven commands
+    runner = CliRunner()
+    result = runner.invoke(cli.cli, "--help")
+    commands = _extract_commands(result.output)
+    assert commands == {
+        "serve",
+        "inspect",
+        "install",
+        "package",
+        "plugins",
+        "publish",
+        "uninstall",
+    }
+
+    # Now install a plugin
+    class VerifyPlugin:
+        __name__ = "VerifyPlugin"
+
+        @hookimpl
+        def register_commands(self, cli):
+            @cli.command()
+            def verify():
+                pass
+
+            @cli.command()
+            def unverify():
+                pass
+
+    pm.register(VerifyPlugin(), name="verify")
+    importlib.reload(cli)
+    result2 = runner.invoke(cli.cli, "--help")
+    commands2 = _extract_commands(result2.output)
+    assert commands2 == {
+        "serve",
+        "inspect",
+        "install",
+        "package",
+        "plugins",
+        "publish",
+        "uninstall",
+        "verify",
+        "unverify",
+    }
+    pm.unregister(name="verify")
+    importlib.reload(cli)

From d3ea36713194e3d92ed4c066337400146c921d0e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 27 Aug 2021 18:55:54 -0700
Subject: [PATCH 1168/2303] Release 0.59a2

Refs #942, #1421, #1423, #1431, #1443, #1446, #1449
---
 datasette/version.py  |  2 +-
 docs/changelog.rst    | 13 +++++++++++++
 docs/plugin_hooks.rst |  2 +-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index f5fbfb3f..87b18fab 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59a1"
+__version__ = "0.59a2"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1406a7ca..737a151b 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,19 @@
 Changelog
 =========
 
+.. _v0_59a2:
+
+0.59a2 (2021-08-27)
+-------------------
+
+- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
+- New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
+- Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
+- Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
+- Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)
+- ``datasette.databases`` property is now a documented API. (:issue:`1443`)
+- Datasette base template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
+
 .. _v0_59a1:
 
 0.59a1 (2021-08-08)
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index a6fe1071..be1624a3 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -587,7 +587,7 @@ See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes u
 
 Examples: `datasette-auth-github `__, `datasette-psutil `__
 
-.. _plugin_register_commands:
+.. _plugin_hook_register_commands:
 
 register_commands(cli)
 ----------------------

From 50c35b66a476c186549167140b6ebc0a6ec43fc1 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 28 Aug 2021 04:14:38 -0700
Subject: [PATCH 1169/2303] Added missing space

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index be1624a3..d61dc727 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -626,7 +626,7 @@ Help text (from the docstring for the function plus any defined Click arguments
 
     datasette verify --help
 
-Plugins can register multiple commands by making multiple calls to the ``@cli.command()`` decorator.Consult the `Click documentation `__ for full details on how to build a CLI command, including how to define arguments and options.
+Plugins can register multiple commands by making multiple calls to the ``@cli.command()`` decorator. Consult the `Click documentation `__ for full details on how to build a CLI command, including how to define arguments and options.
 
 Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-dir mechanism ` - they need to be installed into the same virtual environment as Datasette using ``pip install``. Provided it has a ``setup.py`` file (see :ref:`writing_plugins_packaging`) you can run ``pip install`` directly against the directory in which you are developing your plugin like so::
 

From 67cbf0ae7243431bf13702e6e3ba466b619c4d6f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 28 Aug 2021 04:17:03 -0700
Subject: [PATCH 1170/2303] Example for register_commands, refs #1449

---
 docs/plugin_hooks.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index d61dc727..23f19e38 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -80,7 +80,6 @@ You can now use this filter in your custom templates like so::
 
     Table name: {{ table|uppercase }}
 
-
 .. _plugin_hook_extra_template_vars:
 
 extra_template_vars(template, database, table, columns, view_name, request, datasette)
@@ -632,6 +631,8 @@ Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-d
 
     pip install -e path/to/my/datasette-plugin
 
+Example: `datasette-verify `_
+
 .. _plugin_register_facet_classes:
 
 register_facet_classes()

From 772f9a07ce363869e0aaa7600617454dc00e6966 Mon Sep 17 00:00:00 2001
From: Robert Gieseke 
Date: Sat, 4 Sep 2021 18:31:38 +0200
Subject: [PATCH 1171/2303] Add scientists to target groups (#1455)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 55160afe..95d430cc 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 
 Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API.
 
-Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world.
+Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists and anyone else who has data that they wish to share with the world.
 
 [Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch).
 

From d57ab156b35ec642549fb69d08279850065027d2 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 4 Sep 2021 09:33:20 -0700
Subject: [PATCH 1172/2303] Added researchers too, refs #1455

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 95d430cc..ee9d9a5a 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 
 Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API.
 
-Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists and anyone else who has data that they wish to share with the world.
+Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world.
 
 [Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch).
 

From b28b6cd2fe97f7e193a235877abeec2c8eb0a821 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 12 Sep 2021 13:13:52 -0700
Subject: [PATCH 1173/2303] Warn that execute_write_fn(fn) should be a
 non-async function

---
 docs/internals.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index d5db7ffa..910f2c71 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -664,6 +664,10 @@ This method works like ``.execute_write()``, but instead of a SQL statement you
 
 The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection as long as it is executing.
 
+.. warning::
+
+    ``fn`` needs to be a regular function, not an ``async def`` function.
+
 For example:
 
 .. code-block:: python

From 63886178a649586b403966a27a45881709d2b868 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 22 Sep 2021 15:44:28 -0700
Subject: [PATCH 1174/2303] Describe a common mistake using csrftoken()

---
 docs/internals.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index 910f2c71..411327eb 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -793,6 +793,10 @@ If your plugin implements a ``
    `` anywhere you will need to i
 
     
 
+If you are rendering templates using the :ref:`datasette_render_template` method the ``csrftoken()`` helper will only work if you provide the ``request=`` argument to that method. If you forget to do this you will see the following error::
+
+    form-urlencoded POST field did not match cookie
+
 You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csrf` hook.
 
 .. _internals_internal:

From 98dcabccbbf9c0800efa74df9b7d1fee81c3cd0c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 8 Oct 2021 17:32:11 -0700
Subject: [PATCH 1175/2303] asyncio_run helper to deal with a 3.10 warning,
 refs #1482

---
 datasette/cli.py            | 13 +++++--------
 datasette/utils/__init__.py |  9 +++++++++
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 22e2338a..95ee9495 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -14,6 +14,7 @@ from runpy import run_module
 import webbrowser
 from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm
 from .utils import (
+    asyncio_run,
     StartupError,
     check_connection,
     find_spatialite,
@@ -136,9 +137,7 @@ def cli():
 @click.option("--inspect-file", default="-")
 @sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
-    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
-    loop = asyncio.get_event_loop()
-    inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
+    inspect_data = asyncio_run(inspect_(files, sqlite_extensions))
     if inspect_file == "-":
         sys.stdout.write(json.dumps(inspect_data, indent=2))
     else:
@@ -555,10 +554,10 @@ def serve(
         return ds
 
     # Run the "startup" plugin hooks
-    asyncio.get_event_loop().run_until_complete(ds.invoke_startup())
+    asyncio_run(ds.invoke_startup())
 
     # Run async soundness checks - but only if we're not under pytest
-    asyncio.get_event_loop().run_until_complete(check_databases(ds))
+    asyncio_run(check_databases(ds))
 
     if get:
         client = TestClient(ds)
@@ -578,9 +577,7 @@ def serve(
     if open_browser:
         if url is None:
             # Figure out most convenient URL - to table, database or homepage
-            path = asyncio.get_event_loop().run_until_complete(
-                initial_path_for_datasette(ds)
-            )
+            path = asyncio_run(initial_path_for_datasette(ds))
             url = f"http://{host}:{port}{path}"
         webbrowser.open(url)
     uvicorn_kwargs = dict(
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 70ac8976..1b7bfe05 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1089,3 +1089,12 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
+
+
+def asyncio_run(coro):
+    if hasattr(asyncio, "run"):
+        # Added in Python 3.7
+        return asyncio.run(coro)
+    else:
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(coro)

From 1163da89163f357003007b14f458a2c28f2e0a8e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 8 Oct 2021 17:32:52 -0700
Subject: [PATCH 1176/2303] Update test to handle Python 3.10 error message
 differenc, refs #1482

---
 tests/test_canned_queries.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index 4186a97c..b8c2baec 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -353,4 +353,4 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c
         "/data.json?sql=select+:_header_host&_shape=array"
     )
     assert 400 == response.status
-    assert "You did not supply a value for binding 1." == response.json["error"]
+    assert response.json["error"].startswith("You did not supply a value for binding")

From 875117c343e454221ef023be6ad977fdaea3ceda Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 9 Oct 2021 18:14:56 -0700
Subject: [PATCH 1177/2303] Fix bug with ?_next=x&_sort=rowid, closes #1470

---
 datasette/views/table.py | 6 ++++--
 tests/test_html.py       | 6 ++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 83f7c7cb..e8c66d3c 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -588,13 +588,15 @@ class TableView(RowTableShared):
         _next = _next or special_args.get("_next")
         offset = ""
         if _next:
+            sort_value = None
             if is_view:
                 # _next is an offset
                 offset = f" offset {int(_next)}"
             else:
                 components = urlsafe_components(_next)
-                # If a sort order is applied, the first of these is the sort value
-                if sort or sort_desc:
+                # If a sort order is applied and there are multiple components,
+                # the first of these is the sort value
+                if (sort or sort_desc) and (len(components) > 1):
                     sort_value = components[0]
                     # Special case for if non-urlencoded first token was $null
                     if _next.split(",")[0] == "$null":
diff --git a/tests/test_html.py b/tests/test_html.py
index e73ccd2f..c22d87da 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1817,3 +1817,9 @@ def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
             assert fragment in response.text
         else:
             assert fragment not in response.text
+
+
+def test_sort_rowid_with_next(app_client):
+    # https://github.com/simonw/datasette/issues/1470
+    response = app_client.get("/fixtures/binary_data?_size=1&_next=1&_sort=rowid")
+    assert response.status == 200

From 0d5cc20aeffa3537cfc9296d01ec24b9c6e23dcf Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 9 Oct 2021 18:25:33 -0700
Subject: [PATCH 1178/2303] Revert "asyncio_run helper to deal with a 3.10
 warning, refs #1482"

This reverts commit 98dcabccbbf9c0800efa74df9b7d1fee81c3cd0c.
---
 datasette/cli.py            | 13 ++++++++-----
 datasette/utils/__init__.py |  9 ---------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 95ee9495..22e2338a 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -14,7 +14,6 @@ from runpy import run_module
 import webbrowser
 from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm
 from .utils import (
-    asyncio_run,
     StartupError,
     check_connection,
     find_spatialite,
@@ -137,7 +136,9 @@ def cli():
 @click.option("--inspect-file", default="-")
 @sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
-    inspect_data = asyncio_run(inspect_(files, sqlite_extensions))
+    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
+    loop = asyncio.get_event_loop()
+    inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
     if inspect_file == "-":
         sys.stdout.write(json.dumps(inspect_data, indent=2))
     else:
@@ -554,10 +555,10 @@ def serve(
         return ds
 
     # Run the "startup" plugin hooks
-    asyncio_run(ds.invoke_startup())
+    asyncio.get_event_loop().run_until_complete(ds.invoke_startup())
 
     # Run async soundness checks - but only if we're not under pytest
-    asyncio_run(check_databases(ds))
+    asyncio.get_event_loop().run_until_complete(check_databases(ds))
 
     if get:
         client = TestClient(ds)
@@ -577,7 +578,9 @@ def serve(
     if open_browser:
         if url is None:
             # Figure out most convenient URL - to table, database or homepage
-            path = asyncio_run(initial_path_for_datasette(ds))
+            path = asyncio.get_event_loop().run_until_complete(
+                initial_path_for_datasette(ds)
+            )
             url = f"http://{host}:{port}{path}"
         webbrowser.open(url)
     uvicorn_kwargs = dict(
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 1b7bfe05..70ac8976 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1089,12 +1089,3 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
-
-
-def asyncio_run(coro):
-    if hasattr(asyncio, "run"):
-        # Added in Python 3.7
-        return asyncio.run(coro)
-    else:
-        loop = asyncio.get_event_loop()
-        return loop.run_until_complete(coro)

From b50bf5d13fed10d0b930f62198d8f2658e15e1eb Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 13 Oct 2021 14:08:06 -0700
Subject: [PATCH 1179/2303] Don't persist _next in hidden field, closes #1483

---
 datasette/views/table.py |  2 +-
 tests/test_html.py       | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index e8c66d3c..efcef4d2 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -889,7 +889,7 @@ class TableView(RowTableShared):
 
             form_hidden_args = []
             for key in request.args:
-                if key.startswith("_") and key not in ("_sort", "_search"):
+                if key.startswith("_") and key not in ("_sort", "_search", "_next"):
                     for value in request.args.getlist(key):
                         form_hidden_args.append((key, value))
 
diff --git a/tests/test_html.py b/tests/test_html.py
index c22d87da..151ac5c3 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -563,6 +563,16 @@ def test_facets_persist_through_filter_form(app_client):
     ]
 
 
+def test_next_does_not_persist_in_hidden_field(app_client):
+    response = app_client.get("/fixtures/searchable?_size=1&_next=1")
+    assert response.status == 200
+    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
+        ("_size", "1"),
+    ]
+
+
 @pytest.mark.parametrize(
     "path,expected_classes",
     [

From 68087440b3448633a62807c1623559619584f2ee Mon Sep 17 00:00:00 2001
From: Rhet Turnbull 
Date: Wed, 13 Oct 2021 14:09:10 -0700
Subject: [PATCH 1180/2303] Added instructions for installing plugins via pipx

Closes #1486
---
 docs/installation.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/installation.rst b/docs/installation.rst
index b6881bc0..723f1e3f 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -90,6 +90,25 @@ Once pipx is installed you can use it to install Datasette like this::
 
 Then run ``datasette --version`` to confirm that it has been successfully installed.
 
+Installing plugins using pipx
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can install additional datasette plugins with ``pipx inject`` like so::
+
+    $ pipx inject datasette datasette-json-html
+    injected package datasette-json-html into venv datasette
+    done! ✨ 🌟 ✨
+
+    $ datasette plugins
+    [
+        {
+            "name": "datasette-json-html",
+            "static": false,
+            "templates": false,
+            "version": "0.6"
+        }
+    ]
+
 Upgrading packages using pipx
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 759fd97a54638c1a5e2cac65bac0ac7c07ce2305 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:09:23 -0700
Subject: [PATCH 1181/2303] Update pytest-timeout requirement from <1.5,>=1.4.2
 to >=1.4.2,<2.1 (#1485)

Updates the requirements on [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-timeout/releases)
- [Commits](https://github.com/pytest-dev/pytest-timeout/commits)

---
updated-dependencies:
- dependency-name: pytest-timeout
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 84f32087..da62276e 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,7 @@ setup(
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
             "black==21.7b0",
-            "pytest-timeout>=1.4.2,<1.5",
+            "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
         ],
         "rich": ["rich"],

From 6aab0217f07bff4556cc92885a14279d5b295f84 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:10:03 -0700
Subject: [PATCH 1182/2303] Update pytest-xdist requirement from <2.4,>=2.2.1
 to >=2.2.1,<2.5 (#1476)

Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-xdist/releases)
- [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v2.2.1...v2.4.0)

---
updated-dependencies:
- dependency-name: pytest-xdist
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index da62276e..571bb35a 100644
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@ setup(
         "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
         "test": [
             "pytest>=5.2.2,<6.3.0",
-            "pytest-xdist>=2.2.1,<2.4",
+            "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
             "black==21.7b0",

From 31352914c427162f785d2610222a54a426d5215f Mon Sep 17 00:00:00 2001
From: Michael Tiemann <72577720+MichaelTiemannOSC@users.noreply.github.com>
Date: Wed, 13 Oct 2021 17:10:23 -0400
Subject: [PATCH 1183/2303] Update full_text_search.rst (#1474)

Change "above" to "below" to correct correspondence of reference to example.
---
 docs/full_text_search.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst
index 90b2e8c1..a285417f 100644
--- a/docs/full_text_search.rst
+++ b/docs/full_text_search.rst
@@ -47,7 +47,7 @@ If that option has been specified in the table metadata but you want to over-rid
 Configuring full-text search for a table or view
 ------------------------------------------------
 
-If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown above, Datasette will detect it automatically and add a search interface to the table page for that table.
+If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table.
 
 You can also manually configure which table should be used for full-text search using query string parameters or :ref:`metadata`. You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
 

From a673a93b57e249f06b2d0265ce33f458258feeb0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:11:00 -0700
Subject: [PATCH 1184/2303] Update pluggy requirement from ~=0.13.0 to
 >=0.13,<1.1 (#1448)

Updates the requirements on [pluggy](https://github.com/pytest-dev/pluggy) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pluggy/releases)
- [Changelog](https://github.com/pytest-dev/pluggy/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pluggy/compare/0.13.0...1.0.0)

---
updated-dependencies:
- dependency-name: pluggy
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 571bb35a..cb1cec66 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ setup(
         "hupper~=1.9",
         "httpx>=0.17",
         "pint~=0.9",
-        "pluggy~=0.13.0",
+        "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.8",
         "janus>=0.4,<0.7",

From 763d0a0faabc6b53fa21ea2f0914e83ca12dfb34 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 13 Oct 2021 14:19:53 -0700
Subject: [PATCH 1185/2303] Fix for cog menu default facet bug, closes #1469

---
 datasette/static/table.js      | 9 ++++-----
 datasette/templates/table.html | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/datasette/static/table.js b/datasette/static/table.js
index 85bf073f..3c88cc40 100644
--- a/datasette/static/table.js
+++ b/datasette/static/table.js
@@ -129,16 +129,15 @@ var DROPDOWN_ICON_SVG = ` el.dataset.column);
     var isFirstColumn =
       th.parentElement.querySelector("th:first-of-type") == th;
     var isSinglePk =
       th.getAttribute("data-is-pk") == "1" &&
       document.querySelectorAll('th[data-is-pk="1"]').length == 1;
-    if (
-      isFirstColumn ||
-      params.getAll("_facet").includes(column) ||
-      isSinglePk
-    ) {
+    if (isFirstColumn || displayedFacets.includes(column) || isSinglePk) {
       facetItem.parentNode.style.display = "none";
     } else {
       facetItem.parentNode.style.display = "block";
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index 6ba301b5..4b9df8e1 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -154,7 +154,7 @@
 {% if facet_results %}
     
    
         {% for facet_info in sorted_facet_results %}
-            
    
+            
    
                 
    
                     {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
                         {% if show_facet_counts %} {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}{% endif %}

From e1012e7098056734d9c90f081493991009253390 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:47:42 -0700
Subject: [PATCH 1186/2303] Bump black from 21.7b0 to 21.9b0 (#1471)

Bumps [black](https://github.com/psf/black) from 21.7b0 to 21.9b0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index cb1cec66..20576e72 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
-            "black==21.7b0",
+            "black==21.9b0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
         ],

From 2a8c6690399ee832ee62aafdede1794f5945d911 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 15:35:36 -0700
Subject: [PATCH 1187/2303] Update beautifulsoup4 requirement (#1463)

Updates the requirements on [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) to permit the latest version.

---
updated-dependencies:
- dependency-name: beautifulsoup4
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 20576e72..905fed16 100644
--- a/setup.py
+++ b/setup.py
@@ -70,7 +70,7 @@ setup(
             "pytest>=5.2.2,<6.3.0",
             "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.16",
-            "beautifulsoup4>=4.8.1,<4.10.0",
+            "beautifulsoup4>=4.8.1,<4.11.0",
             "black==21.9b0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",

From b267b5775436577a91a9f9655143908aecff05da Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 11:03:44 -0700
Subject: [PATCH 1188/2303] Upgrade to httpx 0.20

* Upgrade to httpx 0.20, closes #1488
* TestClient.post() should not default to following redirects
---
 datasette/app.py                         |  5 +++-
 datasette/utils/asgi.py                  |  4 +--
 datasette/utils/testing.py               | 20 ++++++-------
 setup.py                                 |  2 +-
 tests/test_api.py                        | 22 ++++++--------
 tests/test_auth.py                       |  5 +---
 tests/test_canned_queries.py             | 12 +-------
 tests/test_custom_pages.py               |  4 +--
 tests/test_html.py                       | 38 ++++++++++++------------
 tests/test_internals_datasette_client.py |  1 -
 tests/test_internals_request.py          |  5 +++-
 tests/test_plugins.py                    | 13 ++++----
 12 files changed, 60 insertions(+), 71 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 06db740e..1f69c2b3 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1139,6 +1139,7 @@ class DatasetteRouter:
         raw_path = scope.get("raw_path")
         if raw_path:
             path = raw_path.decode("ascii")
+        path = path.partition("?")[0]
         return await self.route_path(scope, receive, send, path)
 
     async def route_path(self, scope, receive, send, path):
@@ -1192,7 +1193,9 @@ class DatasetteRouter:
 
     async def handle_404(self, request, send, exception=None):
         # If URL has a trailing slash, redirect to URL without it
-        path = request.scope.get("raw_path", request.scope["path"].encode("utf8"))
+        path = request.scope.get(
+            "raw_path", request.scope["path"].encode("utf8")
+        ).partition(b"?")[0]
         context = {}
         if path.endswith(b"/"):
             path = path.rstrip(b"/")
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 5fa03b0a..696944df 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -75,7 +75,7 @@ class Request:
     @property
     def path(self):
         if self.scope.get("raw_path") is not None:
-            return self.scope["raw_path"].decode("latin-1")
+            return self.scope["raw_path"].decode("latin-1").partition("?")[0]
         else:
             path = self.scope["path"]
             if isinstance(path, str):
@@ -122,7 +122,7 @@ class Request:
             "http_version": "1.1",
             "method": method,
             "path": path,
-            "raw_path": path.encode("latin-1"),
+            "raw_path": path_with_query_string.encode("latin-1"),
             "query_string": query_string.encode("latin-1"),
             "scheme": scheme,
             "type": "http",
diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py
index a169a83d..94750b1f 100644
--- a/datasette/utils/testing.py
+++ b/datasette/utils/testing.py
@@ -55,10 +55,10 @@ class TestClient:
 
     @async_to_sync
     async def get(
-        self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None
+        self, path, follow_redirects=False, redirect_count=0, method="GET", cookies=None
     ):
         return await self._request(
-            path, allow_redirects, redirect_count, method, cookies
+            path, follow_redirects, redirect_count, method, cookies
         )
 
     @async_to_sync
@@ -67,7 +67,7 @@ class TestClient:
         path,
         post_data=None,
         body=None,
-        allow_redirects=True,
+        follow_redirects=False,
         redirect_count=0,
         content_type="application/x-www-form-urlencoded",
         cookies=None,
@@ -90,7 +90,7 @@ class TestClient:
             body = urlencode(post_data, doseq=True)
         return await self._request(
             path=path,
-            allow_redirects=allow_redirects,
+            follow_redirects=follow_redirects,
             redirect_count=redirect_count,
             method="POST",
             cookies=cookies,
@@ -103,7 +103,7 @@ class TestClient:
     async def request(
         self,
         path,
-        allow_redirects=True,
+        follow_redirects=True,
         redirect_count=0,
         method="GET",
         cookies=None,
@@ -113,7 +113,7 @@ class TestClient:
     ):
         return await self._request(
             path,
-            allow_redirects=allow_redirects,
+            follow_redirects=follow_redirects,
             redirect_count=redirect_count,
             method=method,
             cookies=cookies,
@@ -125,7 +125,7 @@ class TestClient:
     async def _request(
         self,
         path,
-        allow_redirects=True,
+        follow_redirects=True,
         redirect_count=0,
         method="GET",
         cookies=None,
@@ -139,19 +139,19 @@ class TestClient:
         httpx_response = await self.ds.client.request(
             method,
             path,
-            allow_redirects=allow_redirects,
+            follow_redirects=follow_redirects,
             avoid_path_rewrites=True,
             cookies=cookies,
             headers=headers,
             content=post_body,
         )
         response = TestResponse(httpx_response)
-        if allow_redirects and response.status in (301, 302):
+        if follow_redirects and response.status in (301, 302):
             assert (
                 redirect_count < self.max_redirects
             ), f"Redirected {redirect_count} times, max_redirects={self.max_redirects}"
             location = response.headers["Location"]
             return await self._request(
-                location, allow_redirects=True, redirect_count=redirect_count + 1
+                location, follow_redirects=True, redirect_count=redirect_count + 1
             )
         return response
diff --git a/setup.py b/setup.py
index 905fed16..45eac040 100644
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,7 @@ setup(
         "click-default-group~=1.2.2",
         "Jinja2>=2.10.3,<3.1.0",
         "hupper~=1.9",
-        "httpx>=0.17",
+        "httpx>=0.20",
         "pint~=0.9",
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
diff --git a/tests/test_api.py b/tests/test_api.py
index 1e93c62e..38d1ba08 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -629,7 +629,7 @@ def test_no_files_uses_memory_database(app_client_no_files):
     ),
 )
 def test_old_memory_urls_redirect(app_client_no_files, path, expected_redirect):
-    response = app_client_no_files.get(path, allow_redirects=False)
+    response = app_client_no_files.get(path)
     assert response.status == 301
     assert response.headers["location"] == expected_redirect
 
@@ -708,12 +708,8 @@ def test_table_not_exists_json(app_client):
 
 
 def test_jsono_redirects_to_shape_objects(app_client_with_hash):
-    response_1 = app_client_with_hash.get(
-        "/fixtures/simple_primary_key.jsono", allow_redirects=False
-    )
-    response = app_client_with_hash.get(
-        response_1.headers["Location"], allow_redirects=False
-    )
+    response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono")
+    response = app_client_with_hash.get(response_1.headers["Location"])
     assert response.status == 302
     assert response.headers["Location"].endswith("?_shape=objects")
 
@@ -1488,7 +1484,7 @@ def test_settings_json(app_client):
     ),
 )
 def test_config_redirects_to_settings(app_client, path, expected_redirect):
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 301
     assert response.headers["Location"] == expected_redirect
 
@@ -1834,9 +1830,7 @@ def test_hash_parameter(
     current_hash = app_client_two_attached_databases_one_immutable.ds.databases[
         "fixtures"
     ].hash[:7]
-    response = app_client_two_attached_databases_one_immutable.get(
-        path, allow_redirects=False
-    )
+    response = app_client_two_attached_databases_one_immutable.get(path)
     assert response.status == 302
     location = response.headers["Location"]
     assert expected_redirect.replace("HASH", current_hash) == location
@@ -1844,7 +1838,7 @@ def test_hash_parameter(
 
 def test_hash_parameter_ignored_for_mutable_databases(app_client):
     path = "/fixtures/facetable.json?_hash=1"
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 200
 
 
@@ -1976,7 +1970,9 @@ def test_cors(app_client_with_cors, path, status_code):
     ),
 )
 def test_database_with_space_in_name(app_client_two_attached_databases, path):
-    response = app_client_two_attached_databases.get("/extra database" + path)
+    response = app_client_two_attached_databases.get(
+        "/extra database" + path, follow_redirects=True
+    )
     assert response.status == 200
 
 
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 16397b7a..974f89ea 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -10,7 +10,6 @@ def test_auth_token(app_client):
     path = f"/-/auth-token?token={app_client.ds._root_token}"
     response = app_client.get(
         path,
-        allow_redirects=False,
     )
     assert 302 == response.status
     assert "/" == response.headers["Location"]
@@ -23,7 +22,6 @@ def test_auth_token(app_client):
         403
         == app_client.get(
             path,
-            allow_redirects=False,
         ).status
     )
 
@@ -78,14 +76,13 @@ def test_logout(app_client):
         in response2.text
     )
     # If logged out you get a redirect to /
-    response3 = app_client.get("/-/logout", allow_redirects=False)
+    response3 = app_client.get("/-/logout")
     assert 302 == response3.status
     # A POST to that page should log the user out
     response4 = app_client.post(
         "/-/logout",
         csrftoken_from=True,
         cookies={"ds_actor": app_client.actor_cookie({"id": "test"})},
-        allow_redirects=False,
     )
     # The ds_actor cookie should have been unset
     assert response4.cookie_was_deleted("ds_actor")
diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index b8c2baec..cea81ec7 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -59,7 +59,6 @@ def test_insert(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name",
         {"name": "Hello"},
-        allow_redirects=False,
         csrftoken_from=True,
         cookies={"foo": "bar"},
     )
@@ -95,16 +94,13 @@ def test_insert_with_cookies_requires_csrf(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name",
         {"name": "Hello"},
-        allow_redirects=False,
         cookies={"foo": "bar"},
     )
     assert 403 == response.status
 
 
 def test_insert_no_cookies_no_csrf(canned_write_client):
-    response = canned_write_client.post(
-        "/data/add_name", {"name": "Hello"}, allow_redirects=False
-    )
+    response = canned_write_client.post("/data/add_name", {"name": "Hello"})
     assert 302 == response.status
     assert "/data/add_name?success" == response.headers["Location"]
 
@@ -114,7 +110,6 @@ def test_custom_success_message(canned_write_client):
         "/data/delete_name",
         {"rowid": 1},
         cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})},
-        allow_redirects=False,
         csrftoken_from=True,
     )
     assert 302 == response.status
@@ -129,7 +124,6 @@ def test_insert_error(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name_specify_id",
         {"rowid": 1, "name": "Should fail"},
-        allow_redirects=False,
         csrftoken_from=True,
     )
     assert 302 == response.status
@@ -145,7 +139,6 @@ def test_insert_error(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name_specify_id",
         {"rowid": 1, "name": "Should fail"},
-        allow_redirects=False,
         csrftoken_from=True,
     )
     assert [["ERROR", 3]] == canned_write_client.ds.unsign(
@@ -168,7 +161,6 @@ def test_json_post_body(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name",
         body=json.dumps({"name": ["Hello", "there"]}),
-        allow_redirects=False,
     )
     assert 302 == response.status
     assert "/data/add_name?success" == response.headers["Location"]
@@ -189,7 +181,6 @@ def test_json_response(canned_write_client, headers, body, querystring):
     response = canned_write_client.post(
         "/data/add_name" + (querystring or ""),
         body=body,
-        allow_redirects=False,
         headers=headers,
     )
     assert 200 == response.status
@@ -331,7 +322,6 @@ def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_js
         f"/data/runme_post{qs}",
         {},
         csrftoken_from=use_csrf or None,
-        allow_redirects=False,
     )
     if return_json:
         assert response.status == 200
diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py
index 76c67397..66b7437a 100644
--- a/tests/test_custom_pages.py
+++ b/tests/test_custom_pages.py
@@ -67,13 +67,13 @@ def test_custom_content_type(custom_pages_client):
 
 
 def test_redirect(custom_pages_client):
-    response = custom_pages_client.get("/redirect", allow_redirects=False)
+    response = custom_pages_client.get("/redirect")
     assert 302 == response.status
     assert "/example" == response.headers["Location"]
 
 
 def test_redirect2(custom_pages_client):
-    response = custom_pages_client.get("/redirect2", allow_redirects=False)
+    response = custom_pages_client.get("/redirect2")
     assert 301 == response.status
     assert "/example" == response.headers["Location"]
 
diff --git a/tests/test_html.py b/tests/test_html.py
index 151ac5c3..5f2ba2f1 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -100,9 +100,9 @@ def test_not_allowed_methods():
 
 
 def test_database_page_redirects_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures", allow_redirects=False)
-    assert response.status == 302
     response = app_client_with_hash.get("/fixtures")
+    assert response.status == 302
+    response = app_client_with_hash.get("/fixtures", follow_redirects=True)
     assert "fixtures" in response.text
 
 
@@ -161,22 +161,22 @@ def test_sql_time_limit(app_client_shorter_time_limit):
 
 
 def test_row_redirects_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get(
-        "/fixtures/simple_primary_key/1", allow_redirects=False
-    )
+    response = app_client_with_hash.get("/fixtures/simple_primary_key/1")
     assert response.status == 302
     assert response.headers["Location"].endswith("/1")
-    response = app_client_with_hash.get("/fixtures/simple_primary_key/1")
+    response = app_client_with_hash.get(
+        "/fixtures/simple_primary_key/1", follow_redirects=True
+    )
     assert response.status == 200
 
 
 def test_row_strange_table_name_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get(
-        "/fixtures/table%2Fwith%2Fslashes.csv/3", allow_redirects=False
-    )
+    response = app_client_with_hash.get("/fixtures/table%2Fwith%2Fslashes.csv/3")
     assert response.status == 302
     assert response.headers["Location"].endswith("/table%2Fwith%2Fslashes.csv/3")
-    response = app_client_with_hash.get("/fixtures/table%2Fwith%2Fslashes.csv/3")
+    response = app_client_with_hash.get(
+        "/fixtures/table%2Fwith%2Fslashes.csv/3", follow_redirects=True
+    )
     assert response.status == 200
 
 
@@ -255,13 +255,13 @@ def test_add_filter_redirects(app_client):
     )
     path_base = "/fixtures/simple_primary_key"
     path = path_base + "?" + filter_args
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?content__startswith=x")
 
     # Adding a redirect to an existing query string:
     path = path_base + "?foo=bar&" + filter_args
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?foo=bar&content__startswith=x")
 
@@ -277,7 +277,7 @@ def test_add_filter_redirects(app_client):
             }
         )
     )
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?content__isnull=5")
 
@@ -299,7 +299,7 @@ def test_existing_filter_redirects(app_client):
     }
     path_base = "/fixtures/simple_primary_key"
     path = path_base + "?" + urllib.parse.urlencode(filter_args)
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert_querystring_equal(
         "name__contains=hello&age__gte=22&age__lt=30&name__contains=world",
@@ -309,7 +309,7 @@ def test_existing_filter_redirects(app_client):
     # Setting _filter_column_3 to empty string should remove *_3 entirely
     filter_args["_filter_column_3"] = ""
     path = path_base + "?" + urllib.parse.urlencode(filter_args)
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert_querystring_equal(
         "name__contains=hello&age__gte=22&name__contains=world",
@@ -317,7 +317,7 @@ def test_existing_filter_redirects(app_client):
     )
 
     # ?_filter_op=exact should be removed if unaccompanied by _fiter_column
-    response = app_client.get(path_base + "?_filter_op=exact", allow_redirects=False)
+    response = app_client.get(path_base + "?_filter_op=exact")
     assert response.status == 302
     assert "?" not in response.headers["Location"]
 
@@ -336,7 +336,7 @@ def test_empty_search_parameter_gets_removed(app_client):
             }
         )
     )
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?name__exact=chidi")
 
@@ -360,7 +360,7 @@ def test_sort_by_desc_redirects(app_client):
         + "?"
         + urllib.parse.urlencode({"_sort": "sortable", "_sort_by_desc": "1"})
     )
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?_sort_desc=sortable")
 
@@ -1148,7 +1148,7 @@ def test_404(app_client, path):
     [("/fixtures/", "/fixtures"), ("/fixtures/simple_view/", "/fixtures/simple_view")],
 )
 def test_404_trailing_slash_redirect(app_client, path, expected_redirect):
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert 302 == response.status
     assert expected_redirect == response.headers["Location"]
 
diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py
index c538bef1..8c5b5bd3 100644
--- a/tests/test_internals_datasette_client.py
+++ b/tests/test_internals_datasette_client.py
@@ -42,7 +42,6 @@ async def test_client_post(datasette, prefix):
             data={
                 "message": "A message",
             },
-            allow_redirects=False,
         )
         assert isinstance(response, httpx.Response)
         assert response.status_code == 302
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index fe273645..c42cfbd3 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -97,11 +97,14 @@ def test_request_url_vars():
     [("/", "", "/"), ("/", "foo=bar", "/?foo=bar"), ("/foo", "bar", "/foo?bar")],
 )
 def test_request_properties(path, query_string, expected_full_path):
+    path_with_query_string = path
+    if query_string:
+        path_with_query_string += "?" + query_string
     scope = {
         "http_version": "1.1",
         "method": "POST",
         "path": path,
-        "raw_path": path.encode("latin-1"),
+        "raw_path": path_with_query_string.encode("latin-1"),
         "query_string": query_string.encode("latin-1"),
         "scheme": "http",
         "type": "http",
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index a024c39b..7dac8002 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -71,7 +71,7 @@ def test_hook_plugin_prepare_connection_arguments(app_client):
             },
         ),
         (
-            "/fixtures/",
+            "/fixtures",
             {
                 "template": "database.html",
                 "database": "fixtures",
@@ -106,6 +106,7 @@ def test_hook_plugin_prepare_connection_arguments(app_client):
 )
 def test_hook_extra_css_urls(app_client, path, expected_decoded_object):
     response = app_client.get(path)
+    assert response.status == 200
     links = Soup(response.body, "html.parser").findAll("link")
     special_href = [
         l for l in links if l.attrs["href"].endswith("/extra-css-urls-demo.css")
@@ -263,7 +264,7 @@ def test_plugin_config_file(app_client):
             },
         ),
         (
-            "/fixtures/",
+            "/fixtures",
             {
                 "template": "database.html",
                 "database": "fixtures",
@@ -640,7 +641,7 @@ async def test_hook_permission_allowed(app_client, action, expected):
 def test_actor_json(app_client):
     assert {"actor": None} == app_client.get("/-/actor.json").json
     assert {"actor": {"id": "bot2", "1+1": 2}} == app_client.get(
-        "/-/actor.json/?_bot2=1"
+        "/-/actor.json?_bot2=1"
     ).json
 
 
@@ -674,7 +675,7 @@ def test_hook_register_routes_with_datasette(configured_path):
         assert configured_path.upper() == response.text
         # Other one should 404
         other_path = [p for p in ("path1", "path2") if configured_path != p][0]
-        assert client.get(f"/{other_path}/").status == 404
+        assert client.get(f"/{other_path}/", follow_redirects=True).status == 404
 
 
 def test_hook_register_routes_post(app_client):
@@ -777,7 +778,7 @@ def test_hook_register_magic_parameters(restore_working_directory):
         },
     ) as client:
         response = client.post("/data/runme", {}, csrftoken_from=True)
-        assert 200 == response.status
+        assert 302 == response.status
         actual = client.get("/data/logs.json?_sort_desc=rowid&_shape=array").json
         assert [{"rowid": 1, "line": "1.1"}] == actual
         # Now try the GET request against get_uuid
@@ -794,7 +795,7 @@ def test_hook_forbidden(restore_working_directory):
     ) as client:
         response = client.get("/")
         assert 403 == response.status
-        response2 = client.get("/data2", allow_redirects=False)
+        response2 = client.get("/data2")
         assert 302 == response2.status
         assert "/login?message=view-database" == response2.headers["Location"]
         assert "view-database" == client.ds._last_forbidden_message

From 827fa823d1b919c445f3141174ecb7a82717d99c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 14 Oct 2021 11:10:42 -0700
Subject: [PATCH 1189/2303] Update pyyaml requirement from ~=5.3 to >=5.3,<7.0
 (#1489)

Updates the requirements on [pyyaml](https://github.com/yaml/pyyaml) to permit the latest version.
- [Release notes](https://github.com/yaml/pyyaml/releases)
- [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES)
- [Commits](https://github.com/yaml/pyyaml/compare/5.3...6.0)

---
updated-dependencies:
- dependency-name: pyyaml
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 45eac040..c1b87bd4 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ setup(
         "aiofiles>=0.4,<0.8",
         "janus>=0.4,<0.7",
         "asgi-csrf>=0.9",
-        "PyYAML~=5.3",
+        "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",
         "itsdangerous>=1.1,<3.0",
         "python-baseconv==1.2.2",

From 0fdbf004843850f200e077a3c87427fe16c18b85 Mon Sep 17 00:00:00 2001
From: "C. Titus Brown" 
Date: Thu, 14 Oct 2021 11:39:55 -0700
Subject: [PATCH 1190/2303] Rework the `--static` documentation

Rework the `--static` documentation to better differentiate between the filesystem and serving locations. Closes #1457

Co-authored-by: Simon Willison 
---
 docs/custom_templates.rst | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst
index efb5b842..3e4eb633 100644
--- a/docs/custom_templates.rst
+++ b/docs/custom_templates.rst
@@ -173,18 +173,18 @@ Datasette can serve static files for you, using the ``--static`` option.
 Consider the following directory structure::
 
     metadata.json
-    static/styles.css
-    static/app.js
+    static-files/styles.css
+    static-files/app.js
 
-You can start Datasette using ``--static static:static/`` to serve those
-files from the ``/static/`` mount point::
+You can start Datasette using ``--static assets:static-files/`` to serve those
+files from the ``/assets/`` mount point::
 
-    $ datasette -m metadata.json --static static:static/ --memory
+    $ datasette -m metadata.json --static assets:static-files/ --memory
 
 The following URLs will now serve the content from those CSS and JS files::
 
-    http://localhost:8001/static/styles.css
-    http://localhost:8001/static/app.js
+    http://localhost:8001/assets/styles.css
+    http://localhost:8001/assets/app.js
 
 You can reference those files from ``metadata.json`` like so:
 
@@ -192,10 +192,10 @@ You can reference those files from ``metadata.json`` like so:
 
     {
         "extra_css_urls": [
-            "/static/styles.css"
+            "/assets/styles.css"
         ],
         "extra_js_urls": [
-            "/static/app.js"
+            "/assets/app.js"
         ]
     }
 
@@ -205,10 +205,10 @@ Publishing static assets
 The :ref:`cli_publish` command can be used to publish your static assets,
 using the same syntax as above::
 
-    $ datasette publish cloudrun mydb.db --static static:static/
+    $ datasette publish cloudrun mydb.db --static assets:static-files/
 
-This will upload the contents of the ``static/`` directory as part of the
-deployment, and configure Datasette to correctly serve the assets.
+This will upload the contents of the ``static-files/`` directory as part of the
+deployment, and configure Datasette to correctly serve the assets from ``/assets/``.
 
 .. _customization_custom_templates:
 

From 85849935292e500ab7a99f8fe0f9546e903baad3 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:03:28 -0700
Subject: [PATCH 1191/2303] --cors Access-Control-Allow-Headers: Authorization

Refs #1467, refs https://github.com/simonw/datasette-auth-tokens/issues/4
---
 datasette/app.py            | 3 ++-
 datasette/utils/__init__.py | 5 +++++
 datasette/views/base.py     | 9 +++++----
 datasette/views/database.py | 3 ++-
 datasette/views/index.py    | 4 ++--
 datasette/views/special.py  | 4 ++--
 tests/test_api.py           | 3 ++-
 7 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 1f69c2b3..52c5e629 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -46,6 +46,7 @@ from .database import Database, QueryInterrupted
 from .utils import (
     PrefixedUrlString,
     StartupError,
+    add_cors_headers,
     async_call_with_supported_arguments,
     await_me_maybe,
     call_with_supported_arguments,
@@ -1321,7 +1322,7 @@ class DatasetteRouter:
         )
         headers = {}
         if self.ds.cors:
-            headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(headers)
         if request.path.split("?")[0].endswith(".json"):
             await asgi_send_json(send, info, status=status, headers=headers)
         else:
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 70ac8976..c339113c 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1089,3 +1089,8 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
+
+
+def add_cors_headers(headers):
+    headers["Access-Control-Allow-Origin"] = "*"
+    headers["Access-Control-Allow-Headers"] = "Authorization"
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 3333781c..01e90220 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -11,6 +11,7 @@ import pint
 from datasette import __version__
 from datasette.database import QueryInterrupted
 from datasette.utils import (
+    add_cors_headers,
     await_me_maybe,
     EscapeHtmlWriter,
     InvalidSql,
@@ -163,7 +164,7 @@ class DataView(BaseView):
     async def options(self, request, *args, **kwargs):
         r = Response.text("ok")
         if self.ds.cors:
-            r.headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(r.headers)
         return r
 
     def redirect(self, request, path, forward_querystring=True, remove_args=None):
@@ -174,7 +175,7 @@ class DataView(BaseView):
         r = Response.redirect(path)
         r.headers["Link"] = f"<{path}>; rel=preload"
         if self.ds.cors:
-            r.headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(r.headers)
         return r
 
     async def data(self, request, database, hash, **kwargs):
@@ -417,7 +418,7 @@ class DataView(BaseView):
 
         headers = {}
         if self.ds.cors:
-            headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(headers)
         if request.args.get("_dl", None):
             if not trace:
                 content_type = "text/csv; charset=utf-8"
@@ -643,5 +644,5 @@ class DataView(BaseView):
             response.headers["Cache-Control"] = ttl_header
         response.headers["Referrer-Policy"] = "no-referrer"
         if self.ds.cors:
-            response.headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(response.headers)
         return response
diff --git a/datasette/views/database.py b/datasette/views/database.py
index e3070ce6..affded9b 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -8,6 +8,7 @@ from urllib.parse import parse_qsl, urlencode
 import markupsafe
 
 from datasette.utils import (
+    add_cors_headers,
     await_me_maybe,
     check_visibility,
     derive_named_parameters,
@@ -176,7 +177,7 @@ class DatabaseDownload(DataView):
         filepath = db.path
         headers = {}
         if self.ds.cors:
-            headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(headers)
         headers["Transfer-Encoding"] = "chunked"
         return AsgiFileDownload(
             filepath,
diff --git a/datasette/views/index.py b/datasette/views/index.py
index e37643f9..18454759 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -1,7 +1,7 @@
 import hashlib
 import json
 
-from datasette.utils import check_visibility, CustomJSONEncoder
+from datasette.utils import add_cors_headers, check_visibility, CustomJSONEncoder
 from datasette.utils.asgi import Response
 from datasette.version import __version__
 
@@ -129,7 +129,7 @@ class IndexView(BaseView):
         if as_format:
             headers = {}
             if self.ds.cors:
-                headers["Access-Control-Allow-Origin"] = "*"
+                add_cors_headers(headers)
             return Response(
                 json.dumps({db["name"]: db for db in databases}, cls=CustomJSONEncoder),
                 content_type="application/json; charset=utf-8",
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 9750dd06..3cb626a5 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -1,6 +1,6 @@
 import json
 from datasette.utils.asgi import Response, Forbidden
-from datasette.utils import actor_matches_allow
+from datasette.utils import actor_matches_allow, add_cors_headers
 from .base import BaseView
 import secrets
 
@@ -23,7 +23,7 @@ class JsonDataView(BaseView):
         if as_format:
             headers = {}
             if self.ds.cors:
-                headers["Access-Control-Allow-Origin"] = "*"
+                add_cors_headers(headers)
             return Response(
                 json.dumps(data),
                 content_type="application/json; charset=utf-8",
diff --git a/tests/test_api.py b/tests/test_api.py
index 38d1ba08..311ae464 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1955,7 +1955,8 @@ def test_trace(trace_debug):
 def test_cors(app_client_with_cors, path, status_code):
     response = app_client_with_cors.get(path)
     assert response.status == status_code
-    assert "*" == response.headers["Access-Control-Allow-Origin"]
+    assert response.headers["Access-Control-Allow-Origin"] == "*"
+    assert response.headers["Access-Control-Allow-Headers"] == "Authorization"
 
 
 @pytest.mark.parametrize(

From 81bf9a9f3cc5b30107a6b1adeee39d5e8312ecfc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:19:03 -0700
Subject: [PATCH 1192/2303] Updated --cors documentation, refs #1467

---
 docs/json_api.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/json_api.rst b/docs/json_api.rst
index 09cac1f9..7d3123b7 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -10,9 +10,10 @@ To access the API for a page, either click on the ``.json`` link on that page or
 edit the URL and add a ``.json`` extension to it.
 
 If you started Datasette with the ``--cors`` option, each JSON endpoint will be
-served with the following additional HTTP header::
+served with the following additional HTTP headers::
 
     Access-Control-Allow-Origin: *
+    Access-Control-Allow-Headers: Authorization
 
 This means JavaScript running on any domain will be able to make cross-origin
 requests to fetch the data.

From 8934507cdc0029a598cf37cdb3818fd31af5e33c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:22:19 -0700
Subject: [PATCH 1193/2303] Release 0.59

Refs #942, #1404, #1405, #1416, #1420, #1421, #1422, #1423, #1425, #1431, #1443, #1446, #1449, #1467, #1469, #1470, #1488
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 24 ++++++++----------------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index 87b18fab..d78b2e90 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59a2"
+__version__ = "0.59"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 737a151b..b2d95c49 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,36 +4,28 @@
 Changelog
 =========
 
-.. _v0_59a2:
+.. _v0_59:
 
-0.59a2 (2021-08-27)
--------------------
+0.59 (2021-10-14)
+-----------------
 
 - Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
 - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
 - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
+- Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defalts to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
+- The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (`#1467 `__)
 - Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
 - Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)
 - ``datasette.databases`` property is now a documented API. (:issue:`1443`)
-- Datasette base template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
-
-.. _v0_59a1:
-
-0.59a1 (2021-08-08)
--------------------
-
+- The ``base.html`` template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
 - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`)
-
-.. _v0_59a0:
-
-0.59a0 (2021-08-06)
--------------------
-
 - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`)
 - New ``hide_sql`` canned query option for defaulting to hiding the SQL quey used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
 - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`)
 - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`)
 - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`)
+- Fixed bug where ``?_next=x&_sort=rowid`` could throw an error. (:issue:`1470`)
+- Column cog menu no longer shows the option to facet by a column that is already selected by the default facets in metadata. (:issue:`1469`)
 
 .. _v0_58_1:
 

From ff9ccfb0310501a3b4b4ca24d73246a8eb3e7914 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:23:43 -0700
Subject: [PATCH 1194/2303] Fixed typo in release notes

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b2d95c49..9a1edfe8 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,7 +12,7 @@ Changelog
 - Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
 - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
 - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
-- Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defalts to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
+- Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
 - The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (`#1467 `__)
 - Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
 - Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)

From e5d01ca5831844a02ef5b1ef0d5b9eb8bad3c9a4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 22 Oct 2021 11:56:27 -0700
Subject: [PATCH 1195/2303] Fixed typo in release notes

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9a1edfe8..24e19baa 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -20,7 +20,7 @@ Changelog
 - The ``base.html`` template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
 - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`)
 - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`)
-- New ``hide_sql`` canned query option for defaulting to hiding the SQL quey used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
+- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
 - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`)
 - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`)
 - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`)

From a1b20852db751acc445dce5f23e988d2c0299655 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 22 Oct 2021 12:00:00 -0700
Subject: [PATCH 1196/2303] Unwrapped some documentation text

---
 docs/sql_queries.rst | 71 +++++++++++---------------------------------
 1 file changed, 17 insertions(+), 54 deletions(-)

diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index 407e4ba2..4958b56a 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -3,34 +3,22 @@
 Running SQL queries
 ===================
 
-Datasette treats SQLite database files as read-only and immutable. This means it
-is not possible to execute INSERT or UPDATE statements using Datasette, which
-allows us to expose SELECT statements to the outside world without needing to
-worry about SQL injection attacks.
+Datasette treats SQLite database files as read-only and immutable. This means it is not possible to execute INSERT or UPDATE statements using Datasette, which allows us to expose SELECT statements to the outside world without needing to worry about SQL injection attacks.
 
-The easiest way to execute custom SQL against Datasette is through the web UI.
-The database index page includes a SQL editor that lets you run any SELECT query
-you like. You can also construct queries using the filter interface on the
-tables page, then click "View and edit SQL" to open that query in the custom
-SQL editor.
+The easiest way to execute custom SQL against Datasette is through the web UI. The database index page includes a SQL editor that lets you run any SELECT query you like. You can also construct queries using the filter interface on the tables page, then click "View and edit SQL" to open that query in the custom SQL editor.
 
-Note that this interface is only available if the :ref:`permissions_execute_sql`
-permission is allowed.
+Note that this interface is only available if the :ref:`permissions_execute_sql` permission is allowed.
 
-Any Datasette SQL query is reflected in the URL of the page, allowing you to
-bookmark them, share them with others and navigate through previous queries
-using your browser back button.
+Any Datasette SQL query is reflected in the URL of the page, allowing you to bookmark them, share them with others and navigate through previous queries using your browser back button.
 
-You can also retrieve the results of any query as JSON by adding ``.json`` to
-the base URL.
+You can also retrieve the results of any query as JSON by adding ``.json`` to the base URL.
 
 .. _sql_parameters:
 
 Named parameters
 ----------------
 
-Datasette has special support for SQLite named parameters. Consider a SQL query
-like this:
+Datasette has special support for SQLite named parameters. Consider a SQL query like this:
 
 .. code-block:: sql
 
@@ -38,17 +26,13 @@ like this:
     where "PermitNotes" like :notes
     and "qSpecies" = :species
 
-If you execute this query using the custom query editor, Datasette will extract
-the two named parameters and use them to construct form fields for you to
-provide values.
+If you execute this query using the custom query editor, Datasette will extract the two named parameters and use them to construct form fields for you to provide values.
 
 You can also provide values for these fields by constructing a URL::
 
     /mydatabase?sql=select...&species=44
 
-SQLite string escaping rules will be applied to values passed using named
-parameters - they will be wrapped in quotes and their content will be correctly
-escaped.
+SQLite string escaping rules will be applied to values passed using named parameters - they will be wrapped in quotes and their content will be correctly escaped.
 
 Datasette disallows custom SQL queries containing the string PRAGMA (with a small number `of exceptions `__) as SQLite pragma statements can be used to change database settings at runtime. If you need to include the string "pragma" in a query you can do so safely using a named parameter.
 
@@ -57,9 +41,7 @@ Datasette disallows custom SQL queries containing the string PRAGMA (with a smal
 Views
 -----
 
-If you want to bundle some pre-written SQL queries with your Datasette-hosted
-database you can do so in two ways. The first is to include SQL views in your
-database - Datasette will then list those views on your database index page.
+If you want to bundle some pre-written SQL queries with your Datasette-hosted database you can do so in two ways. The first is to include SQL views in your database - Datasette will then list those views on your database index page.
 
 The quickest way to create views is with the SQLite command-line interface::
 
@@ -74,8 +56,7 @@ The quickest way to create views is with the SQLite command-line interface::
 Canned queries
 --------------
 
-As an alternative to adding views to your database, you can define canned
-queries inside your ``metadata.json`` file. Here's an example:
+As an alternative to adding views to your database, you can define canned queries inside your ``metadata.json`` file. Here's an example:
 
 .. code-block:: json
 
@@ -95,8 +76,7 @@ Then run Datasette like this::
 
     datasette sf-trees.db -m metadata.json
 
-Each canned query will be listed on the database index page, and will also get
-its own URL at::
+Each canned query will be listed on the database index page, and will also get its own URL at::
 
     /database-name/canned-query-name
 
@@ -104,20 +84,14 @@ For the above example, that URL would be::
 
     /sf-trees/just_species
 
-You can optionally include ``"title"`` and ``"description"`` keys to show a
-title and description on the canned query page. As with regular table metadata
-you can alternatively specify ``"description_html"`` to have your description
-rendered as HTML (rather than having HTML special characters escaped).
+You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped).
 
 .. _canned_queries_named_parameters:
 
 Canned query parameters
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Canned queries support named parameters, so if you include those in the SQL you
-will then be able to enter them using the form fields on the canned query page
-or by adding them to the URL. This means canned queries can be used to create
-custom JSON APIs based on a carefully designed SQL statement.
+Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL statement.
 
 Here's an example of a canned query with a named parameter:
 
@@ -380,14 +354,9 @@ The ``"message"`` and ``"redirect"`` values here will take into account ``on_suc
 Pagination
 ----------
 
-Datasette's default table pagination is designed to be extremely efficient. SQL
-OFFSET/LIMIT pagination can have a significant performance penalty once you get
-into multiple thousands of rows, as each page still requires the database to
-scan through every preceding row to find the correct offset.
+Datasette's default table pagination is designed to be extremely efficient. SQL OFFSET/LIMIT pagination can have a significant performance penalty once you get into multiple thousands of rows, as each page still requires the database to scan through every preceding row to find the correct offset.
 
-When paginating through tables, Datasette instead orders the rows in the table
-by their primary key and performs a WHERE clause against the last seen primary
-key for the previous page. For example:
+When paginating through tables, Datasette instead orders the rows in the table by their primary key and performs a WHERE clause against the last seen primary key for the previous page. For example:
 
 .. code-block:: sql
 
@@ -395,15 +364,9 @@ key for the previous page. For example:
 
 This represents page three for this particular table, with a page size of 100.
 
-Note that we request 101 items in the limit clause rather than 100. This allows
-us to detect if we are on the last page of the results: if the query returns
-less than 101 rows we know we have reached the end of the pagination set.
-Datasette will only return the first 100 rows - the 101st is used purely to
-detect if there should be another page.
+Note that we request 101 items in the limit clause rather than 100. This allows us to detect if we are on the last page of the results: if the query returns less than 101 rows we know we have reached the end of the pagination set. Datasette will only return the first 100 rows - the 101st is used purely to detect if there should be another page.
 
-Since the where clause acts against the index on the primary key, the query is
-extremely fast even for records that are a long way into the overall pagination
-set.
+Since the where clause acts against the index on the primary key, the query is extremely fast even for records that are a long way into the overall pagination set.
 
 .. _cross_database_queries:
 

From 15a9d4abfff0c45dee2a9f851326e1d61b1c678c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 22 Oct 2021 12:34:23 -0700
Subject: [PATCH 1197/2303] Docs on named parameters with cast as real/integer,
 closes #1496

---
 docs/sql_queries.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index 4958b56a..f9a36490 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -34,6 +34,14 @@ You can also provide values for these fields by constructing a URL::
 
 SQLite string escaping rules will be applied to values passed using named parameters - they will be wrapped in quotes and their content will be correctly escaped.
 
+Values from named parameters are treated as SQLite strings. If you need to perform numeric comparisons on them you should cast them to an integer or float first using ``cast(:name as integer)`` or ``cast(:name as real)``, for example:
+
+.. code-block:: sql
+
+    select * from Street_Tree_List
+    where latitude > cast(:min_latitude as real)
+    and latitude < cast(:max_latitude as real)
+
 Datasette disallows custom SQL queries containing the string PRAGMA (with a small number `of exceptions `__) as SQLite pragma statements can be used to change database settings at runtime. If you need to include the string "pragma" in a query you can do so safely using a named parameter.
 
 .. _sql_views:

From 96a823f2834a262ae97a90ebfb6847f14763c415 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 15:19:54 -0700
Subject: [PATCH 1198/2303] Fix compatibility with Python 3.10 (#1481)

* Run tests against Python 3.10
* Upgrade to Janus 0.6.2 for Python 3.10
* Add 3.10 to classifiers

Closes #1482
---
 .github/workflows/test.yml | 2 +-
 setup.py                   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bcb241d3..0b3635fe 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/setup.py b/setup.py
index c1b87bd4..2f5fed49 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ setup(
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.8",
-        "janus>=0.4,<0.7",
+        "janus>=0.6.2,<0.7",
         "asgi-csrf>=0.9",
         "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",
@@ -85,6 +85,7 @@ setup(
         "Intended Audience :: End Users/Desktop",
         "Topic :: Database",
         "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.7",

From 03cc697b6b3d0983618c29ee75b45b5e0ac91139 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 24 Oct 2021 15:22:39 -0700
Subject: [PATCH 1199/2303] Update pytest-asyncio requirement from <0.16,>=0.10
 to >=0.10,<0.17 (#1494)

Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.10.0...v0.16.0)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2f5fed49..17a56a97 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,7 @@ setup(
         "test": [
             "pytest>=5.2.2,<6.3.0",
             "pytest-xdist>=2.2.1,<2.5",
-            "pytest-asyncio>=0.10,<0.16",
+            "pytest-asyncio>=0.10,<0.17",
             "beautifulsoup4>=4.8.1,<4.11.0",
             "black==21.9b0",
             "pytest-timeout>=1.4.2,<2.1",

From e6e44372b34414eac2f36a4c1120af4f755aa423 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 15:29:56 -0700
Subject: [PATCH 1200/2303] Release 0.59.1

Refs #1482, #1496
---
 datasette/version.py | 2 +-
 docs/changelog.rst   | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index d78b2e90..ff1f55e8 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59"
+__version__ = "0.59.1"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 24e19baa..40f49fb2 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,14 @@
 Changelog
 =========
 
+.. _v0_59_1:
+
+0.59.1 (2021-10-24)
+-------------------
+
+- Fix compatibility with Python 3.10. (:issue:`1482`)
+- Documentation on how to use :ref:`sql_parameters` with integer and floating point values. (:issue:`1496`)
+
 .. _v0_59:
 
 0.59 (2021-10-14)

From 3a2ed6300d2d31972a5ac633f4e1e9561e163e29 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 15:37:43 -0700
Subject: [PATCH 1201/2303] Run tests on 3.10 during publish, refs #1482

---
 .github/workflows/publish.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 54e582f0..17c6ae9b 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

From 2c31d1cd9cd3b63458ccbe391866499fa3f44978 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 16:24:41 -0700
Subject: [PATCH 1202/2303] Upgrade Docker base to Debian buster, refs #1497

---
 Dockerfile | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7c56cf56..42f5529b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,18 +1,11 @@
-FROM python:3.9.2-slim-buster as build
+FROM python:3.9.7-slim-bullseye as build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55
 ARG VERSION
 
-# software-properties-common provides add-apt-repository
-# which we need in order to install a more recent release
-# of libsqlite3-mod-spatialite from the sid distribution
 RUN apt-get update && \
-    apt-get -y --no-install-recommends install software-properties-common && \
-    add-apt-repository "deb http://httpredir.debian.org/debian sid main" && \
-    apt-get update && \
-    apt-get -t sid install -y --no-install-recommends libsqlite3-mod-spatialite && \
-    apt-get remove -y software-properties-common && \
+    apt-get install -y --no-install-recommends libsqlite3-mod-spatialite && \
     apt clean && \
     rm -rf /var/lib/apt && \
     rm -rf /var/lib/dpkg/info/*

From c92ab51b3ce0c2df002c0c2f10549a43910dd4be Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 12 Nov 2021 06:18:31 -0800
Subject: [PATCH 1203/2303] Logo at top of README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ee9d9a5a..ce15ccf4 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Datasette
+Datasette
 
 [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/)
 [![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html)

From c306b696de0a582e322f9eb7cb4125c83301e3a9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 20:44:54 -0800
Subject: [PATCH 1204/2303] Correct facet links for columns with a leading
 underscore, closes #1506

---
 datasette/facets.py              |   9 ++-
 tests/fixtures.py                |  10 +--
 tests/test_api.py                |  24 +++---
 tests/test_csv.py                |   2 +-
 tests/test_facets.py             | 133 ++++++++++++++++++++++++++++++-
 tests/test_html.py               |  32 ++++----
 tests/test_internals_database.py |   4 +-
 tests/test_plugins.py            |   8 +-
 8 files changed, 179 insertions(+), 43 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index f74e2d01..94a1d83d 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -237,14 +237,17 @@ class ColumnFacet(Facet):
                 else:
                     expanded = {}
                 for row in facet_rows:
-                    selected = (column, str(row["value"])) in qs_pairs
+                    column_qs = column
+                    if column.startswith("_"):
+                        column_qs = "{}__exact".format(column)
+                    selected = (column_qs, str(row["value"])) in qs_pairs
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {column: str(row["value"])}
+                            self.request, {column_qs: str(row["value"])}
                         )
                     else:
                         toggle_path = path_with_added_args(
-                            self.request, {column: row["value"]}
+                            self.request, {column_qs: row["value"]}
                         )
                     facet_results_values.append(
                         {
diff --git a/tests/fixtures.py b/tests/fixtures.py
index dc22c609..1a879126 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -355,12 +355,12 @@ METADATA = {
                 "neighborhood_search": {
                     "sql": textwrap.dedent(
                         """
-                        select neighborhood, facet_cities.name, state
+                        select _neighborhood, facet_cities.name, state
                         from facetable
                             join facet_cities
                                 on facetable.city_id = facet_cities.id
-                        where neighborhood like '%' || :text || '%'
-                        order by neighborhood;
+                        where _neighborhood like '%' || :text || '%'
+                        order by _neighborhood;
                     """
                     ),
                     "title": "Search neighborhoods",
@@ -559,14 +559,14 @@ CREATE TABLE facetable (
     on_earth integer,
     state text,
     city_id integer,
-    neighborhood text,
+    _neighborhood text,
     tags text,
     complex_array text,
     distinct_some_null,
     FOREIGN KEY ("city_id") REFERENCES [facet_cities](id)
 );
 INSERT INTO facetable
-    (created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null)
+    (created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null)
 VALUES
     ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'),
     ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'),
diff --git a/tests/test_api.py b/tests/test_api.py
index 311ae464..43b52175 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -213,7 +213,7 @@ def test_database_page(app_client):
                 "on_earth",
                 "state",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
@@ -1241,7 +1241,9 @@ def test_table_filter_json_arraynotcontains(app_client):
 
 
 def test_table_filter_extra_where(app_client):
-    response = app_client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'")
+    response = app_client.get(
+        "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
+    )
     assert [
         [
             2,
@@ -1259,14 +1261,16 @@ def test_table_filter_extra_where(app_client):
 
 
 def test_table_filter_extra_where_invalid(app_client):
-    response = app_client.get("/fixtures/facetable.json?_where=neighborhood=Dogpatch'")
+    response = app_client.get("/fixtures/facetable.json?_where=_neighborhood=Dogpatch'")
     assert 400 == response.status
     assert "Invalid SQL" == response.json["title"]
 
 
 def test_table_filter_extra_where_disabled_if_no_sql_allowed():
     with make_app_client(metadata={"allow_sql": {}}) as client:
-        response = client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'")
+        response = client.get(
+            "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
+        )
         assert 403 == response.status
         assert "_where= is not allowed" == response.json["error"]
 
@@ -1696,7 +1700,7 @@ def test_suggested_facets(app_client):
         {"name": "on_earth", "querystring": "_facet=on_earth"},
         {"name": "state", "querystring": "_facet=state"},
         {"name": "city_id", "querystring": "_facet=city_id"},
-        {"name": "neighborhood", "querystring": "_facet=neighborhood"},
+        {"name": "_neighborhood", "querystring": "_facet=_neighborhood"},
         {"name": "tags", "querystring": "_facet=tags"},
         {"name": "complex_array", "querystring": "_facet=complex_array"},
         {"name": "created", "querystring": "_facet_date=created"},
@@ -1752,7 +1756,7 @@ def test_nocount_nofacet_if_shape_is_object(app_client_with_trace):
 def test_expand_labels(app_client):
     response = app_client.get(
         "/fixtures/facetable.json?_shape=object&_labels=1&_size=2"
-        "&neighborhood__contains=c"
+        "&_neighborhood__contains=c"
     )
     assert {
         "2": {
@@ -1762,7 +1766,7 @@ def test_expand_labels(app_client):
             "on_earth": 1,
             "state": "CA",
             "city_id": {"value": 1, "label": "San Francisco"},
-            "neighborhood": "Dogpatch",
+            "_neighborhood": "Dogpatch",
             "tags": '["tag1", "tag3"]',
             "complex_array": "[]",
             "distinct_some_null": "two",
@@ -1774,7 +1778,7 @@ def test_expand_labels(app_client):
             "on_earth": 1,
             "state": "MI",
             "city_id": {"value": 3, "label": "Detroit"},
-            "neighborhood": "Corktown",
+            "_neighborhood": "Corktown",
             "tags": "[]",
             "complex_array": "[]",
             "distinct_some_null": None,
@@ -2125,7 +2129,7 @@ def test_http_options_request(app_client):
                 "on_earth",
                 "state",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
@@ -2152,7 +2156,7 @@ def test_http_options_request(app_client):
                 "planet_int",
                 "on_earth",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 5e9406e7..5902e9db 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -24,7 +24,7 @@ world
 )
 
 EXPECTED_TABLE_WITH_LABELS_CSV = """
-pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,complex_array,distinct_some_null
+pk,created,planet_int,on_earth,state,city_id,city_id_label,_neighborhood,tags,complex_array,distinct_some_null
 1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one
 2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two
 3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[],
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 22927512..01d8b8f5 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -23,7 +23,10 @@ async def test_column_facet_suggest(app_client):
         {"name": "on_earth", "toggle_url": "http://localhost/?_facet=on_earth"},
         {"name": "state", "toggle_url": "http://localhost/?_facet=state"},
         {"name": "city_id", "toggle_url": "http://localhost/?_facet=city_id"},
-        {"name": "neighborhood", "toggle_url": "http://localhost/?_facet=neighborhood"},
+        {
+            "name": "_neighborhood",
+            "toggle_url": "http://localhost/?_facet=_neighborhood",
+        },
         {"name": "tags", "toggle_url": "http://localhost/?_facet=tags"},
         {
             "name": "complex_array",
@@ -56,8 +59,8 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client):
             "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=city_id",
         },
         {
-            "name": "neighborhood",
-            "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=neighborhood",
+            "name": "_neighborhood",
+            "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=_neighborhood",
         },
         {
             "name": "tags",
@@ -86,7 +89,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client):
         "planet_int",
         "on_earth",
         "state",
-        "neighborhood",
+        "_neighborhood",
         "tags",
         "complex_array",
     ] == suggestions
@@ -144,6 +147,128 @@ async def test_column_facet_results(app_client):
     } == buckets
 
 
+@pytest.mark.asyncio
+async def test_column_facet_results_column_starts_with_underscore(app_client):
+    facet = ColumnFacet(
+        app_client.ds,
+        Request.fake("/?_facet=_neighborhood"),
+        database="fixtures",
+        sql="select * from facetable",
+        table="facetable",
+    )
+    buckets, timed_out = await facet.facet_results()
+    assert [] == timed_out
+    assert buckets == {
+        "_neighborhood": {
+            "name": "_neighborhood",
+            "type": "column",
+            "hideable": True,
+            "toggle_url": "/",
+            "results": [
+                {
+                    "value": "Downtown",
+                    "label": "Downtown",
+                    "count": 2,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Downtown",
+                    "selected": False,
+                },
+                {
+                    "value": "Arcadia Planitia",
+                    "label": "Arcadia Planitia",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Arcadia+Planitia",
+                    "selected": False,
+                },
+                {
+                    "value": "Bernal Heights",
+                    "label": "Bernal Heights",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Bernal+Heights",
+                    "selected": False,
+                },
+                {
+                    "value": "Corktown",
+                    "label": "Corktown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Corktown",
+                    "selected": False,
+                },
+                {
+                    "value": "Dogpatch",
+                    "label": "Dogpatch",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Dogpatch",
+                    "selected": False,
+                },
+                {
+                    "value": "Greektown",
+                    "label": "Greektown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Greektown",
+                    "selected": False,
+                },
+                {
+                    "value": "Hayes Valley",
+                    "label": "Hayes Valley",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Hayes+Valley",
+                    "selected": False,
+                },
+                {
+                    "value": "Hollywood",
+                    "label": "Hollywood",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Hollywood",
+                    "selected": False,
+                },
+                {
+                    "value": "Koreatown",
+                    "label": "Koreatown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Koreatown",
+                    "selected": False,
+                },
+                {
+                    "value": "Los Feliz",
+                    "label": "Los Feliz",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Los+Feliz",
+                    "selected": False,
+                },
+                {
+                    "value": "Mexicantown",
+                    "label": "Mexicantown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Mexicantown",
+                    "selected": False,
+                },
+                {
+                    "value": "Mission",
+                    "label": "Mission",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Mission",
+                    "selected": False,
+                },
+                {
+                    "value": "SOMA",
+                    "label": "SOMA",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=SOMA",
+                    "selected": False,
+                },
+                {
+                    "value": "Tenderloin",
+                    "label": "Tenderloin",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Tenderloin",
+                    "selected": False,
+                },
+            ],
+            "truncated": False,
+        }
+    }
+
+
 @pytest.mark.asyncio
 async def test_column_facet_from_metadata_cannot_be_hidden(app_client):
     facet = ColumnFacet(
diff --git a/tests/test_html.py b/tests/test_html.py
index 5f2ba2f1..1955e65b 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -235,7 +235,10 @@ def test_table_cell_truncation():
             "Corkt…",
             "Mexic…",
             "Arcad…",
-        ] == [td.string for td in table.findAll("td", {"class": "col-neighborhood"})]
+        ] == [
+            td.string
+            for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
+        ]
 
 
 def test_row_page_does_not_truncate():
@@ -245,7 +248,8 @@ def test_row_page_does_not_truncate():
         table = Soup(response.body, "html.parser").find("table")
         assert table["class"] == ["rows-and-columns"]
         assert ["Mission"] == [
-            td.string for td in table.findAll("td", {"class": "col-neighborhood"})
+            td.string
+            for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
         ]
 
 
@@ -1312,7 +1316,7 @@ def test_canned_query_show_hide_metadata_option(
 
 def test_extra_where_clauses(app_client):
     response = app_client.get(
-        "/fixtures/facetable?_where=neighborhood='Dogpatch'&_where=city_id=1"
+        "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=city_id=1"
     )
     soup = Soup(response.body, "html.parser")
     div = soup.select(".extra-wheres")[0]
@@ -1320,12 +1324,12 @@ def test_extra_where_clauses(app_client):
     hrefs = [a["href"] for a in div.findAll("a")]
     assert [
         "/fixtures/facetable?_where=city_id%3D1",
-        "/fixtures/facetable?_where=neighborhood%3D%27Dogpatch%27",
+        "/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27",
     ] == hrefs
     # These should also be persisted as hidden fields
     inputs = soup.find("form").findAll("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]
-    assert [("_where", "neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [
+    assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [
         (hidden["name"], hidden["value"]) for hidden in hiddens
     ]
 
@@ -1634,11 +1638,11 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
     [
         (
             "/fixtures/neighborhood_search",
-            "/fixtures?sql=%0Aselect+neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+neighborhood%3B%0A&text=",
+            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=",
         ),
         (
             "/fixtures/neighborhood_search?text=ber",
-            "/fixtures?sql=%0Aselect+neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+neighborhood%3B%0A&text=ber",
+            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber",
         ),
         ("/fixtures/pragma_cache_size", None),
         (
@@ -1716,23 +1720,23 @@ def test_navigation_menu_links(
         (
             5,
             # Default should show 2 facets
-            "/fixtures/facetable?_facet=neighborhood",
+            "/fixtures/facetable?_facet=_neighborhood",
             2,
             True,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
         ),
         # _facet_size above max_returned_rows should show max_returned_rows (5)
         (
             5,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=50",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=50",
             5,
             True,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
         ),
         # If max_returned_rows is high enough, should return all
         (
             20,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
             14,
             False,
             None,
@@ -1741,7 +1745,7 @@ def test_navigation_menu_links(
         # _facet_size above max_returned_rows should show max_returned_rows (5)
         (
             5,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
             5,
             True,
             None,
@@ -1760,7 +1764,7 @@ def test_facet_more_links(
     ) as client:
         response = client.get(path)
         soup = Soup(response.body, "html.parser")
-        lis = soup.select("#facet-neighborhood ul li:not(.facet-truncated)")
+        lis = soup.select("#facet-neighborhood-b352a7 ul li:not(.facet-truncated)")
         facet_truncated = soup.select_one(".facet-truncated")
         assert len(lis) == expected_num_facets
         if not expected_ellipses:
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index ad829751..2d0cae7f 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -82,7 +82,7 @@ async def test_table_exists(db, tables, exists):
                 "on_earth",
                 "state",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
@@ -170,7 +170,7 @@ async def test_table_columns(db, table, expected):
                 ),
                 Column(
                     cid=6,
-                    name="neighborhood",
+                    name="_neighborhood",
                     type="text",
                     notnull=0,
                     default_value=None,
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 7dac8002..7ac6b173 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -437,7 +437,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "on_earth",
             "state",
             "city_id",
-            "neighborhood",
+            "_neighborhood",
             "tags",
             "complex_array",
             "distinct_some_null",
@@ -459,7 +459,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "",
             "",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
@@ -526,12 +526,12 @@ def test_hook_register_output_renderer_can_render(app_client):
             "on_earth",
             "state",
             "city_id",
-            "neighborhood",
+            "_neighborhood",
             "tags",
             "complex_array",
             "distinct_some_null",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",

From c9e3cfecc8e966e5137d72e3f2150be9602d55f5 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 20:53:00 -0800
Subject: [PATCH 1205/2303] Columns in filters now ignore ?_nocol, closes #1503

---
 datasette/views/table.py |  2 +-
 tests/test_html.py       | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index efcef4d2..3e8f38f6 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -922,7 +922,7 @@ class TableView(RowTableShared):
                 "use_rowid": use_rowid,
                 "filters": filters,
                 "display_columns": display_columns,
-                "filter_columns": columns,
+                "filter_columns": table_columns,
                 "display_rows": display_rows,
                 "facets_timed_out": facets_timed_out,
                 "sorted_facet_results": sorted(
diff --git a/tests/test_html.py b/tests/test_html.py
index 1955e65b..eb1e3d20 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -923,6 +923,26 @@ def test_table_html_filter_form_column_options(
     assert expected_column_options == column_options
 
 
+def test_table_html_filter_form_still_shows_nocol_columns(app_client):
+    # https://github.com/simonw/datasette/issues/1503
+    response = app_client.get("/fixtures/sortable?_nocol=sortable")
+    assert response.status == 200
+    form = Soup(response.body, "html.parser").find("form")
+    assert [
+        o.string
+        for o in form.select("select[name='_filter_column']")[0].select("option")
+    ] == [
+        "- column -",
+        "pk1",
+        "pk2",
+        "content",
+        "sortable",
+        "sortable_with_nulls",
+        "sortable_with_nulls_2",
+        "text",
+    ]
+
+
 def test_row_html_compound_primary_key(app_client):
     response = app_client.get("/fixtures/compound_primary_key/a,b")
     assert response.status == 200

From 1c13e1af0664a4dfb1e69714c56523279cae09e4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:08:33 -0800
Subject: [PATCH 1206/2303] Ensure query columns are included too, ref #1503

---
 datasette/views/table.py | 10 +++++++++-
 tests/test_html.py       |  3 ++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 3e8f38f6..e6ae67de 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -915,6 +915,14 @@ class TableView(RowTableShared):
                         links.extend(extra_links)
                 return links
 
+            # filter_columns combine the columns we know are available
+            # in the table with any additional columns (such as rowid)
+            # which are available in the query
+            filter_columns = list(columns) + [
+                table_column
+                for table_column in table_columns
+                if table_column not in columns
+            ]
             return {
                 "table_actions": table_actions,
                 "supports_search": bool(fts_table),
@@ -922,7 +930,7 @@ class TableView(RowTableShared):
                 "use_rowid": use_rowid,
                 "filters": filters,
                 "display_columns": display_columns,
-                "filter_columns": table_columns,
+                "filter_columns": filter_columns,
                 "display_rows": display_rows,
                 "facets_timed_out": facets_timed_out,
                 "sorted_facet_results": sorted(
diff --git a/tests/test_html.py b/tests/test_html.py
index eb1e3d20..f24165bd 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -936,10 +936,11 @@ def test_table_html_filter_form_still_shows_nocol_columns(app_client):
         "pk1",
         "pk2",
         "content",
-        "sortable",
         "sortable_with_nulls",
         "sortable_with_nulls_2",
         "text",
+        # Moved to the end because it is no longer returned by the query:
+        "sortable",
     ]
 
 

From de1e031713f47fbd51eb7239db3e7e6025fbf81a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:14:43 -0800
Subject: [PATCH 1207/2303] Release 0.59.2

Refs #1497, #1503, #1506
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index ff1f55e8..db89b418 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59.1"
+__version__ = "0.59.2"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 40f49fb2..47ca3480 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,16 @@
 Changelog
 =========
 
+.. _v0_59_2:
+
+0.59.2 (2021-11-13)
+-------------------
+
+- Column names with a leading underscore now work correctly when used as a facet. (:issue:`1506`)
+- Applying ``?_nocol=`` to a column no longer removes that column from the filtering interface. (:issue:`1503`)
+- Official Datasette Docker container now uses Debian Bullseye as the base image. (:issue:`1497`)
+- Datasette is four years old today! Here's the `original release announcement `__ from 2017.
+
 .. _v0_59_1:
 
 0.59.1 (2021-10-24)

From 030390fd4abcecf1ab80d0528e32d7dbc50d1b5f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:29:43 -0800
Subject: [PATCH 1208/2303] .readthedocs.yaml configuration, refs #1507

---
 .readthedocs.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 .readthedocs.yaml

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000..816c10e2
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,9 @@
+version: 2
+
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+
+sphinx:
+   configuration: docs/conf.py

From 502c02fa6dde6a8bb840af6c4c8cf858aa1db687 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:37:40 -0800
Subject: [PATCH 1209/2303] Pin to docutils<0.18 in ReadTheDocs, refs #1507

---
 .readthedocs.yaml                 | 4 ++++
 docs/readthedocs-requirements.txt | 1 +
 2 files changed, 5 insertions(+)
 create mode 100644 docs/readthedocs-requirements.txt

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 816c10e2..70db5313 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -7,3 +7,7 @@ build:
 
 sphinx:
    configuration: docs/conf.py
+
+python:
+   install:
+   - requirements: docs/readthedocs-requirements.txt
diff --git a/docs/readthedocs-requirements.txt b/docs/readthedocs-requirements.txt
new file mode 100644
index 00000000..93120e66
--- /dev/null
+++ b/docs/readthedocs-requirements.txt
@@ -0,0 +1 @@
+docutils<0.18

From 07044bd130542870d5eb2e545988d0a24eb573ec Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 15 Nov 2021 15:41:07 -0800
Subject: [PATCH 1210/2303] SQL view-friendly arraycontains/arraynotcontains
 implementation, refs #448

---
 datasette/filters.py  | 10 ++--------
 tests/test_filters.py | 11 +++++++----
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/datasette/filters.py b/datasette/filters.py
index 2b859d99..cbd94415 100644
--- a/datasette/filters.py
+++ b/datasette/filters.py
@@ -149,19 +149,13 @@ class Filters:
                 TemplatedFilter(
                     "arraycontains",
                     "array contains",
-                    """rowid in (
-            select {t}.rowid from {t}, json_each([{t}].[{c}]) j
-            where j.value = :{p}
-        )""",
+                    """:{p} in (select value from json_each([{t}].[{c}]))""",
                     '{c} contains "{v}"',
                 ),
                 TemplatedFilter(
                     "arraynotcontains",
                     "array does not contain",
-                    """rowid not in (
-            select {t}.rowid from {t}, json_each([{t}].[{c}]) j
-            where j.value = :{p}
-        )""",
+                    """:{p} not in (select value from json_each([{t}].[{c}]))""",
                     '{c} does not contain "{v}"',
                 ),
             ]
diff --git a/tests/test_filters.py b/tests/test_filters.py
index f22b7b5c..d05ae80f 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -56,12 +56,15 @@ import pytest
         # Not in, and JSON array not in
         ((("foo__notin", "1,2,3"),), ["foo not in (:p0, :p1, :p2)"], ["1", "2", "3"]),
         ((("foo__notin", "[1,2,3]"),), ["foo not in (:p0, :p1, :p2)"], [1, 2, 3]),
-        # JSON arraycontains
+        # JSON arraycontains, arraynotcontains
         (
             (("Availability+Info__arraycontains", "yes"),),
-            [
-                "rowid in (\n            select table.rowid from table, json_each([table].[Availability+Info]) j\n            where j.value = :p0\n        )"
-            ],
+            [":p0 in (select value from json_each([table].[Availability+Info]))"],
+            ["yes"],
+        ),
+        (
+            (("Availability+Info__arraynotcontains", "yes"),),
+            [":p0 not in (select value from json_each([table].[Availability+Info]))"],
             ["yes"],
         ),
     ],

From 55024b5301892306b786fc37a8ab3c096be5c227 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 15 Nov 2021 17:19:33 -0800
Subject: [PATCH 1211/2303] _facet_array no longer confused by duplicate array
 items, closes #448

---
 datasette/facets.py  | 23 +++++++++++++++---
 tests/test_facets.py | 58 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 94a1d83d..ce2111c9 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -354,11 +354,26 @@ class ArrayFacet(Facet):
             config = source_and_config["config"]
             source = source_and_config["source"]
             column = config.get("column") or config["simple"]
+            # https://github.com/simonw/datasette/issues/448
             facet_sql = """
-                select j.value as value, count(*) as count from (
-                    {sql}
-                ) join json_each({col}) j
-                group by j.value order by count desc, value limit {limit}
+                with inner as ({sql}),
+                deduped_array_items as (
+                    select
+                        distinct j.value,
+                        inner.*
+                    from
+                        json_each([inner].{col}) j
+                        join inner
+                )
+                select
+                    value as value,
+                    count(*) as count
+                from
+                    deduped_array_items
+                group by
+                    value
+                order by
+                    count(*) desc limit {limit}
             """.format(
                 col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
             )
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 01d8b8f5..a20c79c4 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -4,6 +4,7 @@ from datasette.facets import ColumnFacet, ArrayFacet, DateFacet
 from datasette.utils.asgi import Request
 from datasette.utils import detect_json1
 from .fixtures import app_client  # noqa
+import json
 import pytest
 
 
@@ -402,6 +403,63 @@ async def test_array_facet_results(app_client):
     } == buckets
 
 
+@pytest.mark.asyncio
+@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
+async def test_array_facet_handle_duplicate_tags():
+    ds = Datasette([], memory=True)
+    db = ds.add_database(Database(ds, memory_name="test_array_facet"))
+    await db.execute_write("create table otters(name text, tags text)", block=True)
+    for name, tags in (
+        ("Charles", ["friendly", "cunning", "friendly"]),
+        ("Shaun", ["cunning", "empathetic", "friendly"]),
+        ("Tracy", ["empathetic", "eager"]),
+    ):
+        await db.execute_write(
+            "insert into otters (name, tags) values (?, ?)",
+            [name, json.dumps(tags)],
+            block=True,
+        )
+
+    response = await ds.client.get("/test_array_facet/otters.json?_facet_array=tags")
+    assert response.json()["facet_results"]["tags"] == {
+        "name": "tags",
+        "type": "array",
+        "results": [
+            {
+                "value": "cunning",
+                "label": "cunning",
+                "count": 2,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=cunning",
+                "selected": False,
+            },
+            {
+                "value": "empathetic",
+                "label": "empathetic",
+                "count": 2,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=empathetic",
+                "selected": False,
+            },
+            {
+                "value": "friendly",
+                "label": "friendly",
+                "count": 2,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=friendly",
+                "selected": False,
+            },
+            {
+                "value": "eager",
+                "label": "eager",
+                "count": 1,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=eager",
+                "selected": False,
+            },
+        ],
+        "hideable": True,
+        "toggle_url": "/test_array_facet/otters.json",
+        "truncated": False,
+    }
+
+
 @pytest.mark.asyncio
 async def test_date_facet_results(app_client):
     facet = DateFacet(

From 0156c6b5e52d541e93f0d68e9245f20ae83bc933 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 15 Nov 2021 17:31:33 -0800
Subject: [PATCH 1212/2303] Facet in predictable order for tests, refs #448

---
 datasette/facets.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index ce2111c9..62e7775e 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -373,7 +373,7 @@ class ArrayFacet(Facet):
                 group by
                     value
                 order by
-                    count(*) desc limit {limit}
+                    count(*) desc, value limit {limit}
             """.format(
                 col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
             )

From 6e971b4ac175df95ac7fe5dc2b57b53ad7f533fc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Nov 2021 19:07:21 -0800
Subject: [PATCH 1213/2303] Test confirming plugins can over-ride default
 routes, closes #1517

---
 tests/plugins/my_plugin_2.py | 10 +++++++++-
 tests/test_plugins.py        | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index ba298fd4..f5ce36b3 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -176,4 +176,12 @@ def register_routes(datasette):
     if not config:
         return
     path = config["path"]
-    return [(r"/{}/$".format(path), lambda: Response.text(path.upper()))]
+
+    def new_table(request):
+        return Response.text("/db/table: {}".format(sorted(request.url_vars.items())))
+
+    return [
+        (r"/{}/$".format(path), lambda: Response.text(path.upper())),
+        # Also serves to demonstrate over-ride of default paths:
+        (r"/(?P[^/]+)/(?P[^/]+?$)", new_table),
+    ]
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 7ac6b173..c9ff6edb 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -678,6 +678,25 @@ def test_hook_register_routes_with_datasette(configured_path):
         assert client.get(f"/{other_path}/", follow_redirects=True).status == 404
 
 
+def test_hook_register_routes_override():
+    "Plugins can over-ride default paths such as /db/table"
+    with make_app_client(
+        metadata={
+            "plugins": {
+                "register-route-demo": {
+                    "path": "blah",
+                }
+            }
+        }
+    ) as client:
+        response = client.get("/db/table")
+        assert response.status == 200
+        assert (
+            response.text
+            == "/db/table: [('db_name', 'db'), ('table_and_format', 'table')]"
+        )
+
+
 def test_hook_register_routes_post(app_client):
     response = app_client.post("/post/", {"this is": "post data"}, csrftoken_from=True)
     assert 200 == response.status

From 30255055150d7bc0affc8156adc18295495020ff Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Nov 2021 19:19:43 -0800
Subject: [PATCH 1214/2303] functools.wraps to help investigate #1517

---
 datasette/app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/datasette/app.py b/datasette/app.py
index 52c5e629..28268e42 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -2,6 +2,7 @@ import asyncio
 import asgi_csrf
 import collections
 import datetime
+import functools
 import glob
 import hashlib
 import httpx
@@ -1354,6 +1355,7 @@ def _cleaner_task_str(task):
 
 
 def wrap_view(view_fn, datasette):
+    @functools.wraps(view_fn)
     async def async_view_fn(request, send):
         if inspect.iscoroutinefunction(view_fn):
             response = await async_call_with_supported_arguments(

From ff0dd4da38d48c2fa9250ecf336002c9ed724e36 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 12:29:37 -0800
Subject: [PATCH 1215/2303] repr() method for Request, refs #1519

---
 datasette/utils/asgi.py         | 3 +++
 tests/test_internals_request.py | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 696944df..ad137fa9 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -37,6 +37,9 @@ class Request:
         self.scope = scope
         self.receive = receive
 
+    def __repr__(self):
+        return ''.format(self.method, self.url)
+
     @property
     def method(self):
         return self.scope["method"]
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index c42cfbd3..cd956f3f 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -75,6 +75,14 @@ def test_request_args():
         request.args["missing"]
 
 
+def test_request_repr():
+    request = Request.fake("/foo?multi=1&multi=2&single=3")
+    assert (
+        repr(request)
+        == ''
+    )
+
+
 def test_request_url_vars():
     scope = {
         "http_version": "1.1",

From c76bbd40664f789c45564b7796628e5110cd3b17 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 14:50:06 -0800
Subject: [PATCH 1216/2303] New live demo with Apache proxying, refs #1522

---
 demos/apache-proxy/Dockerfile | 42 +++++++++++++++++++++++++++++++++++
 demos/apache-proxy/README.md  |  5 +++++
 demos/apache-proxy/deploy.sh  | 13 +++++++++++
 3 files changed, 60 insertions(+)
 create mode 100644 demos/apache-proxy/Dockerfile
 create mode 100644 demos/apache-proxy/README.md
 create mode 100755 demos/apache-proxy/deploy.sh

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
new file mode 100644
index 00000000..2956f913
--- /dev/null
+++ b/demos/apache-proxy/Dockerfile
@@ -0,0 +1,42 @@
+FROM python:3-alpine
+
+RUN apk add --no-cache \
+	apache2 \
+	apache2-proxy \
+	bash
+
+RUN pip install datasette
+
+ENV TINI_VERSION v0.18.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini
+RUN chmod +x /tini
+
+# Append this to the end of the default httpd.conf file
+RUN echo $'ServerName localhost\n\
+\n\
+\n\
+  Order deny,allow\n\
+  Allow from all\n\
+\n\
+\n\
+ProxyPass        /prefix/ http://localhost:8001/\n\
+Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf
+
+RUN echo $'Datasette' > /var/www/localhost/htdocs/index.html
+
+WORKDIR /app
+
+ADD https://latest.datasette.io/fixtures.db /app/fixtures.db
+
+RUN echo $'#!/usr/bin/env bash\n\
+set -e\n\
+\n\
+httpd -D FOREGROUND &\n\
+datasette fixtures.db --setting base_url "/prefix/" -h 0.0.0.0 -p 8001 &\n\
+\n\
+wait -n' > /app/start.sh
+
+RUN chmod +x /app/start.sh
+
+EXPOSE 80
+ENTRYPOINT ["/tini", "--", "/app/start.sh"]
diff --git a/demos/apache-proxy/README.md b/demos/apache-proxy/README.md
new file mode 100644
index 00000000..9bd3897c
--- /dev/null
+++ b/demos/apache-proxy/README.md
@@ -0,0 +1,5 @@
+# Datasette running behind an Apache proxy
+
+See also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy)
+
+This live demo is running at https://apache-proxy-demo.datasette.io/
diff --git a/demos/apache-proxy/deploy.sh b/demos/apache-proxy/deploy.sh
new file mode 100755
index 00000000..ae33941c
--- /dev/null
+++ b/demos/apache-proxy/deploy.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# https://til.simonwillison.net/cloudrun/ship-dockerfile-to-cloud-run
+
+NAME="datasette-apache-proxy-demo"
+PROJECT=$(gcloud config get-value project)
+IMAGE="gcr.io/$PROJECT/$NAME"
+
+gcloud builds submit --tag $IMAGE
+gcloud run deploy \
+    --allow-unauthenticated \
+    --platform=managed \
+    --image $IMAGE $NAME \
+    --port 80

From c617e1769ea27e045b0f2907ef49a9a1244e577d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 15:13:17 -0800
Subject: [PATCH 1217/2303] Fixed test I broke with new repr() in ##1519

---
 tests/test_plugins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index c9ff6edb..697a6b32 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -463,7 +463,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
-        "request": "",
+        "request": '',
         "view_name": "table",
         "1+1": 2,
     }

From a1ba6cd6bb86d935cdad240de6be6b37aad683f2 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 16:34:35 -0800
Subject: [PATCH 1218/2303] Use build arguments, refs #1522

---
 demos/apache-proxy/Dockerfile | 23 +++++++++++++----------
 demos/apache-proxy/README.md  | 11 +++++++++++
 demos/apache-proxy/deploy.sh  | 31 ++++++++++++++++++++++++-------
 3 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 2956f913..46697c63 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -5,38 +5,41 @@ RUN apk add --no-cache \
 	apache2-proxy \
 	bash
 
-RUN pip install datasette
+ARG DATASETTE_REF
+
+RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip
 
 ENV TINI_VERSION v0.18.0
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini
 RUN chmod +x /tini
 
 # Append this to the end of the default httpd.conf file
-RUN echo $'ServerName localhost\n\
+RUN echo -e 'ServerName localhost\n\
 \n\
 \n\
   Order deny,allow\n\
   Allow from all\n\
 \n\
 \n\
-ProxyPass        /prefix/ http://localhost:8001/\n\
+ProxyPass /prefix/ http://localhost:8001/\n\
 Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf
 
-RUN echo $'Datasette' > /var/www/localhost/htdocs/index.html
+RUN echo 'Datasette' > /var/www/localhost/htdocs/index.html
 
 WORKDIR /app
 
 ADD https://latest.datasette.io/fixtures.db /app/fixtures.db
 
-RUN echo $'#!/usr/bin/env bash\n\
-set -e\n\
+RUN echo -e "#!/usr/bin/env bash\n\
+datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001 &\n\
 \n\
-httpd -D FOREGROUND &\n\
-datasette fixtures.db --setting base_url "/prefix/" -h 0.0.0.0 -p 8001 &\n\
+httpd -D FOREGROUND & \n\
 \n\
-wait -n' > /app/start.sh
+wait -n\n\
+exit $?" > /app/start.sh
 
 RUN chmod +x /app/start.sh
 
 EXPOSE 80
-ENTRYPOINT ["/tini", "--", "/app/start.sh"]
+
+CMD /tini -- /app/start.sh
diff --git a/demos/apache-proxy/README.md b/demos/apache-proxy/README.md
index 9bd3897c..08048512 100644
--- a/demos/apache-proxy/README.md
+++ b/demos/apache-proxy/README.md
@@ -3,3 +3,14 @@
 See also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy)
 
 This live demo is running at https://apache-proxy-demo.datasette.io/
+
+To build locally, passing in a Datasette commit hash (or `main` for the main branch):
+
+    docker build -t datasette-apache-proxy-demo . \
+      --build-arg DATASETTE_REF=c617e1769ea27e045b0f2907ef49a9a1244e577d
+
+Then run it like this:
+
+    docker run -p 5000:80 datasette-apache-proxy-demo
+
+And visit `http://localhost:5000/` or `http://localhost:5000/prefix/`
diff --git a/demos/apache-proxy/deploy.sh b/demos/apache-proxy/deploy.sh
index ae33941c..2846590a 100755
--- a/demos/apache-proxy/deploy.sh
+++ b/demos/apache-proxy/deploy.sh
@@ -1,13 +1,30 @@
 #!/bin/bash
-# https://til.simonwillison.net/cloudrun/ship-dockerfile-to-cloud-run
+# https://til.simonwillison.net/cloudrun/using-build-args-with-cloud-run
+
+if [[ -z "$DATASETTE_REF" ]]; then
+    echo "Must provide DATASETTE_REF environment variable" 1>&2
+    exit 1
+fi
 
 NAME="datasette-apache-proxy-demo"
 PROJECT=$(gcloud config get-value project)
 IMAGE="gcr.io/$PROJECT/$NAME"
 
-gcloud builds submit --tag $IMAGE
-gcloud run deploy \
-    --allow-unauthenticated \
-    --platform=managed \
-    --image $IMAGE $NAME \
-    --port 80
+# Need YAML so we can set --build-arg
+echo "
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '-t', '$IMAGE', '.', '--build-arg', 'DATASETTE_REF=$DATASETTE_REF']
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['push', '$IMAGE']
+" > /tmp/cloudbuild.yml
+
+gcloud builds submit --config /tmp/cloudbuild.yml
+
+rm /tmp/cloudbuild.yml
+
+gcloud run deploy $NAME \
+  --allow-unauthenticated \
+  --platform=managed \
+  --image $IMAGE \
+  --port 80

From fe687fd0207c4c56c4778d3e92e3505fc4b18172 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 16:52:33 -0800
Subject: [PATCH 1219/2303] Fixed a whole bunch of broken base_url links

Refs #1519, #838
---
 datasette/facets.py             | 18 +++++++++++++-----
 datasette/templates/_table.html |  4 ++--
 datasette/views/base.py         | 10 ++++++----
 datasette/views/database.py     |  2 +-
 datasette/views/table.py        |  1 +
 tests/test_html.py              | 24 +++++++++++++++++-------
 6 files changed, 40 insertions(+), 19 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 62e7775e..9a43b95e 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -180,7 +180,11 @@ class ColumnFacet(Facet):
                             "name": column,
                             "toggle_url": self.ds.absolute_url(
                                 self.request,
-                                path_with_added_args(self.request, {"_facet": column}),
+                                self.ds.urls.path(
+                                    path_with_added_args(
+                                        self.request, {"_facet": column}
+                                    )
+                                ),
                             ),
                         }
                     )
@@ -334,8 +338,10 @@ class ArrayFacet(Facet):
                                 "type": "array",
                                 "toggle_url": self.ds.absolute_url(
                                     self.request,
-                                    path_with_added_args(
-                                        self.request, {"_facet_array": column}
+                                    self.ds.urls.path(
+                                        path_with_added_args(
+                                            self.request, {"_facet_array": column}
+                                        )
                                     ),
                                 ),
                             }
@@ -461,8 +467,10 @@ class DateFacet(Facet):
                             "type": "date",
                             "toggle_url": self.ds.absolute_url(
                                 self.request,
-                                path_with_added_args(
-                                    self.request, {"_facet_date": column}
+                                self.ds.urls.path(
+                                    path_with_added_args(
+                                        self.request, {"_facet_date": column}
+                                    )
                                 ),
                             ),
                         }
diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html
index 649f5171..d91a1a57 100644
--- a/datasette/templates/_table.html
+++ b/datasette/templates/_table.html
@@ -9,9 +9,9 @@
                             {{ column.name }}
                         {% else %}
                             {% if column.name == sort %}
-                                {{ column.name }} ▼
+                                {{ column.name }} ▼
                             {% else %}
-                                {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %}
+                                {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %}
                             {% endif %}
                         {% endif %}
                     
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 01e90220..a9953dfd 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -592,13 +592,15 @@ class DataView(BaseView):
                 )
                 it_can_render = await await_me_maybe(it_can_render)
                 if it_can_render:
-                    renderers[key] = path_with_format(
-                        request=request, format=key, extra_qs={**url_labels_extra}
+                    renderers[key] = self.ds.urls.path(
+                        path_with_format(
+                            request=request, format=key, extra_qs={**url_labels_extra}
+                        )
                     )
 
             url_csv_args = {"_size": "max", **url_labels_extra}
-            url_csv = path_with_format(
-                request=request, format="csv", extra_qs=url_csv_args
+            url_csv = self.ds.urls.path(
+                path_with_format(request=request, format="csv", extra_qs=url_csv_args)
             )
             url_csv_path = url_csv.split("?")[0]
             context = {
diff --git a/datasette/views/database.py b/datasette/views/database.py
index affded9b..f1901b34 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -459,7 +459,7 @@ class QueryView(DataView):
                 "metadata": metadata,
                 "settings": self.ds.settings_dict(),
                 "request": request,
-                "show_hide_link": show_hide_link,
+                "show_hide_link": self.ds.urls.path(show_hide_link),
                 "show_hide_text": show_hide_text,
                 "show_hide_hidden": markupsafe.Markup(show_hide_hidden),
                 "hide_sql": hide_sql,
diff --git a/datasette/views/table.py b/datasette/views/table.py
index e6ae67de..296e177f 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -942,6 +942,7 @@ class TableView(RowTableShared):
                 "extra_wheres_for_ui": extra_wheres_for_ui,
                 "form_hidden_args": form_hidden_args,
                 "is_sortable": any(c["sortable"] for c in display_columns),
+                "fix_path": ds.urls.path,
                 "path_with_replaced_args": path_with_replaced_args,
                 "path_with_removed_args": path_with_removed_args,
                 "append_querystring": append_querystring,
diff --git a/tests/test_html.py b/tests/test_html.py
index f24165bd..3301b91d 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1614,11 +1614,16 @@ def test_metadata_sort_desc(app_client):
         "/fixtures/compound_three_primary_keys/a,a,a",
         "/fixtures/paginated_view",
         "/fixtures/facetable",
+        "/fixtures?sql=select+1",
     ],
 )
-def test_base_url_config(app_client_base_url_prefix, path):
+@pytest.mark.parametrize("use_prefix", (True, False))
+def test_base_url_config(app_client_base_url_prefix, path, use_prefix):
     client = app_client_base_url_prefix
-    response = client.get("/prefix/" + path.lstrip("/"))
+    path_to_get = path
+    if use_prefix:
+        path_to_get = "/prefix/" + path.lstrip("/")
+    response = client.get(path_to_get)
     soup = Soup(response.body, "html.parser")
     for el in soup.findAll(["a", "link", "script"]):
         if "href" in el.attrs:
@@ -1642,11 +1647,16 @@ def test_base_url_config(app_client_base_url_prefix, path):
             # If this has been made absolute it may start http://localhost/
             if href.startswith("http://localhost/"):
                 href = href[len("http://localost/") :]
-            assert href.startswith("/prefix/"), {
-                "path": path,
-                "href_or_src": href,
-                "element_parent": str(el.parent),
-            }
+            assert href.startswith("/prefix/"), json.dumps(
+                {
+                    "path": path,
+                    "path_to_get": path_to_get,
+                    "href_or_src": href,
+                    "element_parent": str(el.parent),
+                },
+                indent=4,
+                default=repr,
+            )
 
 
 def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):

From 640031edfd40ba66aee3c4f7008c78c6a78a3e69 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 17:01:17 -0800
Subject: [PATCH 1220/2303] Fixed bug introduced in #1519

---
 datasette/views/table.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 296e177f..66447aa0 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -942,7 +942,7 @@ class TableView(RowTableShared):
                 "extra_wheres_for_ui": extra_wheres_for_ui,
                 "form_hidden_args": form_hidden_args,
                 "is_sortable": any(c["sortable"] for c in display_columns),
-                "fix_path": ds.urls.path,
+                "fix_path": self.ds.urls.path,
                 "path_with_replaced_args": path_with_replaced_args,
                 "path_with_removed_args": path_with_removed_args,
                 "append_querystring": append_querystring,

From 24b5006ad7c316d00a1a963db5bfa82a49fab116 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 17:11:13 -0800
Subject: [PATCH 1221/2303] ProxyPreserveHost On for apache-proxy demo, refs
 #1522

---
 demos/apache-proxy/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 46697c63..59c20433 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -21,6 +21,7 @@ RUN echo -e 'ServerName localhost\n\
   Allow from all\n\
 \n\
 \n\
+ProxyPreserveHost On\n\
 ProxyPass /prefix/ http://localhost:8001/\n\
 Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf
 

From 494f11d5cc88f05df300f6f41bcf083a736487dc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 10:51:14 -0800
Subject: [PATCH 1222/2303] Switch from Alpine to Debian, refs #1522

---
 demos/apache-proxy/Dockerfile | 80 +++++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 31 deletions(-)

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 59c20433..40f5e31d 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -1,46 +1,64 @@
-FROM python:3-alpine
+FROM python:3.9.7-slim-bullseye
 
-RUN apk add --no-cache \
-	apache2 \
-	apache2-proxy \
-	bash
+RUN apt-get update && \
+    apt-get install -y apache2 supervisor && \
+    apt clean && \
+    rm -rf /var/lib/apt && \
+    rm -rf /var/lib/dpkg/info/*
+
+# Apache environment, copied from
+# https://github.com/ijklim/laravel-benfords-law-app/blob/e9bf385dcaddb62ea466a7b245ab6e4ef708c313/docker/os/Dockerfile
+ENV APACHE_DOCUMENT_ROOT=/var/www/html/public
+ENV APACHE_RUN_USER www-data
+ENV APACHE_RUN_GROUP www-data
+ENV APACHE_PID_FILE /var/run/apache2.pid
+ENV APACHE_RUN_DIR /var/run/apache2
+ENV APACHE_LOCK_DIR /var/lock/apache2
+ENV APACHE_LOG_DIR /var/log
+RUN ln -sf /dev/stdout /var/log/apache2-access.log
+RUN ln -sf /dev/stderr /var/log/apache2-error.log
+RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR
+
+RUN a2enmod proxy
+RUN a2enmod proxy_http
+RUN a2enmod headers
 
 ARG DATASETTE_REF
 
 RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip
 
-ENV TINI_VERSION v0.18.0
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini
-RUN chmod +x /tini
-
 # Append this to the end of the default httpd.conf file
-RUN echo -e 'ServerName localhost\n\
+RUN echo '\n\
+\n\
+    Options Indexes FollowSymLinks\n\
+    AllowOverride None\n\
+    Require all granted\n\
+\n\
 \n\
-\n\
-  Order deny,allow\n\
-  Allow from all\n\
-\n\
-\n\
-ProxyPreserveHost On\n\
-ProxyPass /prefix/ http://localhost:8001/\n\
-Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf
-
-RUN echo 'Datasette' > /var/www/localhost/htdocs/index.html
+\n\
+    ServerName localhost\n\
+    DocumentRoot /app/html\n\
+    ProxyPreserveHost On\n\
+    ProxyPass /prefix/ http://127.0.0.1:8001/\n\
+    Header add X-Proxied-By "Apache2 Debian"\n\
+\n\
+' > /etc/apache2/sites-enabled/000-default.conf
 
 WORKDIR /app
+RUN mkdir -p /app/html
+RUN echo 'Datasette' > /app/html/index.html
 
 ADD https://latest.datasette.io/fixtures.db /app/fixtures.db
 
-RUN echo -e "#!/usr/bin/env bash\n\
-datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001 &\n\
-\n\
-httpd -D FOREGROUND & \n\
-\n\
-wait -n\n\
-exit $?" > /app/start.sh
-
-RUN chmod +x /app/start.sh
-
 EXPOSE 80
 
-CMD /tini -- /app/start.sh
+RUN echo "[supervisord]" >> /app/supervisord.conf
+RUN echo "nodaemon=true" >> /app/supervisord.conf
+RUN echo "" >> /app/supervisord.conf
+RUN echo "[program:apache2]" >> /app/supervisord.conf
+RUN echo "command=apache2 -D FOREGROUND" >> /app/supervisord.conf
+RUN echo "" >> /app/supervisord.conf
+RUN echo "[program:datasette]" >> /app/supervisord.conf
+RUN echo "command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/supervisord.conf
+
+CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"]

From 48951e4304cc39b49e26682836d6961e165bddb1 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 10:51:51 -0800
Subject: [PATCH 1223/2303] Switch to hosting demo on Fly, closes #1522

---
 demos/apache-proxy/README.md                  | 24 +++++++++++-
 .../{deploy.sh => deploy-to-cloud-run.sh}     |  0
 demos/apache-proxy/fly.toml                   | 37 +++++++++++++++++++
 3 files changed, 60 insertions(+), 1 deletion(-)
 rename demos/apache-proxy/{deploy.sh => deploy-to-cloud-run.sh} (100%)
 create mode 100644 demos/apache-proxy/fly.toml

diff --git a/demos/apache-proxy/README.md b/demos/apache-proxy/README.md
index 08048512..c76e440d 100644
--- a/demos/apache-proxy/README.md
+++ b/demos/apache-proxy/README.md
@@ -2,7 +2,7 @@
 
 See also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy)
 
-This live demo is running at https://apache-proxy-demo.datasette.io/
+This live demo is running at https://datasette-apache-proxy-demo.fly.dev/prefix/
 
 To build locally, passing in a Datasette commit hash (or `main` for the main branch):
 
@@ -14,3 +14,25 @@ Then run it like this:
     docker run -p 5000:80 datasette-apache-proxy-demo
 
 And visit `http://localhost:5000/` or `http://localhost:5000/prefix/`
+
+## Deployment to Fly
+
+To deploy to [Fly](https://fly.io/) first create an application there by running:
+
+    flyctl apps create --name datasette-apache-proxy-demo
+
+You will need a different name, since I have already taken that one.
+
+Then run this command to deploy:
+
+    flyctl deploy --build-arg DATASETTE_REF=main
+
+This uses `fly.toml` in this directory, which hard-codes the `datasette-apache-proxy-demo` name - so you would need to edit that file to match your application name before running this.
+
+## Deployment to Cloud Run
+
+Deployments to Cloud Run currently result in intermittent 503 errors and I'm not sure why, see [issue #1522](https://github.com/simonw/datasette/issues/1522).
+
+You can deploy like this:
+
+    DATASETTE_REF=main ./deploy-to-cloud-run.sh
diff --git a/demos/apache-proxy/deploy.sh b/demos/apache-proxy/deploy-to-cloud-run.sh
similarity index 100%
rename from demos/apache-proxy/deploy.sh
rename to demos/apache-proxy/deploy-to-cloud-run.sh
diff --git a/demos/apache-proxy/fly.toml b/demos/apache-proxy/fly.toml
new file mode 100644
index 00000000..52e6af5d
--- /dev/null
+++ b/demos/apache-proxy/fly.toml
@@ -0,0 +1,37 @@
+app = "datasette-apache-proxy-demo"
+
+kill_signal = "SIGINT"
+kill_timeout = 5
+processes = []
+
+[env]
+
+[experimental]
+  allowed_public_ports = []
+  auto_rollback = true
+
+[[services]]
+  http_checks = []
+  internal_port = 80
+  processes = ["app"]
+  protocol = "tcp"
+  script_checks = []
+
+  [services.concurrency]
+    hard_limit = 25
+    soft_limit = 20
+    type = "connections"
+
+  [[services.ports]]
+    handlers = ["http"]
+    port = 80
+
+  [[services.ports]]
+    handlers = ["tls", "http"]
+    port = 443
+
+  [[services.tcp_checks]]
+    grace_period = "1s"
+    interval = "15s"
+    restart_limit = 0
+    timeout = "2s"

From 08947fa76433d18988aa1ee1d929bd8320c75fe2 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 11:03:08 -0800
Subject: [PATCH 1224/2303] Fix more broken base_url links

Refs #1519, #838
---
 datasette/facets.py      | 10 +++++-----
 datasette/views/table.py |  2 +-
 tests/test_html.py       |  1 +
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 9a43b95e..29923ef7 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -225,8 +225,8 @@ class ColumnFacet(Facet):
                     "name": column,
                     "type": self.type,
                     "hideable": source != "metadata",
-                    "toggle_url": path_with_removed_args(
-                        self.request, {"_facet": column}
+                    "toggle_url": ds.urls.path(
+                        path_with_removed_args(self.request, {"_facet": column})
                     ),
                     "results": facet_results_values,
                     "truncated": len(facet_rows_results) > facet_size,
@@ -259,7 +259,7 @@ class ColumnFacet(Facet):
                             "label": expanded.get((column, row["value"]), row["value"]),
                             "count": row["count"],
                             "toggle_url": self.ds.absolute_url(
-                                self.request, toggle_path
+                                self.request, self.ds.urls.path(toggle_path)
                             ),
                             "selected": selected,
                         }
@@ -397,8 +397,8 @@ class ArrayFacet(Facet):
                     "type": self.type,
                     "results": facet_results_values,
                     "hideable": source != "metadata",
-                    "toggle_url": path_with_removed_args(
-                        self.request, {"_facet_array": column}
+                    "toggle_url": self.ds.urls.path(
+                        path_with_removed_args(self.request, {"_facet_array": column})
                     ),
                     "truncated": len(facet_rows_results) > facet_size,
                 }
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 66447aa0..1960f455 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -839,7 +839,7 @@ class TableView(RowTableShared):
             else:
                 added_args = {"_next": next_value}
             next_url = self.ds.absolute_url(
-                request, path_with_replaced_args(request, added_args)
+                request, self.ds.urls.path(path_with_replaced_args(request, added_args))
             )
             rows = rows[:page_size]
 
diff --git a/tests/test_html.py b/tests/test_html.py
index 3301b91d..68508d75 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1614,6 +1614,7 @@ def test_metadata_sort_desc(app_client):
         "/fixtures/compound_three_primary_keys/a,a,a",
         "/fixtures/paginated_view",
         "/fixtures/facetable",
+        "/fixtures/facetable?_facet=state",
         "/fixtures?sql=select+1",
     ],
 )

From 250db8192cb8aba5eb8cd301ccc2a49525bc3d24 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 11:09:05 -0800
Subject: [PATCH 1225/2303] Hopefully last fix relating to #1519, #838

---
 datasette/facets.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 29923ef7..8fd2177a 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -225,7 +225,7 @@ class ColumnFacet(Facet):
                     "name": column,
                     "type": self.type,
                     "hideable": source != "metadata",
-                    "toggle_url": ds.urls.path(
+                    "toggle_url": self.ds.urls.path(
                         path_with_removed_args(self.request, {"_facet": column})
                     ),
                     "results": facet_results_values,

From f11a13d73f021906f04b495cd589915e9a926bc5 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 12:23:40 -0800
Subject: [PATCH 1226/2303] Extract out Apache config to separate file, refs
 #1524

---
 demos/apache-proxy/000-default.conf | 13 +++++++++++++
 demos/apache-proxy/Dockerfile       | 20 +++-----------------
 2 files changed, 16 insertions(+), 17 deletions(-)
 create mode 100644 demos/apache-proxy/000-default.conf

diff --git a/demos/apache-proxy/000-default.conf b/demos/apache-proxy/000-default.conf
new file mode 100644
index 00000000..5b6607a3
--- /dev/null
+++ b/demos/apache-proxy/000-default.conf
@@ -0,0 +1,13 @@
+
+    Options Indexes FollowSymLinks
+    AllowOverride None
+    Require all granted
+
+
+
+    ServerName localhost
+    DocumentRoot /app/html
+    ProxyPreserveHost On
+    ProxyPass /prefix/ http://127.0.0.1:8001/
+    Header add X-Proxied-By "Apache2 Debian"
+
diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 40f5e31d..0854b552 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -27,31 +27,17 @@ ARG DATASETTE_REF
 
 RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip
 
-# Append this to the end of the default httpd.conf file
-RUN echo '\n\
-\n\
-    Options Indexes FollowSymLinks\n\
-    AllowOverride None\n\
-    Require all granted\n\
-\n\
-\n\
-\n\
-    ServerName localhost\n\
-    DocumentRoot /app/html\n\
-    ProxyPreserveHost On\n\
-    ProxyPass /prefix/ http://127.0.0.1:8001/\n\
-    Header add X-Proxied-By "Apache2 Debian"\n\
-\n\
-' > /etc/apache2/sites-enabled/000-default.conf
+ADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf
 
 WORKDIR /app
 RUN mkdir -p /app/html
-RUN echo 'Datasette' > /app/html/index.html
+RUN echo '
    Demo is at /prefix/
    ' > /app/html/index.html
 
 ADD https://latest.datasette.io/fixtures.db /app/fixtures.db
 
 EXPOSE 80
 
+# Dynamically build supervisord config since it includes $DATASETTE_REF:
 RUN echo "[supervisord]" >> /app/supervisord.conf
 RUN echo "nodaemon=true" >> /app/supervisord.conf
 RUN echo "" >> /app/supervisord.conf

From ed77eda6d8f10c63fc0670c7150fc974f786ade5 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 15:30:25 -0800
Subject: [PATCH 1227/2303] Add datasette-redirect-to-https plugin

Also configured suprvisord children to log to stdout, so that I
can see them with flyctly logs -a datasette-apache-proxy-demo

Refs #1524
---
 demos/apache-proxy/Dockerfile | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 0854b552..ab7b9d16 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -25,7 +25,9 @@ RUN a2enmod headers
 
 ARG DATASETTE_REF
 
-RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip
+RUN pip install \
+    https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip \
+    datasette-redirect-to-https
 
 ADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf
 
@@ -43,8 +45,12 @@ RUN echo "nodaemon=true" >> /app/supervisord.conf
 RUN echo "" >> /app/supervisord.conf
 RUN echo "[program:apache2]" >> /app/supervisord.conf
 RUN echo "command=apache2 -D FOREGROUND" >> /app/supervisord.conf
+RUN echo "stdout_logfile=/dev/stdout" >> /app/supervisord.conf
+RUN echo "stdout_logfile_maxbytes=0" >> /app/supervisord.conf
 RUN echo "" >> /app/supervisord.conf
 RUN echo "[program:datasette]" >> /app/supervisord.conf
 RUN echo "command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/supervisord.conf
+RUN echo "stdout_logfile=/dev/stdout" >> /app/supervisord.conf
+RUN echo "stdout_logfile_maxbytes=0" >> /app/supervisord.conf
 
 CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"]

From d8c79b1340ceb742077587fb7f76ed8699d4e402 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 15:33:58 -0800
Subject: [PATCH 1228/2303] Link to Apache proxy demo from documentation,
 closes #1524

---
 docs/deploying.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/deploying.rst b/docs/deploying.rst
index 83d9e4dd..d4ad8836 100644
--- a/docs/deploying.rst
+++ b/docs/deploying.rst
@@ -188,6 +188,8 @@ Then add these directives to proxy traffic::
     ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/
     ProxyPreserveHost On
 
+A live demo of Datasette running behind Apache using this proxy setup can be seen at `datasette-apache-proxy-demo.datasette.io/prefix/ `__. The code for that demo can be found in the `demos/apache-proxy `__ directory.
+
 Using ``--uds`` you can use Unix domain sockets similar to the nginx example::
 
     ProxyPass /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/

From 48f11998b73350057b74fe6ab464d4ac3071637c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 20 Nov 2021 15:40:21 -0800
Subject: [PATCH 1229/2303] Release 0.59.3

Refs #448, #838, #1519
---
 datasette/version.py | 2 +-
 docs/changelog.rst   | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index db89b418..0ba55573 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59.2"
+__version__ = "0.59.3"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 47ca3480..449ce412 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,15 @@
 Changelog
 =========
 
+.. _v0_59_3:
+
+0.59.3 (2021-11-20)
+-------------------
+
+- Fixed numerous bugs when running Datasette :ref:`behind a proxy ` with a prefix URL path using the :ref:`setting_base_url` setting. A live demo of this mode is now available at `datasette-apache-proxy-demo.datasette.io/prefix/ `__. (:issue:`1519`, :issue:`838`)
+- ``?column__arraycontains=`` and ``?column__arraynotcontains=`` table parameters now also work against SQL views. (:issue:`448`)
+- ``?_facet_array=column`` no longer returns incorrect counts if columns contain the same value more than once.
+
 .. _v0_59_2:
 
 0.59.2 (2021-11-13)

From 1beb7d939999da79bb77c4d3c777657c8a16bcd9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Nov 2021 18:29:54 -0800
Subject: [PATCH 1230/2303] Update aiofiles requirement from <0.8,>=0.4 to
 >=0.4,<0.9 (#1537)

Updates the requirements on [aiofiles](https://github.com/Tinche/aiofiles) to permit the latest version.
- [Release notes](https://github.com/Tinche/aiofiles/releases)
- [Commits](https://github.com/Tinche/aiofiles/compare/v0.4.0...v0.8.0)

---
updated-dependencies:
- dependency-name: aiofiles
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 17a56a97..9e205ce2 100644
--- a/setup.py
+++ b/setup.py
@@ -51,7 +51,7 @@ setup(
         "pint~=0.9",
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
-        "aiofiles>=0.4,<0.8",
+        "aiofiles>=0.4,<0.9",
         "janus>=0.6.2,<0.7",
         "asgi-csrf>=0.9",
         "PyYAML>=5.3,<7.0",

From 3303514a52b7170f2f1e598cd9c5f82c22f26e6c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Nov 2021 18:35:18 -0800
Subject: [PATCH 1231/2303] Update docutils requirement from <0.18 to <0.19
 (#1508)

Updates the requirements on [docutils](http://docutils.sourceforge.net/) to permit the latest version.

---
updated-dependencies:
- dependency-name: docutils
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs/readthedocs-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/readthedocs-requirements.txt b/docs/readthedocs-requirements.txt
index 93120e66..db1851ad 100644
--- a/docs/readthedocs-requirements.txt
+++ b/docs/readthedocs-requirements.txt
@@ -1 +1 @@
-docutils<0.18
+docutils<0.19

From cc4c70b3670ce2a85bb883b8d5626574590efe14 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Nov 2021 18:35:28 -0800
Subject: [PATCH 1232/2303] Bump black from 21.9b0 to 21.11b1 (#1516)

Bumps [black](https://github.com/psf/black) from 21.9b0 to 21.11b1.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 9e205ce2..71422d87 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.17",
             "beautifulsoup4>=4.8.1,<4.11.0",
-            "black==21.9b0",
+            "black==21.11b1",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
         ],

From 83eb29deced2430f40c3374ff9085d65d86d8281 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Nov 2021 18:37:13 -0800
Subject: [PATCH 1233/2303] Update janus requirement from <0.7,>=0.6.2 to
 >=0.6.2,<0.8 (#1529)

Updates the requirements on [janus](https://github.com/aio-libs/janus) to permit the latest version.
- [Release notes](https://github.com/aio-libs/janus/releases)
- [Changelog](https://github.com/aio-libs/janus/blob/master/CHANGES.rst)
- [Commits](https://github.com/aio-libs/janus/compare/v0.6.2...v0.7.0)

---
updated-dependencies:
- dependency-name: janus
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 71422d87..3cb657e3 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ setup(
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.9",
-        "janus>=0.6.2,<0.7",
+        "janus>=0.6.2,<0.8",
         "asgi-csrf>=0.9",
         "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",

From 06762776f712526fdb40a18ed26f259be62bb214 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 29 Nov 2021 19:04:20 -0800
Subject: [PATCH 1234/2303] Fix for incorrect hidden for fields for _columns,
 refs #1527

---
 datasette/views/table.py |  6 +++++-
 tests/test_html.py       | 13 +++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 1960f455..9fc6afcf 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -889,7 +889,11 @@ class TableView(RowTableShared):
 
             form_hidden_args = []
             for key in request.args:
-                if key.startswith("_") and key not in ("_sort", "_search", "_next"):
+                if (
+                    key.startswith("_")
+                    and key not in ("_sort", "_search", "_next")
+                    and not key.endswith("__exact")
+                ):
                     for value in request.args.getlist(key):
                         form_hidden_args.append((key, value))
 
diff --git a/tests/test_html.py b/tests/test_html.py
index 68508d75..179c3f09 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -326,6 +326,19 @@ def test_existing_filter_redirects(app_client):
     assert "?" not in response.headers["Location"]
 
 
+def test_exact_parameter_results_in_correct_hidden_fields(app_client):
+    # https://github.com/simonw/datasette/issues/1527
+    response = app_client.get(
+        "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown"
+    )
+    # In this case we should NOT have a hidden _neighborhood__exact=Downtown field
+    form = Soup(response.body, "html.parser").find("form")
+    hidden_inputs = {
+        input["name"]: input["value"] for input in form.select("input[type=hidden]")
+    }
+    assert hidden_inputs == {"_facet": "_neighborhood"}
+
+
 def test_empty_search_parameter_gets_removed(app_client):
     path_base = "/fixtures/simple_primary_key"
     path = (

From 69244a617b1118dcbd04a8f102173f04680cf08c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 29 Nov 2021 22:17:27 -0800
Subject: [PATCH 1235/2303] Rename city_id to _city_id in fixtures, refs #1525

---
 tests/fixtures.py                |  8 ++---
 tests/test_api.py                | 50 ++++++++++++++++----------------
 tests/test_facets.py             | 36 +++++++++++------------
 tests/test_html.py               | 34 +++++++++++-----------
 tests/test_internals_database.py |  4 +--
 tests/test_plugins.py            |  8 ++---
 6 files changed, 70 insertions(+), 70 deletions(-)

diff --git a/tests/fixtures.py b/tests/fixtures.py
index 1a879126..37399da0 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -358,7 +358,7 @@ METADATA = {
                         select _neighborhood, facet_cities.name, state
                         from facetable
                             join facet_cities
-                                on facetable.city_id = facet_cities.id
+                                on facetable._city_id = facet_cities.id
                         where _neighborhood like '%' || :text || '%'
                         order by _neighborhood;
                     """
@@ -558,15 +558,15 @@ CREATE TABLE facetable (
     planet_int integer,
     on_earth integer,
     state text,
-    city_id integer,
+    _city_id integer,
     _neighborhood text,
     tags text,
     complex_array text,
     distinct_some_null,
-    FOREIGN KEY ("city_id") REFERENCES [facet_cities](id)
+    FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
 );
 INSERT INTO facetable
-    (created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null)
+    (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null)
 VALUES
     ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'),
     ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'),
diff --git a/tests/test_api.py b/tests/test_api.py
index 43b52175..8b3fcd75 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -197,7 +197,7 @@ def test_database_page(app_client):
                     {
                         "other_table": "facetable",
                         "column": "id",
-                        "other_column": "city_id",
+                        "other_column": "_city_id",
                     }
                 ],
                 "outgoing": [],
@@ -212,7 +212,7 @@ def test_database_page(app_client):
                 "planet_int",
                 "on_earth",
                 "state",
-                "city_id",
+                "_city_id",
                 "_neighborhood",
                 "tags",
                 "complex_array",
@@ -227,7 +227,7 @@ def test_database_page(app_client):
                 "outgoing": [
                     {
                         "other_table": "facet_cities",
-                        "column": "city_id",
+                        "column": "_city_id",
                         "other_column": "id",
                     }
                 ],
@@ -1512,40 +1512,40 @@ def test_page_size_matching_max_returned_rows(
     "path,expected_facet_results",
     [
         (
-            "/fixtures/facetable.json?_facet=state&_facet=city_id",
+            "/fixtures/facetable.json?_facet=state&_facet=_city_id",
             {
                 "state": {
                     "name": "state",
                     "hideable": True,
                     "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?_facet=city_id",
+                    "toggle_url": "/fixtures/facetable.json?_facet=_city_id",
                     "results": [
                         {
                             "value": "CA",
                             "label": "CA",
                             "count": 10,
-                            "toggle_url": "_facet=state&_facet=city_id&state=CA",
+                            "toggle_url": "_facet=state&_facet=_city_id&state=CA",
                             "selected": False,
                         },
                         {
                             "value": "MI",
                             "label": "MI",
                             "count": 4,
-                            "toggle_url": "_facet=state&_facet=city_id&state=MI",
+                            "toggle_url": "_facet=state&_facet=_city_id&state=MI",
                             "selected": False,
                         },
                         {
                             "value": "MC",
                             "label": "MC",
                             "count": 1,
-                            "toggle_url": "_facet=state&_facet=city_id&state=MC",
+                            "toggle_url": "_facet=state&_facet=_city_id&state=MC",
                             "selected": False,
                         },
                     ],
                     "truncated": False,
                 },
-                "city_id": {
-                    "name": "city_id",
+                "_city_id": {
+                    "name": "_city_id",
                     "hideable": True,
                     "type": "column",
                     "toggle_url": "/fixtures/facetable.json?_facet=state",
@@ -1554,28 +1554,28 @@ def test_page_size_matching_max_returned_rows(
                             "value": 1,
                             "label": "San Francisco",
                             "count": 6,
-                            "toggle_url": "_facet=state&_facet=city_id&city_id=1",
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=1",
                             "selected": False,
                         },
                         {
                             "value": 2,
                             "label": "Los Angeles",
                             "count": 4,
-                            "toggle_url": "_facet=state&_facet=city_id&city_id=2",
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=2",
                             "selected": False,
                         },
                         {
                             "value": 3,
                             "label": "Detroit",
                             "count": 4,
-                            "toggle_url": "_facet=state&_facet=city_id&city_id=3",
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=3",
                             "selected": False,
                         },
                         {
                             "value": 4,
                             "label": "Memnonia",
                             "count": 1,
-                            "toggle_url": "_facet=state&_facet=city_id&city_id=4",
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=4",
                             "selected": False,
                         },
                     ],
@@ -1584,26 +1584,26 @@ def test_page_size_matching_max_returned_rows(
             },
         ),
         (
-            "/fixtures/facetable.json?_facet=state&_facet=city_id&state=MI",
+            "/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI",
             {
                 "state": {
                     "name": "state",
                     "hideable": True,
                     "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?_facet=city_id&state=MI",
+                    "toggle_url": "/fixtures/facetable.json?_facet=_city_id&state=MI",
                     "results": [
                         {
                             "value": "MI",
                             "label": "MI",
                             "count": 4,
                             "selected": True,
-                            "toggle_url": "_facet=state&_facet=city_id",
+                            "toggle_url": "_facet=state&_facet=_city_id",
                         }
                     ],
                     "truncated": False,
                 },
-                "city_id": {
-                    "name": "city_id",
+                "_city_id": {
+                    "name": "_city_id",
                     "hideable": True,
                     "type": "column",
                     "toggle_url": "/fixtures/facetable.json?_facet=state&state=MI",
@@ -1613,7 +1613,7 @@ def test_page_size_matching_max_returned_rows(
                             "label": "Detroit",
                             "count": 4,
                             "selected": False,
-                            "toggle_url": "_facet=state&_facet=city_id&state=MI&city_id=3",
+                            "toggle_url": "_facet=state&_facet=_city_id&state=MI&_city_id__exact=3",
                         }
                     ],
                     "truncated": False,
@@ -1699,7 +1699,7 @@ def test_suggested_facets(app_client):
         {"name": "planet_int", "querystring": "_facet=planet_int"},
         {"name": "on_earth", "querystring": "_facet=on_earth"},
         {"name": "state", "querystring": "_facet=state"},
-        {"name": "city_id", "querystring": "_facet=city_id"},
+        {"name": "_city_id", "querystring": "_facet=_city_id"},
         {"name": "_neighborhood", "querystring": "_facet=_neighborhood"},
         {"name": "tags", "querystring": "_facet=tags"},
         {"name": "complex_array", "querystring": "_facet=complex_array"},
@@ -1765,7 +1765,7 @@ def test_expand_labels(app_client):
             "planet_int": 1,
             "on_earth": 1,
             "state": "CA",
-            "city_id": {"value": 1, "label": "San Francisco"},
+            "_city_id": {"value": 1, "label": "San Francisco"},
             "_neighborhood": "Dogpatch",
             "tags": '["tag1", "tag3"]',
             "complex_array": "[]",
@@ -1777,7 +1777,7 @@ def test_expand_labels(app_client):
             "planet_int": 1,
             "on_earth": 1,
             "state": "MI",
-            "city_id": {"value": 3, "label": "Detroit"},
+            "_city_id": {"value": 3, "label": "Detroit"},
             "_neighborhood": "Corktown",
             "tags": "[]",
             "complex_array": "[]",
@@ -2128,7 +2128,7 @@ def test_http_options_request(app_client):
                 "planet_int",
                 "on_earth",
                 "state",
-                "city_id",
+                "_city_id",
                 "_neighborhood",
                 "tags",
                 "complex_array",
@@ -2155,7 +2155,7 @@ def test_http_options_request(app_client):
                 "created",
                 "planet_int",
                 "on_earth",
-                "city_id",
+                "_city_id",
                 "_neighborhood",
                 "tags",
                 "complex_array",
diff --git a/tests/test_facets.py b/tests/test_facets.py
index a20c79c4..429117cb 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -23,7 +23,7 @@ async def test_column_facet_suggest(app_client):
         {"name": "planet_int", "toggle_url": "http://localhost/?_facet=planet_int"},
         {"name": "on_earth", "toggle_url": "http://localhost/?_facet=on_earth"},
         {"name": "state", "toggle_url": "http://localhost/?_facet=state"},
-        {"name": "city_id", "toggle_url": "http://localhost/?_facet=city_id"},
+        {"name": "_city_id", "toggle_url": "http://localhost/?_facet=_city_id"},
         {
             "name": "_neighborhood",
             "toggle_url": "http://localhost/?_facet=_neighborhood",
@@ -56,8 +56,8 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client):
             "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=state",
         },
         {
-            "name": "city_id",
-            "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=city_id",
+            "name": "_city_id",
+            "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=_city_id",
         },
         {
             "name": "_neighborhood",
@@ -82,7 +82,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client):
         database="fixtures",
         sql="select * from facetable",
         table="facetable",
-        metadata={"facets": ["city_id"]},
+        metadata={"facets": ["_city_id"]},
     )
     suggestions = [s["name"] for s in await facet.suggest()]
     assert [
@@ -100,7 +100,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client):
 async def test_column_facet_results(app_client):
     facet = ColumnFacet(
         app_client.ds,
-        Request.fake("/?_facet=city_id"),
+        Request.fake("/?_facet=_city_id"),
         database="fixtures",
         sql="select * from facetable",
         table="facetable",
@@ -108,8 +108,8 @@ async def test_column_facet_results(app_client):
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
     assert {
-        "city_id": {
-            "name": "city_id",
+        "_city_id": {
+            "name": "_city_id",
             "type": "column",
             "hideable": True,
             "toggle_url": "/",
@@ -118,28 +118,28 @@ async def test_column_facet_results(app_client):
                     "value": 1,
                     "label": "San Francisco",
                     "count": 6,
-                    "toggle_url": "http://localhost/?_facet=city_id&city_id=1",
+                    "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=1",
                     "selected": False,
                 },
                 {
                     "value": 2,
                     "label": "Los Angeles",
                     "count": 4,
-                    "toggle_url": "http://localhost/?_facet=city_id&city_id=2",
+                    "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=2",
                     "selected": False,
                 },
                 {
                     "value": 3,
                     "label": "Detroit",
                     "count": 4,
-                    "toggle_url": "http://localhost/?_facet=city_id&city_id=3",
+                    "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=3",
                     "selected": False,
                 },
                 {
                     "value": 4,
                     "label": "Memnonia",
                     "count": 1,
-                    "toggle_url": "http://localhost/?_facet=city_id&city_id=4",
+                    "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=4",
                     "selected": False,
                 },
             ],
@@ -278,13 +278,13 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client):
         database="fixtures",
         sql="select * from facetable",
         table="facetable",
-        metadata={"facets": ["city_id"]},
+        metadata={"facets": ["_city_id"]},
     )
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
     assert {
-        "city_id": {
-            "name": "city_id",
+        "_city_id": {
+            "name": "_city_id",
             "type": "column",
             "hideable": False,
             "toggle_url": "/",
@@ -293,28 +293,28 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client):
                     "value": 1,
                     "label": "San Francisco",
                     "count": 6,
-                    "toggle_url": "http://localhost/?city_id=1",
+                    "toggle_url": "http://localhost/?_city_id__exact=1",
                     "selected": False,
                 },
                 {
                     "value": 2,
                     "label": "Los Angeles",
                     "count": 4,
-                    "toggle_url": "http://localhost/?city_id=2",
+                    "toggle_url": "http://localhost/?_city_id__exact=2",
                     "selected": False,
                 },
                 {
                     "value": 3,
                     "label": "Detroit",
                     "count": 4,
-                    "toggle_url": "http://localhost/?city_id=3",
+                    "toggle_url": "http://localhost/?_city_id__exact=3",
                     "selected": False,
                 },
                 {
                     "value": 4,
                     "label": "Memnonia",
                     "count": 1,
-                    "toggle_url": "http://localhost/?city_id=4",
+                    "toggle_url": "http://localhost/?_city_id__exact=4",
                     "selected": False,
                 },
             ],
diff --git a/tests/test_html.py b/tests/test_html.py
index 179c3f09..aaf7da09 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -487,7 +487,7 @@ def test_sort_links(app_client):
 
 def test_facet_display(app_client):
     response = app_client.get(
-        "/fixtures/facetable?_facet=planet_int&_facet=city_id&_facet=on_earth"
+        "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth"
     )
     assert response.status == 200
     soup = Soup(response.body, "html.parser")
@@ -509,26 +509,26 @@ def test_facet_display(app_client):
         )
     assert actual == [
         {
-            "name": "city_id",
+            "name": "_city_id",
             "items": [
                 {
                     "name": "San Francisco",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=1",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1",
                     "count": 6,
                 },
                 {
                     "name": "Los Angeles",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=2",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2",
                     "count": 4,
                 },
                 {
                     "name": "Detroit",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=3",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3",
                     "count": 4,
                 },
                 {
                     "name": "Memnonia",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=4",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4",
                     "count": 1,
                 },
             ],
@@ -538,12 +538,12 @@ def test_facet_display(app_client):
             "items": [
                 {
                     "name": "1",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&planet_int=1",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1",
                     "count": 14,
                 },
                 {
                     "name": "2",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&planet_int=2",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2",
                     "count": 1,
                 },
             ],
@@ -553,12 +553,12 @@ def test_facet_display(app_client):
             "items": [
                 {
                     "name": "1",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&on_earth=1",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1",
                     "count": 14,
                 },
                 {
                     "name": "0",
-                    "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&on_earth=0",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0",
                     "count": 1,
                 },
             ],
@@ -568,14 +568,14 @@ def test_facet_display(app_client):
 
 def test_facets_persist_through_filter_form(app_client):
     response = app_client.get(
-        "/fixtures/facetable?_facet=planet_int&_facet=city_id&_facet_array=tags"
+        "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags"
     )
     assert response.status == 200
     inputs = Soup(response.body, "html.parser").find("form").findAll("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]
     assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
         ("_facet", "planet_int"),
-        ("_facet", "city_id"),
+        ("_facet", "_city_id"),
         ("_facet_array", "tags"),
     ]
 
@@ -1350,20 +1350,20 @@ def test_canned_query_show_hide_metadata_option(
 
 def test_extra_where_clauses(app_client):
     response = app_client.get(
-        "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=city_id=1"
+        "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=_city_id=1"
     )
     soup = Soup(response.body, "html.parser")
     div = soup.select(".extra-wheres")[0]
     assert "2 extra where clauses" == div.find("h3").text
     hrefs = [a["href"] for a in div.findAll("a")]
     assert [
-        "/fixtures/facetable?_where=city_id%3D1",
+        "/fixtures/facetable?_where=_city_id%3D1",
         "/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27",
     ] == hrefs
     # These should also be persisted as hidden fields
     inputs = soup.find("form").findAll("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]
-    assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [
+    assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "_city_id=1")] == [
         (hidden["name"], hidden["value"]) for hidden in hiddens
     ]
 
@@ -1683,11 +1683,11 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
     [
         (
             "/fixtures/neighborhood_search",
-            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=",
+            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=",
         ),
         (
             "/fixtures/neighborhood_search?text=ber",
-            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber",
+            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber",
         ),
         ("/fixtures/pragma_cache_size", None),
         (
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 2d0cae7f..a00fe447 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -81,7 +81,7 @@ async def test_table_exists(db, tables, exists):
                 "planet_int",
                 "on_earth",
                 "state",
-                "city_id",
+                "_city_id",
                 "_neighborhood",
                 "tags",
                 "complex_array",
@@ -161,7 +161,7 @@ async def test_table_columns(db, table, expected):
                 ),
                 Column(
                     cid=5,
-                    name="city_id",
+                    name="_city_id",
                     type="integer",
                     notnull=0,
                     default_value=None,
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 697a6b32..1da28453 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -436,7 +436,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "planet_int",
             "on_earth",
             "state",
-            "city_id",
+            "_city_id",
             "_neighborhood",
             "tags",
             "complex_array",
@@ -459,7 +459,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "",
             "",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
@@ -525,13 +525,13 @@ def test_hook_register_output_renderer_can_render(app_client):
             "planet_int",
             "on_earth",
             "state",
-            "city_id",
+            "_city_id",
             "_neighborhood",
             "tags",
             "complex_array",
             "distinct_some_null",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",

From a37ee74891f14898d5810127c7ca3355e77ff57d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 29 Nov 2021 22:34:31 -0800
Subject: [PATCH 1236/2303] Correct link to _ prefix on row page, closes #1525

---
 datasette/templates/row.html |  2 +-
 datasette/views/table.py     | 10 +++++++++-
 tests/test_html.py           | 28 ++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/datasette/templates/row.html b/datasette/templates/row.html
index 916980b6..c86e979d 100644
--- a/datasette/templates/row.html
+++ b/datasette/templates/row.html
@@ -38,7 +38,7 @@
     
      
         {% for other in foreign_key_tables %}
             
    • 
-                
+                
                     {{ "{:,}".format(other.count) }} row{% if other.count == 1 %}{% else %}s{% endif %}
                 from {{ other.other_column }} in {{ other.other_table }}
             
      • 
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 9fc6afcf..f58b78f5 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -1120,5 +1120,13 @@ class RowView(RowTableShared):
             count = (
                 foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0
             )
-            foreign_key_tables.append({**fk, **{"count": count}})
+            key = fk["other_column"]
+            if key.startswith("_"):
+                key += "__exact"
+            link = "{}?{}={}".format(
+                self.ds.urls.table(database, fk["other_table"]),
+                key,
+                ",".join(pk_values),
+            )
+            foreign_key_tables.append({**fk, **{"count": count, "link": link}})
         return foreign_key_tables
diff --git a/tests/test_html.py b/tests/test_html.py
index aaf7da09..a7cb105c 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -820,6 +820,34 @@ def test_row_html_no_primary_key(app_client):
     ]
 
 
+@pytest.mark.parametrize(
+    "path,expected_text,expected_link",
+    (
+        (
+            "/fixtures/facet_cities/1",
+            "6 rows from _city_id in facetable",
+            "/fixtures/facetable?_city_id__exact=1",
+        ),
+        (
+            "/fixtures/attraction_characteristic/2",
+            "3 rows from characteristic_id in roadside_attraction_characteristics",
+            "/fixtures/roadside_attraction_characteristics?characteristic_id=2",
+        ),
+    ),
+)
+def test_row_links_from_other_tables(app_client, path, expected_text, expected_link):
+    response = app_client.get(path)
+    assert response.status == 200
+    soup = Soup(response.body, "html.parser")
+    h2 = soup.find("h2")
+    assert h2.text == "Links from other tables"
+    li = h2.findNext("ul").find("li")
+    text = re.sub(r"\s+", " ", li.text.strip())
+    assert text == expected_text
+    link = li.find("a")["href"]
+    assert link == expected_link
+
+
 def test_table_html_compound_primary_key(app_client):
     response = app_client.get("/fixtures/compound_primary_key")
     assert response.status == 200

From 35b12746ba2bf9f254791bddac03d25b19be9b77 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 29 Nov 2021 22:37:22 -0800
Subject: [PATCH 1237/2303] Fixed CSV test I broke in #1525

---
 tests/test_csv.py | 44 +++++++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/tests/test_csv.py b/tests/test_csv.py
index 5902e9db..8749cd8b 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -24,7 +24,7 @@ world
 )
 
 EXPECTED_TABLE_WITH_LABELS_CSV = """
-pk,created,planet_int,on_earth,state,city_id,city_id_label,_neighborhood,tags,complex_array,distinct_some_null
+pk,created,planet_int,on_earth,state,_city_id,_city_id_label,_neighborhood,tags,complex_array,distinct_some_null
 1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one
 2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two
 3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[],
@@ -57,42 +57,42 @@ def test_table_csv(app_client):
     response = app_client.get("/fixtures/simple_primary_key.csv?_oh=1")
     assert response.status == 200
     assert not response.headers.get("Access-Control-Allow-Origin")
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
-    assert EXPECTED_TABLE_CSV == response.text
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == EXPECTED_TABLE_CSV
 
 
 def test_table_csv_cors_headers(app_client_with_cors):
     response = app_client_with_cors.get("/fixtures/simple_primary_key.csv")
     assert response.status == 200
-    assert "*" == response.headers["Access-Control-Allow-Origin"]
+    assert response.headers["Access-Control-Allow-Origin"] == "*"
 
 
 def test_table_csv_no_header(app_client):
     response = app_client.get("/fixtures/simple_primary_key.csv?_header=off")
     assert response.status == 200
     assert not response.headers.get("Access-Control-Allow-Origin")
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
-    assert EXPECTED_TABLE_CSV.split("\r\n", 1)[1] == response.text
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == EXPECTED_TABLE_CSV.split("\r\n", 1)[1]
 
 
 def test_table_csv_with_labels(app_client):
     response = app_client.get("/fixtures/facetable.csv?_labels=1")
     assert response.status == 200
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
-    assert EXPECTED_TABLE_WITH_LABELS_CSV == response.text
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == EXPECTED_TABLE_WITH_LABELS_CSV
 
 
 def test_table_csv_with_nullable_labels(app_client):
     response = app_client.get("/fixtures/foreign_key_references.csv?_labels=1")
     assert response.status == 200
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
-    assert EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV == response.text
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV
 
 
 def test_table_csv_blob_columns(app_client):
     response = app_client.get("/fixtures/binary_data.csv")
     assert response.status == 200
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
     assert response.text == (
         "rowid,data\r\n"
         "1,http://localhost/fixtures/binary_data/1.blob?_blob_column=data\r\n"
@@ -104,7 +104,7 @@ def test_table_csv_blob_columns(app_client):
 def test_custom_sql_csv_blob_columns(app_client):
     response = app_client.get("/fixtures.csv?sql=select+rowid,+data+from+binary_data")
     assert response.status == 200
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
     assert response.text == (
         "rowid,data\r\n"
         '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n'
@@ -118,16 +118,18 @@ def test_custom_sql_csv(app_client):
         "/fixtures.csv?sql=select+content+from+simple_primary_key+limit+2"
     )
     assert response.status == 200
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
-    assert EXPECTED_CUSTOM_CSV == response.text
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == EXPECTED_CUSTOM_CSV
 
 
 def test_table_csv_download(app_client):
     response = app_client.get("/fixtures/simple_primary_key.csv?_dl=1")
     assert response.status == 200
-    assert "text/csv; charset=utf-8" == response.headers["content-type"]
-    expected_disposition = 'attachment; filename="simple_primary_key.csv"'
-    assert expected_disposition == response.headers["content-disposition"]
+    assert response.headers["content-type"] == "text/csv; charset=utf-8"
+    assert (
+        response.headers["content-disposition"]
+        == 'attachment; filename="simple_primary_key.csv"'
+    )
 
 
 def test_csv_with_non_ascii_characters(app_client):
@@ -135,8 +137,8 @@ def test_csv_with_non_ascii_characters(app_client):
         "/fixtures.csv?sql=select%0D%0A++%27%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC%27+as+text%2C%0D%0A++1+as+number%0D%0Aunion%0D%0Aselect%0D%0A++%27bob%27+as+text%2C%0D%0A++2+as+number%0D%0Aorder+by%0D%0A++number"
     )
     assert response.status == 200
-    assert "text/plain; charset=utf-8" == response.headers["content-type"]
-    assert "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" == response.text
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n"
 
 
 def test_max_csv_mb(app_client_csv_max_mb_one):
@@ -156,10 +158,10 @@ def test_max_csv_mb(app_client_csv_max_mb_one):
 def test_table_csv_stream(app_client):
     # Without _stream should return header + 100 rows:
     response = app_client.get("/fixtures/compound_three_primary_keys.csv?_size=max")
-    assert 101 == len([b for b in response.body.split(b"\r\n") if b])
+    assert len([b for b in response.body.split(b"\r\n") if b]) == 101
     # With _stream=1 should return header + 1001 rows
     response = app_client.get("/fixtures/compound_three_primary_keys.csv?_stream=1")
-    assert 1002 == len([b for b in response.body.split(b"\r\n") if b])
+    assert len([b for b in response.body.split(b"\r\n") if b]) == 1002
 
 
 def test_csv_trace(app_client_with_trace):

From ca6624643842f4b80644b83c3f4ad7c2265c15d8 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 29 Nov 2021 22:45:04 -0800
Subject: [PATCH 1238/2303] Updated JSON foreign key tables test for #1525

---
 tests/test_api.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_api.py b/tests/test_api.py
index 8b3fcd75..400dae7e 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1357,30 +1357,35 @@ def test_row_foreign_key_tables(app_client):
             "column": "id",
             "other_column": "foreign_key_with_blank_label",
             "count": 0,
+            "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1",
         },
         {
             "other_table": "foreign_key_references",
             "column": "id",
             "other_column": "foreign_key_with_label",
             "count": 1,
+            "link": "/fixtures/foreign_key_references?foreign_key_with_label=1",
         },
         {
             "other_table": "complex_foreign_keys",
             "column": "id",
             "other_column": "f3",
             "count": 1,
+            "link": "/fixtures/complex_foreign_keys?f3=1",
         },
         {
             "other_table": "complex_foreign_keys",
             "column": "id",
             "other_column": "f2",
             "count": 0,
+            "link": "/fixtures/complex_foreign_keys?f2=1",
         },
         {
             "other_table": "complex_foreign_keys",
             "column": "id",
             "other_column": "f1",
             "count": 1,
+            "link": "/fixtures/complex_foreign_keys?f1=1",
         },
     ]
 

From 7c02be2ee94cc64b120cc58b7a72cd387031f287 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 29 Nov 2021 22:45:37 -0800
Subject: [PATCH 1239/2303] Release 0.59.4

Refs #1525, #1527
---
 datasette/version.py | 2 +-
 docs/changelog.rst   | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 0ba55573..9c85b763 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59.3"
+__version__ = "0.59.4"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 449ce412..9ddc2794 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,15 @@
 Changelog
 =========
 
+.. _v0_59_4:
+
+0.59.4 (2021-11-29)
+-------------------
+
+- Fixed bug where columns with a leading underscore could not be removed from the interactive filters list. (:issue:`1527`)
+- Fixed bug where columns with a leading underscore were not correctly linked to by the "Links from other tables" interface on the row page. (:issue:`1525`)
+- Upgraded dependencies ``aiofiles``, ``black`` and ``janus``.
+
 .. _v0_59_3:
 
 0.59.3 (2021-11-20)

From 36b596e3832f6126bb0e4e90cf9257b9e9c9a55e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 7 Dec 2021 11:41:56 -0800
Subject: [PATCH 1240/2303] Framework :: Datasette Trove classifier

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 3cb657e3..9b5bab61 100644
--- a/setup.py
+++ b/setup.py
@@ -80,6 +80,7 @@ setup(
     tests_require=["datasette[test]"],
     classifiers=[
         "Development Status :: 4 - Beta",
+        "Framework :: Datasette",
         "Intended Audience :: Developers",
         "Intended Audience :: Science/Research",
         "Intended Audience :: End Users/Desktop",

From 737115ea14cd51ffb55dea886e6a684c148db2c9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 7 Dec 2021 12:03:42 -0800
Subject: [PATCH 1241/2303] Label column finder is now case-insensitive

Closes #1544
---
 datasette/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/database.py b/datasette/database.py
index 9f3bbddc..d1217e18 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -286,7 +286,7 @@ class Database:
             return explicit_label_column
         column_names = await self.execute_fn(lambda conn: table_columns(conn, table))
         # Is there a name or title column?
-        name_or_title = [c for c in column_names if c in ("name", "title")]
+        name_or_title = [c for c in column_names if c.lower() in ("name", "title")]
         if name_or_title:
             return name_or_title[0]
         # If a table has two columns, one of which is ID, then label_column is the other one

From 1876975e3b120298cec2ff14825260f4a19a0568 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 11 Dec 2021 19:06:45 -0800
Subject: [PATCH 1242/2303] Refactor table view HTML tests to
 test_table_html.py

Refs #1518
---
 tests/test_html.py       | 1064 +-------------------------------------
 tests/test_table_html.py | 1045 +++++++++++++++++++++++++++++++++++++
 tests/utils.py           |   24 +
 3 files changed, 1070 insertions(+), 1063 deletions(-)
 create mode 100644 tests/test_table_html.py
 create mode 100644 tests/utils.py

diff --git a/tests/test_html.py b/tests/test_html.py
index a7cb105c..bfe5c8f9 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -9,11 +9,11 @@ from .fixtures import (  # noqa
     make_app_client,
     METADATA,
 )
+from .utils import assert_footer_links, inner_html
 import json
 import pathlib
 import pytest
 import re
-import textwrap
 import urllib.parse
 
 
@@ -180,67 +180,6 @@ def test_row_strange_table_name_with_url_hash(app_client_with_hash):
     assert response.status == 200
 
 
-@pytest.mark.parametrize(
-    "path,expected_definition_sql",
-    [
-        (
-            "/fixtures/facet_cities",
-            """
-CREATE TABLE facet_cities (
-    id integer primary key,
-    name text
-);
-        """.strip(),
-        ),
-        (
-            "/fixtures/compound_three_primary_keys",
-            """
-CREATE TABLE compound_three_primary_keys (
-  pk1 varchar(30),
-  pk2 varchar(30),
-  pk3 varchar(30),
-  content text,
-  PRIMARY KEY (pk1, pk2, pk3)
-);
-CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
-            """.strip(),
-        ),
-    ],
-)
-def test_definition_sql(path, expected_definition_sql, app_client):
-    response = app_client.get(path)
-    pre = Soup(response.body, "html.parser").select_one("pre.wrapped-sql")
-    assert expected_definition_sql == pre.string
-
-
-def test_table_cell_truncation():
-    with make_app_client(settings={"truncate_cells_html": 5}) as client:
-        response = client.get("/fixtures/facetable")
-        assert response.status == 200
-        table = Soup(response.body, "html.parser").find("table")
-        assert table["class"] == ["rows-and-columns"]
-        assert [
-            "Missi…",
-            "Dogpa…",
-            "SOMA",
-            "Tende…",
-            "Berna…",
-            "Hayes…",
-            "Holly…",
-            "Downt…",
-            "Los F…",
-            "Korea…",
-            "Downt…",
-            "Greek…",
-            "Corkt…",
-            "Mexic…",
-            "Arcad…",
-        ] == [
-            td.string
-            for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
-        ]
-
-
 def test_row_page_does_not_truncate():
     with make_app_client(settings={"truncate_cells_html": 5}) as client:
         response = client.get("/fixtures/facetable/1")
@@ -253,343 +192,6 @@ def test_row_page_does_not_truncate():
         ]
 
 
-def test_add_filter_redirects(app_client):
-    filter_args = urllib.parse.urlencode(
-        {"_filter_column": "content", "_filter_op": "startswith", "_filter_value": "x"}
-    )
-    path_base = "/fixtures/simple_primary_key"
-    path = path_base + "?" + filter_args
-    response = app_client.get(path)
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?content__startswith=x")
-
-    # Adding a redirect to an existing query string:
-    path = path_base + "?foo=bar&" + filter_args
-    response = app_client.get(path)
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?foo=bar&content__startswith=x")
-
-    # Test that op with a __x suffix overrides the filter value
-    path = (
-        path_base
-        + "?"
-        + urllib.parse.urlencode(
-            {
-                "_filter_column": "content",
-                "_filter_op": "isnull__5",
-                "_filter_value": "x",
-            }
-        )
-    )
-    response = app_client.get(path)
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?content__isnull=5")
-
-
-def test_existing_filter_redirects(app_client):
-    filter_args = {
-        "_filter_column_1": "name",
-        "_filter_op_1": "contains",
-        "_filter_value_1": "hello",
-        "_filter_column_2": "age",
-        "_filter_op_2": "gte",
-        "_filter_value_2": "22",
-        "_filter_column_3": "age",
-        "_filter_op_3": "lt",
-        "_filter_value_3": "30",
-        "_filter_column_4": "name",
-        "_filter_op_4": "contains",
-        "_filter_value_4": "world",
-    }
-    path_base = "/fixtures/simple_primary_key"
-    path = path_base + "?" + urllib.parse.urlencode(filter_args)
-    response = app_client.get(path)
-    assert response.status == 302
-    assert_querystring_equal(
-        "name__contains=hello&age__gte=22&age__lt=30&name__contains=world",
-        response.headers["Location"].split("?")[1],
-    )
-
-    # Setting _filter_column_3 to empty string should remove *_3 entirely
-    filter_args["_filter_column_3"] = ""
-    path = path_base + "?" + urllib.parse.urlencode(filter_args)
-    response = app_client.get(path)
-    assert response.status == 302
-    assert_querystring_equal(
-        "name__contains=hello&age__gte=22&name__contains=world",
-        response.headers["Location"].split("?")[1],
-    )
-
-    # ?_filter_op=exact should be removed if unaccompanied by _fiter_column
-    response = app_client.get(path_base + "?_filter_op=exact")
-    assert response.status == 302
-    assert "?" not in response.headers["Location"]
-
-
-def test_exact_parameter_results_in_correct_hidden_fields(app_client):
-    # https://github.com/simonw/datasette/issues/1527
-    response = app_client.get(
-        "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown"
-    )
-    # In this case we should NOT have a hidden _neighborhood__exact=Downtown field
-    form = Soup(response.body, "html.parser").find("form")
-    hidden_inputs = {
-        input["name"]: input["value"] for input in form.select("input[type=hidden]")
-    }
-    assert hidden_inputs == {"_facet": "_neighborhood"}
-
-
-def test_empty_search_parameter_gets_removed(app_client):
-    path_base = "/fixtures/simple_primary_key"
-    path = (
-        path_base
-        + "?"
-        + urllib.parse.urlencode(
-            {
-                "_search": "",
-                "_filter_column": "name",
-                "_filter_op": "exact",
-                "_filter_value": "chidi",
-            }
-        )
-    )
-    response = app_client.get(path)
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?name__exact=chidi")
-
-
-def test_searchable_view_persists_fts_table(app_client):
-    # The search form should persist ?_fts_table as a hidden field
-    response = app_client.get(
-        "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk"
-    )
-    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
-    hiddens = [i for i in inputs if i["type"] == "hidden"]
-    assert [("_fts_table", "searchable_fts"), ("_fts_pk", "pk")] == [
-        (hidden["name"], hidden["value"]) for hidden in hiddens
-    ]
-
-
-def test_sort_by_desc_redirects(app_client):
-    path_base = "/fixtures/sortable"
-    path = (
-        path_base
-        + "?"
-        + urllib.parse.urlencode({"_sort": "sortable", "_sort_by_desc": "1"})
-    )
-    response = app_client.get(path)
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?_sort_desc=sortable")
-
-
-def test_sort_links(app_client):
-    response = app_client.get("/fixtures/sortable?_sort=sortable")
-    assert response.status == 200
-    ths = Soup(response.body, "html.parser").findAll("th")
-    attrs_and_link_attrs = [
-        {
-            "attrs": th.attrs,
-            "a_href": (th.find("a")["href"] if th.find("a") else None),
-        }
-        for th in ths
-    ]
-    assert attrs_and_link_attrs == [
-        {
-            "attrs": {
-                "class": ["col-Link"],
-                "scope": "col",
-                "data-column": "Link",
-                "data-column-type": "",
-                "data-column-not-null": "0",
-                "data-is-pk": "0",
-            },
-            "a_href": None,
-        },
-        {
-            "attrs": {
-                "class": ["col-pk1"],
-                "scope": "col",
-                "data-column": "pk1",
-                "data-column-type": "varchar(30)",
-                "data-column-not-null": "0",
-                "data-is-pk": "1",
-            },
-            "a_href": None,
-        },
-        {
-            "attrs": {
-                "class": ["col-pk2"],
-                "scope": "col",
-                "data-column": "pk2",
-                "data-column-type": "varchar(30)",
-                "data-column-not-null": "0",
-                "data-is-pk": "1",
-            },
-            "a_href": None,
-        },
-        {
-            "attrs": {
-                "class": ["col-content"],
-                "scope": "col",
-                "data-column": "content",
-                "data-column-type": "text",
-                "data-column-not-null": "0",
-                "data-is-pk": "0",
-            },
-            "a_href": None,
-        },
-        {
-            "attrs": {
-                "class": ["col-sortable"],
-                "scope": "col",
-                "data-column": "sortable",
-                "data-column-type": "integer",
-                "data-column-not-null": "0",
-                "data-is-pk": "0",
-            },
-            "a_href": "/fixtures/sortable?_sort_desc=sortable",
-        },
-        {
-            "attrs": {
-                "class": ["col-sortable_with_nulls"],
-                "scope": "col",
-                "data-column": "sortable_with_nulls",
-                "data-column-type": "real",
-                "data-column-not-null": "0",
-                "data-is-pk": "0",
-            },
-            "a_href": "/fixtures/sortable?_sort=sortable_with_nulls",
-        },
-        {
-            "attrs": {
-                "class": ["col-sortable_with_nulls_2"],
-                "scope": "col",
-                "data-column": "sortable_with_nulls_2",
-                "data-column-type": "real",
-                "data-column-not-null": "0",
-                "data-is-pk": "0",
-            },
-            "a_href": "/fixtures/sortable?_sort=sortable_with_nulls_2",
-        },
-        {
-            "attrs": {
-                "class": ["col-text"],
-                "scope": "col",
-                "data-column": "text",
-                "data-column-type": "text",
-                "data-column-not-null": "0",
-                "data-is-pk": "0",
-            },
-            "a_href": "/fixtures/sortable?_sort=text",
-        },
-    ]
-
-
-def test_facet_display(app_client):
-    response = app_client.get(
-        "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth"
-    )
-    assert response.status == 200
-    soup = Soup(response.body, "html.parser")
-    divs = soup.find("div", {"class": "facet-results"}).findAll("div")
-    actual = []
-    for div in divs:
-        actual.append(
-            {
-                "name": div.find("strong").text.split()[0],
-                "items": [
-                    {
-                        "name": a.text,
-                        "qs": a["href"].split("?")[-1],
-                        "count": int(str(a.parent).split("")[1].split("<")[0]),
-                    }
-                    for a in div.find("ul").findAll("a")
-                ],
-            }
-        )
-    assert actual == [
-        {
-            "name": "_city_id",
-            "items": [
-                {
-                    "name": "San Francisco",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1",
-                    "count": 6,
-                },
-                {
-                    "name": "Los Angeles",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2",
-                    "count": 4,
-                },
-                {
-                    "name": "Detroit",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3",
-                    "count": 4,
-                },
-                {
-                    "name": "Memnonia",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4",
-                    "count": 1,
-                },
-            ],
-        },
-        {
-            "name": "planet_int",
-            "items": [
-                {
-                    "name": "1",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1",
-                    "count": 14,
-                },
-                {
-                    "name": "2",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2",
-                    "count": 1,
-                },
-            ],
-        },
-        {
-            "name": "on_earth",
-            "items": [
-                {
-                    "name": "1",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1",
-                    "count": 14,
-                },
-                {
-                    "name": "0",
-                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0",
-                    "count": 1,
-                },
-            ],
-        },
-    ]
-
-
-def test_facets_persist_through_filter_form(app_client):
-    response = app_client.get(
-        "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags"
-    )
-    assert response.status == 200
-    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
-    hiddens = [i for i in inputs if i["type"] == "hidden"]
-    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
-        ("_facet", "planet_int"),
-        ("_facet", "_city_id"),
-        ("_facet_array", "tags"),
-    ]
-
-
-def test_next_does_not_persist_in_hidden_field(app_client):
-    response = app_client.get("/fixtures/searchable?_size=1&_next=1")
-    assert response.status == 200
-    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
-    hiddens = [i for i in inputs if i["type"] == "hidden"]
-    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
-        ("_size", "1"),
-    ]
-
-
 @pytest.mark.parametrize(
     "path,expected_classes",
     [
@@ -646,74 +248,6 @@ def test_templates_considered(app_client, path, expected_considered):
     assert f"" in response.text
 
 
-def test_table_html_simple_primary_key(app_client):
-    response = app_client.get("/fixtures/simple_primary_key?_size=3")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    assert table["class"] == ["rows-and-columns"]
-    ths = table.findAll("th")
-    assert "id\xa0▼" == ths[0].find("a").string.strip()
-    for expected_col, th in zip(("content",), ths[1:]):
-        a = th.find("a")
-        assert expected_col == a.string
-        assert a["href"].endswith(f"/simple_primary_key?_size=3&_sort={expected_col}")
-        assert ["nofollow"] == a["rel"]
-    assert [
-        [
-            '1',
-            'hello',
-        ],
-        [
-            '2',
-            'world',
-        ],
-        [
-            '3',
-            '\xa0',
-        ],
-    ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-
-
-def test_table_csv_json_export_interface(app_client):
-    response = app_client.get("/fixtures/simple_primary_key?id__gt=2")
-    assert response.status == 200
-    # The links at the top of the page
-    links = (
-        Soup(response.body, "html.parser")
-        .find("p", {"class": "export-links"})
-        .findAll("a")
-    )
-    actual = [l["href"] for l in links]
-    expected = [
-        "/fixtures/simple_primary_key.json?id__gt=2",
-        "/fixtures/simple_primary_key.testall?id__gt=2",
-        "/fixtures/simple_primary_key.testnone?id__gt=2",
-        "/fixtures/simple_primary_key.testresponse?id__gt=2",
-        "/fixtures/simple_primary_key.csv?id__gt=2&_size=max",
-        "#export",
-    ]
-    assert expected == actual
-    # And the advaced export box at the bottom:
-    div = Soup(response.body, "html.parser").find("div", {"class": "advanced-export"})
-    json_links = [a["href"] for a in div.find("p").findAll("a")]
-    assert [
-        "/fixtures/simple_primary_key.json?id__gt=2",
-        "/fixtures/simple_primary_key.json?id__gt=2&_shape=array",
-        "/fixtures/simple_primary_key.json?id__gt=2&_shape=array&_nl=on",
-        "/fixtures/simple_primary_key.json?id__gt=2&_shape=object",
-    ] == json_links
-    # And the CSV form
-    form = div.find("form")
-    assert form["action"].endswith("/simple_primary_key.csv")
-    inputs = [str(input) for input in form.findAll("input")]
-    assert [
-        '',
-        '',
-        '',
-        '',
-    ] == inputs
-
-
 def test_row_json_export_link(app_client):
     response = app_client.get("/fixtures/simple_primary_key/1")
     assert response.status == 200
@@ -727,26 +261,6 @@ def test_query_json_csv_export_links(app_client):
     assert 'CSV' in response.text
 
 
-def test_csv_json_export_links_include_labels_if_foreign_keys(app_client):
-    response = app_client.get("/fixtures/facetable")
-    assert response.status == 200
-    links = (
-        Soup(response.body, "html.parser")
-        .find("p", {"class": "export-links"})
-        .findAll("a")
-    )
-    actual = [l["href"] for l in links]
-    expected = [
-        "/fixtures/facetable.json?_labels=on",
-        "/fixtures/facetable.testall?_labels=on",
-        "/fixtures/facetable.testnone?_labels=on",
-        "/fixtures/facetable.testresponse?_labels=on",
-        "/fixtures/facetable.csv?_labels=on&_size=max",
-        "#export",
-    ]
-    assert expected == actual
-
-
 def test_row_html_simple_primary_key(app_client):
     response = app_client.get("/fixtures/simple_primary_key/1")
     assert response.status == 200
@@ -760,45 +274,6 @@ def test_row_html_simple_primary_key(app_client):
     ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
 
 
-def test_table_not_exists(app_client):
-    assert "Table not found: blah" in app_client.get("/fixtures/blah").text
-
-
-def test_table_html_no_primary_key(app_client):
-    response = app_client.get("/fixtures/no_primary_key")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    # We have disabled sorting for this table using metadata.json
-    assert ["content", "a", "b", "c"] == [
-        th.string.strip() for th in table.select("thead th")[2:]
-    ]
-    expected = [
-        [
-            '{}'.format(
-                i, i
-            ),
-            f'{i}',
-            f'{i}',
-            f'a{i}',
-            f'b{i}',
-            f'c{i}',
-        ]
-        for i in range(1, 51)
-    ]
-    assert expected == [
-        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
-
-
-def test_rowid_sortable_no_primary_key(app_client):
-    response = app_client.get("/fixtures/no_primary_key")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    assert table["class"] == ["rows-and-columns"]
-    ths = table.findAll("th")
-    assert "rowid\xa0▼" == ths[1].find("a").string.strip()
-
-
 def test_row_html_no_primary_key(app_client):
     response = app_client.get("/fixtures/no_primary_key/1")
     assert response.status == 200
@@ -848,143 +323,6 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l
     assert link == expected_link
 
 
-def test_table_html_compound_primary_key(app_client):
-    response = app_client.get("/fixtures/compound_primary_key")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    ths = table.findAll("th")
-    assert "Link" == ths[0].string.strip()
-    for expected_col, th in zip(("pk1", "pk2", "content"), ths[1:]):
-        a = th.find("a")
-        assert expected_col == a.string
-        assert th["class"] == [f"col-{expected_col}"]
-        assert a["href"].endswith(f"/compound_primary_key?_sort={expected_col}")
-    expected = [
-        [
-            'a,b',
-            'a',
-            'b',
-            'c',
-        ]
-    ]
-    assert expected == [
-        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
-
-
-def test_table_html_foreign_key_links(app_client):
-    response = app_client.get("/fixtures/foreign_key_references")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-    assert actual == [
-        [
-            '1',
-            'hello\xa01',
-            '-\xa03',
-            '1',
-            'a',
-            'b',
-        ],
-        [
-            '2',
-            '\xa0',
-            '\xa0',
-            '\xa0',
-            '\xa0',
-            '\xa0',
-        ],
-    ]
-
-
-def test_table_html_foreign_key_facets(app_client):
-    response = app_client.get(
-        "/fixtures/foreign_key_references?_facet=foreign_key_with_blank_label"
-    )
-    assert response.status == 200
-    assert (
-        '
    • '
-        "- 1
      • "
-    ) in response.text
-
-
-def test_table_html_disable_foreign_key_links_with_labels(app_client):
-    response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-    assert actual == [
-        [
-            '1',
-            '1',
-            '3',
-            '1',
-            'a',
-            'b',
-        ]
-    ]
-
-
-def test_table_html_foreign_key_custom_label_column(app_client):
-    response = app_client.get("/fixtures/custom_foreign_key_label")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    expected = [
-        [
-            '1',
-            'world2\xa01',
-        ]
-    ]
-    assert expected == [
-        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
-
-
-@pytest.mark.parametrize(
-    "path,expected_column_options",
-    [
-        ("/fixtures/infinity", ["- column -", "rowid", "value"]),
-        (
-            "/fixtures/primary_key_multiple_columns",
-            ["- column -", "id", "content", "content2"],
-        ),
-        ("/fixtures/compound_primary_key", ["- column -", "pk1", "pk2", "content"]),
-    ],
-)
-def test_table_html_filter_form_column_options(
-    path, expected_column_options, app_client
-):
-    response = app_client.get(path)
-    assert response.status == 200
-    form = Soup(response.body, "html.parser").find("form")
-    column_options = [
-        o.attrs.get("value") or o.string
-        for o in form.select("select[name=_filter_column] option")
-    ]
-    assert expected_column_options == column_options
-
-
-def test_table_html_filter_form_still_shows_nocol_columns(app_client):
-    # https://github.com/simonw/datasette/issues/1503
-    response = app_client.get("/fixtures/sortable?_nocol=sortable")
-    assert response.status == 200
-    form = Soup(response.body, "html.parser").find("form")
-    assert [
-        o.string
-        for o in form.select("select[name='_filter_column']")[0].select("option")
-    ] == [
-        "- column -",
-        "pk1",
-        "pk2",
-        "content",
-        "sortable_with_nulls",
-        "sortable_with_nulls_2",
-        "text",
-        # Moved to the end because it is no longer returned by the query:
-        "sortable",
-    ]
-
-
 def test_row_html_compound_primary_key(app_client):
     response = app_client.get("/fixtures/compound_primary_key/a,b")
     assert response.status == 200
@@ -1004,58 +342,6 @@ def test_row_html_compound_primary_key(app_client):
     ]
 
 
-def test_compound_primary_key_with_foreign_key_references(app_client):
-    # e.g. a many-to-many table with a compound primary key on the two columns
-    response = app_client.get("/fixtures/searchable_tags")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    expected = [
-        [
-            '1,feline',
-            '1\xa01',
-            'feline',
-        ],
-        [
-            '2,canine',
-            '2\xa02',
-            'canine',
-        ],
-    ]
-    assert expected == [
-        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
-
-
-def test_view_html(app_client):
-    response = app_client.get("/fixtures/simple_view?_size=3")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    ths = table.select("thead th")
-    assert 2 == len(ths)
-    assert ths[0].find("a") is not None
-    assert ths[0].find("a")["href"].endswith("/simple_view?_size=3&_sort=content")
-    assert ths[0].find("a").string.strip() == "content"
-    assert ths[1].find("a") is None
-    assert ths[1].string.strip() == "upper_content"
-    expected = [
-        [
-            'hello',
-            'HELLO',
-        ],
-        [
-            'world',
-            'WORLD',
-        ],
-        [
-            '\xa0',
-            '\xa0',
-        ],
-    ]
-    assert expected == [
-        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
-
-
 def test_index_metadata(app_client):
     response = app_client.get("/")
     assert response.status == 200
@@ -1094,20 +380,6 @@ def test_database_metadata_with_custom_sql(app_client):
     assert_footer_links(soup)
 
 
-def test_table_metadata(app_client):
-    response = app_client.get("/fixtures/simple_primary_key")
-    assert response.status == 200
-    soup = Soup(response.body, "html.parser")
-    # Page title should be custom and should be HTML escaped
-    assert "This <em>HTML</em> is escaped" == inner_html(soup.find("h1"))
-    # Description should be custom and NOT escaped (we used description_html)
-    assert "Simple primary key" == inner_html(
-        soup.find("div", {"class": "metadata-description"})
-    )
-    # The source/license should be inherited
-    assert_footer_links(soup)
-
-
 def test_database_download_for_immutable():
     with make_app_client(is_immutable=True) as client:
         assert not client.ds.databases["fixtures"].is_mutable
@@ -1169,36 +441,6 @@ def test_allow_sql_off():
         assert b"View and edit SQL" not in response.body
 
 
-def assert_querystring_equal(expected, actual):
-    assert sorted(expected.split("&")) == sorted(actual.split("&"))
-
-
-def assert_footer_links(soup):
-    footer_links = soup.find("footer").findAll("a")
-    assert 4 == len(footer_links)
-    datasette_link, license_link, source_link, about_link = footer_links
-    assert "Datasette" == datasette_link.text.strip()
-    assert "tests/fixtures.py" == source_link.text.strip()
-    assert "Apache License 2.0" == license_link.text.strip()
-    assert "About Datasette" == about_link.text.strip()
-    assert "https://datasette.io/" == datasette_link["href"]
-    assert (
-        "https://github.com/simonw/datasette/blob/main/tests/fixtures.py"
-        == source_link["href"]
-    )
-    assert (
-        "https://github.com/simonw/datasette/blob/main/LICENSE" == license_link["href"]
-    )
-    assert "https://github.com/simonw/datasette" == about_link["href"]
-
-
-def inner_html(soup):
-    html = str(soup)
-    # This includes the parent tag - so remove that
-    inner_html = html.split(">", 1)[1].rsplit("<", 1)[0]
-    return inner_html.strip()
-
-
 @pytest.mark.parametrize("path", ["/404", "/fixtures/404"])
 def test_404(app_client, path):
     response = app_client.get(path)
@@ -1249,31 +491,6 @@ def test_canned_query_with_custom_metadata(app_client):
     )
 
 
-@pytest.mark.parametrize(
-    "path,has_object,has_stream,has_expand",
-    [
-        ("/fixtures/no_primary_key", False, True, False),
-        ("/fixtures/complex_foreign_keys", True, False, True),
-    ],
-)
-def test_advanced_export_box(app_client, path, has_object, has_stream, has_expand):
-    response = app_client.get(path)
-    assert response.status == 200
-    soup = Soup(response.body, "html.parser")
-    # JSON shape options
-    expected_json_shapes = ["default", "array", "newline-delimited"]
-    if has_object:
-        expected_json_shapes.append("object")
-    div = soup.find("div", {"class": "advanced-export"})
-    assert expected_json_shapes == [a.text for a in div.find("p").findAll("a")]
-    # "stream all rows" option
-    if has_stream:
-        assert "stream all rows" in str(div)
-    # "expand labels" option
-    if has_expand:
-        assert "expand labels" in str(div)
-
-
 def test_urlify_custom_queries(app_client):
     path = "/fixtures?" + urllib.parse.urlencode(
         {"sql": "select ('https://twitter.com/' || 'simonw') as user_url;"}
@@ -1376,91 +593,6 @@ def test_canned_query_show_hide_metadata_option(
             assert '1',
-            '1',
-            '<Binary:\xa07\xa0bytes>',
-        ],
-        [
-            '2',
-            '2',
-            '<Binary:\xa07\xa0bytes>',
-        ],
-        [
-            '3',
-            '3',
-            '\xa0',
-        ],
-    ]
-    assert expected_tds == [
-        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
-
-
 def test_binary_data_display_in_query(app_client):
     response = app_client.get("/fixtures?sql=select+*+from+binary_data")
     assert response.status == 200
@@ -1525,19 +657,6 @@ def test_metadata_json_html(app_client):
     assert METADATA == json.loads(pre.text)
 
 
-def test_custom_table_include():
-    with make_app_client(
-        template_dir=str(pathlib.Path(__file__).parent / "test_templates")
-    ) as client:
-        response = client.get("/fixtures/complex_foreign_keys")
-        assert response.status == 200
-        assert (
-            '
      '
-            '1 - 2 - hello 1'
-            "
      "
-        ) == str(Soup(response.text, "html.parser").select_one("div.custom-table-row"))
-
-
 @pytest.mark.parametrize(
     "path",
     [
@@ -1584,68 +703,6 @@ def test_debug_context_includes_extra_template_vars():
         assert "scope_path" in response.text
 
 
-def test_metadata_sort(app_client):
-    response = app_client.get("/fixtures/facet_cities")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    assert table["class"] == ["rows-and-columns"]
-    ths = table.findAll("th")
-    assert ["id", "name\xa0▼"] == [th.find("a").string.strip() for th in ths]
-    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-    expected = [
-        [
-            '3',
-            'Detroit',
-        ],
-        [
-            '2',
-            'Los Angeles',
-        ],
-        [
-            '4',
-            'Memnonia',
-        ],
-        [
-            '1',
-            'San Francisco',
-        ],
-    ]
-    assert expected == rows
-    # Make sure you can reverse that sort order
-    response = app_client.get("/fixtures/facet_cities?_sort_desc=name")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-    assert list(reversed(expected)) == rows
-
-
-def test_metadata_sort_desc(app_client):
-    response = app_client.get("/fixtures/attraction_characteristic")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    assert table["class"] == ["rows-and-columns"]
-    ths = table.findAll("th")
-    assert ["pk\xa0▲", "name"] == [th.find("a").string.strip() for th in ths]
-    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-    expected = [
-        [
-            '2',
-            'Paranormal',
-        ],
-        [
-            '1',
-            'Museum',
-        ],
-    ]
-    assert expected == rows
-    # Make sure you can reverse that sort order
-    response = app_client.get("/fixtures/attraction_characteristic?_sort=pk")
-    assert response.status == 200
-    table = Soup(response.body, "html.parser").find("table")
-    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
-    assert list(reversed(expected)) == rows
-
-
 @pytest.mark.parametrize(
     "path",
     [
@@ -1787,126 +844,7 @@ def test_navigation_menu_links(
             ), f"{link} found but should not have been in nav menu"
 
 
-@pytest.mark.parametrize(
-    "max_returned_rows,path,expected_num_facets,expected_ellipses,expected_ellipses_url",
-    (
-        (
-            5,
-            # Default should show 2 facets
-            "/fixtures/facetable?_facet=_neighborhood",
-            2,
-            True,
-            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
-        ),
-        # _facet_size above max_returned_rows should show max_returned_rows (5)
-        (
-            5,
-            "/fixtures/facetable?_facet=_neighborhood&_facet_size=50",
-            5,
-            True,
-            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
-        ),
-        # If max_returned_rows is high enough, should return all
-        (
-            20,
-            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
-            14,
-            False,
-            None,
-        ),
-        # If num facets > max_returned_rows, show ... without a link
-        # _facet_size above max_returned_rows should show max_returned_rows (5)
-        (
-            5,
-            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
-            5,
-            True,
-            None,
-        ),
-    ),
-)
-def test_facet_more_links(
-    max_returned_rows,
-    path,
-    expected_num_facets,
-    expected_ellipses,
-    expected_ellipses_url,
-):
-    with make_app_client(
-        settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
-    ) as client:
-        response = client.get(path)
-        soup = Soup(response.body, "html.parser")
-        lis = soup.select("#facet-neighborhood-b352a7 ul li:not(.facet-truncated)")
-        facet_truncated = soup.select_one(".facet-truncated")
-        assert len(lis) == expected_num_facets
-        if not expected_ellipses:
-            assert facet_truncated is None
-        else:
-            if expected_ellipses_url:
-                assert facet_truncated.find("a")["href"] == expected_ellipses_url
-            else:
-                assert facet_truncated.find("a") is None
-
-
-def test_unavailable_table_does_not_break_sort_relationships():
-    # https://github.com/simonw/datasette/issues/1305
-    with make_app_client(
-        metadata={
-            "databases": {
-                "fixtures": {"tables": {"foreign_key_references": {"allow": False}}}
-            }
-        }
-    ) as client:
-        response = client.get("/?_sort=relationships")
-        assert response.status == 200
-
-
 def test_trace_correctly_escaped(app_client):
     response = app_client.get("/fixtures?sql=select+'
      Hello'&_trace=1")
     assert "select '
      Hello" not in response.text
     assert "select '<h1>Hello" in response.text
-
-
-def test_column_metadata(app_client):
-    response = app_client.get("/fixtures/roadside_attractions")
-    soup = Soup(response.body, "html.parser")
-    dl = soup.find("dl")
-    assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [
-        ("name", "The name of the attraction"),
-        ("address", "The street address for the attraction"),
-    ]
-    assert (
-        soup.select("th[data-column=name]")[0]["data-column-description"]
-        == "The name of the attraction"
-    )
-    assert (
-        soup.select("th[data-column=address]")[0]["data-column-description"]
-        == "The street address for the attraction"
-    )
-
-
-@pytest.mark.parametrize("use_facet_size_max", (True, False))
-def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
-    # https://github.com/simonw/datasette/issues/1423
-    with make_app_client(settings={"max_returned_rows": 100}) as client:
-        path = "/fixtures/sortable?_facet=content&_facet=pk1"
-        if use_facet_size_max:
-            path += "&_facet_size=max"
-        response = client.get(path)
-        assert response.status == 200
-    fragments = (
-        '>100',
-        '8',
-    )
-    for fragment in fragments:
-        if use_facet_size_max:
-            assert fragment in response.text
-        else:
-            assert fragment not in response.text
-
-
-def test_sort_rowid_with_next(app_client):
-    # https://github.com/simonw/datasette/issues/1470
-    response = app_client.get("/fixtures/binary_data?_size=1&_next=1&_sort=rowid")
-    assert response.status == 200
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
new file mode 100644
index 00000000..2fbb53bd
--- /dev/null
+++ b/tests/test_table_html.py
@@ -0,0 +1,1045 @@
+from bs4 import BeautifulSoup as Soup
+from .fixtures import (  # noqa
+    app_client,
+    make_app_client,
+)
+import pathlib
+import pytest
+import urllib.parse
+from .utils import assert_footer_links, inner_html
+
+
+@pytest.mark.parametrize(
+    "path,expected_definition_sql",
+    [
+        (
+            "/fixtures/facet_cities",
+            """
+CREATE TABLE facet_cities (
+    id integer primary key,
+    name text
+);
+        """.strip(),
+        ),
+        (
+            "/fixtures/compound_three_primary_keys",
+            """
+CREATE TABLE compound_three_primary_keys (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  pk3 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2, pk3)
+);
+CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
+            """.strip(),
+        ),
+    ],
+)
+def test_table_definition_sql(path, expected_definition_sql, app_client):
+    response = app_client.get(path)
+    pre = Soup(response.body, "html.parser").select_one("pre.wrapped-sql")
+    assert expected_definition_sql == pre.string
+
+
+def test_table_cell_truncation():
+    with make_app_client(settings={"truncate_cells_html": 5}) as client:
+        response = client.get("/fixtures/facetable")
+        assert response.status == 200
+        table = Soup(response.body, "html.parser").find("table")
+        assert table["class"] == ["rows-and-columns"]
+        assert [
+            "Missi…",
+            "Dogpa…",
+            "SOMA",
+            "Tende…",
+            "Berna…",
+            "Hayes…",
+            "Holly…",
+            "Downt…",
+            "Los F…",
+            "Korea…",
+            "Downt…",
+            "Greek…",
+            "Corkt…",
+            "Mexic…",
+            "Arcad…",
+        ] == [
+            td.string
+            for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
+        ]
+
+
+def test_add_filter_redirects(app_client):
+    filter_args = urllib.parse.urlencode(
+        {"_filter_column": "content", "_filter_op": "startswith", "_filter_value": "x"}
+    )
+    path_base = "/fixtures/simple_primary_key"
+    path = path_base + "?" + filter_args
+    response = app_client.get(path)
+    assert response.status == 302
+    assert response.headers["Location"].endswith("?content__startswith=x")
+
+    # Adding a redirect to an existing query string:
+    path = path_base + "?foo=bar&" + filter_args
+    response = app_client.get(path)
+    assert response.status == 302
+    assert response.headers["Location"].endswith("?foo=bar&content__startswith=x")
+
+    # Test that op with a __x suffix overrides the filter value
+    path = (
+        path_base
+        + "?"
+        + urllib.parse.urlencode(
+            {
+                "_filter_column": "content",
+                "_filter_op": "isnull__5",
+                "_filter_value": "x",
+            }
+        )
+    )
+    response = app_client.get(path)
+    assert response.status == 302
+    assert response.headers["Location"].endswith("?content__isnull=5")
+
+
+def test_existing_filter_redirects(app_client):
+    filter_args = {
+        "_filter_column_1": "name",
+        "_filter_op_1": "contains",
+        "_filter_value_1": "hello",
+        "_filter_column_2": "age",
+        "_filter_op_2": "gte",
+        "_filter_value_2": "22",
+        "_filter_column_3": "age",
+        "_filter_op_3": "lt",
+        "_filter_value_3": "30",
+        "_filter_column_4": "name",
+        "_filter_op_4": "contains",
+        "_filter_value_4": "world",
+    }
+    path_base = "/fixtures/simple_primary_key"
+    path = path_base + "?" + urllib.parse.urlencode(filter_args)
+    response = app_client.get(path)
+    assert response.status == 302
+    assert_querystring_equal(
+        "name__contains=hello&age__gte=22&age__lt=30&name__contains=world",
+        response.headers["Location"].split("?")[1],
+    )
+
+    # Setting _filter_column_3 to empty string should remove *_3 entirely
+    filter_args["_filter_column_3"] = ""
+    path = path_base + "?" + urllib.parse.urlencode(filter_args)
+    response = app_client.get(path)
+    assert response.status == 302
+    assert_querystring_equal(
+        "name__contains=hello&age__gte=22&name__contains=world",
+        response.headers["Location"].split("?")[1],
+    )
+
+    # ?_filter_op=exact should be removed if unaccompanied by _fiter_column
+    response = app_client.get(path_base + "?_filter_op=exact")
+    assert response.status == 302
+    assert "?" not in response.headers["Location"]
+
+
+def test_exact_parameter_results_in_correct_hidden_fields(app_client):
+    # https://github.com/simonw/datasette/issues/1527
+    response = app_client.get(
+        "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown"
+    )
+    # In this case we should NOT have a hidden _neighborhood__exact=Downtown field
+    form = Soup(response.body, "html.parser").find("form")
+    hidden_inputs = {
+        input["name"]: input["value"] for input in form.select("input[type=hidden]")
+    }
+    assert hidden_inputs == {"_facet": "_neighborhood"}
+
+
+def test_empty_search_parameter_gets_removed(app_client):
+    path_base = "/fixtures/simple_primary_key"
+    path = (
+        path_base
+        + "?"
+        + urllib.parse.urlencode(
+            {
+                "_search": "",
+                "_filter_column": "name",
+                "_filter_op": "exact",
+                "_filter_value": "chidi",
+            }
+        )
+    )
+    response = app_client.get(path)
+    assert response.status == 302
+    assert response.headers["Location"].endswith("?name__exact=chidi")
+
+
+def test_searchable_view_persists_fts_table(app_client):
+    # The search form should persist ?_fts_table as a hidden field
+    response = app_client.get(
+        "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk"
+    )
+    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [("_fts_table", "searchable_fts"), ("_fts_pk", "pk")] == [
+        (hidden["name"], hidden["value"]) for hidden in hiddens
+    ]
+
+
+def test_sort_by_desc_redirects(app_client):
+    path_base = "/fixtures/sortable"
+    path = (
+        path_base
+        + "?"
+        + urllib.parse.urlencode({"_sort": "sortable", "_sort_by_desc": "1"})
+    )
+    response = app_client.get(path)
+    assert response.status == 302
+    assert response.headers["Location"].endswith("?_sort_desc=sortable")
+
+
+def test_sort_links(app_client):
+    response = app_client.get("/fixtures/sortable?_sort=sortable")
+    assert response.status == 200
+    ths = Soup(response.body, "html.parser").findAll("th")
+    attrs_and_link_attrs = [
+        {
+            "attrs": th.attrs,
+            "a_href": (th.find("a")["href"] if th.find("a") else None),
+        }
+        for th in ths
+    ]
+    assert attrs_and_link_attrs == [
+        {
+            "attrs": {
+                "class": ["col-Link"],
+                "scope": "col",
+                "data-column": "Link",
+                "data-column-type": "",
+                "data-column-not-null": "0",
+                "data-is-pk": "0",
+            },
+            "a_href": None,
+        },
+        {
+            "attrs": {
+                "class": ["col-pk1"],
+                "scope": "col",
+                "data-column": "pk1",
+                "data-column-type": "varchar(30)",
+                "data-column-not-null": "0",
+                "data-is-pk": "1",
+            },
+            "a_href": None,
+        },
+        {
+            "attrs": {
+                "class": ["col-pk2"],
+                "scope": "col",
+                "data-column": "pk2",
+                "data-column-type": "varchar(30)",
+                "data-column-not-null": "0",
+                "data-is-pk": "1",
+            },
+            "a_href": None,
+        },
+        {
+            "attrs": {
+                "class": ["col-content"],
+                "scope": "col",
+                "data-column": "content",
+                "data-column-type": "text",
+                "data-column-not-null": "0",
+                "data-is-pk": "0",
+            },
+            "a_href": None,
+        },
+        {
+            "attrs": {
+                "class": ["col-sortable"],
+                "scope": "col",
+                "data-column": "sortable",
+                "data-column-type": "integer",
+                "data-column-not-null": "0",
+                "data-is-pk": "0",
+            },
+            "a_href": "/fixtures/sortable?_sort_desc=sortable",
+        },
+        {
+            "attrs": {
+                "class": ["col-sortable_with_nulls"],
+                "scope": "col",
+                "data-column": "sortable_with_nulls",
+                "data-column-type": "real",
+                "data-column-not-null": "0",
+                "data-is-pk": "0",
+            },
+            "a_href": "/fixtures/sortable?_sort=sortable_with_nulls",
+        },
+        {
+            "attrs": {
+                "class": ["col-sortable_with_nulls_2"],
+                "scope": "col",
+                "data-column": "sortable_with_nulls_2",
+                "data-column-type": "real",
+                "data-column-not-null": "0",
+                "data-is-pk": "0",
+            },
+            "a_href": "/fixtures/sortable?_sort=sortable_with_nulls_2",
+        },
+        {
+            "attrs": {
+                "class": ["col-text"],
+                "scope": "col",
+                "data-column": "text",
+                "data-column-type": "text",
+                "data-column-not-null": "0",
+                "data-is-pk": "0",
+            },
+            "a_href": "/fixtures/sortable?_sort=text",
+        },
+    ]
+
+
+def test_facet_display(app_client):
+    response = app_client.get(
+        "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth"
+    )
+    assert response.status == 200
+    soup = Soup(response.body, "html.parser")
+    divs = soup.find("div", {"class": "facet-results"}).findAll("div")
+    actual = []
+    for div in divs:
+        actual.append(
+            {
+                "name": div.find("strong").text.split()[0],
+                "items": [
+                    {
+                        "name": a.text,
+                        "qs": a["href"].split("?")[-1],
+                        "count": int(str(a.parent).split("")[1].split("<")[0]),
+                    }
+                    for a in div.find("ul").findAll("a")
+                ],
+            }
+        )
+    assert actual == [
+        {
+            "name": "_city_id",
+            "items": [
+                {
+                    "name": "San Francisco",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1",
+                    "count": 6,
+                },
+                {
+                    "name": "Los Angeles",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2",
+                    "count": 4,
+                },
+                {
+                    "name": "Detroit",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3",
+                    "count": 4,
+                },
+                {
+                    "name": "Memnonia",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4",
+                    "count": 1,
+                },
+            ],
+        },
+        {
+            "name": "planet_int",
+            "items": [
+                {
+                    "name": "1",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1",
+                    "count": 14,
+                },
+                {
+                    "name": "2",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2",
+                    "count": 1,
+                },
+            ],
+        },
+        {
+            "name": "on_earth",
+            "items": [
+                {
+                    "name": "1",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1",
+                    "count": 14,
+                },
+                {
+                    "name": "0",
+                    "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0",
+                    "count": 1,
+                },
+            ],
+        },
+    ]
+
+
+def test_facets_persist_through_filter_form(app_client):
+    response = app_client.get(
+        "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags"
+    )
+    assert response.status == 200
+    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
+        ("_facet", "planet_int"),
+        ("_facet", "_city_id"),
+        ("_facet_array", "tags"),
+    ]
+
+
+def test_next_does_not_persist_in_hidden_field(app_client):
+    response = app_client.get("/fixtures/searchable?_size=1&_next=1")
+    assert response.status == 200
+    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
+        ("_size", "1"),
+    ]
+
+
+def test_table_html_simple_primary_key(app_client):
+    response = app_client.get("/fixtures/simple_primary_key?_size=3")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    assert table["class"] == ["rows-and-columns"]
+    ths = table.findAll("th")
+    assert "id\xa0▼" == ths[0].find("a").string.strip()
+    for expected_col, th in zip(("content",), ths[1:]):
+        a = th.find("a")
+        assert expected_col == a.string
+        assert a["href"].endswith(f"/simple_primary_key?_size=3&_sort={expected_col}")
+        assert ["nofollow"] == a["rel"]
+    assert [
+        [
+            '1',
+            'hello',
+        ],
+        [
+            '2',
+            'world',
+        ],
+        [
+            '3',
+            '\xa0',
+        ],
+    ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+
+
+def test_table_csv_json_export_interface(app_client):
+    response = app_client.get("/fixtures/simple_primary_key?id__gt=2")
+    assert response.status == 200
+    # The links at the top of the page
+    links = (
+        Soup(response.body, "html.parser")
+        .find("p", {"class": "export-links"})
+        .findAll("a")
+    )
+    actual = [l["href"] for l in links]
+    expected = [
+        "/fixtures/simple_primary_key.json?id__gt=2",
+        "/fixtures/simple_primary_key.testall?id__gt=2",
+        "/fixtures/simple_primary_key.testnone?id__gt=2",
+        "/fixtures/simple_primary_key.testresponse?id__gt=2",
+        "/fixtures/simple_primary_key.csv?id__gt=2&_size=max",
+        "#export",
+    ]
+    assert expected == actual
+    # And the advaced export box at the bottom:
+    div = Soup(response.body, "html.parser").find("div", {"class": "advanced-export"})
+    json_links = [a["href"] for a in div.find("p").findAll("a")]
+    assert [
+        "/fixtures/simple_primary_key.json?id__gt=2",
+        "/fixtures/simple_primary_key.json?id__gt=2&_shape=array",
+        "/fixtures/simple_primary_key.json?id__gt=2&_shape=array&_nl=on",
+        "/fixtures/simple_primary_key.json?id__gt=2&_shape=object",
+    ] == json_links
+    # And the CSV form
+    form = div.find("form")
+    assert form["action"].endswith("/simple_primary_key.csv")
+    inputs = [str(input) for input in form.findAll("input")]
+    assert [
+        '',
+        '',
+        '',
+        '',
+    ] == inputs
+
+
+def test_csv_json_export_links_include_labels_if_foreign_keys(app_client):
+    response = app_client.get("/fixtures/facetable")
+    assert response.status == 200
+    links = (
+        Soup(response.body, "html.parser")
+        .find("p", {"class": "export-links"})
+        .findAll("a")
+    )
+    actual = [l["href"] for l in links]
+    expected = [
+        "/fixtures/facetable.json?_labels=on",
+        "/fixtures/facetable.testall?_labels=on",
+        "/fixtures/facetable.testnone?_labels=on",
+        "/fixtures/facetable.testresponse?_labels=on",
+        "/fixtures/facetable.csv?_labels=on&_size=max",
+        "#export",
+    ]
+    assert expected == actual
+
+
+def test_table_not_exists(app_client):
+    assert "Table not found: blah" in app_client.get("/fixtures/blah").text
+
+
+def test_table_html_no_primary_key(app_client):
+    response = app_client.get("/fixtures/no_primary_key")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    # We have disabled sorting for this table using metadata.json
+    assert ["content", "a", "b", "c"] == [
+        th.string.strip() for th in table.select("thead th")[2:]
+    ]
+    expected = [
+        [
+            '{}'.format(
+                i, i
+            ),
+            f'{i}',
+            f'{i}',
+            f'a{i}',
+            f'b{i}',
+            f'c{i}',
+        ]
+        for i in range(1, 51)
+    ]
+    assert expected == [
+        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
+    ]
+
+
+def test_rowid_sortable_no_primary_key(app_client):
+    response = app_client.get("/fixtures/no_primary_key")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    assert table["class"] == ["rows-and-columns"]
+    ths = table.findAll("th")
+    assert "rowid\xa0▼" == ths[1].find("a").string.strip()
+
+
+def test_table_html_compound_primary_key(app_client):
+    response = app_client.get("/fixtures/compound_primary_key")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    ths = table.findAll("th")
+    assert "Link" == ths[0].string.strip()
+    for expected_col, th in zip(("pk1", "pk2", "content"), ths[1:]):
+        a = th.find("a")
+        assert expected_col == a.string
+        assert th["class"] == [f"col-{expected_col}"]
+        assert a["href"].endswith(f"/compound_primary_key?_sort={expected_col}")
+    expected = [
+        [
+            'a,b',
+            'a',
+            'b',
+            'c',
+        ]
+    ]
+    assert expected == [
+        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
+    ]
+
+
+def test_table_html_foreign_key_links(app_client):
+    response = app_client.get("/fixtures/foreign_key_references")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+    assert actual == [
+        [
+            '1',
+            'hello\xa01',
+            '-\xa03',
+            '1',
+            'a',
+            'b',
+        ],
+        [
+            '2',
+            '\xa0',
+            '\xa0',
+            '\xa0',
+            '\xa0',
+            '\xa0',
+        ],
+    ]
+
+
+def test_table_html_foreign_key_facets(app_client):
+    response = app_client.get(
+        "/fixtures/foreign_key_references?_facet=foreign_key_with_blank_label"
+    )
+    assert response.status == 200
+    assert (
+        '
    • '
+        "- 1
      • "
+    ) in response.text
+
+
+def test_table_html_disable_foreign_key_links_with_labels(app_client):
+    response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+    assert actual == [
+        [
+            '1',
+            '1',
+            '3',
+            '1',
+            'a',
+            'b',
+        ]
+    ]
+
+
+def test_table_html_foreign_key_custom_label_column(app_client):
+    response = app_client.get("/fixtures/custom_foreign_key_label")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    expected = [
+        [
+            '1',
+            'world2\xa01',
+        ]
+    ]
+    assert expected == [
+        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
+    ]
+
+
+@pytest.mark.parametrize(
+    "path,expected_column_options",
+    [
+        ("/fixtures/infinity", ["- column -", "rowid", "value"]),
+        (
+            "/fixtures/primary_key_multiple_columns",
+            ["- column -", "id", "content", "content2"],
+        ),
+        ("/fixtures/compound_primary_key", ["- column -", "pk1", "pk2", "content"]),
+    ],
+)
+def test_table_html_filter_form_column_options(
+    path, expected_column_options, app_client
+):
+    response = app_client.get(path)
+    assert response.status == 200
+    form = Soup(response.body, "html.parser").find("form")
+    column_options = [
+        o.attrs.get("value") or o.string
+        for o in form.select("select[name=_filter_column] option")
+    ]
+    assert expected_column_options == column_options
+
+
+def test_table_html_filter_form_still_shows_nocol_columns(app_client):
+    # https://github.com/simonw/datasette/issues/1503
+    response = app_client.get("/fixtures/sortable?_nocol=sortable")
+    assert response.status == 200
+    form = Soup(response.body, "html.parser").find("form")
+    assert [
+        o.string
+        for o in form.select("select[name='_filter_column']")[0].select("option")
+    ] == [
+        "- column -",
+        "pk1",
+        "pk2",
+        "content",
+        "sortable_with_nulls",
+        "sortable_with_nulls_2",
+        "text",
+        # Moved to the end because it is no longer returned by the query:
+        "sortable",
+    ]
+
+
+def test_compound_primary_key_with_foreign_key_references(app_client):
+    # e.g. a many-to-many table with a compound primary key on the two columns
+    response = app_client.get("/fixtures/searchable_tags")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    expected = [
+        [
+            '1,feline',
+            '1\xa01',
+            'feline',
+        ],
+        [
+            '2,canine',
+            '2\xa02',
+            'canine',
+        ],
+    ]
+    assert expected == [
+        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
+    ]
+
+
+def test_view_html(app_client):
+    response = app_client.get("/fixtures/simple_view?_size=3")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    ths = table.select("thead th")
+    assert 2 == len(ths)
+    assert ths[0].find("a") is not None
+    assert ths[0].find("a")["href"].endswith("/simple_view?_size=3&_sort=content")
+    assert ths[0].find("a").string.strip() == "content"
+    assert ths[1].find("a") is None
+    assert ths[1].string.strip() == "upper_content"
+    expected = [
+        [
+            'hello',
+            'HELLO',
+        ],
+        [
+            'world',
+            'WORLD',
+        ],
+        [
+            '\xa0',
+            '\xa0',
+        ],
+    ]
+    assert expected == [
+        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
+    ]
+
+
+def test_table_metadata(app_client):
+    response = app_client.get("/fixtures/simple_primary_key")
+    assert response.status == 200
+    soup = Soup(response.body, "html.parser")
+    # Page title should be custom and should be HTML escaped
+    assert "This <em>HTML</em> is escaped" == inner_html(soup.find("h1"))
+    # Description should be custom and NOT escaped (we used description_html)
+    assert "Simple primary key" == inner_html(
+        soup.find("div", {"class": "metadata-description"})
+    )
+    # The source/license should be inherited
+    assert_footer_links(soup)
+
+
+@pytest.mark.parametrize(
+    "path,has_object,has_stream,has_expand",
+    [
+        ("/fixtures/no_primary_key", False, True, False),
+        ("/fixtures/complex_foreign_keys", True, False, True),
+    ],
+)
+def test_advanced_export_box(app_client, path, has_object, has_stream, has_expand):
+    response = app_client.get(path)
+    assert response.status == 200
+    soup = Soup(response.body, "html.parser")
+    # JSON shape options
+    expected_json_shapes = ["default", "array", "newline-delimited"]
+    if has_object:
+        expected_json_shapes.append("object")
+    div = soup.find("div", {"class": "advanced-export"})
+    assert expected_json_shapes == [a.text for a in div.find("p").findAll("a")]
+    # "stream all rows" option
+    if has_stream:
+        assert "stream all rows" in str(div)
+    # "expand labels" option
+    if has_expand:
+        assert "expand labels" in str(div)
+
+
+def test_extra_where_clauses(app_client):
+    response = app_client.get(
+        "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=_city_id=1"
+    )
+    soup = Soup(response.body, "html.parser")
+    div = soup.select(".extra-wheres")[0]
+    assert "2 extra where clauses" == div.find("h3").text
+    hrefs = [a["href"] for a in div.findAll("a")]
+    assert [
+        "/fixtures/facetable?_where=_city_id%3D1",
+        "/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27",
+    ] == hrefs
+    # These should also be persisted as hidden fields
+    inputs = soup.find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "_city_id=1")] == [
+        (hidden["name"], hidden["value"]) for hidden in hiddens
+    ]
+
+
+@pytest.mark.parametrize(
+    "path,expected_hidden",
+    [
+        ("/fixtures/facetable?_size=10", [("_size", "10")]),
+        (
+            "/fixtures/facetable?_size=10&_ignore=1&_ignore=2",
+            [
+                ("_size", "10"),
+                ("_ignore", "1"),
+                ("_ignore", "2"),
+            ],
+        ),
+    ],
+)
+def test_other_hidden_form_fields(app_client, path, expected_hidden):
+    response = app_client.get(path)
+    soup = Soup(response.body, "html.parser")
+    inputs = soup.find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == expected_hidden
+
+
+@pytest.mark.parametrize(
+    "path,expected_hidden",
+    [
+        ("/fixtures/searchable?_search=terry", []),
+        ("/fixtures/searchable?_sort=text2", []),
+        ("/fixtures/searchable?_sort=text2&_where=1", [("_where", "1")]),
+    ],
+)
+def test_search_and_sort_fields_not_duplicated(app_client, path, expected_hidden):
+    # https://github.com/simonw/datasette/issues/1214
+    response = app_client.get(path)
+    soup = Soup(response.body, "html.parser")
+    inputs = soup.find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == expected_hidden
+
+
+def test_binary_data_display_in_table(app_client):
+    response = app_client.get("/fixtures/binary_data")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    expected_tds = [
+        [
+            '1',
+            '1',
+            '<Binary:\xa07\xa0bytes>',
+        ],
+        [
+            '2',
+            '2',
+            '<Binary:\xa07\xa0bytes>',
+        ],
+        [
+            '3',
+            '3',
+            '\xa0',
+        ],
+    ]
+    assert expected_tds == [
+        [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
+    ]
+
+
+def test_custom_table_include():
+    with make_app_client(
+        template_dir=str(pathlib.Path(__file__).parent / "test_templates")
+    ) as client:
+        response = client.get("/fixtures/complex_foreign_keys")
+        assert response.status == 200
+        assert (
+            '
      '
+            '1 - 2 - hello 1'
+            "
      "
+        ) == str(Soup(response.text, "html.parser").select_one("div.custom-table-row"))
+
+
+def test_metadata_sort(app_client):
+    response = app_client.get("/fixtures/facet_cities")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    assert table["class"] == ["rows-and-columns"]
+    ths = table.findAll("th")
+    assert ["id", "name\xa0▼"] == [th.find("a").string.strip() for th in ths]
+    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+    expected = [
+        [
+            '3',
+            'Detroit',
+        ],
+        [
+            '2',
+            'Los Angeles',
+        ],
+        [
+            '4',
+            'Memnonia',
+        ],
+        [
+            '1',
+            'San Francisco',
+        ],
+    ]
+    assert expected == rows
+    # Make sure you can reverse that sort order
+    response = app_client.get("/fixtures/facet_cities?_sort_desc=name")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+    assert list(reversed(expected)) == rows
+
+
+def test_metadata_sort_desc(app_client):
+    response = app_client.get("/fixtures/attraction_characteristic")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    assert table["class"] == ["rows-and-columns"]
+    ths = table.findAll("th")
+    assert ["pk\xa0▲", "name"] == [th.find("a").string.strip() for th in ths]
+    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+    expected = [
+        [
+            '2',
+            'Paranormal',
+        ],
+        [
+            '1',
+            'Museum',
+        ],
+    ]
+    assert expected == rows
+    # Make sure you can reverse that sort order
+    response = app_client.get("/fixtures/attraction_characteristic?_sort=pk")
+    assert response.status == 200
+    table = Soup(response.body, "html.parser").find("table")
+    rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")]
+    assert list(reversed(expected)) == rows
+
+
+@pytest.mark.parametrize(
+    "max_returned_rows,path,expected_num_facets,expected_ellipses,expected_ellipses_url",
+    (
+        (
+            5,
+            # Default should show 2 facets
+            "/fixtures/facetable?_facet=_neighborhood",
+            2,
+            True,
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
+        ),
+        # _facet_size above max_returned_rows should show max_returned_rows (5)
+        (
+            5,
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=50",
+            5,
+            True,
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
+        ),
+        # If max_returned_rows is high enough, should return all
+        (
+            20,
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
+            14,
+            False,
+            None,
+        ),
+        # If num facets > max_returned_rows, show ... without a link
+        # _facet_size above max_returned_rows should show max_returned_rows (5)
+        (
+            5,
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
+            5,
+            True,
+            None,
+        ),
+    ),
+)
+def test_facet_more_links(
+    max_returned_rows,
+    path,
+    expected_num_facets,
+    expected_ellipses,
+    expected_ellipses_url,
+):
+    with make_app_client(
+        settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
+    ) as client:
+        response = client.get(path)
+        soup = Soup(response.body, "html.parser")
+        lis = soup.select("#facet-neighborhood-b352a7 ul li:not(.facet-truncated)")
+        facet_truncated = soup.select_one(".facet-truncated")
+        assert len(lis) == expected_num_facets
+        if not expected_ellipses:
+            assert facet_truncated is None
+        else:
+            if expected_ellipses_url:
+                assert facet_truncated.find("a")["href"] == expected_ellipses_url
+            else:
+                assert facet_truncated.find("a") is None
+
+
+def test_unavailable_table_does_not_break_sort_relationships():
+    # https://github.com/simonw/datasette/issues/1305
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {"tables": {"foreign_key_references": {"allow": False}}}
+            }
+        }
+    ) as client:
+        response = client.get("/?_sort=relationships")
+        assert response.status == 200
+
+
+def test_column_metadata(app_client):
+    response = app_client.get("/fixtures/roadside_attractions")
+    soup = Soup(response.body, "html.parser")
+    dl = soup.find("dl")
+    assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [
+        ("name", "The name of the attraction"),
+        ("address", "The street address for the attraction"),
+    ]
+    assert (
+        soup.select("th[data-column=name]")[0]["data-column-description"]
+        == "The name of the attraction"
+    )
+    assert (
+        soup.select("th[data-column=address]")[0]["data-column-description"]
+        == "The street address for the attraction"
+    )
+
+
+@pytest.mark.parametrize("use_facet_size_max", (True, False))
+def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
+    # https://github.com/simonw/datasette/issues/1423
+    with make_app_client(settings={"max_returned_rows": 100}) as client:
+        path = "/fixtures/sortable?_facet=content&_facet=pk1"
+        if use_facet_size_max:
+            path += "&_facet_size=max"
+        response = client.get(path)
+        assert response.status == 200
+    fragments = (
+        '>100',
+        '8',
+    )
+    for fragment in fragments:
+        if use_facet_size_max:
+            assert fragment in response.text
+        else:
+            assert fragment not in response.text
+
+
+def test_sort_rowid_with_next(app_client):
+    # https://github.com/simonw/datasette/issues/1470
+    response = app_client.get("/fixtures/binary_data?_size=1&_next=1&_sort=rowid")
+    assert response.status == 200
+
+
+def assert_querystring_equal(expected, actual):
+    assert sorted(expected.split("&")) == sorted(actual.split("&"))
diff --git a/tests/utils.py b/tests/utils.py
new file mode 100644
index 00000000..972300db
--- /dev/null
+++ b/tests/utils.py
@@ -0,0 +1,24 @@
+def assert_footer_links(soup):
+    footer_links = soup.find("footer").findAll("a")
+    assert 4 == len(footer_links)
+    datasette_link, license_link, source_link, about_link = footer_links
+    assert "Datasette" == datasette_link.text.strip()
+    assert "tests/fixtures.py" == source_link.text.strip()
+    assert "Apache License 2.0" == license_link.text.strip()
+    assert "About Datasette" == about_link.text.strip()
+    assert "https://datasette.io/" == datasette_link["href"]
+    assert (
+        "https://github.com/simonw/datasette/blob/main/tests/fixtures.py"
+        == source_link["href"]
+    )
+    assert (
+        "https://github.com/simonw/datasette/blob/main/LICENSE" == license_link["href"]
+    )
+    assert "https://github.com/simonw/datasette" == about_link["href"]
+
+
+def inner_html(soup):
+    html = str(soup)
+    # This includes the parent tag - so remove that
+    inner_html = html.split(">", 1)[1].rsplit("<", 1)[0]
+    return inner_html.strip()

From 492f9835aa7e90540dd0c6324282b109f73df71b Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 11 Dec 2021 19:07:19 -0800
Subject: [PATCH 1243/2303] Refactor table view API tests to test_table_api.py

Refs #1518
---
 tests/test_api.py       | 1215 +--------------------------------------
 tests/test_table_api.py | 1206 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 1214 insertions(+), 1207 deletions(-)
 create mode 100644 tests/test_table_api.py

diff --git a/tests/test_api.py b/tests/test_api.py
index 400dae7e..df9e0fc4 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1,29 +1,22 @@
 from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
-from datasette.utils import detect_json1
-from datasette.utils.sqlite import sqlite3, sqlite_version, supports_table_xinfo
+from datasette.utils.sqlite import supports_table_xinfo
 from datasette.version import __version__
 from .fixtures import (  # noqa
     app_client,
     app_client_no_files,
-    app_client_with_hash,
-    app_client_shorter_time_limit,
-    app_client_larger_cache_size,
-    app_client_returned_rows_matches_page_size,
-    app_client_two_attached_databases,
-    app_client_two_attached_databases_one_immutable,
-    app_client_conflicting_database_names,
-    app_client_with_cors,
     app_client_with_dot,
-    app_client_with_trace,
+    app_client_shorter_time_limit,
+    app_client_two_attached_databases_one_immutable,
+    app_client_larger_cache_size,
+    app_client_with_cors,
+    app_client_two_attached_databases,
+    app_client_conflicting_database_names,
     app_client_immutable_and_inspect_file,
-    generate_compound_rows,
-    generate_sortable_rows,
     make_app_client,
     EXPECTED_PLUGINS,
     METADATA,
 )
-import json
 import pathlib
 import pytest
 import sys
@@ -680,649 +673,6 @@ def test_invalid_custom_sql(app_client):
     assert "Statement must be a SELECT" == response.json["error"]
 
 
-def test_table_json(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects")
-    assert response.status == 200
-    data = response.json
-    assert (
-        data["query"]["sql"]
-        == "select id, content from simple_primary_key order by id limit 51"
-    )
-    assert data["query"]["params"] == {}
-    assert data["rows"] == [
-        {"id": "1", "content": "hello"},
-        {"id": "2", "content": "world"},
-        {"id": "3", "content": ""},
-        {"id": "4", "content": "RENDER_CELL_DEMO"},
-        {"id": "5", "content": "RENDER_CELL_ASYNC"},
-    ]
-
-
-def test_table_not_exists_json(app_client):
-    assert {
-        "ok": False,
-        "error": "Table not found: blah",
-        "status": 404,
-        "title": None,
-    } == app_client.get("/fixtures/blah.json").json
-
-
-def test_jsono_redirects_to_shape_objects(app_client_with_hash):
-    response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono")
-    response = app_client_with_hash.get(response_1.headers["Location"])
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?_shape=objects")
-
-
-def test_table_shape_arrays(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=arrays")
-    assert [
-        ["1", "hello"],
-        ["2", "world"],
-        ["3", ""],
-        ["4", "RENDER_CELL_DEMO"],
-        ["5", "RENDER_CELL_ASYNC"],
-    ] == response.json["rows"]
-
-
-def test_table_shape_arrayfirst(app_client):
-    response = app_client.get(
-        "/fixtures.json?"
-        + urllib.parse.urlencode(
-            {
-                "sql": "select content from simple_primary_key order by id",
-                "_shape": "arrayfirst",
-            }
-        )
-    )
-    assert [
-        "hello",
-        "world",
-        "",
-        "RENDER_CELL_DEMO",
-        "RENDER_CELL_ASYNC",
-    ] == response.json
-
-
-def test_table_shape_objects(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects")
-    assert [
-        {"id": "1", "content": "hello"},
-        {"id": "2", "content": "world"},
-        {"id": "3", "content": ""},
-        {"id": "4", "content": "RENDER_CELL_DEMO"},
-        {"id": "5", "content": "RENDER_CELL_ASYNC"},
-    ] == response.json["rows"]
-
-
-def test_table_shape_array(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=array")
-    assert [
-        {"id": "1", "content": "hello"},
-        {"id": "2", "content": "world"},
-        {"id": "3", "content": ""},
-        {"id": "4", "content": "RENDER_CELL_DEMO"},
-        {"id": "5", "content": "RENDER_CELL_ASYNC"},
-    ] == response.json
-
-
-def test_table_shape_array_nl(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=array&_nl=on")
-    lines = response.text.split("\n")
-    results = [json.loads(line) for line in lines]
-    assert [
-        {"id": "1", "content": "hello"},
-        {"id": "2", "content": "world"},
-        {"id": "3", "content": ""},
-        {"id": "4", "content": "RENDER_CELL_DEMO"},
-        {"id": "5", "content": "RENDER_CELL_ASYNC"},
-    ] == results
-
-
-def test_table_shape_invalid(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=invalid")
-    assert {
-        "ok": False,
-        "error": "Invalid _shape: invalid",
-        "status": 400,
-        "title": None,
-    } == response.json
-
-
-def test_table_shape_object(app_client):
-    response = app_client.get("/fixtures/simple_primary_key.json?_shape=object")
-    assert {
-        "1": {"id": "1", "content": "hello"},
-        "2": {"id": "2", "content": "world"},
-        "3": {"id": "3", "content": ""},
-        "4": {"id": "4", "content": "RENDER_CELL_DEMO"},
-        "5": {"id": "5", "content": "RENDER_CELL_ASYNC"},
-    } == response.json
-
-
-def test_table_shape_object_compound_primary_key(app_client):
-    response = app_client.get("/fixtures/compound_primary_key.json?_shape=object")
-    assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json
-
-
-def test_table_with_slashes_in_name(app_client):
-    response = app_client.get(
-        "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json"
-    )
-    assert response.status == 200
-    data = response.json
-    assert data["rows"] == [{"pk": "3", "content": "hey"}]
-
-
-def test_table_with_reserved_word_name(app_client):
-    response = app_client.get("/fixtures/select.json?_shape=objects")
-    assert response.status == 200
-    data = response.json
-    assert data["rows"] == [
-        {
-            "rowid": 1,
-            "group": "group",
-            "having": "having",
-            "and": "and",
-            "json": '{"href": "http://example.com/", "label":"Example"}',
-        }
-    ]
-
-
-@pytest.mark.parametrize(
-    "path,expected_rows,expected_pages",
-    [
-        ("/fixtures/no_primary_key.json", 201, 5),
-        ("/fixtures/paginated_view.json", 201, 9),
-        ("/fixtures/no_primary_key.json?_size=25", 201, 9),
-        ("/fixtures/paginated_view.json?_size=50", 201, 5),
-        ("/fixtures/paginated_view.json?_size=max", 201, 3),
-        ("/fixtures/123_starts_with_digits.json", 0, 1),
-        # Ensure faceting doesn't break pagination:
-        ("/fixtures/compound_three_primary_keys.json?_facet=pk1", 1001, 21),
-        # Paginating while sorted by an expanded foreign key should work
-        (
-            "/fixtures/roadside_attraction_characteristics.json?_size=2&_sort=attraction_id&_labels=on",
-            5,
-            3,
-        ),
-    ],
-)
-def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages):
-    fetched = []
-    count = 0
-    while path:
-        response = app_client.get(path)
-        assert 200 == response.status
-        count += 1
-        fetched.extend(response.json["rows"])
-        path = response.json["next_url"]
-        if path:
-            assert urllib.parse.urlencode({"_next": response.json["next"]}) in path
-            path = path.replace("http://localhost", "")
-        assert count < 30, "Possible infinite loop detected"
-
-    assert expected_rows == len(fetched)
-    assert expected_pages == count
-
-
-@pytest.mark.parametrize(
-    "path,expected_error",
-    [
-        ("/fixtures/no_primary_key.json?_size=-4", "_size must be a positive integer"),
-        ("/fixtures/no_primary_key.json?_size=dog", "_size must be a positive integer"),
-        ("/fixtures/no_primary_key.json?_size=1001", "_size must be <= 100"),
-    ],
-)
-def test_validate_page_size(app_client, path, expected_error):
-    response = app_client.get(path)
-    assert expected_error == response.json["error"]
-    assert 400 == response.status
-
-
-def test_page_size_zero(app_client):
-    """For _size=0 we return the counts, empty rows and no continuation token"""
-    response = app_client.get("/fixtures/no_primary_key.json?_size=0")
-    assert 200 == response.status
-    assert [] == response.json["rows"]
-    assert 201 == response.json["filtered_table_rows_count"]
-    assert None is response.json["next"]
-    assert None is response.json["next_url"]
-
-
-def test_paginate_compound_keys(app_client):
-    fetched = []
-    path = "/fixtures/compound_three_primary_keys.json?_shape=objects"
-    page = 0
-    while path:
-        page += 1
-        response = app_client.get(path)
-        fetched.extend(response.json["rows"])
-        path = response.json["next_url"]
-        if path:
-            path = path.replace("http://localhost", "")
-        assert page < 100
-    assert 1001 == len(fetched)
-    assert 21 == page
-    # Should be correctly ordered
-    contents = [f["content"] for f in fetched]
-    expected = [r[3] for r in generate_compound_rows(1001)]
-    assert expected == contents
-
-
-def test_paginate_compound_keys_with_extra_filters(app_client):
-    fetched = []
-    path = (
-        "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects"
-    )
-    page = 0
-    while path:
-        page += 1
-        assert page < 100
-        response = app_client.get(path)
-        fetched.extend(response.json["rows"])
-        path = response.json["next_url"]
-        if path:
-            path = path.replace("http://localhost", "")
-    assert 2 == page
-    expected = [r[3] for r in generate_compound_rows(1001) if "d" in r[3]]
-    assert expected == [f["content"] for f in fetched]
-
-
-@pytest.mark.parametrize(
-    "query_string,sort_key,human_description_en",
-    [
-        ("_sort=sortable", lambda row: row["sortable"], "sorted by sortable"),
-        (
-            "_sort_desc=sortable",
-            lambda row: -row["sortable"],
-            "sorted by sortable descending",
-        ),
-        (
-            "_sort=sortable_with_nulls",
-            lambda row: (
-                1 if row["sortable_with_nulls"] is not None else 0,
-                row["sortable_with_nulls"],
-            ),
-            "sorted by sortable_with_nulls",
-        ),
-        (
-            "_sort_desc=sortable_with_nulls",
-            lambda row: (
-                1 if row["sortable_with_nulls"] is None else 0,
-                -row["sortable_with_nulls"]
-                if row["sortable_with_nulls"] is not None
-                else 0,
-                row["content"],
-            ),
-            "sorted by sortable_with_nulls descending",
-        ),
-        # text column contains '$null' - ensure it doesn't confuse pagination:
-        ("_sort=text", lambda row: row["text"], "sorted by text"),
-    ],
-)
-def test_sortable(app_client, query_string, sort_key, human_description_en):
-    path = f"/fixtures/sortable.json?_shape=objects&{query_string}"
-    fetched = []
-    page = 0
-    while path:
-        page += 1
-        assert page < 100
-        response = app_client.get(path)
-        assert human_description_en == response.json["human_description_en"]
-        fetched.extend(response.json["rows"])
-        path = response.json["next_url"]
-        if path:
-            path = path.replace("http://localhost", "")
-    assert 5 == page
-    expected = list(generate_sortable_rows(201))
-    expected.sort(key=sort_key)
-    assert [r["content"] for r in expected] == [r["content"] for r in fetched]
-
-
-def test_sortable_and_filtered(app_client):
-    path = (
-        "/fixtures/sortable.json"
-        "?content__contains=d&_sort_desc=sortable&_shape=objects"
-    )
-    response = app_client.get(path)
-    fetched = response.json["rows"]
-    assert (
-        'where content contains "d" sorted by sortable descending'
-        == response.json["human_description_en"]
-    )
-    expected = [row for row in generate_sortable_rows(201) if "d" in row["content"]]
-    assert len(expected) == response.json["filtered_table_rows_count"]
-    expected.sort(key=lambda row: -row["sortable"])
-    assert [r["content"] for r in expected] == [r["content"] for r in fetched]
-
-
-def test_sortable_argument_errors(app_client):
-    response = app_client.get("/fixtures/sortable.json?_sort=badcolumn")
-    assert "Cannot sort table by badcolumn" == response.json["error"]
-    response = app_client.get("/fixtures/sortable.json?_sort_desc=badcolumn2")
-    assert "Cannot sort table by badcolumn2" == response.json["error"]
-    response = app_client.get(
-        "/fixtures/sortable.json?_sort=sortable_with_nulls&_sort_desc=sortable"
-    )
-    assert "Cannot use _sort and _sort_desc at the same time" == response.json["error"]
-
-
-def test_sortable_columns_metadata(app_client):
-    response = app_client.get("/fixtures/sortable.json?_sort=content")
-    assert "Cannot sort table by content" == response.json["error"]
-    # no_primary_key has ALL sort options disabled
-    for column in ("content", "a", "b", "c"):
-        response = app_client.get(f"/fixtures/sortable.json?_sort={column}")
-        assert f"Cannot sort table by {column}" == response.json["error"]
-
-
-@pytest.mark.parametrize(
-    "path,expected_rows",
-    [
-        (
-            "/fixtures/searchable.json?_search=dog",
-            [
-                [1, "barry cat", "terry dog", "panther"],
-                [2, "terry dog", "sara weasel", "puma"],
-            ],
-        ),
-        (
-            # Special keyword shouldn't break FTS query
-            "/fixtures/searchable.json?_search=AND",
-            [],
-        ),
-        (
-            # Without _searchmode=raw this should return no results
-            "/fixtures/searchable.json?_search=te*+AND+do*",
-            [],
-        ),
-        (
-            # _searchmode=raw
-            "/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw",
-            [
-                [1, "barry cat", "terry dog", "panther"],
-                [2, "terry dog", "sara weasel", "puma"],
-            ],
-        ),
-        (
-            # _searchmode=raw combined with _search_COLUMN
-            "/fixtures/searchable.json?_search_text2=te*&_searchmode=raw",
-            [
-                [1, "barry cat", "terry dog", "panther"],
-            ],
-        ),
-        (
-            "/fixtures/searchable.json?_search=weasel",
-            [[2, "terry dog", "sara weasel", "puma"]],
-        ),
-        (
-            "/fixtures/searchable.json?_search_text2=dog",
-            [[1, "barry cat", "terry dog", "panther"]],
-        ),
-        (
-            "/fixtures/searchable.json?_search_name%20with%20.%20and%20spaces=panther",
-            [[1, "barry cat", "terry dog", "panther"]],
-        ),
-    ],
-)
-def test_searchable(app_client, path, expected_rows):
-    response = app_client.get(path)
-    assert expected_rows == response.json["rows"]
-
-
-_SEARCHMODE_RAW_RESULTS = [
-    [1, "barry cat", "terry dog", "panther"],
-    [2, "terry dog", "sara weasel", "puma"],
-]
-
-
-@pytest.mark.parametrize(
-    "table_metadata,querystring,expected_rows",
-    [
-        (
-            {},
-            "_search=te*+AND+do*",
-            [],
-        ),
-        (
-            {"searchmode": "raw"},
-            "_search=te*+AND+do*",
-            _SEARCHMODE_RAW_RESULTS,
-        ),
-        (
-            {},
-            "_search=te*+AND+do*&_searchmode=raw",
-            _SEARCHMODE_RAW_RESULTS,
-        ),
-        # Can be over-ridden with _searchmode=escaped
-        (
-            {"searchmode": "raw"},
-            "_search=te*+AND+do*&_searchmode=escaped",
-            [],
-        ),
-    ],
-)
-def test_searchmode(table_metadata, querystring, expected_rows):
-    with make_app_client(
-        metadata={"databases": {"fixtures": {"tables": {"searchable": table_metadata}}}}
-    ) as client:
-        response = client.get("/fixtures/searchable.json?" + querystring)
-        assert expected_rows == response.json["rows"]
-
-
-@pytest.mark.parametrize(
-    "path,expected_rows",
-    [
-        (
-            "/fixtures/searchable_view_configured_by_metadata.json?_search=weasel",
-            [[2, "terry dog", "sara weasel", "puma"]],
-        ),
-        # This should return all results because search is not configured:
-        (
-            "/fixtures/searchable_view.json?_search=weasel",
-            [
-                [1, "barry cat", "terry dog", "panther"],
-                [2, "terry dog", "sara weasel", "puma"],
-            ],
-        ),
-        (
-            "/fixtures/searchable_view.json?_search=weasel&_fts_table=searchable_fts&_fts_pk=pk",
-            [[2, "terry dog", "sara weasel", "puma"]],
-        ),
-    ],
-)
-def test_searchable_views(app_client, path, expected_rows):
-    response = app_client.get(path)
-    assert expected_rows == response.json["rows"]
-
-
-def test_searchable_invalid_column(app_client):
-    response = app_client.get("/fixtures/searchable.json?_search_invalid=x")
-    assert 400 == response.status
-    assert {
-        "ok": False,
-        "error": "Cannot search by that column",
-        "status": 400,
-        "title": None,
-    } == response.json
-
-
-@pytest.mark.parametrize(
-    "path,expected_rows",
-    [
-        ("/fixtures/simple_primary_key.json?content=hello", [["1", "hello"]]),
-        (
-            "/fixtures/simple_primary_key.json?content__contains=o",
-            [
-                ["1", "hello"],
-                ["2", "world"],
-                ["4", "RENDER_CELL_DEMO"],
-            ],
-        ),
-        ("/fixtures/simple_primary_key.json?content__exact=", [["3", ""]]),
-        (
-            "/fixtures/simple_primary_key.json?content__not=world",
-            [
-                ["1", "hello"],
-                ["3", ""],
-                ["4", "RENDER_CELL_DEMO"],
-                ["5", "RENDER_CELL_ASYNC"],
-            ],
-        ),
-    ],
-)
-def test_table_filter_queries(app_client, path, expected_rows):
-    response = app_client.get(path)
-    assert expected_rows == response.json["rows"]
-
-
-def test_table_filter_queries_multiple_of_same_type(app_client):
-    response = app_client.get(
-        "/fixtures/simple_primary_key.json?content__not=world&content__not=hello"
-    )
-    assert [
-        ["3", ""],
-        ["4", "RENDER_CELL_DEMO"],
-        ["5", "RENDER_CELL_ASYNC"],
-    ] == response.json["rows"]
-
-
-@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
-def test_table_filter_json_arraycontains(app_client):
-    response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1")
-    assert response.json["rows"] == [
-        [
-            1,
-            "2019-01-14 08:00:00",
-            1,
-            1,
-            "CA",
-            1,
-            "Mission",
-            '["tag1", "tag2"]',
-            '[{"foo": "bar"}]',
-            "one",
-        ],
-        [
-            2,
-            "2019-01-14 08:00:00",
-            1,
-            1,
-            "CA",
-            1,
-            "Dogpatch",
-            '["tag1", "tag3"]',
-            "[]",
-            "two",
-        ],
-    ]
-
-
-@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
-def test_table_filter_json_arraynotcontains(app_client):
-    response = app_client.get(
-        "/fixtures/facetable.json?tags__arraynotcontains=tag3&tags__not=[]"
-    )
-    assert response.json["rows"] == [
-        [
-            1,
-            "2019-01-14 08:00:00",
-            1,
-            1,
-            "CA",
-            1,
-            "Mission",
-            '["tag1", "tag2"]',
-            '[{"foo": "bar"}]',
-            "one",
-        ]
-    ]
-
-
-def test_table_filter_extra_where(app_client):
-    response = app_client.get(
-        "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
-    )
-    assert [
-        [
-            2,
-            "2019-01-14 08:00:00",
-            1,
-            1,
-            "CA",
-            1,
-            "Dogpatch",
-            '["tag1", "tag3"]',
-            "[]",
-            "two",
-        ]
-    ] == response.json["rows"]
-
-
-def test_table_filter_extra_where_invalid(app_client):
-    response = app_client.get("/fixtures/facetable.json?_where=_neighborhood=Dogpatch'")
-    assert 400 == response.status
-    assert "Invalid SQL" == response.json["title"]
-
-
-def test_table_filter_extra_where_disabled_if_no_sql_allowed():
-    with make_app_client(metadata={"allow_sql": {}}) as client:
-        response = client.get(
-            "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
-        )
-        assert 403 == response.status
-        assert "_where= is not allowed" == response.json["error"]
-
-
-def test_table_through(app_client):
-    # Just the museums:
-    response = app_client.get(
-        '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
-    )
-    assert [
-        [
-            3,
-            "Burlingame Museum of PEZ Memorabilia",
-            "214 California Drive, Burlingame, CA 94010",
-            37.5793,
-            -122.3442,
-        ],
-        [
-            4,
-            "Bigfoot Discovery Museum",
-            "5497 Highway 9, Felton, CA 95018",
-            37.0414,
-            -122.0725,
-        ],
-    ] == response.json["rows"]
-    assert (
-        'where roadside_attraction_characteristics.characteristic_id = "1"'
-        == response.json["human_description_en"]
-    )
-
-
-def test_max_returned_rows(app_client):
-    response = app_client.get("/fixtures.json?sql=select+content+from+no_primary_key")
-    data = response.json
-    assert {"sql": "select content from no_primary_key", "params": {}} == data["query"]
-    assert data["truncated"]
-    assert 100 == len(data["rows"])
-
-
-def test_view(app_client):
-    response = app_client.get("/fixtures/simple_view.json?_shape=objects")
-    assert response.status == 200
-    data = response.json
-    assert data["rows"] == [
-        {"upper_content": "HELLO", "content": "hello"},
-        {"upper_content": "WORLD", "content": "world"},
-        {"upper_content": "", "content": ""},
-        {"upper_content": "RENDER_CELL_DEMO", "content": "RENDER_CELL_DEMO"},
-        {"upper_content": "RENDER_CELL_ASYNC", "content": "RENDER_CELL_ASYNC"},
-    ]
-
-
 def test_row(app_client):
     response = app_client.get("/fixtures/simple_primary_key/1.json?_shape=objects")
     assert response.status == 200
@@ -1390,20 +740,6 @@ def test_row_foreign_key_tables(app_client):
     ]
 
 
-def test_unit_filters(app_client):
-    response = app_client.get(
-        "/fixtures/units.json?distance__lt=75km&frequency__gt=1kHz"
-    )
-    assert response.status == 200
-    data = response.json
-
-    assert data["units"]["distance"] == "m"
-    assert data["units"]["frequency"] == "Hz"
-
-    assert len(data["rows"]) == 1
-    assert data["rows"][0][0] == 2
-
-
 def test_databases_json(app_client_two_attached_databases_one_immutable):
     response = app_client_two_attached_databases_one_immutable.get("/-/databases.json")
     databases = response.json
@@ -1498,330 +834,6 @@ def test_config_redirects_to_settings(app_client, path, expected_redirect):
     assert response.headers["Location"] == expected_redirect
 
 
-def test_page_size_matching_max_returned_rows(
-    app_client_returned_rows_matches_page_size,
-):
-    fetched = []
-    path = "/fixtures/no_primary_key.json"
-    while path:
-        response = app_client_returned_rows_matches_page_size.get(path)
-        fetched.extend(response.json["rows"])
-        assert len(response.json["rows"]) in (1, 50)
-        path = response.json["next_url"]
-        if path:
-            path = path.replace("http://localhost", "")
-    assert 201 == len(fetched)
-
-
-@pytest.mark.parametrize(
-    "path,expected_facet_results",
-    [
-        (
-            "/fixtures/facetable.json?_facet=state&_facet=_city_id",
-            {
-                "state": {
-                    "name": "state",
-                    "hideable": True,
-                    "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?_facet=_city_id",
-                    "results": [
-                        {
-                            "value": "CA",
-                            "label": "CA",
-                            "count": 10,
-                            "toggle_url": "_facet=state&_facet=_city_id&state=CA",
-                            "selected": False,
-                        },
-                        {
-                            "value": "MI",
-                            "label": "MI",
-                            "count": 4,
-                            "toggle_url": "_facet=state&_facet=_city_id&state=MI",
-                            "selected": False,
-                        },
-                        {
-                            "value": "MC",
-                            "label": "MC",
-                            "count": 1,
-                            "toggle_url": "_facet=state&_facet=_city_id&state=MC",
-                            "selected": False,
-                        },
-                    ],
-                    "truncated": False,
-                },
-                "_city_id": {
-                    "name": "_city_id",
-                    "hideable": True,
-                    "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?_facet=state",
-                    "results": [
-                        {
-                            "value": 1,
-                            "label": "San Francisco",
-                            "count": 6,
-                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=1",
-                            "selected": False,
-                        },
-                        {
-                            "value": 2,
-                            "label": "Los Angeles",
-                            "count": 4,
-                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=2",
-                            "selected": False,
-                        },
-                        {
-                            "value": 3,
-                            "label": "Detroit",
-                            "count": 4,
-                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=3",
-                            "selected": False,
-                        },
-                        {
-                            "value": 4,
-                            "label": "Memnonia",
-                            "count": 1,
-                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=4",
-                            "selected": False,
-                        },
-                    ],
-                    "truncated": False,
-                },
-            },
-        ),
-        (
-            "/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI",
-            {
-                "state": {
-                    "name": "state",
-                    "hideable": True,
-                    "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?_facet=_city_id&state=MI",
-                    "results": [
-                        {
-                            "value": "MI",
-                            "label": "MI",
-                            "count": 4,
-                            "selected": True,
-                            "toggle_url": "_facet=state&_facet=_city_id",
-                        }
-                    ],
-                    "truncated": False,
-                },
-                "_city_id": {
-                    "name": "_city_id",
-                    "hideable": True,
-                    "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?_facet=state&state=MI",
-                    "results": [
-                        {
-                            "value": 3,
-                            "label": "Detroit",
-                            "count": 4,
-                            "selected": False,
-                            "toggle_url": "_facet=state&_facet=_city_id&state=MI&_city_id__exact=3",
-                        }
-                    ],
-                    "truncated": False,
-                },
-            },
-        ),
-        (
-            "/fixtures/facetable.json?_facet=planet_int",
-            {
-                "planet_int": {
-                    "name": "planet_int",
-                    "hideable": True,
-                    "type": "column",
-                    "toggle_url": "/fixtures/facetable.json",
-                    "results": [
-                        {
-                            "value": 1,
-                            "label": 1,
-                            "count": 14,
-                            "selected": False,
-                            "toggle_url": "_facet=planet_int&planet_int=1",
-                        },
-                        {
-                            "value": 2,
-                            "label": 2,
-                            "count": 1,
-                            "selected": False,
-                            "toggle_url": "_facet=planet_int&planet_int=2",
-                        },
-                    ],
-                    "truncated": False,
-                }
-            },
-        ),
-        (
-            # planet_int is an integer field:
-            "/fixtures/facetable.json?_facet=planet_int&planet_int=1",
-            {
-                "planet_int": {
-                    "name": "planet_int",
-                    "hideable": True,
-                    "type": "column",
-                    "toggle_url": "/fixtures/facetable.json?planet_int=1",
-                    "results": [
-                        {
-                            "value": 1,
-                            "label": 1,
-                            "count": 14,
-                            "selected": True,
-                            "toggle_url": "_facet=planet_int",
-                        }
-                    ],
-                    "truncated": False,
-                }
-            },
-        ),
-    ],
-)
-def test_facets(app_client, path, expected_facet_results):
-    response = app_client.get(path)
-    facet_results = response.json["facet_results"]
-    # We only compare the querystring portion of the taggle_url
-    for facet_name, facet_info in facet_results.items():
-        assert facet_name == facet_info["name"]
-        assert False is facet_info["truncated"]
-        for facet_value in facet_info["results"]:
-            facet_value["toggle_url"] = facet_value["toggle_url"].split("?")[1]
-    assert expected_facet_results == facet_results
-
-
-def test_suggested_facets(app_client):
-    suggestions = [
-        {
-            "name": suggestion["name"],
-            "querystring": suggestion["toggle_url"].split("?")[-1],
-        }
-        for suggestion in app_client.get("/fixtures/facetable.json").json[
-            "suggested_facets"
-        ]
-    ]
-    expected = [
-        {"name": "created", "querystring": "_facet=created"},
-        {"name": "planet_int", "querystring": "_facet=planet_int"},
-        {"name": "on_earth", "querystring": "_facet=on_earth"},
-        {"name": "state", "querystring": "_facet=state"},
-        {"name": "_city_id", "querystring": "_facet=_city_id"},
-        {"name": "_neighborhood", "querystring": "_facet=_neighborhood"},
-        {"name": "tags", "querystring": "_facet=tags"},
-        {"name": "complex_array", "querystring": "_facet=complex_array"},
-        {"name": "created", "querystring": "_facet_date=created"},
-    ]
-    if detect_json1():
-        expected.append({"name": "tags", "querystring": "_facet_array=tags"})
-    assert expected == suggestions
-
-
-def test_allow_facet_off():
-    with make_app_client(settings={"allow_facet": False}) as client:
-        assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status
-        # Should not suggest any facets either:
-        assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
-
-
-def test_suggest_facets_off():
-    with make_app_client(settings={"suggest_facets": False}) as client:
-        # Now suggested_facets should be []
-        assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
-
-
-@pytest.mark.parametrize("nofacet", (True, False))
-def test_nofacet(app_client, nofacet):
-    path = "/fixtures/facetable.json?_facet=state"
-    if nofacet:
-        path += "&_nofacet=1"
-    response = app_client.get(path)
-    if nofacet:
-        assert response.json["suggested_facets"] == []
-        assert response.json["facet_results"] == {}
-    else:
-        assert response.json["suggested_facets"] != []
-        assert response.json["facet_results"] != {}
-
-
-@pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15)))
-def test_nocount(app_client, nocount, expected_count):
-    path = "/fixtures/facetable.json"
-    if nocount:
-        path += "?_nocount=1"
-    response = app_client.get(path)
-    assert response.json["filtered_table_rows_count"] == expected_count
-
-
-def test_nocount_nofacet_if_shape_is_object(app_client_with_trace):
-    response = app_client_with_trace.get(
-        "/fixtures/facetable.json?_trace=1&_shape=object"
-    )
-    assert "count(*)" not in response.text
-
-
-def test_expand_labels(app_client):
-    response = app_client.get(
-        "/fixtures/facetable.json?_shape=object&_labels=1&_size=2"
-        "&_neighborhood__contains=c"
-    )
-    assert {
-        "2": {
-            "pk": 2,
-            "created": "2019-01-14 08:00:00",
-            "planet_int": 1,
-            "on_earth": 1,
-            "state": "CA",
-            "_city_id": {"value": 1, "label": "San Francisco"},
-            "_neighborhood": "Dogpatch",
-            "tags": '["tag1", "tag3"]',
-            "complex_array": "[]",
-            "distinct_some_null": "two",
-        },
-        "13": {
-            "pk": 13,
-            "created": "2019-01-17 08:00:00",
-            "planet_int": 1,
-            "on_earth": 1,
-            "state": "MI",
-            "_city_id": {"value": 3, "label": "Detroit"},
-            "_neighborhood": "Corktown",
-            "tags": "[]",
-            "complex_array": "[]",
-            "distinct_some_null": None,
-        },
-    } == response.json
-
-
-def test_expand_label(app_client):
-    response = app_client.get(
-        "/fixtures/foreign_key_references.json?_shape=object"
-        "&_label=foreign_key_with_label&_size=1"
-    )
-    assert response.json == {
-        "1": {
-            "pk": "1",
-            "foreign_key_with_label": {"value": "1", "label": "hello"},
-            "foreign_key_with_blank_label": "3",
-            "foreign_key_with_no_label": "1",
-            "foreign_key_compound_pk1": "a",
-            "foreign_key_compound_pk2": "b",
-        }
-    }
-
-
-@pytest.mark.parametrize(
-    "path,expected_cache_control",
-    [
-        ("/fixtures/facetable.json", "max-age=5"),
-        ("/fixtures/facetable.json?_ttl=invalid", "max-age=5"),
-        ("/fixtures/facetable.json?_ttl=10", "max-age=10"),
-        ("/fixtures/facetable.json?_ttl=0", "no-cache"),
-    ],
-)
-def test_ttl_parameter(app_client, path, expected_cache_control):
-    response = app_client.get(path)
-    assert expected_cache_control == response.headers["Cache-Control"]
-
-
 @pytest.mark.parametrize(
     "path,expected_redirect",
     [
@@ -1899,29 +911,6 @@ def test_config_force_https_urls():
         assert client.ds._last_request.scheme == "https"
 
 
-def test_infinity_returned_as_null(app_client):
-    response = app_client.get("/fixtures/infinity.json?_shape=array")
-    assert [
-        {"rowid": 1, "value": None},
-        {"rowid": 2, "value": None},
-        {"rowid": 3, "value": 1.5},
-    ] == response.json
-
-
-def test_infinity_returned_as_invalid_json_if_requested(app_client):
-    response = app_client.get("/fixtures/infinity.json?_shape=array&_json_infinity=1")
-    assert [
-        {"rowid": 1, "value": float("inf")},
-        {"rowid": 2, "value": float("-inf")},
-        {"rowid": 3, "value": 1.5},
-    ] == response.json
-
-
-def test_custom_query_with_unicode_characters(app_client):
-    response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array")
-    assert [{"id": 1, "name": "San Francisco"}] == response.json
-
-
 @pytest.mark.parametrize("trace_debug", (True, False))
 def test_trace(trace_debug):
     with make_app_client(settings={"trace_debug": trace_debug}) as client:
@@ -1997,205 +986,17 @@ def test_common_prefix_database_names(app_client_conflicting_database_names):
         assert db_name == data["database"]
 
 
-def test_null_and_compound_foreign_keys_are_not_expanded(app_client):
-    response = app_client.get(
-        "/fixtures/foreign_key_references.json?_shape=array&_labels=on"
-    )
-    assert response.json == [
-        {
-            "pk": "1",
-            "foreign_key_with_label": {"value": "1", "label": "hello"},
-            "foreign_key_with_blank_label": {"value": "3", "label": ""},
-            "foreign_key_with_no_label": {"value": "1", "label": "1"},
-            "foreign_key_compound_pk1": "a",
-            "foreign_key_compound_pk2": "b",
-        },
-        {
-            "pk": "2",
-            "foreign_key_with_label": None,
-            "foreign_key_with_blank_label": None,
-            "foreign_key_with_no_label": None,
-            "foreign_key_compound_pk1": None,
-            "foreign_key_compound_pk2": None,
-        },
-    ]
-
-
 def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file):
     response = app_client_immutable_and_inspect_file.get("/fixtures/sortable.json")
     assert response.json["filtered_table_rows_count"] == 100
 
 
-@pytest.mark.parametrize(
-    "path,expected_json,expected_text",
-    [
-        (
-            "/fixtures/binary_data.json?_shape=array",
-            [
-                {"rowid": 1, "data": {"$base64": True, "encoded": "FRwCx60F/g=="}},
-                {"rowid": 2, "data": {"$base64": True, "encoded": "FRwDx60F/g=="}},
-                {"rowid": 3, "data": None},
-            ],
-            None,
-        ),
-        (
-            "/fixtures/binary_data.json?_shape=array&_nl=on",
-            None,
-            (
-                '{"rowid": 1, "data": {"$base64": true, "encoded": "FRwCx60F/g=="}}\n'
-                '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}\n'
-                '{"rowid": 3, "data": null}'
-            ),
-        ),
-    ],
-)
-def test_binary_data_in_json(app_client, path, expected_json, expected_text):
-    response = app_client.get(path)
-    if expected_json:
-        assert response.json == expected_json
-    else:
-        assert response.text == expected_text
-
-
-@pytest.mark.parametrize(
-    "qs",
-    [
-        "",
-        "?_shape=arrays",
-        "?_shape=arrayfirst",
-        "?_shape=object",
-        "?_shape=objects",
-        "?_shape=array",
-        "?_shape=array&_nl=on",
-    ],
-)
-def test_paginate_using_link_header(app_client, qs):
-    path = f"/fixtures/compound_three_primary_keys.json{qs}"
-    num_pages = 0
-    while path:
-        response = app_client.get(path)
-        assert response.status == 200
-        num_pages += 1
-        link = response.headers.get("link")
-        if link:
-            assert link.startswith("<")
-            assert link.endswith('>; rel="next"')
-            path = link[1:].split(">")[0]
-            path = path.replace("http://localhost", "")
-        else:
-            path = None
-    assert num_pages == 21
-
-
-@pytest.mark.skipif(
-    sqlite_version() < (3, 31, 0),
-    reason="generated columns were added in SQLite 3.31.0",
-)
-def test_generated_columns_are_visible_in_datasette():
-    with make_app_client(
-        extra_databases={
-            "generated.db": """
-                CREATE TABLE generated_columns (
-                    body TEXT,
-                    id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED,
-                    consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED
-                );
-                INSERT INTO generated_columns (body) VALUES (
-                    '{"number": 1, "string": "This is a string"}'
-                );"""
-        }
-    ) as client:
-        response = client.get("/generated/generated_columns.json?_shape=array")
-        assert response.json == [
-            {
-                "rowid": 1,
-                "body": '{"number": 1, "string": "This is a string"}',
-                "id": 1,
-                "consideration": "This is a string",
-            }
-        ]
-
-
 def test_http_options_request(app_client):
     response = app_client.request("/fixtures", method="OPTIONS")
     assert response.status == 200
     assert response.text == "ok"
 
 
-@pytest.mark.parametrize(
-    "path,expected_columns",
-    (
-        ("/fixtures/facetable.json?_col=created", ["pk", "created"]),
-        (
-            "/fixtures/facetable.json?_nocol=created",
-            [
-                "pk",
-                "planet_int",
-                "on_earth",
-                "state",
-                "_city_id",
-                "_neighborhood",
-                "tags",
-                "complex_array",
-                "distinct_some_null",
-            ],
-        ),
-        (
-            "/fixtures/facetable.json?_col=state&_col=created",
-            ["pk", "state", "created"],
-        ),
-        (
-            "/fixtures/facetable.json?_col=state&_col=state",
-            ["pk", "state"],
-        ),
-        (
-            "/fixtures/facetable.json?_col=state&_col=created&_nocol=created",
-            ["pk", "state"],
-        ),
-        (
-            # Ensure faceting doesn't break, https://github.com/simonw/datasette/issues/1345
-            "/fixtures/facetable.json?_nocol=state&_facet=state",
-            [
-                "pk",
-                "created",
-                "planet_int",
-                "on_earth",
-                "_city_id",
-                "_neighborhood",
-                "tags",
-                "complex_array",
-                "distinct_some_null",
-            ],
-        ),
-        (
-            "/fixtures/simple_view.json?_nocol=content",
-            ["upper_content"],
-        ),
-        ("/fixtures/simple_view.json?_col=content", ["content"]),
-    ),
-)
-def test_col_nocol(app_client, path, expected_columns):
-    response = app_client.get(path)
-    assert response.status == 200
-    columns = response.json["columns"]
-    assert columns == expected_columns
-
-
-@pytest.mark.parametrize(
-    "path,expected_error",
-    (
-        ("/fixtures/facetable.json?_col=bad", "_col=bad - invalid columns"),
-        ("/fixtures/facetable.json?_nocol=bad", "_nocol=bad - invalid columns"),
-        ("/fixtures/facetable.json?_nocol=pk", "_nocol=pk - invalid columns"),
-        ("/fixtures/simple_view.json?_col=bad", "_col=bad - invalid columns"),
-    ),
-)
-def test_col_nocol_errors(app_client, path, expected_error):
-    response = app_client.get(path)
-    assert response.status == 400
-    assert response.json["error"] == expected_error
-
-
 @pytest.mark.asyncio
 async def test_db_path(app_client):
     db = app_client.ds.get_database()
@@ -2205,5 +1006,5 @@ async def test_db_path(app_client):
 
     datasette = Datasette([path])
 
-    # this will break with a path
+    # Previously this broke if path was a pathlib.Path:
     await datasette.refresh_schemas()
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
new file mode 100644
index 00000000..a530de44
--- /dev/null
+++ b/tests/test_table_api.py
@@ -0,0 +1,1206 @@
+from datasette.utils import detect_json1
+from datasette.utils.sqlite import sqlite_version
+from .fixtures import (  # noqa
+    app_client,
+    app_client_with_hash,
+    app_client_with_trace,
+    app_client_returned_rows_matches_page_size,
+    generate_compound_rows,
+    generate_sortable_rows,
+    make_app_client,
+)
+import json
+import pytest
+import urllib
+
+
+def test_table_json(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects")
+    assert response.status == 200
+    data = response.json
+    assert (
+        data["query"]["sql"]
+        == "select id, content from simple_primary_key order by id limit 51"
+    )
+    assert data["query"]["params"] == {}
+    assert data["rows"] == [
+        {"id": "1", "content": "hello"},
+        {"id": "2", "content": "world"},
+        {"id": "3", "content": ""},
+        {"id": "4", "content": "RENDER_CELL_DEMO"},
+        {"id": "5", "content": "RENDER_CELL_ASYNC"},
+    ]
+
+
+def test_table_not_exists_json(app_client):
+    assert {
+        "ok": False,
+        "error": "Table not found: blah",
+        "status": 404,
+        "title": None,
+    } == app_client.get("/fixtures/blah.json").json
+
+
+def test_jsono_redirects_to_shape_objects(app_client_with_hash):
+    response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono")
+    response = app_client_with_hash.get(response_1.headers["Location"])
+    assert response.status == 302
+    assert response.headers["Location"].endswith("?_shape=objects")
+
+
+def test_table_shape_arrays(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=arrays")
+    assert [
+        ["1", "hello"],
+        ["2", "world"],
+        ["3", ""],
+        ["4", "RENDER_CELL_DEMO"],
+        ["5", "RENDER_CELL_ASYNC"],
+    ] == response.json["rows"]
+
+
+def test_table_shape_arrayfirst(app_client):
+    response = app_client.get(
+        "/fixtures.json?"
+        + urllib.parse.urlencode(
+            {
+                "sql": "select content from simple_primary_key order by id",
+                "_shape": "arrayfirst",
+            }
+        )
+    )
+    assert [
+        "hello",
+        "world",
+        "",
+        "RENDER_CELL_DEMO",
+        "RENDER_CELL_ASYNC",
+    ] == response.json
+
+
+def test_table_shape_objects(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects")
+    assert [
+        {"id": "1", "content": "hello"},
+        {"id": "2", "content": "world"},
+        {"id": "3", "content": ""},
+        {"id": "4", "content": "RENDER_CELL_DEMO"},
+        {"id": "5", "content": "RENDER_CELL_ASYNC"},
+    ] == response.json["rows"]
+
+
+def test_table_shape_array(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=array")
+    assert [
+        {"id": "1", "content": "hello"},
+        {"id": "2", "content": "world"},
+        {"id": "3", "content": ""},
+        {"id": "4", "content": "RENDER_CELL_DEMO"},
+        {"id": "5", "content": "RENDER_CELL_ASYNC"},
+    ] == response.json
+
+
+def test_table_shape_array_nl(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=array&_nl=on")
+    lines = response.text.split("\n")
+    results = [json.loads(line) for line in lines]
+    assert [
+        {"id": "1", "content": "hello"},
+        {"id": "2", "content": "world"},
+        {"id": "3", "content": ""},
+        {"id": "4", "content": "RENDER_CELL_DEMO"},
+        {"id": "5", "content": "RENDER_CELL_ASYNC"},
+    ] == results
+
+
+def test_table_shape_invalid(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=invalid")
+    assert {
+        "ok": False,
+        "error": "Invalid _shape: invalid",
+        "status": 400,
+        "title": None,
+    } == response.json
+
+
+def test_table_shape_object(app_client):
+    response = app_client.get("/fixtures/simple_primary_key.json?_shape=object")
+    assert {
+        "1": {"id": "1", "content": "hello"},
+        "2": {"id": "2", "content": "world"},
+        "3": {"id": "3", "content": ""},
+        "4": {"id": "4", "content": "RENDER_CELL_DEMO"},
+        "5": {"id": "5", "content": "RENDER_CELL_ASYNC"},
+    } == response.json
+
+
+def test_table_shape_object_compound_primary_key(app_client):
+    response = app_client.get("/fixtures/compound_primary_key.json?_shape=object")
+    assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json
+
+
+def test_table_with_slashes_in_name(app_client):
+    response = app_client.get(
+        "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json"
+    )
+    assert response.status == 200
+    data = response.json
+    assert data["rows"] == [{"pk": "3", "content": "hey"}]
+
+
+def test_table_with_reserved_word_name(app_client):
+    response = app_client.get("/fixtures/select.json?_shape=objects")
+    assert response.status == 200
+    data = response.json
+    assert data["rows"] == [
+        {
+            "rowid": 1,
+            "group": "group",
+            "having": "having",
+            "and": "and",
+            "json": '{"href": "http://example.com/", "label":"Example"}',
+        }
+    ]
+
+
+@pytest.mark.parametrize(
+    "path,expected_rows,expected_pages",
+    [
+        ("/fixtures/no_primary_key.json", 201, 5),
+        ("/fixtures/paginated_view.json", 201, 9),
+        ("/fixtures/no_primary_key.json?_size=25", 201, 9),
+        ("/fixtures/paginated_view.json?_size=50", 201, 5),
+        ("/fixtures/paginated_view.json?_size=max", 201, 3),
+        ("/fixtures/123_starts_with_digits.json", 0, 1),
+        # Ensure faceting doesn't break pagination:
+        ("/fixtures/compound_three_primary_keys.json?_facet=pk1", 1001, 21),
+        # Paginating while sorted by an expanded foreign key should work
+        (
+            "/fixtures/roadside_attraction_characteristics.json?_size=2&_sort=attraction_id&_labels=on",
+            5,
+            3,
+        ),
+    ],
+)
+def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages):
+    fetched = []
+    count = 0
+    while path:
+        response = app_client.get(path)
+        assert 200 == response.status
+        count += 1
+        fetched.extend(response.json["rows"])
+        path = response.json["next_url"]
+        if path:
+            assert urllib.parse.urlencode({"_next": response.json["next"]}) in path
+            path = path.replace("http://localhost", "")
+        assert count < 30, "Possible infinite loop detected"
+
+    assert expected_rows == len(fetched)
+    assert expected_pages == count
+
+
+@pytest.mark.parametrize(
+    "path,expected_error",
+    [
+        ("/fixtures/no_primary_key.json?_size=-4", "_size must be a positive integer"),
+        ("/fixtures/no_primary_key.json?_size=dog", "_size must be a positive integer"),
+        ("/fixtures/no_primary_key.json?_size=1001", "_size must be <= 100"),
+    ],
+)
+def test_validate_page_size(app_client, path, expected_error):
+    response = app_client.get(path)
+    assert expected_error == response.json["error"]
+    assert 400 == response.status
+
+
+def test_page_size_zero(app_client):
+    """For _size=0 we return the counts, empty rows and no continuation token"""
+    response = app_client.get("/fixtures/no_primary_key.json?_size=0")
+    assert 200 == response.status
+    assert [] == response.json["rows"]
+    assert 201 == response.json["filtered_table_rows_count"]
+    assert None is response.json["next"]
+    assert None is response.json["next_url"]
+
+
+def test_paginate_compound_keys(app_client):
+    fetched = []
+    path = "/fixtures/compound_three_primary_keys.json?_shape=objects"
+    page = 0
+    while path:
+        page += 1
+        response = app_client.get(path)
+        fetched.extend(response.json["rows"])
+        path = response.json["next_url"]
+        if path:
+            path = path.replace("http://localhost", "")
+        assert page < 100
+    assert 1001 == len(fetched)
+    assert 21 == page
+    # Should be correctly ordered
+    contents = [f["content"] for f in fetched]
+    expected = [r[3] for r in generate_compound_rows(1001)]
+    assert expected == contents
+
+
+def test_paginate_compound_keys_with_extra_filters(app_client):
+    fetched = []
+    path = (
+        "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects"
+    )
+    page = 0
+    while path:
+        page += 1
+        assert page < 100
+        response = app_client.get(path)
+        fetched.extend(response.json["rows"])
+        path = response.json["next_url"]
+        if path:
+            path = path.replace("http://localhost", "")
+    assert 2 == page
+    expected = [r[3] for r in generate_compound_rows(1001) if "d" in r[3]]
+    assert expected == [f["content"] for f in fetched]
+
+
+@pytest.mark.parametrize(
+    "query_string,sort_key,human_description_en",
+    [
+        ("_sort=sortable", lambda row: row["sortable"], "sorted by sortable"),
+        (
+            "_sort_desc=sortable",
+            lambda row: -row["sortable"],
+            "sorted by sortable descending",
+        ),
+        (
+            "_sort=sortable_with_nulls",
+            lambda row: (
+                1 if row["sortable_with_nulls"] is not None else 0,
+                row["sortable_with_nulls"],
+            ),
+            "sorted by sortable_with_nulls",
+        ),
+        (
+            "_sort_desc=sortable_with_nulls",
+            lambda row: (
+                1 if row["sortable_with_nulls"] is None else 0,
+                -row["sortable_with_nulls"]
+                if row["sortable_with_nulls"] is not None
+                else 0,
+                row["content"],
+            ),
+            "sorted by sortable_with_nulls descending",
+        ),
+        # text column contains '$null' - ensure it doesn't confuse pagination:
+        ("_sort=text", lambda row: row["text"], "sorted by text"),
+    ],
+)
+def test_sortable(app_client, query_string, sort_key, human_description_en):
+    path = f"/fixtures/sortable.json?_shape=objects&{query_string}"
+    fetched = []
+    page = 0
+    while path:
+        page += 1
+        assert page < 100
+        response = app_client.get(path)
+        assert human_description_en == response.json["human_description_en"]
+        fetched.extend(response.json["rows"])
+        path = response.json["next_url"]
+        if path:
+            path = path.replace("http://localhost", "")
+    assert 5 == page
+    expected = list(generate_sortable_rows(201))
+    expected.sort(key=sort_key)
+    assert [r["content"] for r in expected] == [r["content"] for r in fetched]
+
+
+def test_sortable_and_filtered(app_client):
+    path = (
+        "/fixtures/sortable.json"
+        "?content__contains=d&_sort_desc=sortable&_shape=objects"
+    )
+    response = app_client.get(path)
+    fetched = response.json["rows"]
+    assert (
+        'where content contains "d" sorted by sortable descending'
+        == response.json["human_description_en"]
+    )
+    expected = [row for row in generate_sortable_rows(201) if "d" in row["content"]]
+    assert len(expected) == response.json["filtered_table_rows_count"]
+    expected.sort(key=lambda row: -row["sortable"])
+    assert [r["content"] for r in expected] == [r["content"] for r in fetched]
+
+
+def test_sortable_argument_errors(app_client):
+    response = app_client.get("/fixtures/sortable.json?_sort=badcolumn")
+    assert "Cannot sort table by badcolumn" == response.json["error"]
+    response = app_client.get("/fixtures/sortable.json?_sort_desc=badcolumn2")
+    assert "Cannot sort table by badcolumn2" == response.json["error"]
+    response = app_client.get(
+        "/fixtures/sortable.json?_sort=sortable_with_nulls&_sort_desc=sortable"
+    )
+    assert "Cannot use _sort and _sort_desc at the same time" == response.json["error"]
+
+
+def test_sortable_columns_metadata(app_client):
+    response = app_client.get("/fixtures/sortable.json?_sort=content")
+    assert "Cannot sort table by content" == response.json["error"]
+    # no_primary_key has ALL sort options disabled
+    for column in ("content", "a", "b", "c"):
+        response = app_client.get(f"/fixtures/sortable.json?_sort={column}")
+        assert f"Cannot sort table by {column}" == response.json["error"]
+
+
+@pytest.mark.parametrize(
+    "path,expected_rows",
+    [
+        (
+            "/fixtures/searchable.json?_search=dog",
+            [
+                [1, "barry cat", "terry dog", "panther"],
+                [2, "terry dog", "sara weasel", "puma"],
+            ],
+        ),
+        (
+            # Special keyword shouldn't break FTS query
+            "/fixtures/searchable.json?_search=AND",
+            [],
+        ),
+        (
+            # Without _searchmode=raw this should return no results
+            "/fixtures/searchable.json?_search=te*+AND+do*",
+            [],
+        ),
+        (
+            # _searchmode=raw
+            "/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw",
+            [
+                [1, "barry cat", "terry dog", "panther"],
+                [2, "terry dog", "sara weasel", "puma"],
+            ],
+        ),
+        (
+            # _searchmode=raw combined with _search_COLUMN
+            "/fixtures/searchable.json?_search_text2=te*&_searchmode=raw",
+            [
+                [1, "barry cat", "terry dog", "panther"],
+            ],
+        ),
+        (
+            "/fixtures/searchable.json?_search=weasel",
+            [[2, "terry dog", "sara weasel", "puma"]],
+        ),
+        (
+            "/fixtures/searchable.json?_search_text2=dog",
+            [[1, "barry cat", "terry dog", "panther"]],
+        ),
+        (
+            "/fixtures/searchable.json?_search_name%20with%20.%20and%20spaces=panther",
+            [[1, "barry cat", "terry dog", "panther"]],
+        ),
+    ],
+)
+def test_searchable(app_client, path, expected_rows):
+    response = app_client.get(path)
+    assert expected_rows == response.json["rows"]
+
+
+_SEARCHMODE_RAW_RESULTS = [
+    [1, "barry cat", "terry dog", "panther"],
+    [2, "terry dog", "sara weasel", "puma"],
+]
+
+
+@pytest.mark.parametrize(
+    "table_metadata,querystring,expected_rows",
+    [
+        (
+            {},
+            "_search=te*+AND+do*",
+            [],
+        ),
+        (
+            {"searchmode": "raw"},
+            "_search=te*+AND+do*",
+            _SEARCHMODE_RAW_RESULTS,
+        ),
+        (
+            {},
+            "_search=te*+AND+do*&_searchmode=raw",
+            _SEARCHMODE_RAW_RESULTS,
+        ),
+        # Can be over-ridden with _searchmode=escaped
+        (
+            {"searchmode": "raw"},
+            "_search=te*+AND+do*&_searchmode=escaped",
+            [],
+        ),
+    ],
+)
+def test_searchmode(table_metadata, querystring, expected_rows):
+    with make_app_client(
+        metadata={"databases": {"fixtures": {"tables": {"searchable": table_metadata}}}}
+    ) as client:
+        response = client.get("/fixtures/searchable.json?" + querystring)
+        assert expected_rows == response.json["rows"]
+
+
+@pytest.mark.parametrize(
+    "path,expected_rows",
+    [
+        (
+            "/fixtures/searchable_view_configured_by_metadata.json?_search=weasel",
+            [[2, "terry dog", "sara weasel", "puma"]],
+        ),
+        # This should return all results because search is not configured:
+        (
+            "/fixtures/searchable_view.json?_search=weasel",
+            [
+                [1, "barry cat", "terry dog", "panther"],
+                [2, "terry dog", "sara weasel", "puma"],
+            ],
+        ),
+        (
+            "/fixtures/searchable_view.json?_search=weasel&_fts_table=searchable_fts&_fts_pk=pk",
+            [[2, "terry dog", "sara weasel", "puma"]],
+        ),
+    ],
+)
+def test_searchable_views(app_client, path, expected_rows):
+    response = app_client.get(path)
+    assert expected_rows == response.json["rows"]
+
+
+def test_searchable_invalid_column(app_client):
+    response = app_client.get("/fixtures/searchable.json?_search_invalid=x")
+    assert 400 == response.status
+    assert {
+        "ok": False,
+        "error": "Cannot search by that column",
+        "status": 400,
+        "title": None,
+    } == response.json
+
+
+@pytest.mark.parametrize(
+    "path,expected_rows",
+    [
+        ("/fixtures/simple_primary_key.json?content=hello", [["1", "hello"]]),
+        (
+            "/fixtures/simple_primary_key.json?content__contains=o",
+            [
+                ["1", "hello"],
+                ["2", "world"],
+                ["4", "RENDER_CELL_DEMO"],
+            ],
+        ),
+        ("/fixtures/simple_primary_key.json?content__exact=", [["3", ""]]),
+        (
+            "/fixtures/simple_primary_key.json?content__not=world",
+            [
+                ["1", "hello"],
+                ["3", ""],
+                ["4", "RENDER_CELL_DEMO"],
+                ["5", "RENDER_CELL_ASYNC"],
+            ],
+        ),
+    ],
+)
+def test_table_filter_queries(app_client, path, expected_rows):
+    response = app_client.get(path)
+    assert expected_rows == response.json["rows"]
+
+
+def test_table_filter_queries_multiple_of_same_type(app_client):
+    response = app_client.get(
+        "/fixtures/simple_primary_key.json?content__not=world&content__not=hello"
+    )
+    assert [
+        ["3", ""],
+        ["4", "RENDER_CELL_DEMO"],
+        ["5", "RENDER_CELL_ASYNC"],
+    ] == response.json["rows"]
+
+
+@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
+def test_table_filter_json_arraycontains(app_client):
+    response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1")
+    assert response.json["rows"] == [
+        [
+            1,
+            "2019-01-14 08:00:00",
+            1,
+            1,
+            "CA",
+            1,
+            "Mission",
+            '["tag1", "tag2"]',
+            '[{"foo": "bar"}]',
+            "one",
+        ],
+        [
+            2,
+            "2019-01-14 08:00:00",
+            1,
+            1,
+            "CA",
+            1,
+            "Dogpatch",
+            '["tag1", "tag3"]',
+            "[]",
+            "two",
+        ],
+    ]
+
+
+@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
+def test_table_filter_json_arraynotcontains(app_client):
+    response = app_client.get(
+        "/fixtures/facetable.json?tags__arraynotcontains=tag3&tags__not=[]"
+    )
+    assert response.json["rows"] == [
+        [
+            1,
+            "2019-01-14 08:00:00",
+            1,
+            1,
+            "CA",
+            1,
+            "Mission",
+            '["tag1", "tag2"]',
+            '[{"foo": "bar"}]',
+            "one",
+        ]
+    ]
+
+
+def test_table_filter_extra_where(app_client):
+    response = app_client.get(
+        "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
+    )
+    assert [
+        [
+            2,
+            "2019-01-14 08:00:00",
+            1,
+            1,
+            "CA",
+            1,
+            "Dogpatch",
+            '["tag1", "tag3"]',
+            "[]",
+            "two",
+        ]
+    ] == response.json["rows"]
+
+
+def test_table_filter_extra_where_invalid(app_client):
+    response = app_client.get("/fixtures/facetable.json?_where=_neighborhood=Dogpatch'")
+    assert 400 == response.status
+    assert "Invalid SQL" == response.json["title"]
+
+
+def test_table_filter_extra_where_disabled_if_no_sql_allowed():
+    with make_app_client(metadata={"allow_sql": {}}) as client:
+        response = client.get(
+            "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
+        )
+        assert 403 == response.status
+        assert "_where= is not allowed" == response.json["error"]
+
+
+def test_table_through(app_client):
+    # Just the museums:
+    response = app_client.get(
+        '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
+    )
+    assert [
+        [
+            3,
+            "Burlingame Museum of PEZ Memorabilia",
+            "214 California Drive, Burlingame, CA 94010",
+            37.5793,
+            -122.3442,
+        ],
+        [
+            4,
+            "Bigfoot Discovery Museum",
+            "5497 Highway 9, Felton, CA 95018",
+            37.0414,
+            -122.0725,
+        ],
+    ] == response.json["rows"]
+    assert (
+        'where roadside_attraction_characteristics.characteristic_id = "1"'
+        == response.json["human_description_en"]
+    )
+
+
+def test_max_returned_rows(app_client):
+    response = app_client.get("/fixtures.json?sql=select+content+from+no_primary_key")
+    data = response.json
+    assert {"sql": "select content from no_primary_key", "params": {}} == data["query"]
+    assert data["truncated"]
+    assert 100 == len(data["rows"])
+
+
+def test_view(app_client):
+    response = app_client.get("/fixtures/simple_view.json?_shape=objects")
+    assert response.status == 200
+    data = response.json
+    assert data["rows"] == [
+        {"upper_content": "HELLO", "content": "hello"},
+        {"upper_content": "WORLD", "content": "world"},
+        {"upper_content": "", "content": ""},
+        {"upper_content": "RENDER_CELL_DEMO", "content": "RENDER_CELL_DEMO"},
+        {"upper_content": "RENDER_CELL_ASYNC", "content": "RENDER_CELL_ASYNC"},
+    ]
+
+
+def test_unit_filters(app_client):
+    response = app_client.get(
+        "/fixtures/units.json?distance__lt=75km&frequency__gt=1kHz"
+    )
+    assert response.status == 200
+    data = response.json
+
+    assert data["units"]["distance"] == "m"
+    assert data["units"]["frequency"] == "Hz"
+
+    assert len(data["rows"]) == 1
+    assert data["rows"][0][0] == 2
+
+
+def test_page_size_matching_max_returned_rows(
+    app_client_returned_rows_matches_page_size,
+):
+    fetched = []
+    path = "/fixtures/no_primary_key.json"
+    while path:
+        response = app_client_returned_rows_matches_page_size.get(path)
+        fetched.extend(response.json["rows"])
+        assert len(response.json["rows"]) in (1, 50)
+        path = response.json["next_url"]
+        if path:
+            path = path.replace("http://localhost", "")
+    assert 201 == len(fetched)
+
+
+@pytest.mark.parametrize(
+    "path,expected_facet_results",
+    [
+        (
+            "/fixtures/facetable.json?_facet=state&_facet=_city_id",
+            {
+                "state": {
+                    "name": "state",
+                    "hideable": True,
+                    "type": "column",
+                    "toggle_url": "/fixtures/facetable.json?_facet=_city_id",
+                    "results": [
+                        {
+                            "value": "CA",
+                            "label": "CA",
+                            "count": 10,
+                            "toggle_url": "_facet=state&_facet=_city_id&state=CA",
+                            "selected": False,
+                        },
+                        {
+                            "value": "MI",
+                            "label": "MI",
+                            "count": 4,
+                            "toggle_url": "_facet=state&_facet=_city_id&state=MI",
+                            "selected": False,
+                        },
+                        {
+                            "value": "MC",
+                            "label": "MC",
+                            "count": 1,
+                            "toggle_url": "_facet=state&_facet=_city_id&state=MC",
+                            "selected": False,
+                        },
+                    ],
+                    "truncated": False,
+                },
+                "_city_id": {
+                    "name": "_city_id",
+                    "hideable": True,
+                    "type": "column",
+                    "toggle_url": "/fixtures/facetable.json?_facet=state",
+                    "results": [
+                        {
+                            "value": 1,
+                            "label": "San Francisco",
+                            "count": 6,
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=1",
+                            "selected": False,
+                        },
+                        {
+                            "value": 2,
+                            "label": "Los Angeles",
+                            "count": 4,
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=2",
+                            "selected": False,
+                        },
+                        {
+                            "value": 3,
+                            "label": "Detroit",
+                            "count": 4,
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=3",
+                            "selected": False,
+                        },
+                        {
+                            "value": 4,
+                            "label": "Memnonia",
+                            "count": 1,
+                            "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=4",
+                            "selected": False,
+                        },
+                    ],
+                    "truncated": False,
+                },
+            },
+        ),
+        (
+            "/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI",
+            {
+                "state": {
+                    "name": "state",
+                    "hideable": True,
+                    "type": "column",
+                    "toggle_url": "/fixtures/facetable.json?_facet=_city_id&state=MI",
+                    "results": [
+                        {
+                            "value": "MI",
+                            "label": "MI",
+                            "count": 4,
+                            "selected": True,
+                            "toggle_url": "_facet=state&_facet=_city_id",
+                        }
+                    ],
+                    "truncated": False,
+                },
+                "_city_id": {
+                    "name": "_city_id",
+                    "hideable": True,
+                    "type": "column",
+                    "toggle_url": "/fixtures/facetable.json?_facet=state&state=MI",
+                    "results": [
+                        {
+                            "value": 3,
+                            "label": "Detroit",
+                            "count": 4,
+                            "selected": False,
+                            "toggle_url": "_facet=state&_facet=_city_id&state=MI&_city_id__exact=3",
+                        }
+                    ],
+                    "truncated": False,
+                },
+            },
+        ),
+        (
+            "/fixtures/facetable.json?_facet=planet_int",
+            {
+                "planet_int": {
+                    "name": "planet_int",
+                    "hideable": True,
+                    "type": "column",
+                    "toggle_url": "/fixtures/facetable.json",
+                    "results": [
+                        {
+                            "value": 1,
+                            "label": 1,
+                            "count": 14,
+                            "selected": False,
+                            "toggle_url": "_facet=planet_int&planet_int=1",
+                        },
+                        {
+                            "value": 2,
+                            "label": 2,
+                            "count": 1,
+                            "selected": False,
+                            "toggle_url": "_facet=planet_int&planet_int=2",
+                        },
+                    ],
+                    "truncated": False,
+                }
+            },
+        ),
+        (
+            # planet_int is an integer field:
+            "/fixtures/facetable.json?_facet=planet_int&planet_int=1",
+            {
+                "planet_int": {
+                    "name": "planet_int",
+                    "hideable": True,
+                    "type": "column",
+                    "toggle_url": "/fixtures/facetable.json?planet_int=1",
+                    "results": [
+                        {
+                            "value": 1,
+                            "label": 1,
+                            "count": 14,
+                            "selected": True,
+                            "toggle_url": "_facet=planet_int",
+                        }
+                    ],
+                    "truncated": False,
+                }
+            },
+        ),
+    ],
+)
+def test_facets(app_client, path, expected_facet_results):
+    response = app_client.get(path)
+    facet_results = response.json["facet_results"]
+    # We only compare the querystring portion of the taggle_url
+    for facet_name, facet_info in facet_results.items():
+        assert facet_name == facet_info["name"]
+        assert False is facet_info["truncated"]
+        for facet_value in facet_info["results"]:
+            facet_value["toggle_url"] = facet_value["toggle_url"].split("?")[1]
+    assert expected_facet_results == facet_results
+
+
+def test_suggested_facets(app_client):
+    suggestions = [
+        {
+            "name": suggestion["name"],
+            "querystring": suggestion["toggle_url"].split("?")[-1],
+        }
+        for suggestion in app_client.get("/fixtures/facetable.json").json[
+            "suggested_facets"
+        ]
+    ]
+    expected = [
+        {"name": "created", "querystring": "_facet=created"},
+        {"name": "planet_int", "querystring": "_facet=planet_int"},
+        {"name": "on_earth", "querystring": "_facet=on_earth"},
+        {"name": "state", "querystring": "_facet=state"},
+        {"name": "_city_id", "querystring": "_facet=_city_id"},
+        {"name": "_neighborhood", "querystring": "_facet=_neighborhood"},
+        {"name": "tags", "querystring": "_facet=tags"},
+        {"name": "complex_array", "querystring": "_facet=complex_array"},
+        {"name": "created", "querystring": "_facet_date=created"},
+    ]
+    if detect_json1():
+        expected.append({"name": "tags", "querystring": "_facet_array=tags"})
+    assert expected == suggestions
+
+
+def test_allow_facet_off():
+    with make_app_client(settings={"allow_facet": False}) as client:
+        assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status
+        # Should not suggest any facets either:
+        assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
+
+
+def test_suggest_facets_off():
+    with make_app_client(settings={"suggest_facets": False}) as client:
+        # Now suggested_facets should be []
+        assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
+
+
+@pytest.mark.parametrize("nofacet", (True, False))
+def test_nofacet(app_client, nofacet):
+    path = "/fixtures/facetable.json?_facet=state"
+    if nofacet:
+        path += "&_nofacet=1"
+    response = app_client.get(path)
+    if nofacet:
+        assert response.json["suggested_facets"] == []
+        assert response.json["facet_results"] == {}
+    else:
+        assert response.json["suggested_facets"] != []
+        assert response.json["facet_results"] != {}
+
+
+@pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15)))
+def test_nocount(app_client, nocount, expected_count):
+    path = "/fixtures/facetable.json"
+    if nocount:
+        path += "?_nocount=1"
+    response = app_client.get(path)
+    assert response.json["filtered_table_rows_count"] == expected_count
+
+
+def test_nocount_nofacet_if_shape_is_object(app_client_with_trace):
+    response = app_client_with_trace.get(
+        "/fixtures/facetable.json?_trace=1&_shape=object"
+    )
+    assert "count(*)" not in response.text
+
+
+def test_expand_labels(app_client):
+    response = app_client.get(
+        "/fixtures/facetable.json?_shape=object&_labels=1&_size=2"
+        "&_neighborhood__contains=c"
+    )
+    assert {
+        "2": {
+            "pk": 2,
+            "created": "2019-01-14 08:00:00",
+            "planet_int": 1,
+            "on_earth": 1,
+            "state": "CA",
+            "_city_id": {"value": 1, "label": "San Francisco"},
+            "_neighborhood": "Dogpatch",
+            "tags": '["tag1", "tag3"]',
+            "complex_array": "[]",
+            "distinct_some_null": "two",
+        },
+        "13": {
+            "pk": 13,
+            "created": "2019-01-17 08:00:00",
+            "planet_int": 1,
+            "on_earth": 1,
+            "state": "MI",
+            "_city_id": {"value": 3, "label": "Detroit"},
+            "_neighborhood": "Corktown",
+            "tags": "[]",
+            "complex_array": "[]",
+            "distinct_some_null": None,
+        },
+    } == response.json
+
+
+def test_expand_label(app_client):
+    response = app_client.get(
+        "/fixtures/foreign_key_references.json?_shape=object"
+        "&_label=foreign_key_with_label&_size=1"
+    )
+    assert response.json == {
+        "1": {
+            "pk": "1",
+            "foreign_key_with_label": {"value": "1", "label": "hello"},
+            "foreign_key_with_blank_label": "3",
+            "foreign_key_with_no_label": "1",
+            "foreign_key_compound_pk1": "a",
+            "foreign_key_compound_pk2": "b",
+        }
+    }
+
+
+@pytest.mark.parametrize(
+    "path,expected_cache_control",
+    [
+        ("/fixtures/facetable.json", "max-age=5"),
+        ("/fixtures/facetable.json?_ttl=invalid", "max-age=5"),
+        ("/fixtures/facetable.json?_ttl=10", "max-age=10"),
+        ("/fixtures/facetable.json?_ttl=0", "no-cache"),
+    ],
+)
+def test_ttl_parameter(app_client, path, expected_cache_control):
+    response = app_client.get(path)
+    assert expected_cache_control == response.headers["Cache-Control"]
+
+
+def test_infinity_returned_as_null(app_client):
+    response = app_client.get("/fixtures/infinity.json?_shape=array")
+    assert [
+        {"rowid": 1, "value": None},
+        {"rowid": 2, "value": None},
+        {"rowid": 3, "value": 1.5},
+    ] == response.json
+
+
+def test_infinity_returned_as_invalid_json_if_requested(app_client):
+    response = app_client.get("/fixtures/infinity.json?_shape=array&_json_infinity=1")
+    assert [
+        {"rowid": 1, "value": float("inf")},
+        {"rowid": 2, "value": float("-inf")},
+        {"rowid": 3, "value": 1.5},
+    ] == response.json
+
+
+def test_custom_query_with_unicode_characters(app_client):
+    response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array")
+    assert [{"id": 1, "name": "San Francisco"}] == response.json
+
+
+def test_null_and_compound_foreign_keys_are_not_expanded(app_client):
+    response = app_client.get(
+        "/fixtures/foreign_key_references.json?_shape=array&_labels=on"
+    )
+    assert response.json == [
+        {
+            "pk": "1",
+            "foreign_key_with_label": {"value": "1", "label": "hello"},
+            "foreign_key_with_blank_label": {"value": "3", "label": ""},
+            "foreign_key_with_no_label": {"value": "1", "label": "1"},
+            "foreign_key_compound_pk1": "a",
+            "foreign_key_compound_pk2": "b",
+        },
+        {
+            "pk": "2",
+            "foreign_key_with_label": None,
+            "foreign_key_with_blank_label": None,
+            "foreign_key_with_no_label": None,
+            "foreign_key_compound_pk1": None,
+            "foreign_key_compound_pk2": None,
+        },
+    ]
+
+
+@pytest.mark.parametrize(
+    "path,expected_json,expected_text",
+    [
+        (
+            "/fixtures/binary_data.json?_shape=array",
+            [
+                {"rowid": 1, "data": {"$base64": True, "encoded": "FRwCx60F/g=="}},
+                {"rowid": 2, "data": {"$base64": True, "encoded": "FRwDx60F/g=="}},
+                {"rowid": 3, "data": None},
+            ],
+            None,
+        ),
+        (
+            "/fixtures/binary_data.json?_shape=array&_nl=on",
+            None,
+            (
+                '{"rowid": 1, "data": {"$base64": true, "encoded": "FRwCx60F/g=="}}\n'
+                '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}\n'
+                '{"rowid": 3, "data": null}'
+            ),
+        ),
+    ],
+)
+def test_binary_data_in_json(app_client, path, expected_json, expected_text):
+    response = app_client.get(path)
+    if expected_json:
+        assert response.json == expected_json
+    else:
+        assert response.text == expected_text
+
+
+@pytest.mark.parametrize(
+    "qs",
+    [
+        "",
+        "?_shape=arrays",
+        "?_shape=arrayfirst",
+        "?_shape=object",
+        "?_shape=objects",
+        "?_shape=array",
+        "?_shape=array&_nl=on",
+    ],
+)
+def test_paginate_using_link_header(app_client, qs):
+    path = f"/fixtures/compound_three_primary_keys.json{qs}"
+    num_pages = 0
+    while path:
+        response = app_client.get(path)
+        assert response.status == 200
+        num_pages += 1
+        link = response.headers.get("link")
+        if link:
+            assert link.startswith("<")
+            assert link.endswith('>; rel="next"')
+            path = link[1:].split(">")[0]
+            path = path.replace("http://localhost", "")
+        else:
+            path = None
+    assert num_pages == 21
+
+
+@pytest.mark.skipif(
+    sqlite_version() < (3, 31, 0),
+    reason="generated columns were added in SQLite 3.31.0",
+)
+def test_generated_columns_are_visible_in_datasette():
+    with make_app_client(
+        extra_databases={
+            "generated.db": """
+                CREATE TABLE generated_columns (
+                    body TEXT,
+                    id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED,
+                    consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED
+                );
+                INSERT INTO generated_columns (body) VALUES (
+                    '{"number": 1, "string": "This is a string"}'
+                );"""
+        }
+    ) as client:
+        response = client.get("/generated/generated_columns.json?_shape=array")
+        assert response.json == [
+            {
+                "rowid": 1,
+                "body": '{"number": 1, "string": "This is a string"}',
+                "id": 1,
+                "consideration": "This is a string",
+            }
+        ]
+
+
+@pytest.mark.parametrize(
+    "path,expected_columns",
+    (
+        ("/fixtures/facetable.json?_col=created", ["pk", "created"]),
+        (
+            "/fixtures/facetable.json?_nocol=created",
+            [
+                "pk",
+                "planet_int",
+                "on_earth",
+                "state",
+                "_city_id",
+                "_neighborhood",
+                "tags",
+                "complex_array",
+                "distinct_some_null",
+            ],
+        ),
+        (
+            "/fixtures/facetable.json?_col=state&_col=created",
+            ["pk", "state", "created"],
+        ),
+        (
+            "/fixtures/facetable.json?_col=state&_col=state",
+            ["pk", "state"],
+        ),
+        (
+            "/fixtures/facetable.json?_col=state&_col=created&_nocol=created",
+            ["pk", "state"],
+        ),
+        (
+            # Ensure faceting doesn't break, https://github.com/simonw/datasette/issues/1345
+            "/fixtures/facetable.json?_nocol=state&_facet=state",
+            [
+                "pk",
+                "created",
+                "planet_int",
+                "on_earth",
+                "_city_id",
+                "_neighborhood",
+                "tags",
+                "complex_array",
+                "distinct_some_null",
+            ],
+        ),
+        (
+            "/fixtures/simple_view.json?_nocol=content",
+            ["upper_content"],
+        ),
+        ("/fixtures/simple_view.json?_col=content", ["content"]),
+    ),
+)
+def test_col_nocol(app_client, path, expected_columns):
+    response = app_client.get(path)
+    assert response.status == 200
+    columns = response.json["columns"]
+    assert columns == expected_columns
+
+
+@pytest.mark.parametrize(
+    "path,expected_error",
+    (
+        ("/fixtures/facetable.json?_col=bad", "_col=bad - invalid columns"),
+        ("/fixtures/facetable.json?_nocol=bad", "_nocol=bad - invalid columns"),
+        ("/fixtures/facetable.json?_nocol=pk", "_nocol=pk - invalid columns"),
+        ("/fixtures/simple_view.json?_col=bad", "_col=bad - invalid columns"),
+    ),
+)
+def test_col_nocol_errors(app_client, path, expected_error):
+    response = app_client.get(path)
+    assert response.status == 400
+    assert response.json["error"] == expected_error

From a6ff123de5464806441f6a6f95145c9a83b7f20b Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 12 Dec 2021 12:01:51 -0800
Subject: [PATCH 1244/2303] keep_blank_values=True when parsing query_string,
 closes #1551

Refs #1518
---
 datasette/utils/asgi.py         |  2 +-
 datasette/views/table.py        | 11 +++--------
 tests/test_internals_request.py | 16 ++++++++++++++++
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index ad137fa9..cd3ec654 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -97,7 +97,7 @@ class Request:
 
     @property
     def args(self):
-        return MultiParams(parse_qs(qs=self.query_string))
+        return MultiParams(parse_qs(qs=self.query_string, keep_blank_values=True))
 
     @property
     def actor(self):
diff --git a/datasette/views/table.py b/datasette/views/table.py
index f58b78f5..59010723 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -393,21 +393,16 @@ class TableView(RowTableShared):
             nocount = True
             nofacet = True
 
-        # Ensure we don't drop anything with an empty value e.g. ?name__exact=
-        args = MultiParams(
-            urllib.parse.parse_qs(request.query_string, keep_blank_values=True)
-        )
-
         # Special args start with _ and do not contain a __
         # That's so if there is a column that starts with _
         # it can still be queried using ?_col__exact=blah
         special_args = {}
         other_args = []
-        for key in args:
+        for key in request.args:
             if key.startswith("_") and "__" not in key:
-                special_args[key] = args[key]
+                special_args[key] = request.args[key]
             else:
-                for v in args.getlist(key):
+                for v in request.args.getlist(key):
                     other_args.append((key, v))
 
         # Handle ?_filter_column and redirect, if present
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index cd956f3f..01c93eec 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -121,3 +121,19 @@ def test_request_properties(path, query_string, expected_full_path):
     assert request.path == path
     assert request.query_string == query_string
     assert request.full_path == expected_full_path
+
+
+def test_request_blank_values():
+    query_string = "a=b&foo=bar&foo=bar2&baz="
+    path_with_query_string = "/?" + query_string
+    scope = {
+        "http_version": "1.1",
+        "method": "POST",
+        "path": "/",
+        "raw_path": path_with_query_string.encode("latin-1"),
+        "query_string": query_string.encode("latin-1"),
+        "scheme": "http",
+        "type": "http",
+    }
+    request = Request(scope, None)
+    assert request.args._data == {"a": ["b"], "foo": ["bar", "bar2"], "baz": [""]}

From 8b411a6b70e93e044820d613a28607ba5d6fe416 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Dec 2021 15:22:21 -0800
Subject: [PATCH 1245/2303] Update pytest-xdist requirement from <2.5,>=2.2.1
 to >=2.2.1,<2.6 (#1548)

Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-xdist/releases)
- [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v2.2.1...v2.5.0)

---
updated-dependencies:
- dependency-name: pytest-xdist
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 9b5bab61..da8dea49 100644
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@ setup(
         "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
         "test": [
             "pytest>=5.2.2,<6.3.0",
-            "pytest-xdist>=2.2.1,<2.5",
+            "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.10,<0.17",
             "beautifulsoup4>=4.8.1,<4.11.0",
             "black==21.11b1",

From f5538e7161cce92a4dfaa7c5b71fcb6755d96c05 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Dec 2021 15:22:29 -0800
Subject: [PATCH 1246/2303] Bump black from 21.11b1 to 21.12b0 (#1543)

Bumps [black](https://github.com/psf/black) from 21.11b1 to 21.12b0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index da8dea49..534265c2 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.10,<0.17",
             "beautifulsoup4>=4.8.1,<4.11.0",
-            "black==21.11b1",
+            "black==21.12b0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
         ],

From 4f02c8d4d7f8672cc98e5f8d435b5dc8fb5211dc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 14 Dec 2021 12:28:34 -0800
Subject: [PATCH 1247/2303] Test for JSON in query_string name, refs #621

Plus simplified implementation of test_request_blank_values
---
 tests/test_internals_request.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index 01c93eec..44aaa153 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -124,16 +124,18 @@ def test_request_properties(path, query_string, expected_full_path):
 
 
 def test_request_blank_values():
-    query_string = "a=b&foo=bar&foo=bar2&baz="
-    path_with_query_string = "/?" + query_string
-    scope = {
-        "http_version": "1.1",
-        "method": "POST",
-        "path": "/",
-        "raw_path": path_with_query_string.encode("latin-1"),
-        "query_string": query_string.encode("latin-1"),
-        "scheme": "http",
-        "type": "http",
-    }
-    request = Request(scope, None)
+    request = Request.fake("/?a=b&foo=bar&foo=bar2&baz=")
     assert request.args._data == {"a": ["b"], "foo": ["bar", "bar2"], "baz": [""]}
+
+
+def test_json_in_query_string_name():
+    query_string = (
+        '?_through.["roadside_attraction_characteristics"%2C"characteristic_id"]=1'
+    )
+    request = Request.fake("/" + query_string)
+    assert (
+        request.args[
+            '_through.["roadside_attraction_characteristics","characteristic_id"]'
+        ]
+        == "1"
+    )

From eb53837d2aeacaffd8d37f81a6639139c6a0b4d4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 15 Dec 2021 09:58:01 -0800
Subject: [PATCH 1248/2303] Always show count of distinct facet values, closes
 #1556

Refs #1423
---
 datasette/templates/table.html |  2 +-
 datasette/views/table.py       |  1 -
 tests/test_table_html.py       | 13 ++++---------
 3 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index 4b9df8e1..f3749b57 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -157,7 +157,7 @@
             
      
                 
      
                     {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
-                        {% if show_facet_counts %} {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}{% endif %}
+                        {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}
                     
                     {% if facet_info.hideable %}
                         ✖
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 59010723..bb5876cc 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -937,7 +937,6 @@ class TableView(RowTableShared):
                     key=lambda f: (len(f["results"]), f["name"]),
                     reverse=True,
                 ),
-                "show_facet_counts": special_args.get("_facet_size") == "max",
                 "extra_wheres_for_ui": extra_wheres_for_ui,
                 "form_hidden_args": form_hidden_args,
                 "is_sortable": any(c["sortable"] for c in display_columns),
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 2fbb53bd..50d679a0 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1015,24 +1015,19 @@ def test_column_metadata(app_client):
     )
 
 
-@pytest.mark.parametrize("use_facet_size_max", (True, False))
-def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
+def test_facet_total():
     # https://github.com/simonw/datasette/issues/1423
+    # https://github.com/simonw/datasette/issues/1556
     with make_app_client(settings={"max_returned_rows": 100}) as client:
         path = "/fixtures/sortable?_facet=content&_facet=pk1"
-        if use_facet_size_max:
-            path += "&_facet_size=max"
         response = client.get(path)
         assert response.status == 200
     fragments = (
-        '>100',
+        '>30',
         '8',
     )
     for fragment in fragments:
-        if use_facet_size_max:
-            assert fragment in response.text
-        else:
-            assert fragment not in response.text
+        assert fragment in response.text
 
 
 def test_sort_rowid_with_next(app_client):

From 40e5b0a5b5cbbe7ec9b1a525d61f58227061597e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 10:03:10 -0800
Subject: [PATCH 1249/2303] How to create indexes with sqlite-utils

---
 docs/facets.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/facets.rst b/docs/facets.rst
index 7730e4ac..4bbfa16f 100644
--- a/docs/facets.rst
+++ b/docs/facets.rst
@@ -133,6 +133,10 @@ The performance of facets can be greatly improved by adding indexes on the colum
     Enter ".help" for usage hints.
     sqlite> CREATE INDEX Food_Trucks_state ON Food_Trucks("state");
 
+Or using the `sqlite-utils `__ command-line utility::
+
+    $ sqlite-utils create-index mydatabase.db Food_Trucks state
+
 .. _facet_by_json_array:
 
 Facet by JSON array

From 20a2ed6bec367d2f6759be4a879364a72780b59d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 10:47:22 -0800
Subject: [PATCH 1250/2303] Fixed bug with metadata config of array/date
 facets, closes #1552

Thanks @davidbgk for spotting the fix for the bug.
---
 datasette/facets.py  |  2 +-
 docs/facets.rst      | 20 ++++++++++++++++++--
 tests/test_facets.py | 25 ++++++++++++++++++++++++-
 3 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 8fd2177a..51fccb01 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -30,7 +30,7 @@ def load_facet_configs(request, table_metadata):
             assert (
                 len(metadata_config.values()) == 1
             ), "Metadata config dicts should be {type: config}"
-            type, metadata_config = metadata_config.items()[0]
+            type, metadata_config = list(metadata_config.items())[0]
             if isinstance(metadata_config, str):
                 metadata_config = {"simple": metadata_config}
         facet_configs.setdefault(type, []).append(
diff --git a/docs/facets.rst b/docs/facets.rst
index 4bbfa16f..0228aa84 100644
--- a/docs/facets.rst
+++ b/docs/facets.rst
@@ -16,7 +16,9 @@ To turn on faceting for specific columns on a Datasette table view, add one or m
 
     /dbname/tablename?_facet=state&_facet=city_id
 
-This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this::
+This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this:
+
+.. code-block:: json
 
     {
       "state": {
@@ -93,7 +95,9 @@ Facets in metadata.json
 
 You can turn facets on by default for specific tables by adding them to a ``"facets"`` key in a Datasette :ref:`metadata` file.
 
-Here's an example that turns on faceting by default for the ``qLegalStatus`` column in the ``Street_Tree_List`` table in the ``sf-trees`` database::
+Here's an example that turns on faceting by default for the ``qLegalStatus`` column in the ``Street_Tree_List`` table in the ``sf-trees`` database:
+
+.. code-block:: json
 
     {
       "databases": {
@@ -109,6 +113,18 @@ Here's an example that turns on faceting by default for the ``qLegalStatus`` col
 
 Facets defined in this way will always be shown in the interface and returned in the API, regardless of the ``_facet`` arguments passed to the view.
 
+You can specify :ref:`array ` or :ref:`date ` facets in metadata using JSON objects with a single key of ``array`` or ``date`` and a value specifying the column, like this:
+
+.. code-block:: json
+
+  {
+    "facets": [
+      {"array": "tags"},
+      {"date": "created"}
+    ]
+  }
+
+
 Suggested facets
 ----------------
 
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 429117cb..5b1aa935 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -3,7 +3,7 @@ from datasette.database import Database
 from datasette.facets import ColumnFacet, ArrayFacet, DateFacet
 from datasette.utils.asgi import Request
 from datasette.utils import detect_json1
-from .fixtures import app_client  # noqa
+from .fixtures import app_client, make_app_client  # noqa
 import json
 import pytest
 
@@ -588,3 +588,26 @@ async def test_facet_size():
     )
     data5 = response5.json()
     assert len(data5["facet_results"]["city"]["results"]) == 20
+
+
+def test_other_types_of_facet_in_metadata():
+    with make_app_client(
+        metadata={
+            "databases": {
+                "fixtures": {
+                    "tables": {
+                        "facetable": {
+                            "facets": ["state", {"array": "tags"}, {"date": "created"}]
+                        }
+                    }
+                }
+            }
+        }
+    ) as client:
+        response = client.get("/fixtures/facetable")
+        for fragment in (
+            "created (date)\n",
+            "tags (array)\n",
+            "state\n",
+        ):
+            assert fragment in response.text

From 992496f2611a72bd51e94bfd0b17c1d84e732487 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 11:24:54 -0800
Subject: [PATCH 1251/2303] ?_nosuggest=1 parameter for table views, closes
 #1557

---
 datasette/views/table.py |  2 ++
 docs/json_api.rst        |  3 +++
 tests/test_table_api.py  | 15 +++++++++++++++
 3 files changed, 20 insertions(+)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index bb5876cc..f294ffb1 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -388,6 +388,7 @@ class TableView(RowTableShared):
 
         nocount = request.args.get("_nocount")
         nofacet = request.args.get("_nofacet")
+        nosuggest = request.args.get("_nosuggest")
 
         if request.args.get("_shape") in ("array", "object"):
             nocount = True
@@ -846,6 +847,7 @@ class TableView(RowTableShared):
             and self.ds.setting("allow_facet")
             and not _next
             and not nofacet
+            and not nosuggest
         ):
             for facet in facet_instances:
                 suggested_facets.extend(await facet.suggest())
diff --git a/docs/json_api.rst b/docs/json_api.rst
index 7d3123b7..bd55c163 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -397,6 +397,9 @@ Special table arguments
 ``?_nofacet=1``
     Disable all facets and facet suggestions for this page, including any defined by :ref:`facets_metadata`.
 
+``?_nosuggest=1``
+    Disable facet suggestions for this page.
+
 ``?_nocount=1``
     Disable the ``select count(*)`` query used on this page - a count of ``None`` will be returned instead.
 
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index a530de44..6a6daed5 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -915,6 +915,21 @@ def test_nofacet(app_client, nofacet):
         assert response.json["facet_results"] != {}
 
 
+@pytest.mark.parametrize("nosuggest", (True, False))
+def test_nosuggest(app_client, nosuggest):
+    path = "/fixtures/facetable.json?_facet=state"
+    if nosuggest:
+        path += "&_nosuggest=1"
+    response = app_client.get(path)
+    if nosuggest:
+        assert response.json["suggested_facets"] == []
+        # But facets should still be returned:
+        assert response.json["facet_results"] != {}
+    else:
+        assert response.json["suggested_facets"] != []
+        assert response.json["facet_results"] != {}
+
+
 @pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15)))
 def test_nocount(app_client, nocount, expected_count):
     path = "/fixtures/facetable.json"

From 95d0dd7a1cf6be6b7da41e1404184217eb93f64a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 12:12:04 -0800
Subject: [PATCH 1252/2303] Fix for colliding facet types bug, closes #625

Refs #830
---
 datasette/facets.py      | 74 ++++++++++++++++++++++------------------
 datasette/views/table.py |  9 ++++-
 docs/plugin_hooks.rst    |  6 ++--
 tests/test_facets.py     | 30 ++++++++--------
 4 files changed, 67 insertions(+), 52 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 51fccb01..a1bb4a5f 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -193,7 +193,7 @@ class ColumnFacet(Facet):
         return suggested_facets
 
     async def facet_results(self):
-        facet_results = {}
+        facet_results = []
         facets_timed_out = []
 
         qs_pairs = self.get_querystring_pairs()
@@ -221,16 +221,18 @@ class ColumnFacet(Facet):
                     custom_time_limit=self.ds.setting("facet_time_limit_ms"),
                 )
                 facet_results_values = []
-                facet_results[column] = {
-                    "name": column,
-                    "type": self.type,
-                    "hideable": source != "metadata",
-                    "toggle_url": self.ds.urls.path(
-                        path_with_removed_args(self.request, {"_facet": column})
-                    ),
-                    "results": facet_results_values,
-                    "truncated": len(facet_rows_results) > facet_size,
-                }
+                facet_results.append(
+                    {
+                        "name": column,
+                        "type": self.type,
+                        "hideable": source != "metadata",
+                        "toggle_url": self.ds.urls.path(
+                            path_with_removed_args(self.request, {"_facet": column})
+                        ),
+                        "results": facet_results_values,
+                        "truncated": len(facet_rows_results) > facet_size,
+                    }
+                )
                 facet_rows = facet_rows_results.rows[:facet_size]
                 if self.table:
                     # Attempt to expand foreign keys into labels
@@ -352,7 +354,7 @@ class ArrayFacet(Facet):
 
     async def facet_results(self):
         # self.configs should be a plain list of columns
-        facet_results = {}
+        facet_results = []
         facets_timed_out = []
 
         facet_size = self.get_facet_size()
@@ -392,16 +394,20 @@ class ArrayFacet(Facet):
                     custom_time_limit=self.ds.setting("facet_time_limit_ms"),
                 )
                 facet_results_values = []
-                facet_results[column] = {
-                    "name": column,
-                    "type": self.type,
-                    "results": facet_results_values,
-                    "hideable": source != "metadata",
-                    "toggle_url": self.ds.urls.path(
-                        path_with_removed_args(self.request, {"_facet_array": column})
-                    ),
-                    "truncated": len(facet_rows_results) > facet_size,
-                }
+                facet_results.append(
+                    {
+                        "name": column,
+                        "type": self.type,
+                        "results": facet_results_values,
+                        "hideable": source != "metadata",
+                        "toggle_url": self.ds.urls.path(
+                            path_with_removed_args(
+                                self.request, {"_facet_array": column}
+                            )
+                        ),
+                        "truncated": len(facet_rows_results) > facet_size,
+                    }
+                )
                 facet_rows = facet_rows_results.rows[:facet_size]
                 pairs = self.get_querystring_pairs()
                 for row in facet_rows:
@@ -480,7 +486,7 @@ class DateFacet(Facet):
         return suggested_facets
 
     async def facet_results(self):
-        facet_results = {}
+        facet_results = []
         facets_timed_out = []
         args = dict(self.get_querystring_pairs())
         facet_size = self.get_facet_size()
@@ -507,16 +513,18 @@ class DateFacet(Facet):
                     custom_time_limit=self.ds.setting("facet_time_limit_ms"),
                 )
                 facet_results_values = []
-                facet_results[column] = {
-                    "name": column,
-                    "type": self.type,
-                    "results": facet_results_values,
-                    "hideable": source != "metadata",
-                    "toggle_url": path_with_removed_args(
-                        self.request, {"_facet_date": column}
-                    ),
-                    "truncated": len(facet_rows_results) > facet_size,
-                }
+                facet_results.append(
+                    {
+                        "name": column,
+                        "type": self.type,
+                        "results": facet_results_values,
+                        "hideable": source != "metadata",
+                        "toggle_url": path_with_removed_args(
+                            self.request, {"_facet_date": column}
+                        ),
+                        "truncated": len(facet_rows_results) > facet_size,
+                    }
+                )
                 facet_rows = facet_rows_results.rows[:facet_size]
                 for row in facet_rows:
                     selected = str(args.get(f"{column}__date")) == str(row["value"])
diff --git a/datasette/views/table.py b/datasette/views/table.py
index f294ffb1..3d0e27cb 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -754,7 +754,14 @@ class TableView(RowTableShared):
                     instance_facet_results,
                     instance_facets_timed_out,
                 ) = await facet.facet_results()
-                facet_results.update(instance_facet_results)
+                for facet_info in instance_facet_results:
+                    base_key = facet_info["name"]
+                    key = base_key
+                    i = 1
+                    while key in facet_results:
+                        i += 1
+                        key = f"{base_key}_{i}"
+                    facet_results[key] = facet_info
                 facets_timed_out.extend(instance_facets_timed_out)
 
         # Figure out columns and rows for the query
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 23f19e38..4a7c36c3 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -668,7 +668,7 @@ Each Facet subclass implements a new type of facet operation. The class should l
         async def facet_results(self):
             # This should execute the facet operation and return results, again
             # using self.sql and self.params as the starting point
-            facet_results = {}
+            facet_results = []
             facets_timed_out = []
             facet_size = self.get_facet_size()
             # Do some calculations here...
@@ -683,11 +683,11 @@ Each Facet subclass implements a new type of facet operation. The class should l
                         "toggle_url": self.ds.absolute_url(self.request, toggle_path),
                         "selected": selected,
                     })
-                    facet_results[column] = {
+                    facet_results.append({
                         "name": column,
                         "results": facet_results_values,
                         "truncated": len(facet_rows_results) > facet_size,
-                    }
+                    })
                 except QueryInterrupted:
                     facets_timed_out.append(column)
 
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 5b1aa935..a99979d3 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -107,8 +107,8 @@ async def test_column_facet_results(app_client):
     )
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
-    assert {
-        "_city_id": {
+    assert [
+        {
             "name": "_city_id",
             "type": "column",
             "hideable": True,
@@ -145,7 +145,7 @@ async def test_column_facet_results(app_client):
             ],
             "truncated": False,
         }
-    } == buckets
+    ] == buckets
 
 
 @pytest.mark.asyncio
@@ -159,8 +159,8 @@ async def test_column_facet_results_column_starts_with_underscore(app_client):
     )
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
-    assert buckets == {
-        "_neighborhood": {
+    assert buckets == [
+        {
             "name": "_neighborhood",
             "type": "column",
             "hideable": True,
@@ -267,7 +267,7 @@ async def test_column_facet_results_column_starts_with_underscore(app_client):
             ],
             "truncated": False,
         }
-    }
+    ]
 
 
 @pytest.mark.asyncio
@@ -282,8 +282,8 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client):
     )
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
-    assert {
-        "_city_id": {
+    assert [
+        {
             "name": "_city_id",
             "type": "column",
             "hideable": False,
@@ -320,7 +320,7 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client):
             ],
             "truncated": False,
         }
-    } == buckets
+    ] == buckets
 
 
 @pytest.mark.asyncio
@@ -369,8 +369,8 @@ async def test_array_facet_results(app_client):
     )
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
-    assert {
-        "tags": {
+    assert [
+        {
             "name": "tags",
             "type": "array",
             "results": [
@@ -400,7 +400,7 @@ async def test_array_facet_results(app_client):
             "toggle_url": "/",
             "truncated": False,
         }
-    } == buckets
+    ] == buckets
 
 
 @pytest.mark.asyncio
@@ -471,8 +471,8 @@ async def test_date_facet_results(app_client):
     )
     buckets, timed_out = await facet.facet_results()
     assert [] == timed_out
-    assert {
-        "created": {
+    assert [
+        {
             "name": "created",
             "type": "date",
             "results": [
@@ -509,7 +509,7 @@ async def test_date_facet_results(app_client):
             "toggle_url": "/",
             "truncated": False,
         }
-    } == buckets
+    ] == buckets
 
 
 @pytest.mark.asyncio

From 0d4145d0f4d8b2a7edc1ba4aac1be56cd536a10a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 12:30:31 -0800
Subject: [PATCH 1253/2303] Additional test for #625

---
 tests/test_facets.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_facets.py b/tests/test_facets.py
index a99979d3..3f292a3b 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -611,3 +611,16 @@ def test_other_types_of_facet_in_metadata():
             "state\n",
         ):
             assert fragment in response.text
+
+
+def test_conflicting_facet_names_json(app_client):
+    response = app_client.get(
+        "/fixtures/facetable.json?_facet=created&_facet_date=created"
+        "&_facet=tags&_facet_array=tags"
+    )
+    assert set(response.json["facet_results"].keys()) == {
+        "created",
+        "tags",
+        "created_2",
+        "tags_2",
+    }

From 2c07327d23d9c5cf939ada9ba4091c1b8b2ba42d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 13:43:44 -0800
Subject: [PATCH 1254/2303] Move columns_to_select to TableView class, add lots
 of comments, refs #1518

---
 datasette/views/table.py | 117 +++++++++++++++++++++------------------
 1 file changed, 63 insertions(+), 54 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 3d0e27cb..d37a3066 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -64,41 +64,6 @@ class Row:
 
 
 class RowTableShared(DataView):
-    async def columns_to_select(self, db, table, request):
-        table_columns = await db.table_columns(table)
-        pks = await db.primary_keys(table)
-        columns = list(table_columns)
-        if "_col" in request.args:
-            columns = list(pks)
-            _cols = request.args.getlist("_col")
-            bad_columns = [column for column in _cols if column not in table_columns]
-            if bad_columns:
-                raise DatasetteError(
-                    "_col={} - invalid columns".format(", ".join(bad_columns)),
-                    status=400,
-                )
-            # De-duplicate maintaining order:
-            columns.extend(dict.fromkeys(_cols))
-        if "_nocol" in request.args:
-            # Return all columns EXCEPT these
-            bad_columns = [
-                column
-                for column in request.args.getlist("_nocol")
-                if (column not in table_columns) or (column in pks)
-            ]
-            if bad_columns:
-                raise DatasetteError(
-                    "_nocol={} - invalid columns".format(", ".join(bad_columns)),
-                    status=400,
-                )
-            tmp_columns = [
-                column
-                for column in columns
-                if column not in request.args.getlist("_nocol")
-            ]
-            columns = tmp_columns
-        return columns
-
     async def sortable_columns_for_table(self, database, table, use_rowid):
         db = self.ds.databases[database]
         table_metadata = self.ds.table_metadata(database, table)
@@ -321,6 +286,39 @@ class TableView(RowTableShared):
             write=bool(canned_query.get("write")),
         )
 
+    async def columns_to_select(self, table_columns, pks, request):
+        columns = list(table_columns)
+        if "_col" in request.args:
+            columns = list(pks)
+            _cols = request.args.getlist("_col")
+            bad_columns = [column for column in _cols if column not in table_columns]
+            if bad_columns:
+                raise DatasetteError(
+                    "_col={} - invalid columns".format(", ".join(bad_columns)),
+                    status=400,
+                )
+            # De-duplicate maintaining order:
+            columns.extend(dict.fromkeys(_cols))
+        if "_nocol" in request.args:
+            # Return all columns EXCEPT these
+            bad_columns = [
+                column
+                for column in request.args.getlist("_nocol")
+                if (column not in table_columns) or (column in pks)
+            ]
+            if bad_columns:
+                raise DatasetteError(
+                    "_nocol={} - invalid columns".format(", ".join(bad_columns)),
+                    status=400,
+                )
+            tmp_columns = [
+                column
+                for column in columns
+                if column not in request.args.getlist("_nocol")
+            ]
+            columns = tmp_columns
+        return columns
+
     async def data(
         self,
         request,
@@ -331,6 +329,7 @@ class TableView(RowTableShared):
         _next=None,
         _size=None,
     ):
+        # If this is a canned query, not a table, then dispatch to QueryView instead
         canned_query = await self.ds.get_canned_query(database, table, request.actor)
         if canned_query:
             return await QueryView(self.ds).data(
@@ -348,9 +347,12 @@ class TableView(RowTableShared):
         db = self.ds.databases[database]
         is_view = bool(await db.get_view_definition(table))
         table_exists = bool(await db.table_exists(table))
+
+        # If table or view not found, return 404
         if not is_view and not table_exists:
             raise NotFound(f"Table not found: {table}")
 
+        # Ensure user has permission to view this table
         await self.check_permissions(
             request,
             [
@@ -364,15 +366,18 @@ class TableView(RowTableShared):
             None, "view-table", (database, table), default=True
         )
 
+        # Introspect columns and primary keys for table
         pks = await db.primary_keys(table)
         table_columns = await db.table_columns(table)
 
-        specified_columns = await self.columns_to_select(db, table, request)
+        # Take ?_col= and ?_nocol= into account
+        specified_columns = await self.columns_to_select(table_columns, pks, request)
         select_specified_columns = ", ".join(
             escape_sqlite(t) for t in specified_columns
         )
         select_all_columns = ", ".join(escape_sqlite(t) for t in table_columns)
 
+        # rowid tables (no specified primary key) need a different SELECT
         use_rowid = not pks and not is_view
         if use_rowid:
             select_specified_columns = f"rowid, {select_specified_columns}"
@@ -487,7 +492,7 @@ class TableView(RowTableShared):
                     f'{through_table}.{other_column} = "{value}"'
                 )
 
-        # _search support:
+        # _search= support:
         fts_table = special_args.get("_fts_table")
         fts_table = fts_table or table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
@@ -541,8 +546,6 @@ class TableView(RowTableShared):
                     )
                     params[f"search_{i}"] = search_text
 
-        sortable_columns = set()
-
         sortable_columns = await self.sortable_columns_for_table(
             database, table, use_rowid
         )
@@ -581,6 +584,7 @@ class TableView(RowTableShared):
 
         count_sql = f"select count(*) {from_sql}"
 
+        # Handl pagination driven by ?_next=
         _next = _next or special_args.get("_next")
         offset = ""
         if _next:
@@ -679,6 +683,7 @@ class TableView(RowTableShared):
         else:
             page_size = self.ds.page_size
 
+        # Facets are calculated against SQL without order by or limit
         sql_no_order_no_limit = (
             "select {select_all_columns} from {table_name} {where}".format(
                 select_all_columns=select_all_columns,
@@ -686,6 +691,8 @@ class TableView(RowTableShared):
                 where=where_clause,
             )
         )
+
+        # This is the SQL that populates the main table on the page
         sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format(
             select_specified_columns=select_specified_columns,
             table_name=escape_sqlite(table),
@@ -698,15 +705,17 @@ class TableView(RowTableShared):
         if request.args.get("_timelimit"):
             extra_args["custom_time_limit"] = int(request.args.get("_timelimit"))
 
+        # Execute the main query!
         results = await db.execute(sql, params, truncate=True, **extra_args)
 
-        # Number of filtered rows in whole set:
+        # Calculate the total count for this query
         filtered_table_rows_count = None
         if (
             not db.is_mutable
             and self.ds.inspect_data
             and count_sql == f"select count(*) from {table} "
         ):
+            # We can use a previously cached table row count
             try:
                 filtered_table_rows_count = self.ds.inspect_data[database]["tables"][
                     table
@@ -714,6 +723,7 @@ class TableView(RowTableShared):
             except KeyError:
                 pass
 
+        # Otherwise run a select count(*) ...
         if count_sql and filtered_table_rows_count is None and not nocount:
             try:
                 count_rows = list(await db.execute(count_sql, from_sql_params))
@@ -721,7 +731,7 @@ class TableView(RowTableShared):
             except QueryInterrupted:
                 pass
 
-        # facets support
+        # Faceting
         if not self.ds.setting("allow_facet") and any(
             arg.startswith("_facet") for arg in request.args
         ):
@@ -764,6 +774,18 @@ class TableView(RowTableShared):
                     facet_results[key] = facet_info
                 facets_timed_out.extend(instance_facets_timed_out)
 
+        # Calculate suggested facets
+        suggested_facets = []
+        if (
+            self.ds.setting("suggest_facets")
+            and self.ds.setting("allow_facet")
+            and not _next
+            and not nofacet
+            and not nosuggest
+        ):
+            for facet in facet_instances:
+                suggested_facets.extend(await facet.suggest())
+
         # Figure out columns and rows for the query
         columns = [r[0] for r in results.description]
         rows = list(results.rows)
@@ -846,19 +868,6 @@ class TableView(RowTableShared):
             )
             rows = rows[:page_size]
 
-        # Detect suggested facets
-        suggested_facets = []
-
-        if (
-            self.ds.setting("suggest_facets")
-            and self.ds.setting("allow_facet")
-            and not _next
-            and not nofacet
-            and not nosuggest
-        ):
-            for facet in facet_instances:
-                suggested_facets.extend(await facet.suggest())
-
         # human_description_en combines filters AND search, if provided
         human_description_en = filters.human_description_en(
             extra=extra_human_descriptions

From 0663d5525cc41e9260ac7d1f6386d3a6eb5ad2a9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 16 Dec 2021 14:00:29 -0800
Subject: [PATCH 1255/2303] More comments in TableView.data(), refs #1518

---
 datasette/views/table.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index d37a3066..da263966 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -437,6 +437,8 @@ class TableView(RowTableShared):
 
         table_metadata = self.ds.table_metadata(database, table)
         units = table_metadata.get("units", {})
+
+        # Build where clauses from query string arguments
         filters = Filters(sorted(other_args), units, ureg)
         where_clauses, params = filters.build_where_clauses(table)
 
@@ -584,7 +586,7 @@ class TableView(RowTableShared):
 
         count_sql = f"select count(*) {from_sql}"
 
-        # Handl pagination driven by ?_next=
+        # Handle pagination driven by ?_next=
         _next = _next or special_args.get("_next")
         offset = ""
         if _next:

From aa7f0037a46eb76ae6fe9bf2a1f616c58738ecdf Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 17 Dec 2021 11:02:14 -0800
Subject: [PATCH 1256/2303] filters_from_request plugin hook, now used in
 TableView

- New `filters_from_request` plugin hook, closes #473
- Used it to extract the logic from TableView that handles `_search` and
`_through` and `_where` - refs #1518

Also needed for this plugin work: https://github.com/simonw/datasette-leaflet-freedraw/issues/7
---
 datasette/filters.py     | 167 ++++++++++++++++++++++++++++++++++++++-
 datasette/hookspecs.py   |  11 +++
 datasette/plugins.py     |   1 +
 datasette/views/table.py | 127 +++++------------------------
 docs/plugin_hooks.rst    |  53 +++++++++++++
 tests/test_filters.py    |  87 +++++++++++++++++++-
 tests/test_plugins.py    |  18 +++++
 7 files changed, 353 insertions(+), 111 deletions(-)

diff --git a/datasette/filters.py b/datasette/filters.py
index cbd94415..5ea3488b 100644
--- a/datasette/filters.py
+++ b/datasette/filters.py
@@ -1,7 +1,172 @@
+from datasette import hookimpl
+from datasette.views.base import DatasetteError
+from datasette.utils.asgi import BadRequest
 import json
 import numbers
+from .utils import detect_json1, escape_sqlite, path_with_removed_args
 
-from .utils import detect_json1, escape_sqlite
+
+@hookimpl(specname="filters_from_request")
+def where_filters(request, database, datasette):
+    # This one deals with ?_where=
+    async def inner():
+        where_clauses = []
+        extra_wheres_for_ui = []
+        if "_where" in request.args:
+            if not await datasette.permission_allowed(
+                request.actor,
+                "execute-sql",
+                resource=database,
+                default=True,
+            ):
+                raise DatasetteError("_where= is not allowed", status=403)
+            else:
+                where_clauses.extend(request.args.getlist("_where"))
+                extra_wheres_for_ui = [
+                    {
+                        "text": text,
+                        "remove_url": path_with_removed_args(request, {"_where": text}),
+                    }
+                    for text in request.args.getlist("_where")
+                ]
+
+        return FilterArguments(
+            where_clauses,
+            extra_context={
+                "extra_wheres_for_ui": extra_wheres_for_ui,
+            },
+        )
+
+    return inner
+
+
+@hookimpl(specname="filters_from_request")
+def search_filters(request, database, table, datasette):
+    # ?_search= and _search_colname=
+    async def inner():
+        where_clauses = []
+        params = {}
+        human_descriptions = []
+        extra_context = {}
+
+        # Figure out which fts_table to use
+        table_metadata = datasette.table_metadata(database, table)
+        db = datasette.get_database(database)
+        fts_table = request.args.get("_fts_table")
+        fts_table = fts_table or table_metadata.get("fts_table")
+        fts_table = fts_table or await db.fts_table(table)
+        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        search_args = {
+            key: request.args[key]
+            for key in request.args
+            if key.startswith("_search") and key != "_searchmode"
+        }
+        search = ""
+        search_mode_raw = table_metadata.get("searchmode") == "raw"
+        # Or set search mode from the querystring
+        qs_searchmode = request.args.get("_searchmode")
+        if qs_searchmode == "escaped":
+            search_mode_raw = False
+        if qs_searchmode == "raw":
+            search_mode_raw = True
+
+        extra_context["supports_search"] = bool(fts_table)
+
+        if fts_table and search_args:
+            if "_search" in search_args:
+                # Simple ?_search=xxx
+                search = search_args["_search"]
+                where_clauses.append(
+                    "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format(
+                        fts_table=escape_sqlite(fts_table),
+                        fts_pk=escape_sqlite(fts_pk),
+                        match_clause=":search"
+                        if search_mode_raw
+                        else "escape_fts(:search)",
+                    )
+                )
+                human_descriptions.append(f'search matches "{search}"')
+                params["search"] = search
+                extra_context["search"] = search
+            else:
+                # More complex: search against specific columns
+                for i, (key, search_text) in enumerate(search_args.items()):
+                    search_col = key.split("_search_", 1)[1]
+                    if search_col not in await db.table_columns(fts_table):
+                        raise BadRequest("Cannot search by that column")
+
+                    where_clauses.append(
+                        "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format(
+                            fts_table=escape_sqlite(fts_table),
+                            search_col=escape_sqlite(search_col),
+                            match_clause=":search_{}".format(i)
+                            if search_mode_raw
+                            else "escape_fts(:search_{})".format(i),
+                        )
+                    )
+                    human_descriptions.append(
+                        f'search column "{search_col}" matches "{search_text}"'
+                    )
+                    params[f"search_{i}"] = search_text
+                    extra_context["search"] = search_text
+
+        return FilterArguments(where_clauses, params, human_descriptions, extra_context)
+
+    return inner
+
+
+@hookimpl(specname="filters_from_request")
+def through_filters(request, database, table, datasette):
+    # ?_search= and _search_colname=
+    async def inner():
+        where_clauses = []
+        params = {}
+        human_descriptions = []
+        extra_context = {}
+
+        # Support for ?_through={table, column, value}
+        if "_through" in request.args:
+            for through in request.args.getlist("_through"):
+                through_data = json.loads(through)
+                through_table = through_data["table"]
+                other_column = through_data["column"]
+                value = through_data["value"]
+                db = datasette.get_database(database)
+                outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
+                try:
+                    fk_to_us = [
+                        fk for fk in outgoing_foreign_keys if fk["other_table"] == table
+                    ][0]
+                except IndexError:
+                    raise DatasetteError(
+                        "Invalid _through - could not find corresponding foreign key"
+                    )
+                param = f"p{len(params)}"
+                where_clauses.append(
+                    "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format(
+                        through_table=escape_sqlite(through_table),
+                        our_pk=escape_sqlite(fk_to_us["other_column"]),
+                        our_column=escape_sqlite(fk_to_us["column"]),
+                        other_column=escape_sqlite(other_column),
+                        param=param,
+                    )
+                )
+                params[param] = value
+                human_descriptions.append(f'{through_table}.{other_column} = "{value}"')
+
+        return FilterArguments(where_clauses, params, human_descriptions, extra_context)
+
+    return inner
+
+
+class FilterArguments:
+    def __init__(
+        self, where_clauses, params=None, human_descriptions=None, extra_context=None
+    ):
+        self.where_clauses = where_clauses
+        self.params = params or {}
+        self.human_descriptions = human_descriptions or []
+        self.extra_context = extra_context or {}
 
 
 class Filter:
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 1d4e3b27..8f4fecab 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -89,6 +89,17 @@ def actor_from_request(datasette, request):
     """Return an actor dictionary based on the incoming request"""
 
 
+@hookspec
+def filters_from_request(request, database, table, datasette):
+    """
+    Return datasette.filters.FilterArguments(
+        where_clauses=[str, str, str],
+        params={},
+        human_descriptions=[str, str, str],
+        extra_context={}
+    ) based on the request"""
+
+
 @hookspec
 def permission_allowed(datasette, actor, action, resource):
     """Check if actor is allowed to perform this action - return True, False or None"""
diff --git a/datasette/plugins.py b/datasette/plugins.py
index 50791988..76b46a47 100644
--- a/datasette/plugins.py
+++ b/datasette/plugins.py
@@ -8,6 +8,7 @@ DEFAULT_PLUGINS = (
     "datasette.publish.heroku",
     "datasette.publish.cloudrun",
     "datasette.facets",
+    "datasette.filters",
     "datasette.sql_functions",
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
diff --git a/datasette/views/table.py b/datasette/views/table.py
index da263966..cfd31bd3 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -442,117 +442,27 @@ class TableView(RowTableShared):
         filters = Filters(sorted(other_args), units, ureg)
         where_clauses, params = filters.build_where_clauses(table)
 
-        extra_wheres_for_ui = []
-        # Add _where= from querystring
-        if "_where" in request.args:
-            if not await self.ds.permission_allowed(
-                request.actor,
-                "execute-sql",
-                resource=database,
-                default=True,
-            ):
-                raise DatasetteError("_where= is not allowed", status=403)
-            else:
-                where_clauses.extend(request.args.getlist("_where"))
-                extra_wheres_for_ui = [
-                    {
-                        "text": text,
-                        "remove_url": path_with_removed_args(request, {"_where": text}),
-                    }
-                    for text in request.args.getlist("_where")
-                ]
-
-        # Support for ?_through={table, column, value}
+        # Execute filters_from_request plugin hooks
+        extra_context_from_filters = {}
         extra_human_descriptions = []
-        if "_through" in request.args:
-            for through in request.args.getlist("_through"):
-                through_data = json.loads(through)
-                through_table = through_data["table"]
-                other_column = through_data["column"]
-                value = through_data["value"]
-                outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
-                try:
-                    fk_to_us = [
-                        fk for fk in outgoing_foreign_keys if fk["other_table"] == table
-                    ][0]
-                except IndexError:
-                    raise DatasetteError(
-                        "Invalid _through - could not find corresponding foreign key"
-                    )
-                param = f"p{len(params)}"
-                where_clauses.append(
-                    "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format(
-                        through_table=escape_sqlite(through_table),
-                        our_pk=escape_sqlite(fk_to_us["other_column"]),
-                        our_column=escape_sqlite(fk_to_us["column"]),
-                        other_column=escape_sqlite(other_column),
-                        param=param,
-                    )
-                )
-                params[param] = value
-                extra_human_descriptions.append(
-                    f'{through_table}.{other_column} = "{value}"'
-                )
 
-        # _search= support:
-        fts_table = special_args.get("_fts_table")
-        fts_table = fts_table or table_metadata.get("fts_table")
-        fts_table = fts_table or await db.fts_table(table)
-        fts_pk = special_args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
-        search_args = dict(
-            pair
-            for pair in special_args.items()
-            if pair[0].startswith("_search") and pair[0] != "_searchmode"
-        )
-        search = ""
-        search_mode_raw = table_metadata.get("searchmode") == "raw"
-        # Or set it from the querystring
-        qs_searchmode = special_args.get("_searchmode")
-        if qs_searchmode == "escaped":
-            search_mode_raw = False
-        if qs_searchmode == "raw":
-            search_mode_raw = True
-        if fts_table and search_args:
-            if "_search" in search_args:
-                # Simple ?_search=xxx
-                search = search_args["_search"]
-                where_clauses.append(
-                    "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format(
-                        fts_table=escape_sqlite(fts_table),
-                        fts_pk=escape_sqlite(fts_pk),
-                        match_clause=":search"
-                        if search_mode_raw
-                        else "escape_fts(:search)",
-                    )
-                )
-                extra_human_descriptions.append(f'search matches "{search}"')
-                params["search"] = search
-            else:
-                # More complex: search against specific columns
-                for i, (key, search_text) in enumerate(search_args.items()):
-                    search_col = key.split("_search_", 1)[1]
-                    if search_col not in await db.table_columns(fts_table):
-                        raise BadRequest("Cannot search by that column")
-
-                    where_clauses.append(
-                        "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format(
-                            fts_table=escape_sqlite(fts_table),
-                            search_col=escape_sqlite(search_col),
-                            match_clause=":search_{}".format(i)
-                            if search_mode_raw
-                            else "escape_fts(:search_{})".format(i),
-                        )
-                    )
-                    extra_human_descriptions.append(
-                        f'search column "{search_col}" matches "{search_text}"'
-                    )
-                    params[f"search_{i}"] = search_text
+        for hook in pm.hook.filters_from_request(
+            request=request,
+            table=table,
+            database=database,
+            datasette=self.ds,
+        ):
+            filter_arguments = await await_me_maybe(hook)
+            if filter_arguments:
+                where_clauses.extend(filter_arguments.where_clauses)
+                params.update(filter_arguments.params)
+                extra_human_descriptions.extend(filter_arguments.human_descriptions)
+                extra_context_from_filters.update(filter_arguments.extra_context)
 
+        # Deal with custom sort orders
         sortable_columns = await self.sortable_columns_for_table(
             database, table, use_rowid
         )
-
-        # Allow for custom sort order
         sort = special_args.get("_sort")
         sort_desc = special_args.get("_sort_desc")
 
@@ -942,10 +852,8 @@ class TableView(RowTableShared):
                 for table_column in table_columns
                 if table_column not in columns
             ]
-            return {
+            d = {
                 "table_actions": table_actions,
-                "supports_search": bool(fts_table),
-                "search": search or "",
                 "use_rowid": use_rowid,
                 "filters": filters,
                 "display_columns": display_columns,
@@ -957,7 +865,6 @@ class TableView(RowTableShared):
                     key=lambda f: (len(f["results"]), f["name"]),
                     reverse=True,
                 ),
-                "extra_wheres_for_ui": extra_wheres_for_ui,
                 "form_hidden_args": form_hidden_args,
                 "is_sortable": any(c["sortable"] for c in display_columns),
                 "fix_path": self.ds.urls.path,
@@ -977,6 +884,8 @@ class TableView(RowTableShared):
                 "view_definition": await db.get_view_definition(table),
                 "table_definition": await db.get_table_definition(table),
             }
+            d.update(extra_context_from_filters)
+            return d
 
         return (
             {
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 4a7c36c3..d76f70e5 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -923,6 +923,59 @@ Instead of returning a dictionary, this function can return an awaitable functio
 
 Example: `datasette-auth-tokens `_
 
+.. _plugin_hook_filters_from_request:
+
+filters_from_request(request, database, table, datasette)
+---------------------------------------------------------
+
+``request`` - object
+    The current HTTP :ref:`internals_request`.
+
+``database`` - string
+    The name of the database.
+
+``table`` - string
+    The name of the table.
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
+
+This hook runs on the :ref:`table ` page, and can influence the ``where`` clause of the SQL query used to populate that page, based on query string arguments on the incoming request.
+
+The hook should return an instance of ``datasette.filters.FilterArguments`` which has one required and three optional arguments:
+
+.. code-block:: python
+
+    return FilterArguments(
+        where_clauses=["id > :max_id"],
+        params={"max_id": 5},
+        human_descriptions=["max_id is greater than 5"],
+        extra_context={}
+    )
+
+The arguments to the ``FilterArguments`` class constructor are as follows:
+
+``where_clauses`` - list of strings, required
+    A list of SQL fragments that will be inserted into the SQL query, joined by the ``and`` operator. These can include ``:named`` parameters which will be populated using data in ``params``.
+``params`` - dictionary, optional
+    Additional keyword arguments to be used when the query is executed. These should match any ``:arguments`` in the where clauses.
+``human_descriptions`` - list of strings, optional
+    These strings will be included in the human-readable description at the top of the page and the page ````.
+``extra_context`` - dictionary, optional
+    Additional context variables that should be made available to the ``table.html`` template when it is rendered.
+
+This example plugin causes 0 results to be returned if ``?_nothing=1`` is added to the URL:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette.filters import FilterArguments
+
+    @hookimpl
+    def filters_from_request(self, request):
+        if request.args.get("_nothing"):
+            return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"])
+
 .. _plugin_hook_permission_allowed:
 
 permission_allowed(datasette, actor, action, resource)
diff --git a/tests/test_filters.py b/tests/test_filters.py
index d05ae80f..2ff57489 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -1,4 +1,6 @@
-from datasette.filters import Filters
+from datasette.filters import Filters, through_filters, where_filters, search_filters
+from datasette.utils.asgi import Request
+from .fixtures import app_client
 import pytest
 
 
@@ -74,3 +76,86 @@ def test_build_where(args, expected_where, expected_params):
     sql_bits, actual_params = f.build_where_clauses("table")
     assert expected_where == sql_bits
     assert {f"p{i}": param for i, param in enumerate(expected_params)} == actual_params
+
+
+@pytest.mark.asyncio
+async def test_through_filters_from_request(app_client):
+    request = Request.fake(
+        '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
+    )
+    filter_args = await (
+        through_filters(
+            request=request,
+            datasette=app_client.ds,
+            table="roadside_attractions",
+            database="fixtures",
+        )
+    )()
+    assert filter_args.where_clauses == [
+        "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)"
+    ]
+    assert filter_args.params == {"p0": "1"}
+    assert filter_args.human_descriptions == [
+        'roadside_attraction_characteristics.characteristic_id = "1"'
+    ]
+    assert filter_args.extra_context == {}
+
+
+@pytest.mark.asyncio
+async def test_through_filters_from_request(app_client):
+    request = Request.fake(
+        '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
+    )
+    filter_args = await (
+        through_filters(
+            request=request,
+            datasette=app_client.ds,
+            table="roadside_attractions",
+            database="fixtures",
+        )
+    )()
+    assert filter_args.where_clauses == [
+        "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)"
+    ]
+    assert filter_args.params == {"p0": "1"}
+    assert filter_args.human_descriptions == [
+        'roadside_attraction_characteristics.characteristic_id = "1"'
+    ]
+    assert filter_args.extra_context == {}
+
+
+@pytest.mark.asyncio
+async def test_where_filters_from_request(app_client):
+    request = Request.fake("/?_where=pk+>+3")
+    filter_args = await (
+        where_filters(
+            request=request,
+            datasette=app_client.ds,
+            database="fixtures",
+        )
+    )()
+    assert filter_args.where_clauses == ["pk > 3"]
+    assert filter_args.params == {}
+    assert filter_args.human_descriptions == []
+    assert filter_args.extra_context == {
+        "extra_wheres_for_ui": [{"text": "pk > 3", "remove_url": "/"}]
+    }
+
+
+@pytest.mark.asyncio
+async def test_search_filters_from_request(app_client):
+    request = Request.fake("/?_search=bobcat")
+    filter_args = await (
+        search_filters(
+            request=request,
+            datasette=app_client.ds,
+            database="fixtures",
+            table="searchable",
+        )
+    )()
+    assert filter_args.where_clauses == [
+        "rowid in (select rowid from searchable_fts where searchable_fts match escape_fts(:search))"
+    ]
+    assert filter_args.params == {"search": "bobcat"}
+    assert filter_args.human_descriptions == ['search matches "bobcat"']
+    assert filter_args.extra_context == {"supports_search": True, "search": "bobcat"}
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 1da28453..656f39e4 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -9,6 +9,7 @@ from .fixtures import (
 from click.testing import CliRunner
 from datasette.app import Datasette
 from datasette import cli, hookimpl
+from datasette.filters import FilterArguments
 from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
 from datasette.utils.sqlite import sqlite3
 from datasette.utils import CustomRow
@@ -977,3 +978,20 @@ def test_hook_register_commands():
     }
     pm.unregister(name="verify")
     importlib.reload(cli)
+
+
+def test_hook_filters_from_request(app_client):
+    class ReturnNothingPlugin:
+        __name__ = "ReturnNothingPlugin"
+
+        @hookimpl
+        def filters_from_request(self, request):
+            if request.args.get("_nothing"):
+                return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"])
+
+    pm.register(ReturnNothingPlugin(), name="ReturnNothingPlugin")
+    response = app_client.get("/fixtures/facetable?_nothing=1")
+    assert "0 rows\n        where NOTHING" in response.text
+    json_response = app_client.get("/fixtures/facetable.json?_nothing=1")
+    assert json_response.json["rows"] == []
+    pm.unregister(name="ReturnNothingPlugin")

From 92a5280d2e75c39424a75ad6226fc74400ae984f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 11:13:02 -0800
Subject: [PATCH 1257/2303] Release 0.60a0

Refs #473, #625, #1544, #1551, #1552, #1556, #1557
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 9c85b763..2fce006c 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59.4"
+__version__ = "0.60a0"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9ddc2794..92a9d941 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,19 @@
 Changelog
 =========
 
+.. _v0_60a0:
+
+0.60a0 (2021-12-17)
+-------------------
+
+- New plugin hook: :ref:`plugin_hook_filters_from_request`, which runs on the table page and can be used to support new custom query string parameters that modify the SQL query. (:issue:`473`)
+- The number of unique values in a facet is now always displayed. Previously it was only displayed if the user specified ``?_facet_size=max``. (:issue:`1556`)
+- Fixed bug where ``?_facet_array=tags&_facet=tags`` would only display one of the two selected facets. (:issue:`625`)
+- Facets of type ``date`` or ``array`` can now be configured in ``metadata.json``, see :ref:`facets_metadata`. Thanks, David Larlet. (:issue:`1552`)
+- New ``?_nosuggest=1`` parameter for table views, which disables facet suggestion. (:issue:`1557`)
+- Label columns detected for foreign keys are now case-insensitive, so ``Name`` or ``TITLE`` will be detected in the same way as ``name`` or ``title``. (:issue:`1544`)
+- The query string variables exposed by ``request.args`` will now include blank strings for arguments such as ``foo`` in ``?foo=&bar=1`` rather than ignoring those parameters entirely. (:issue:`1551`)
+
 .. _v0_59_4:
 
 0.59.4 (2021-11-29)

From f000a7bd75ac512478070f2e2a09c8fb9604c82d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 12:15:29 -0800
Subject: [PATCH 1258/2303] Use load_extension(?) instead of fstring

---
 datasette/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index 28268e42..715506bd 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -553,7 +553,7 @@ class Datasette:
         if self.sqlite_extensions:
             conn.enable_load_extension(True)
             for extension in self.sqlite_extensions:
-                conn.execute(f"SELECT load_extension('{extension}')")
+                conn.execute("SELECT load_extension(?)", [extension])
         if self.setting("cache_size_kb"):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member

From 35cba9e85a574cebf2986b64107fa84d02bd86ad Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 17 Dec 2021 15:08:28 -0800
Subject: [PATCH 1259/2303] Update janus requirement from <0.8,>=0.6.2 to
 >=0.6.2,<1.1 (#1562)

Updates the requirements on [janus](https://github.com/aio-libs/janus) to permit the latest version.
- [Release notes](https://github.com/aio-libs/janus/releases)
- [Changelog](https://github.com/aio-libs/janus/blob/master/CHANGES.rst)
- [Commits](https://github.com/aio-libs/janus/compare/v0.6.2...v1.0.0)

---
updated-dependencies:
- dependency-name: janus
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 534265c2..f8cd3e5b 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ setup(
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.9",
-        "janus>=0.6.2,<0.8",
+        "janus>=0.6.2,<1.1",
         "asgi-csrf>=0.9",
         "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",

From d0f24f9bbc596873f261ed4e0267c4aa5a0bac2b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 15:28:26 -0800
Subject: [PATCH 1260/2303] Clarifying comment

The new filters stuff is a little bit action-at-a-distance
---
 datasette/views/table.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index cfd31bd3..c3bcf01d 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -442,7 +442,8 @@ class TableView(RowTableShared):
         filters = Filters(sorted(other_args), units, ureg)
         where_clauses, params = filters.build_where_clauses(table)
 
-        # Execute filters_from_request plugin hooks
+        # Execute filters_from_request plugin hooks - including the default
+        # ones that live in datasette/filters.py
         extra_context_from_filters = {}
         extra_human_descriptions = []
 

From 0c91e59d2bbfc08884cfcf5d1b902a2f4968b7ff Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 15:28:44 -0800
Subject: [PATCH 1261/2303] datasette-leaflet-freedraw is an example of
 filters_from_request

---
 docs/plugin_hooks.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index d76f70e5..cbaf4c54 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -976,6 +976,8 @@ This example plugin causes 0 results to be returned if ``?_nothing=1`` is added
         if request.args.get("_nothing"):
             return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"])
 
+Example: `datasette-leaflet-freedraw <https://datasette.io/plugins/datasette-leaflet-freedraw>`_
+
 .. _plugin_hook_permission_allowed:
 
 permission_allowed(datasette, actor, action, resource)

From c35b84a2aabe2f14aeacf6cda4110ae1e94d6059 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 17:54:39 -0800
Subject: [PATCH 1262/2303] Remove undocumented sqlite_functions mechanism,
 closes #1567

---
 datasette/app.py                    | 3 ---
 tests/fixtures.py                   | 1 -
 tests/plugins/sleep_sql_function.py | 7 +++++++
 3 files changed, 7 insertions(+), 4 deletions(-)
 create mode 100644 tests/plugins/sleep_sql_function.py

diff --git a/datasette/app.py b/datasette/app.py
index 715506bd..d94cd5a2 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -259,7 +259,6 @@ class Datasette:
             with metadata_files[0].open() as fp:
                 metadata = parse_metadata(fp.read())
         self._metadata_local = metadata or {}
-        self.sqlite_functions = []
         self.sqlite_extensions = []
         for extension in sqlite_extensions or []:
             # Resolve spatialite, if requested
@@ -548,8 +547,6 @@ class Datasette:
     def _prepare_connection(self, conn, database):
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
-        for name, num_args, func in self.sqlite_functions:
-            conn.create_function(name, num_args, func)
         if self.sqlite_extensions:
             conn.enable_load_extension(True)
             for extension in self.sqlite_extensions:
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 37399da0..76f794c6 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -153,7 +153,6 @@ def make_app_client(
             template_dir=template_dir,
             crossdb=crossdb,
         )
-        ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n))))
         yield TestClient(ds)
 
 
diff --git a/tests/plugins/sleep_sql_function.py b/tests/plugins/sleep_sql_function.py
new file mode 100644
index 00000000..d4b32a09
--- /dev/null
+++ b/tests/plugins/sleep_sql_function.py
@@ -0,0 +1,7 @@
+from datasette import hookimpl
+import time
+
+
+@hookimpl
+def prepare_connection(conn):
+    conn.create_function("sleep", 1, lambda n: time.sleep(float(n)))

From 83bacfa9452babe7bd66e3579e23af988d00f6ac Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 17:58:39 -0800
Subject: [PATCH 1263/2303] Call _prepare_connection() on write connections,
 closes #1564

---
 datasette/database.py            | 1 +
 tests/test_internals_database.py | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/datasette/database.py b/datasette/database.py
index d1217e18..0a0c104a 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -128,6 +128,7 @@ class Database:
         conn = None
         try:
             conn = self.connect(write=True)
+            self.ds._prepare_connection(conn, self.name)
         except Exception as e:
             conn_exception = e
         while True:
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index a00fe447..609caabf 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -396,6 +396,12 @@ async def test_execute_write_block_false(db):
     assert "Mystery!" == rows.rows[0][0]
 
 
+@pytest.mark.asyncio
+async def test_execute_write_has_correctly_prepared_connection(db):
+    # The sleep() function is only available if ds._prepare_connection() was called
+    await db.execute_write("select sleep(0.01)", block=True)
+
+
 @pytest.mark.asyncio
 async def test_execute_write_fn_block_false(db):
     def write_fn(conn):

From 359140cedaf69242d6356479fb8a9d3aa591e618 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 18:09:00 -0800
Subject: [PATCH 1264/2303] Datasette() constructor no longer requires files=,
 closes #1563

---
 datasette/app.py                  |  4 ++--
 tests/test_internals_datasette.py | 17 +++++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index d94cd5a2..17fa06a5 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -191,7 +191,7 @@ class Datasette:
 
     def __init__(
         self,
-        files,
+        files=None,
         immutables=None,
         cache_headers=True,
         cors=False,
@@ -214,7 +214,7 @@ class Datasette:
         ), "config_dir= should be a pathlib.Path"
         self.pdb = pdb
         self._secret = secret or secrets.token_hex(32)
-        self.files = tuple(files) + tuple(immutables or [])
+        self.files = tuple(files or []) + tuple(immutables or [])
         if config_dir:
             self.files += tuple([str(p) for p in config_dir.glob("*.db")])
         if (
diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py
index 56bc2fb4..692312a7 100644
--- a/tests/test_internals_datasette.py
+++ b/tests/test_internals_datasette.py
@@ -1,6 +1,7 @@
 """
 Tests for the datasette.app.Datasette class
 """
+from datasette.app import Datasette
 from itsdangerous import BadSignature
 from .fixtures import app_client
 import pytest
@@ -45,3 +46,19 @@ def test_sign_unsign(datasette, value, namespace):
 )
 def test_datasette_setting(datasette, setting, expected):
     assert datasette.setting(setting) == expected
+
+
+@pytest.mark.asyncio
+async def test_datasette_constructor():
+    ds = Datasette(memory=True)
+    databases = (await ds.client.get("/-/databases.json")).json()
+    assert databases == [
+        {
+            "name": "_memory",
+            "path": None,
+            "size": 0,
+            "is_mutable": False,
+            "is_memory": True,
+            "hash": None,
+        }
+    ]

From 3a0cae4d7f77b5c2a103ea74ca7fa7a0d9ff2e66 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 18:19:09 -0800
Subject: [PATCH 1265/2303] Fix bug introduced by refactor in
 c35b84a2aabe2f14aeacf6cda4110ae1e94d6059

---
 tests/fixtures.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/fixtures.py b/tests/fixtures.py
index 76f794c6..26f0cf7b 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -83,6 +83,13 @@ EXPECTED_PLUGINS = [
         "version": None,
         "hooks": ["register_output_renderer"],
     },
+    {
+        "name": "sleep_sql_function.py",
+        "static": False,
+        "templates": False,
+        "version": None,
+        "hooks": ["prepare_connection"],
+    },
     {
         "name": "view_name.py",
         "static": False,

From 7c8f8aa209e4ba7bf83976f8495d67c28fbfca24 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 18:19:36 -0800
Subject: [PATCH 1266/2303] Documentation for Datasette() constructor, closes
 #1563

---
 docs/internals.rst                | 21 +++++++++++++++++++++
 tests/test_internals_datasette.py |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index 411327eb..c706031b 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -196,6 +196,27 @@ Datasette class
 
 This object is an instance of the ``Datasette`` class, passed to many plugin hooks as an argument called ``datasette``.
 
+You can create your own instance of this - for example to help write tests for a plugin - like so:
+
+.. code-block:: python
+
+    from datasette.app import Datasette
+
+    # With no arguments a single in-memory database will be attached
+    datasette = Datasette()
+
+    # The files= argument can load files from disk
+    datasette = Datasette(files="/path/to/my-database.db")
+
+    # Pass metadata as a JSON dictionary like this
+    datasette = Datasette(files="/path/to/my-database.db", metadata={
+        "databases": {
+            "my-database": {
+                "description": "This is my database"
+            }
+        }
+    })
+
 .. _datasette_databases:
 
 .databases
diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py
index 692312a7..adf84be9 100644
--- a/tests/test_internals_datasette.py
+++ b/tests/test_internals_datasette.py
@@ -50,7 +50,7 @@ def test_datasette_setting(datasette, setting, expected):
 
 @pytest.mark.asyncio
 async def test_datasette_constructor():
-    ds = Datasette(memory=True)
+    ds = Datasette()
     databases = (await ds.client.get("/-/databases.json")).json()
     assert databases == [
         {

From f81d9d0cd9f567e73a1a54be34b653db8ae2c1cf Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 17 Dec 2021 18:42:29 -0800
Subject: [PATCH 1267/2303] Trace write SQL queries in addition to read ones,
 closes #1568

---
 datasette/database.py |  4 +++-
 tests/test_api.py     | 13 +++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/datasette/database.py b/datasette/database.py
index 0a0c104a..468e9360 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -99,7 +99,9 @@ class Database:
             with conn:
                 return conn.execute(sql, params or [])
 
-        return await self.execute_write_fn(_inner, block=block)
+        with trace("sql", database=self.name, sql=sql.strip(), params=params):
+            results = await self.execute_write_fn(_inner, block=block)
+        return results
 
     async def execute_write_fn(self, fn, block=False):
         task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
diff --git a/tests/test_api.py b/tests/test_api.py
index df9e0fc4..9ad7d569 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -939,6 +939,19 @@ def test_trace(trace_debug):
         assert isinstance(trace["sql"], str)
         assert isinstance(trace["params"], (list, dict, None.__class__))
 
+    sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace]
+    # There should be a mix of different types of SQL statement
+    expected = (
+        "CREATE TABLE ",
+        "PRAGMA ",
+        "INSERT OR REPLACE INTO ",
+        "DELETE FROM ",
+        "INSERT INTO",
+        "select ",
+    )
+    for prefix in expected:
+        assert any(sql.startswith(prefix) for sql in sqls)
+
 
 @pytest.mark.parametrize(
     "path,status_code",

From 85c22f4fbccb7b35fbc16d3ef035ca71b1a5a20a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 10:10:37 -0800
Subject: [PATCH 1268/2303] Corrected Datasette(files=) example from #1563

---
 docs/internals.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index c706031b..8788b26a 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -206,10 +206,10 @@ You can create your own instance of this - for example to help write tests for a
     datasette = Datasette()
 
     # The files= argument can load files from disk
-    datasette = Datasette(files="/path/to/my-database.db")
+    datasette = Datasette(files=["/path/to/my-database.db"])
 
     # Pass metadata as a JSON dictionary like this
-    datasette = Datasette(files="/path/to/my-database.db", metadata={
+    datasette = Datasette(files=["/path/to/my-database.db"], metadata={
         "databases": {
             "my-database": {
                 "description": "This is my database"

From 9e094b7c9d575320a2f0c956eb547bfcf6d64d39 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 10:28:25 -0800
Subject: [PATCH 1269/2303] db.execute_write(executescript=True) option, closes
 #1569

---
 datasette/database.py            |  8 ++++++--
 docs/internals.rst               |  6 ++++--
 tests/test_internals_database.py | 21 +++++++++++++++++++++
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 468e9360..350c4e9c 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -94,10 +94,14 @@ class Database:
             f"file:{self.path}{qs}", uri=True, check_same_thread=False
         )
 
-    async def execute_write(self, sql, params=None, block=False):
+    async def execute_write(self, sql, params=None, executescript=False, block=False):
+        assert not (executescript and params), "Cannot use params with executescript=True"
         def _inner(conn):
             with conn:
-                return conn.execute(sql, params or [])
+                if executescript:
+                    return conn.executescript(sql)
+                else:
+                    return conn.execute(sql, params or [])
 
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
             results = await self.execute_write_fn(_inner, block=block)
diff --git a/docs/internals.rst b/docs/internals.rst
index 8788b26a..d40e679b 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -663,8 +663,8 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=False)
------------------------------------------------------
+await db.execute_write(sql, params=None, executescript=False, block=False)
+--------------------------------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -676,6 +676,8 @@ By default queries are considered to be "fire and forget" - they will be added t
 
 If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library.
 
+If you pass ``executescript=True`` your SQL will be executed using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method. This allows multiple SQL statements to be separated by semicolons, but cannot be used with the ``params=`` option.
+
 .. _database_execute_write_fn:
 
 await db.execute_write_fn(fn, block=False)
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 609caabf..0a5c01a3 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -396,6 +396,27 @@ async def test_execute_write_block_false(db):
     assert "Mystery!" == rows.rows[0][0]
 
 
+@pytest.mark.asyncio
+async def test_execute_write_executescript(db):
+    await db.execute_write(
+        "create table foo (id integer primary key); create table bar (id integer primary key); ",
+        executescript=True,
+        block=True
+    )
+    table_names = await db.table_names()
+    assert {"foo", "bar"}.issubset(table_names)
+
+
+@pytest.mark.asyncio
+async def test_execute_write_executescript_not_allowed_with_params(db):
+    with pytest.raises(AssertionError):
+        await db.execute_write(
+            "update roadside_attractions set name = ? where pk = ?",
+            ["Mystery!", 1],
+            executescript=True
+        )
+
+
 @pytest.mark.asyncio
 async def test_execute_write_has_correctly_prepared_connection(db):
     # The sleep() function is only available if ds._prepare_connection() was called

From 2e4ba71b53a45a7d2273afd30e400002c7f39755 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 10:30:53 -0800
Subject: [PATCH 1270/2303] Optimize create table calls using
 executescript=True

Refs #1555, #1569
---
 datasette/database.py            |  5 +++-
 datasette/utils/internal_db.py   | 48 ++++++--------------------------
 tests/test_api.py                |  4 ++-
 tests/test_internals_database.py |  4 +--
 4 files changed, 18 insertions(+), 43 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 350c4e9c..f8365f5c 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -95,7 +95,10 @@ class Database:
         )
 
     async def execute_write(self, sql, params=None, executescript=False, block=False):
-        assert not (executescript and params), "Cannot use params with executescript=True"
+        assert not (
+            executescript and params
+        ), "Cannot use params with executescript=True"
+
         def _inner(conn):
             with conn:
                 if executescript:
diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 40fe719e..80babff8 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -2,22 +2,14 @@ import textwrap
 
 
 async def init_internal_db(db):
-    await db.execute_write(
-        textwrap.dedent(
-            """
+    create_tables_sql = textwrap.dedent(
+        """
     CREATE TABLE IF NOT EXISTS databases (
         database_name TEXT PRIMARY KEY,
         path TEXT,
         is_memory INTEGER,
         schema_version INTEGER
-    )
-    """
-        ),
-        block=True,
-    )
-    await db.execute_write(
-        textwrap.dedent(
-            """
+    );
     CREATE TABLE IF NOT EXISTS tables (
         database_name TEXT,
         table_name TEXT,
@@ -25,14 +17,7 @@ async def init_internal_db(db):
         sql TEXT,
         PRIMARY KEY (database_name, table_name),
         FOREIGN KEY (database_name) REFERENCES databases(database_name)
-    )
-    """
-        ),
-        block=True,
-    )
-    await db.execute_write(
-        textwrap.dedent(
-            """
+    );
     CREATE TABLE IF NOT EXISTS columns (
         database_name TEXT,
         table_name TEXT,
@@ -46,14 +31,7 @@ async def init_internal_db(db):
         PRIMARY KEY (database_name, table_name, name),
         FOREIGN KEY (database_name) REFERENCES databases(database_name),
         FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
-    )
-    """
-        ),
-        block=True,
-    )
-    await db.execute_write(
-        textwrap.dedent(
-            """
+    );
     CREATE TABLE IF NOT EXISTS indexes (
         database_name TEXT,
         table_name TEXT,
@@ -65,14 +43,7 @@ async def init_internal_db(db):
         PRIMARY KEY (database_name, table_name, name),
         FOREIGN KEY (database_name) REFERENCES databases(database_name),
         FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
-    )
-    """
-        ),
-        block=True,
-    )
-    await db.execute_write(
-        textwrap.dedent(
-            """
+    );
     CREATE TABLE IF NOT EXISTS foreign_keys (
         database_name TEXT,
         table_name TEXT,
@@ -87,11 +58,10 @@ async def init_internal_db(db):
         PRIMARY KEY (database_name, table_name, id, seq),
         FOREIGN KEY (database_name) REFERENCES databases(database_name),
         FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
-    )
+    );
     """
-        ),
-        block=True,
-    )
+    ).strip()
+    await db.execute_write(create_tables_sql, block=True, executescript=True)
 
 
 async def populate_schema_tables(internal_db, db):
diff --git a/tests/test_api.py b/tests/test_api.py
index 9ad7d569..29c92920 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -950,7 +950,9 @@ def test_trace(trace_debug):
         "select ",
     )
     for prefix in expected:
-        assert any(sql.startswith(prefix) for sql in sqls)
+        assert any(
+            sql.startswith(prefix) for sql in sqls
+        ), "No trace beginning with: {}".format(prefix)
 
 
 @pytest.mark.parametrize(
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 0a5c01a3..aa5676e7 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -401,7 +401,7 @@ async def test_execute_write_executescript(db):
     await db.execute_write(
         "create table foo (id integer primary key); create table bar (id integer primary key); ",
         executescript=True,
-        block=True
+        block=True,
     )
     table_names = await db.table_names()
     assert {"foo", "bar"}.issubset(table_names)
@@ -413,7 +413,7 @@ async def test_execute_write_executescript_not_allowed_with_params(db):
         await db.execute_write(
             "update roadside_attractions set name = ? where pk = ?",
             ["Mystery!", 1],
-            executescript=True
+            executescript=True,
         )
 
 

From 5cadc244895fc47e0534c6e90df976d34293921e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 10:57:22 -0800
Subject: [PATCH 1271/2303] db.execute_write_script() and
 db.execute_write_many(), closes #1570

Refs #1555
---
 datasette/database.py            | 29 ++++++++++++++++++++---------
 datasette/utils/internal_db.py   |  2 +-
 docs/internals.rst               | 26 +++++++++++++++++++++++---
 tests/test_internals_database.py | 24 ++++++++++++++----------
 4 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index f8365f5c..1de1d5ec 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -94,22 +94,33 @@ class Database:
             f"file:{self.path}{qs}", uri=True, check_same_thread=False
         )
 
-    async def execute_write(self, sql, params=None, executescript=False, block=False):
-        assert not (
-            executescript and params
-        ), "Cannot use params with executescript=True"
-
+    async def execute_write(self, sql, params=None, block=False):
         def _inner(conn):
             with conn:
-                if executescript:
-                    return conn.executescript(sql)
-                else:
-                    return conn.execute(sql, params or [])
+                return conn.execute(sql, params or [])
 
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
             results = await self.execute_write_fn(_inner, block=block)
         return results
 
+    async def execute_write_script(self, sql, block=False):
+        def _inner(conn):
+            with conn:
+                return conn.executescript(sql)
+
+        with trace("sql", database=self.name, sql=sql.strip(), executescript=True):
+            results = await self.execute_write_fn(_inner, block=block)
+        return results
+
+    async def execute_write_many(self, sql, params_seq, block=False):
+        def _inner(conn):
+            with conn:
+                return conn.executemany(sql, params_seq)
+
+        with trace("sql", database=self.name, sql=sql.strip(), executemany=True):
+            results = await self.execute_write_fn(_inner, block=block)
+        return results
+
     async def execute_write_fn(self, fn, block=False):
         task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
         if self._write_queue is None:
diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 80babff8..8a145767 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -61,7 +61,7 @@ async def init_internal_db(db):
     );
     """
     ).strip()
-    await db.execute_write(create_tables_sql, block=True, executescript=True)
+    await db.execute_write_script(create_tables_sql, block=True)
 
 
 async def populate_schema_tables(internal_db, db):
diff --git a/docs/internals.rst b/docs/internals.rst
index d40e679b..bc0174a8 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -663,8 +663,8 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, executescript=False, block=False)
---------------------------------------------------------------------------
+await db.execute_write(sql, params=None, block=False)
+-----------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -676,7 +676,27 @@ By default queries are considered to be "fire and forget" - they will be added t
 
 If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library.
 
-If you pass ``executescript=True`` your SQL will be executed using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method. This allows multiple SQL statements to be separated by semicolons, but cannot be used with the ``params=`` option.
+.. _database_execute_write_script:
+
+await db.execute_write_script(sql, block=False)
+-----------------------------------------------
+
+Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method.
+
+.. _database_execute_write_many:
+
+await db.execute_write_many(sql, params_seq, block=False)
+---------------------------------------------------------
+
+Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executemany>`__ method. This will efficiently execute the same SQL statement against each of the parameters in the ``params_seq`` iterator, for example:
+
+.. code-block:: python
+
+    await db.execute_write_many(
+        "insert into characters (id, name) values (?, ?)",
+        [(1, "Melanie"), (2, "Selma"), (2, "Viktor")],
+        block=True,
+    )
 
 .. _database_execute_write_fn:
 
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index aa5676e7..f751bf9a 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -397,10 +397,9 @@ async def test_execute_write_block_false(db):
 
 
 @pytest.mark.asyncio
-async def test_execute_write_executescript(db):
-    await db.execute_write(
+async def test_execute_write_script(db):
+    await db.execute_write_script(
         "create table foo (id integer primary key); create table bar (id integer primary key); ",
-        executescript=True,
         block=True,
     )
     table_names = await db.table_names()
@@ -408,13 +407,18 @@ async def test_execute_write_executescript(db):
 
 
 @pytest.mark.asyncio
-async def test_execute_write_executescript_not_allowed_with_params(db):
-    with pytest.raises(AssertionError):
-        await db.execute_write(
-            "update roadside_attractions set name = ? where pk = ?",
-            ["Mystery!", 1],
-            executescript=True,
-        )
+async def test_execute_write_many(db):
+    await db.execute_write_script(
+        "create table foomany (id integer primary key)",
+        block=True,
+    )
+    await db.execute_write_many(
+        "insert into foomany (id) values (?)",
+        [(1,), (10,), (100,)],
+        block=True,
+    )
+    result = await db.execute("select * from foomany")
+    assert [r[0] for r in result.rows] == [1, 10, 100]
 
 
 @pytest.mark.asyncio

From d637ed46762fdbbd8e32b86f258cd9a53c1cfdc7 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 11:11:08 -0800
Subject: [PATCH 1272/2303] Use execute_write_many to optimize internal DB,
 refs #1555, #1570

---
 datasette/utils/internal_db.py | 142 +++++++++++++++++++--------------
 tests/test_api.py              |   2 +-
 2 files changed, 81 insertions(+), 63 deletions(-)

diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 8a145767..95055d8b 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -70,84 +70,102 @@ async def populate_schema_tables(internal_db, db):
         "DELETE FROM tables WHERE database_name = ?", [database_name], block=True
     )
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
+    tables_to_insert = []
+    columns_to_delete = []
+    columns_to_insert = []
+    foreign_keys_to_delete = []
+    foreign_keys_to_insert = []
+    indexes_to_delete = []
+    indexes_to_insert = []
+
     for table in tables:
         table_name = table["name"]
-        await internal_db.execute_write(
-            """
-            INSERT INTO tables (database_name, table_name, rootpage, sql)
-            values (?, ?, ?, ?)
-        """,
-            [database_name, table_name, table["rootpage"], table["sql"]],
-            block=True,
-        )
-        # And the columns
-        await internal_db.execute_write(
-            "DELETE FROM columns WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
-            block=True,
+        tables_to_insert.append(
+            (database_name, table_name, table["rootpage"], table["sql"])
         )
+        columns_to_delete.append((database_name, table_name))
         columns = await db.table_column_details(table_name)
-        for column in columns:
-            params = {
+        columns_to_insert.extend(
+            {
                 **{"database_name": database_name, "table_name": table_name},
                 **column._asdict(),
             }
-            await internal_db.execute_write(
-                """
-                INSERT INTO columns (
-                    database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
-                ) VALUES (
-                    :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
-                )
-            """,
-                params,
-                block=True,
-            )
-        # And the foreign_keys
-        await internal_db.execute_write(
-            "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
-            block=True,
+            for column in columns
         )
+        foreign_keys_to_delete.append((database_name, table_name))
         foreign_keys = (
             await db.execute(f"PRAGMA foreign_key_list([{table_name}])")
         ).rows
-        for foreign_key in foreign_keys:
-            params = {
+        foreign_keys_to_insert.extend(
+            {
                 **{"database_name": database_name, "table_name": table_name},
                 **dict(foreign_key),
             }
-            await internal_db.execute_write(
-                """
-                INSERT INTO foreign_keys (
-                    database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
-                ) VALUES (
-                    :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
-                )
-            """,
-                params,
-                block=True,
-            )
-        # And the indexes
-        await internal_db.execute_write(
-            "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
-            block=True,
+            for foreign_key in foreign_keys
         )
+        indexes_to_delete.append((database_name, table_name))
         indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows
-        for index in indexes:
-            params = {
+        indexes_to_insert.extend(
+            {
                 **{"database_name": database_name, "table_name": table_name},
                 **dict(index),
             }
-            await internal_db.execute_write(
-                """
-                INSERT INTO indexes (
-                    database_name, table_name, seq, name, "unique", origin, partial
-                ) VALUES (
-                    :database_name, :table_name, :seq, :name, :unique, :origin, :partial
-                )
-            """,
-                params,
-                block=True,
-            )
+            for index in indexes
+        )
+
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO tables (database_name, table_name, rootpage, sql)
+        values (?, ?, ?, ?)
+    """,
+        tables_to_insert,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        "DELETE FROM columns WHERE database_name = ? and table_name = ?",
+        columns_to_delete,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO columns (
+            database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
+        ) VALUES (
+            :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
+        )
+    """,
+        columns_to_insert,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
+        foreign_keys_to_delete,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO foreign_keys (
+            database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
+        ) VALUES (
+            :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
+        )
+    """,
+        foreign_keys_to_insert,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
+        indexes_to_delete,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO indexes (
+            database_name, table_name, seq, name, "unique", origin, partial
+        ) VALUES (
+            :database_name, :table_name, :seq, :name, :unique, :origin, :partial
+        )
+    """,
+        indexes_to_insert,
+        block=True,
+    )
diff --git a/tests/test_api.py b/tests/test_api.py
index 29c92920..f198c1f9 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -937,7 +937,7 @@ def test_trace(trace_debug):
         assert isinstance(trace["traceback"], list)
         assert isinstance(trace["database"], str)
         assert isinstance(trace["sql"], str)
-        assert isinstance(trace["params"], (list, dict, None.__class__))
+        assert isinstance(trace.get("params"), (list, dict, None.__class__))
 
     sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace]
     # There should be a mix of different types of SQL statement

From 97b1723dd09cf000485d4e050efc5bb4f5184a06 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 19:49:11 -0800
Subject: [PATCH 1273/2303] Optimize init_internal_db by running PRAGMA in a
 single function

Refs #1555
---
 datasette/utils/internal_db.py | 102 ++++++++++++++++++++-------------
 1 file changed, 62 insertions(+), 40 deletions(-)

diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 95055d8b..58f99825 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -1,4 +1,5 @@
 import textwrap
+from datasette.utils import table_column_details
 
 
 async def init_internal_db(db):
@@ -70,49 +71,70 @@ async def populate_schema_tables(internal_db, db):
         "DELETE FROM tables WHERE database_name = ?", [database_name], block=True
     )
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
-    tables_to_insert = []
-    columns_to_delete = []
-    columns_to_insert = []
-    foreign_keys_to_delete = []
-    foreign_keys_to_insert = []
-    indexes_to_delete = []
-    indexes_to_insert = []
 
-    for table in tables:
-        table_name = table["name"]
-        tables_to_insert.append(
-            (database_name, table_name, table["rootpage"], table["sql"])
-        )
-        columns_to_delete.append((database_name, table_name))
-        columns = await db.table_column_details(table_name)
-        columns_to_insert.extend(
-            {
-                **{"database_name": database_name, "table_name": table_name},
-                **column._asdict(),
-            }
-            for column in columns
-        )
-        foreign_keys_to_delete.append((database_name, table_name))
-        foreign_keys = (
-            await db.execute(f"PRAGMA foreign_key_list([{table_name}])")
-        ).rows
-        foreign_keys_to_insert.extend(
-            {
-                **{"database_name": database_name, "table_name": table_name},
-                **dict(foreign_key),
-            }
-            for foreign_key in foreign_keys
-        )
-        indexes_to_delete.append((database_name, table_name))
-        indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows
-        indexes_to_insert.extend(
-            {
-                **{"database_name": database_name, "table_name": table_name},
-                **dict(index),
-            }
-            for index in indexes
+    def collect_info(conn):
+        tables_to_insert = []
+        columns_to_delete = []
+        columns_to_insert = []
+        foreign_keys_to_delete = []
+        foreign_keys_to_insert = []
+        indexes_to_delete = []
+        indexes_to_insert = []
+
+        for table in tables:
+            table_name = table["name"]
+            tables_to_insert.append(
+                (database_name, table_name, table["rootpage"], table["sql"])
+            )
+            columns_to_delete.append((database_name, table_name))
+            columns = table_column_details(conn, table_name)
+            columns_to_insert.extend(
+                {
+                    **{"database_name": database_name, "table_name": table_name},
+                    **column._asdict(),
+                }
+                for column in columns
+            )
+            foreign_keys_to_delete.append((database_name, table_name))
+            foreign_keys = conn.execute(
+                f"PRAGMA foreign_key_list([{table_name}])"
+            ).fetchall()
+            foreign_keys_to_insert.extend(
+                {
+                    **{"database_name": database_name, "table_name": table_name},
+                    **dict(foreign_key),
+                }
+                for foreign_key in foreign_keys
+            )
+            indexes_to_delete.append((database_name, table_name))
+            indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall()
+            indexes_to_insert.extend(
+                {
+                    **{"database_name": database_name, "table_name": table_name},
+                    **dict(index),
+                }
+                for index in indexes
+            )
+        return (
+            tables_to_insert,
+            columns_to_delete,
+            columns_to_insert,
+            foreign_keys_to_delete,
+            foreign_keys_to_insert,
+            indexes_to_delete,
+            indexes_to_insert,
         )
 
+    (
+        tables_to_insert,
+        columns_to_delete,
+        columns_to_insert,
+        foreign_keys_to_delete,
+        foreign_keys_to_insert,
+        indexes_to_delete,
+        indexes_to_insert,
+    ) = await db.execute_fn(collect_info)
+
     await internal_db.execute_write_many(
         """
         INSERT INTO tables (database_name, table_name, rootpage, sql)

From c6ff1f23e6a0b26dde8f5b30be3b868b031b6ecf Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 18 Dec 2021 20:03:21 -0800
Subject: [PATCH 1274/2303] Queries took rather than query took, closes #1572

---
 datasette/templates/_footer.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/templates/_footer.html b/datasette/templates/_footer.html
index b1380ae9..074270f1 100644
--- a/datasette/templates/_footer.html
+++ b/datasette/templates/_footer.html
@@ -1,5 +1,5 @@
 Powered by <a href="https://datasette.io/" title="Datasette v{{ datasette_version }}">Datasette</a>
-{% if query_ms %}· Query took {{ query_ms|round(3) }}ms{% endif %}
+{% if query_ms %}· Queries took {{ query_ms|round(3) }}ms{% endif %}
 {% if metadata %}
     {% if metadata.license or metadata.license_url %}· Data license:
         {% if metadata.license_url %}

From f65817000fdf87ce8a0c23edc40784ebe33b5842 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 19 Dec 2021 12:30:34 -0800
Subject: [PATCH 1275/2303] Include count in execute_write_many traces, closes
 #1571

---
 datasette/database.py | 19 +++++++++++++++----
 datasette/tracer.py   |  6 +++---
 tests/test_api.py     | 14 +++++++++++---
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 1de1d5ec..0e41ff32 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -114,11 +114,22 @@ class Database:
 
     async def execute_write_many(self, sql, params_seq, block=False):
         def _inner(conn):
-            with conn:
-                return conn.executemany(sql, params_seq)
+            count = 0
 
-        with trace("sql", database=self.name, sql=sql.strip(), executemany=True):
-            results = await self.execute_write_fn(_inner, block=block)
+            def count_params(params):
+                nonlocal count
+                for param in params:
+                    count += 1
+                    yield param
+
+            with conn:
+                return conn.executemany(sql, count_params(params_seq)), count
+
+        with trace(
+            "sql", database=self.name, sql=sql.strip(), executemany=True
+        ) as kwargs:
+            results, count = await self.execute_write_fn(_inner, block=block)
+            kwargs["count"] = count
         return results
 
     async def execute_write_fn(self, fn, block=False):
diff --git a/datasette/tracer.py b/datasette/tracer.py
index 62c3c90c..6703f060 100644
--- a/datasette/tracer.py
+++ b/datasette/tracer.py
@@ -32,14 +32,14 @@ def trace(type, **kwargs):
     ), f".trace() keyword parameters cannot include {TRACE_RESERVED_KEYS}"
     task_id = get_task_id()
     if task_id is None:
-        yield
+        yield kwargs
         return
     tracer = tracers.get(task_id)
     if tracer is None:
-        yield
+        yield kwargs
         return
     start = time.perf_counter()
-    yield
+    yield kwargs
     end = time.perf_counter()
     trace_info = {
         "type": type,
diff --git a/tests/test_api.py b/tests/test_api.py
index f198c1f9..8ecaef43 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -928,8 +928,9 @@ def test_trace(trace_debug):
     assert isinstance(trace_info["sum_trace_duration_ms"], float)
     assert isinstance(trace_info["num_traces"], int)
     assert isinstance(trace_info["traces"], list)
-    assert len(trace_info["traces"]) == trace_info["num_traces"]
-    for trace in trace_info["traces"]:
+    traces = trace_info["traces"]
+    assert len(traces) == trace_info["num_traces"]
+    for trace in traces:
         assert isinstance(trace["type"], str)
         assert isinstance(trace["start"], float)
         assert isinstance(trace["end"], float)
@@ -939,7 +940,7 @@ def test_trace(trace_debug):
         assert isinstance(trace["sql"], str)
         assert isinstance(trace.get("params"), (list, dict, None.__class__))
 
-    sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace]
+    sqls = [trace["sql"] for trace in traces if "sql" in trace]
     # There should be a mix of different types of SQL statement
     expected = (
         "CREATE TABLE ",
@@ -954,6 +955,13 @@ def test_trace(trace_debug):
             sql.startswith(prefix) for sql in sqls
         ), "No trace beginning with: {}".format(prefix)
 
+    # Should be at least one executescript
+    assert any(trace for trace in traces if trace.get("executescript"))
+    # And at least one executemany
+    execute_manys = [trace for trace in traces if trace.get("executemany")]
+    assert execute_manys
+    assert all(isinstance(trace["count"], int) for trace in execute_manys)
+
 
 @pytest.mark.parametrize(
     "path,status_code",

From 5fac26aa221a111d7633f2dd92014641f7c0ade9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 19 Dec 2021 12:54:12 -0800
Subject: [PATCH 1276/2303] Another populate_schema_tables optimization, refs
 #1555

---
 datasette/utils/internal_db.py | 41 +++++++++-------------------------
 tests/test_api.py              |  1 -
 2 files changed, 11 insertions(+), 31 deletions(-)

diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 58f99825..ed589a7a 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -67,18 +67,23 @@ async def init_internal_db(db):
 
 async def populate_schema_tables(internal_db, db):
     database_name = db.name
-    await internal_db.execute_write(
-        "DELETE FROM tables WHERE database_name = ?", [database_name], block=True
-    )
+
+    def delete_everything(conn):
+        conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name])
+        conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name])
+        conn.execute(
+            "DELETE FROM foreign_keys WHERE database_name = ?", [database_name]
+        )
+        conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name])
+
+    await internal_db.execute_write_fn(delete_everything, block=True)
+
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
 
     def collect_info(conn):
         tables_to_insert = []
-        columns_to_delete = []
         columns_to_insert = []
-        foreign_keys_to_delete = []
         foreign_keys_to_insert = []
-        indexes_to_delete = []
         indexes_to_insert = []
 
         for table in tables:
@@ -86,7 +91,6 @@ async def populate_schema_tables(internal_db, db):
             tables_to_insert.append(
                 (database_name, table_name, table["rootpage"], table["sql"])
             )
-            columns_to_delete.append((database_name, table_name))
             columns = table_column_details(conn, table_name)
             columns_to_insert.extend(
                 {
@@ -95,7 +99,6 @@ async def populate_schema_tables(internal_db, db):
                 }
                 for column in columns
             )
-            foreign_keys_to_delete.append((database_name, table_name))
             foreign_keys = conn.execute(
                 f"PRAGMA foreign_key_list([{table_name}])"
             ).fetchall()
@@ -106,7 +109,6 @@ async def populate_schema_tables(internal_db, db):
                 }
                 for foreign_key in foreign_keys
             )
-            indexes_to_delete.append((database_name, table_name))
             indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall()
             indexes_to_insert.extend(
                 {
@@ -117,21 +119,15 @@ async def populate_schema_tables(internal_db, db):
             )
         return (
             tables_to_insert,
-            columns_to_delete,
             columns_to_insert,
-            foreign_keys_to_delete,
             foreign_keys_to_insert,
-            indexes_to_delete,
             indexes_to_insert,
         )
 
     (
         tables_to_insert,
-        columns_to_delete,
         columns_to_insert,
-        foreign_keys_to_delete,
         foreign_keys_to_insert,
-        indexes_to_delete,
         indexes_to_insert,
     ) = await db.execute_fn(collect_info)
 
@@ -143,11 +139,6 @@ async def populate_schema_tables(internal_db, db):
         tables_to_insert,
         block=True,
     )
-    await internal_db.execute_write_many(
-        "DELETE FROM columns WHERE database_name = ? and table_name = ?",
-        columns_to_delete,
-        block=True,
-    )
     await internal_db.execute_write_many(
         """
         INSERT INTO columns (
@@ -159,11 +150,6 @@ async def populate_schema_tables(internal_db, db):
         columns_to_insert,
         block=True,
     )
-    await internal_db.execute_write_many(
-        "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
-        foreign_keys_to_delete,
-        block=True,
-    )
     await internal_db.execute_write_many(
         """
         INSERT INTO foreign_keys (
@@ -175,11 +161,6 @@ async def populate_schema_tables(internal_db, db):
         foreign_keys_to_insert,
         block=True,
     )
-    await internal_db.execute_write_many(
-        "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
-        indexes_to_delete,
-        block=True,
-    )
     await internal_db.execute_write_many(
         """
         INSERT INTO indexes (
diff --git a/tests/test_api.py b/tests/test_api.py
index 8ecaef43..574ebb41 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -946,7 +946,6 @@ def test_trace(trace_debug):
         "CREATE TABLE ",
         "PRAGMA ",
         "INSERT OR REPLACE INTO ",
-        "DELETE FROM ",
         "INSERT INTO",
         "select ",
     )

From 4094741c2881c2ada3f3f878b532fdaec7914953 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 19 Dec 2021 13:11:57 -0800
Subject: [PATCH 1277/2303] Fixed bug with custom templates for writable canned
 queries, closes #1547

---
 datasette/views/database.py  | 11 +++++------
 tests/test_canned_queries.py | 22 +++++++++++++++++++++-
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index f1901b34..aa8d27ec 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -246,6 +246,11 @@ class QueryView(DataView):
             extra_args["page_size"] = _size
 
         templates = [f"query-{to_css_class(database)}.html", "query.html"]
+        if canned_query:
+            templates.insert(
+                0,
+                f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html",
+            )
 
         query_error = None
 
@@ -340,12 +345,6 @@ class QueryView(DataView):
                 results = None
                 columns = []
 
-        if canned_query:
-            templates.insert(
-                0,
-                f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html",
-            )
-
         allow_execute_sql = await self.ds.permission_allowed(
             request.actor, "execute-sql", database, default=True
         )
diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index cea81ec7..c5ccaf5c 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -6,9 +6,19 @@ from .fixtures import make_app_client, app_client
 
 
 @pytest.fixture
-def canned_write_client():
+def canned_write_client(tmpdir):
+    template_dir = tmpdir / "canned_write_templates"
+    template_dir.mkdir()
+    (template_dir / "query-data-update_name.html").write_text(
+        """
+    {% extends "query.html" %}
+    {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %}
+    """,
+        "utf-8",
+    )
     with make_app_client(
         extra_databases={"data.db": "create table names (name text)"},
+        template_dir=str(template_dir),
         metadata={
             "databases": {
                 "data": {
@@ -344,3 +354,13 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c
     )
     assert 400 == response.status
     assert response.json["error"].startswith("You did not supply a value for binding")
+
+
+def test_canned_write_custom_template(canned_write_client):
+    response = canned_write_client.get("/data/update_name")
+    assert response.status == 200
+    assert (
+        "<!-- Templates considered: *query-data-update_name.html, query-data.html, query.html -->"
+        in response.text
+    )
+    assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text

From dbaac79946034e0b00714e2da39f934d693883d2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 19 Dec 2021 14:08:10 -0800
Subject: [PATCH 1278/2303] Release 0.60a1

Refs #1547, #1555, #1562, #1563, #1564, #1567, #1568, #1569, #1570, #1571, #1572
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 2fce006c..290fbcf3 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.60a0"
+__version__ = "0.60a1"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 92a9d941..99d3315e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,18 @@
 Changelog
 =========
 
+.. _v0_60a1:
+
+0.60a1 (2021-12-19)
+-------------------
+
+- Database write connections now execute the :ref:`plugin_hook_prepare_connection` plugin hook. (:issue:`1564`)
+- The ``Datasette()`` constructor no longer requires the ``files=`` argument, and is now documented at :ref:`internals_datasette`. (:issue:`1563`)
+- The tracing feature now traces write queries, not just read queries. (:issue:`1568`)
+- Added two methods for writing to the database: :ref:`database_execute_write_script` and :ref:`database_execute_write_many`. (:issue:`1570`)
+- Made several performance improvements to the database schema introspection code that runs when Datasette first starts up. (:issue:`1555`)
+- Fixed bug where writable canned queries could not be used with custom templates.  (:issue:`1547`)
+
 .. _v0_60a0:
 
 0.60a0 (2021-12-17)

From f36e010b3b69ada104b79d83c7685caf9359049e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 19 Dec 2021 17:25:40 -0800
Subject: [PATCH 1279/2303] Upgrade to Pluggy>=1.0, refs #1575

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index f8cd3e5b..1f8855cf 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ setup(
         "hupper~=1.9",
         "httpx>=0.20",
         "pint~=0.9",
-        "pluggy>=0.13,<1.1",
+        "pluggy>=1.0,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.9",
         "janus>=0.6.2,<1.1",

From 554aae5c51b7ce1b570e0e9bbe45f4cf9200f2bf Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 20 Dec 2021 09:23:05 -0800
Subject: [PATCH 1280/2303] Plausible analytics for the documentation

---
 docs/_templates/layout.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index b7b6f794..e44f3b56 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -1,5 +1,10 @@
 {%- extends "!layout.html" %}
 
+{% block htmltitle %}
+{{ super() }}
+<script defer data-domain="datasette.io" src="https://plausible.io/js/plausible.js"></script>
+{% endblock %}
+
 {% block sidebartitle %}
 
 <a href="https://datasette.io/">

From 6b1384b2f529134998fb507e63307609a5b7f5c0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 20 Dec 2021 15:55:17 -0800
Subject: [PATCH 1281/2303] Track plausible for docs.datasette.io not
 datasette.io

---
 docs/_templates/layout.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index e44f3b56..db16b428 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -2,7 +2,7 @@
 
 {% block htmltitle %}
 {{ super() }}
-<script defer data-domain="datasette.io" src="https://plausible.io/js/plausible.js"></script>
+<script defer data-domain="docs.datasette.io" src="https://plausible.io/js/plausible.js"></script>
 {% endblock %}
 
 {% block sidebartitle %}

From ace86566b28280091b3844cf5fbecd20158e9004 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 22 Dec 2021 12:22:44 -0800
Subject: [PATCH 1282/2303] Remove concept of special_args, re-arrange
 TableView a bit, refs #1518

---
 datasette/views/table.py | 79 +++++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 41 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index c3bcf01d..9808fd24 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -366,6 +366,30 @@ class TableView(RowTableShared):
             None, "view-table", (database, table), default=True
         )
 
+        # Handle ?_filter_column and redirect, if present
+        redirect_params = filters_should_redirect(request.args)
+        if redirect_params:
+            return self.redirect(
+                request,
+                path_with_added_args(request, redirect_params),
+                forward_querystring=False,
+            )
+
+        # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort)
+        if "_sort_by_desc" in request.args:
+            return self.redirect(
+                request,
+                path_with_added_args(
+                    request,
+                    {
+                        "_sort_desc": request.args.get("_sort"),
+                        "_sort_by_desc": None,
+                        "_sort": None,
+                    },
+                ),
+                forward_querystring=False,
+            )
+
         # Introspect columns and primary keys for table
         pks = await db.primary_keys(table)
         table_columns = await db.table_columns(table)
@@ -399,47 +423,20 @@ class TableView(RowTableShared):
             nocount = True
             nofacet = True
 
-        # Special args start with _ and do not contain a __
-        # That's so if there is a column that starts with _
-        # it can still be queried using ?_col__exact=blah
-        special_args = {}
-        other_args = []
-        for key in request.args:
-            if key.startswith("_") and "__" not in key:
-                special_args[key] = request.args[key]
-            else:
-                for v in request.args.getlist(key):
-                    other_args.append((key, v))
-
-        # Handle ?_filter_column and redirect, if present
-        redirect_params = filters_should_redirect(special_args)
-        if redirect_params:
-            return self.redirect(
-                request,
-                path_with_added_args(request, redirect_params),
-                forward_querystring=False,
-            )
-
-        # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort)
-        if "_sort_by_desc" in special_args:
-            return self.redirect(
-                request,
-                path_with_added_args(
-                    request,
-                    {
-                        "_sort_desc": special_args.get("_sort"),
-                        "_sort_by_desc": None,
-                        "_sort": None,
-                    },
-                ),
-                forward_querystring=False,
-            )
-
         table_metadata = self.ds.table_metadata(database, table)
         units = table_metadata.get("units", {})
 
+        # Arguments that start with _ and don't contain a __ are
+        # special - things like ?_search= - and should not be
+        # treated as filters.
+        filter_args = []
+        for key in request.args:
+            if not (key.startswith("_") and "__" not in key):
+                for v in request.args.getlist(key):
+                    filter_args.append((key, v))
+
         # Build where clauses from query string arguments
-        filters = Filters(sorted(other_args), units, ureg)
+        filters = Filters(sorted(filter_args), units, ureg)
         where_clauses, params = filters.build_where_clauses(table)
 
         # Execute filters_from_request plugin hooks - including the default
@@ -464,8 +461,8 @@ class TableView(RowTableShared):
         sortable_columns = await self.sortable_columns_for_table(
             database, table, use_rowid
         )
-        sort = special_args.get("_sort")
-        sort_desc = special_args.get("_sort_desc")
+        sort = request.args.get("_sort")
+        sort_desc = request.args.get("_sort_desc")
 
         if not sort and not sort_desc:
             sort = table_metadata.get("sort")
@@ -498,7 +495,7 @@ class TableView(RowTableShared):
         count_sql = f"select count(*) {from_sql}"
 
         # Handle pagination driven by ?_next=
-        _next = _next or special_args.get("_next")
+        _next = _next or request.args.get("_next")
         offset = ""
         if _next:
             sort_value = None
@@ -708,7 +705,7 @@ class TableView(RowTableShared):
         expandable_columns = await self.expandable_columns(database, table)
         columns_to_expand = None
         try:
-            all_labels = value_as_boolean(special_args.get("_labels", ""))
+            all_labels = value_as_boolean(request.args.get("_labels", ""))
         except ValueError:
             all_labels = default_labels
         # Check for explicit _label=

From 00a2895cd2dc42c63846216b36b2dc9f41170129 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 23 Dec 2021 11:03:49 -0800
Subject: [PATCH 1283/2303] execute_write defaut is now block=True, closes
 #1579

---
 datasette/database.py            |  8 ++++----
 docs/internals.rst               | 14 +++++++-------
 tests/test_internals_database.py |  2 +-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 0e41ff32..e908d1ea 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -94,7 +94,7 @@ class Database:
             f"file:{self.path}{qs}", uri=True, check_same_thread=False
         )
 
-    async def execute_write(self, sql, params=None, block=False):
+    async def execute_write(self, sql, params=None, block=True):
         def _inner(conn):
             with conn:
                 return conn.execute(sql, params or [])
@@ -103,7 +103,7 @@ class Database:
             results = await self.execute_write_fn(_inner, block=block)
         return results
 
-    async def execute_write_script(self, sql, block=False):
+    async def execute_write_script(self, sql, block=True):
         def _inner(conn):
             with conn:
                 return conn.executescript(sql)
@@ -112,7 +112,7 @@ class Database:
             results = await self.execute_write_fn(_inner, block=block)
         return results
 
-    async def execute_write_many(self, sql, params_seq, block=False):
+    async def execute_write_many(self, sql, params_seq, block=True):
         def _inner(conn):
             count = 0
 
@@ -132,7 +132,7 @@ class Database:
             kwargs["count"] = count
         return results
 
-    async def execute_write_fn(self, fn, block=False):
+    async def execute_write_fn(self, fn, block=True):
         task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
         if self._write_queue is None:
             self._write_queue = queue.Queue()
diff --git a/docs/internals.rst b/docs/internals.rst
index bc0174a8..667ac33a 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -663,7 +663,7 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=False)
+await db.execute_write(sql, params=None, block=True)
 -----------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
@@ -672,20 +672,20 @@ This method can be used to queue up a non-SELECT SQL query to be executed agains
 
 You can pass additional SQL parameters as a tuple or dictionary.
 
-By default queries are considered to be "fire and forget" - they will be added to the queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task.
+The method will block until the operation is completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library.
 
-If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library.
+If you pass ``block=False`` this behaviour changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task.
 
 .. _database_execute_write_script:
 
-await db.execute_write_script(sql, block=False)
+await db.execute_write_script(sql, block=True)
 -----------------------------------------------
 
 Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method.
 
 .. _database_execute_write_many:
 
-await db.execute_write_many(sql, params_seq, block=False)
+await db.execute_write_many(sql, params_seq, block=True)
 ---------------------------------------------------------
 
 Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executemany>`__ method. This will efficiently execute the same SQL statement against each of the parameters in the ``params_seq`` iterator, for example:
@@ -700,7 +700,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d
 
 .. _database_execute_write_fn:
 
-await db.execute_write_fn(fn, block=False)
+await db.execute_write_fn(fn, block=True)
 ------------------------------------------
 
 This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. This function will be queued up and then called when the write connection is available, passing that connection as the argument to the function.
@@ -725,7 +725,7 @@ This method is fire-and-forget, queueing your function to be executed and then a
 
 If you pass ``block=True`` your calling code will block until the function has been executed. The return value to the ``await`` will be the return value of your function.
 
-If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=False`` any exceptions will be silently ignored.
+If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=True`` any exceptions will be silently ignored.
 
 Here's an example of ``block=True`` in action:
 
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index f751bf9a..80f47ab9 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -435,7 +435,7 @@ async def test_execute_write_fn_block_false(db):
             row = conn.execute("select count(*) from roadside_attractions").fetchone()
         return row[0]
 
-    task_id = await db.execute_write_fn(write_fn)
+    task_id = await db.execute_write_fn(write_fn, block=False)
     assert isinstance(task_id, uuid.UUID)
 
 

From 75153ea9b94d09ec3d61f7c6ebdf378e0c0c7a0b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 23 Dec 2021 11:16:31 -0800
Subject: [PATCH 1284/2303] Updated db.execute_write_fn() docs for block=True
 default, refs #1579

---
 docs/internals.rst | 35 ++++++++++++-----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index 667ac33a..6a5666fd 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -331,7 +331,7 @@ This will add a mutable database and serve it at ``/my-new-database``.
 .. code-block:: python
 
     db = datasette.add_database(Database(datasette, memory_name="statistics"))
-    await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True)
+    await db.execute_write("CREATE TABLE foo(id integer primary key)")
 
 .. _datasette_add_memory_database:
 
@@ -694,8 +694,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d
 
     await db.execute_write_many(
         "insert into characters (id, name) values (?, ?)",
-        [(1, "Melanie"), (2, "Selma"), (2, "Viktor")],
-        block=True,
+        [(1, "Melanie"), (2, "Selma"), (2, "Viktor")]
     )
 
 .. _database_execute_write_fn:
@@ -703,9 +702,9 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d
 await db.execute_write_fn(fn, block=True)
 ------------------------------------------
 
-This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. This function will be queued up and then called when the write connection is available, passing that connection as the argument to the function.
+This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. Your function will be queued up and then called when the write connection is available, passing that connection as the argument to the function.
 
-The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection as long as it is executing.
+The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection for as long as it is executing.
 
 .. warning::
 
@@ -715,31 +714,21 @@ For example:
 
 .. code-block:: python
 
-    def my_action(conn):
-        conn.execute("delete from some_table")
-        conn.execute("delete from other_table")
-
-    await database.execute_write_fn(my_action)
-
-This method is fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned.
-
-If you pass ``block=True`` your calling code will block until the function has been executed. The return value to the ``await`` will be the return value of your function.
-
-If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=True`` any exceptions will be silently ignored.
-
-Here's an example of ``block=True`` in action:
-
-.. code-block:: python
-
-    def my_action(conn):
+    def delete_and_return_count(conn):
         conn.execute("delete from some_table where id > 5")
         return conn.execute("select count(*) from some_table").fetchone()[0]
 
     try:
-        num_rows_left = await database.execute_write_fn(my_action, block=True)
+        num_rows_left = await database.execute_write_fn(delete_and_return_count)
     except Exception as e:
         print("An error occurred:", e)
 
+The value returned from ``await database.execute_write_fn(...)`` will be the return value from your function.
+
+If your function raises an exception that exception will be propagated up to the ``await`` line.
+
+If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed.
+
 .. _internals_database_introspection:
 
 Database introspection

From 8c401ee0f054de2f568c3a8302c9223555146407 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 23 Dec 2021 11:18:20 -0800
Subject: [PATCH 1285/2303] Fixed remaining code and docs for new block=True
 default, closes #1579

---
 datasette/app.py                 |  1 -
 datasette/utils/internal_db.py   |  8 ++------
 datasette/views/database.py      |  2 +-
 docs/plugin_hooks.rst            |  2 +-
 tests/test_facets.py             | 17 +++++------------
 tests/test_internals_database.py | 28 ++++++++++------------------
 6 files changed, 19 insertions(+), 39 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 17fa06a5..bd663509 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -367,7 +367,6 @@ class Datasette:
                 VALUES (?, ?, ?, ?)
             """,
                 [database_name, str(db.path), db.is_memory, schema_version],
-                block=True,
             )
             await populate_schema_tables(internal_db, db)
 
diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index ed589a7a..e4b49e80 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -62,7 +62,7 @@ async def init_internal_db(db):
     );
     """
     ).strip()
-    await db.execute_write_script(create_tables_sql, block=True)
+    await db.execute_write_script(create_tables_sql)
 
 
 async def populate_schema_tables(internal_db, db):
@@ -76,7 +76,7 @@ async def populate_schema_tables(internal_db, db):
         )
         conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name])
 
-    await internal_db.execute_write_fn(delete_everything, block=True)
+    await internal_db.execute_write_fn(delete_everything)
 
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
 
@@ -137,7 +137,6 @@ async def populate_schema_tables(internal_db, db):
         values (?, ?, ?, ?)
     """,
         tables_to_insert,
-        block=True,
     )
     await internal_db.execute_write_many(
         """
@@ -148,7 +147,6 @@ async def populate_schema_tables(internal_db, db):
         )
     """,
         columns_to_insert,
-        block=True,
     )
     await internal_db.execute_write_many(
         """
@@ -159,7 +157,6 @@ async def populate_schema_tables(internal_db, db):
         )
     """,
         foreign_keys_to_insert,
-        block=True,
     )
     await internal_db.execute_write_many(
         """
@@ -170,5 +167,4 @@ async def populate_schema_tables(internal_db, db):
         )
     """,
         indexes_to_insert,
-        block=True,
     )
diff --git a/datasette/views/database.py b/datasette/views/database.py
index aa8d27ec..e26706e7 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -279,7 +279,7 @@ class QueryView(DataView):
                 ok = None
                 try:
                     cursor = await self.ds.databases[database].execute_write(
-                        sql, params_for_query, block=True
+                        sql, params_for_query
                     )
                     message = metadata.get(
                         "on_success_message"
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index cbaf4c54..88e1def0 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -770,7 +770,7 @@ Or you can return an async function which will be awaited on startup. Use this o
             if "my_table" not in await db.table_names():
                 await db.execute_write("""
                     create table my_table (mycol text)
-                """, block=True)
+                """)
         return inner
 
 Potential use-cases:
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 3f292a3b..c28dc43c 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -408,16 +408,14 @@ async def test_array_facet_results(app_client):
 async def test_array_facet_handle_duplicate_tags():
     ds = Datasette([], memory=True)
     db = ds.add_database(Database(ds, memory_name="test_array_facet"))
-    await db.execute_write("create table otters(name text, tags text)", block=True)
+    await db.execute_write("create table otters(name text, tags text)")
     for name, tags in (
         ("Charles", ["friendly", "cunning", "friendly"]),
         ("Shaun", ["cunning", "empathetic", "friendly"]),
         ("Tracy", ["empathetic", "eager"]),
     ):
         await db.execute_write(
-            "insert into otters (name, tags) values (?, ?)",
-            [name, json.dumps(tags)],
-            block=True,
+            "insert into otters (name, tags) values (?, ?)", [name, json.dumps(tags)]
         )
 
     response = await ds.client.get("/test_array_facet/otters.json?_facet_array=tags")
@@ -516,11 +514,9 @@ async def test_date_facet_results(app_client):
 async def test_json_array_with_blanks_and_nulls():
     ds = Datasette([], memory=True)
     db = ds.add_database(Database(ds, memory_name="test_json_array"))
-    await db.execute_write("create table foo(json_column text)", block=True)
+    await db.execute_write("create table foo(json_column text)")
     for value in ('["a", "b", "c"]', '["a", "b"]', "", None):
-        await db.execute_write(
-            "insert into foo (json_column) values (?)", [value], block=True
-        )
+        await db.execute_write("insert into foo (json_column) values (?)", [value])
     response = await ds.client.get("/test_json_array/foo.json")
     data = response.json()
     assert data["suggested_facets"] == [
@@ -536,15 +532,12 @@ async def test_json_array_with_blanks_and_nulls():
 async def test_facet_size():
     ds = Datasette([], memory=True, settings={"max_returned_rows": 50})
     db = ds.add_database(Database(ds, memory_name="test_facet_size"))
-    await db.execute_write(
-        "create table neighbourhoods(city text, neighbourhood text)", block=True
-    )
+    await db.execute_write("create table neighbourhoods(city text, neighbourhood text)")
     for i in range(1, 51):
         for j in range(1, 4):
             await db.execute_write(
                 "insert into neighbourhoods (city, neighbourhood) values (?, ?)",
                 ["City {}".format(i), "Neighbourhood {}".format(j)],
-                block=True,
             )
     response = await ds.client.get("/test_facet_size/neighbourhoods.json")
     data = response.json()
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 80f47ab9..bcecb486 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -377,9 +377,7 @@ async def test_table_names(db):
 @pytest.mark.asyncio
 async def test_execute_write_block_true(db):
     await db.execute_write(
-        "update roadside_attractions set name = ? where pk = ?",
-        ["Mystery!", 1],
-        block=True,
+        "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1]
     )
     rows = await db.execute("select name from roadside_attractions where pk = 1")
     assert "Mystery!" == rows.rows[0][0]
@@ -399,8 +397,7 @@ async def test_execute_write_block_false(db):
 @pytest.mark.asyncio
 async def test_execute_write_script(db):
     await db.execute_write_script(
-        "create table foo (id integer primary key); create table bar (id integer primary key); ",
-        block=True,
+        "create table foo (id integer primary key); create table bar (id integer primary key);"
     )
     table_names = await db.table_names()
     assert {"foo", "bar"}.issubset(table_names)
@@ -408,14 +405,9 @@ async def test_execute_write_script(db):
 
 @pytest.mark.asyncio
 async def test_execute_write_many(db):
-    await db.execute_write_script(
-        "create table foomany (id integer primary key)",
-        block=True,
-    )
+    await db.execute_write_script("create table foomany (id integer primary key)")
     await db.execute_write_many(
-        "insert into foomany (id) values (?)",
-        [(1,), (10,), (100,)],
-        block=True,
+        "insert into foomany (id) values (?)", [(1,), (10,), (100,)]
     )
     result = await db.execute("select * from foomany")
     assert [r[0] for r in result.rows] == [1, 10, 100]
@@ -424,7 +416,7 @@ async def test_execute_write_many(db):
 @pytest.mark.asyncio
 async def test_execute_write_has_correctly_prepared_connection(db):
     # The sleep() function is only available if ds._prepare_connection() was called
-    await db.execute_write("select sleep(0.01)", block=True)
+    await db.execute_write("select sleep(0.01)")
 
 
 @pytest.mark.asyncio
@@ -447,7 +439,7 @@ async def test_execute_write_fn_block_true(db):
             row = conn.execute("select count(*) from roadside_attractions").fetchone()
         return row[0]
 
-    new_count = await db.execute_write_fn(write_fn, block=True)
+    new_count = await db.execute_write_fn(write_fn)
     assert 3 == new_count
 
 
@@ -457,7 +449,7 @@ async def test_execute_write_fn_exception(db):
         assert False
 
     with pytest.raises(AssertionError):
-        await db.execute_write_fn(write_fn, block=True)
+        await db.execute_write_fn(write_fn)
 
 
 @pytest.mark.asyncio
@@ -472,7 +464,7 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client):
         assert False
 
     with pytest.raises(AssertionError):
-        await db.execute_write_fn(write_fn, block=True)
+        await db.execute_write_fn(write_fn)
 
     app_client.ds.remove_database("immutable-db")
 
@@ -513,7 +505,7 @@ async def test_database_memory_name(app_client):
         table_names = await db.table_names()
         assert table_names == []
     # Now create a table in foo
-    await foo1.execute_write("create table foo (t text)", block=True)
+    await foo1.execute_write("create table foo (t text)")
     assert await foo1.table_names() == ["foo"]
     assert await foo2.table_names() == ["foo"]
     assert await bar1.table_names() == []
@@ -528,5 +520,5 @@ async def test_in_memory_databases_forbid_writes(app_client):
         await db.execute("create table foo (t text)")
     assert await db.table_names() == []
     # Using db.execute_write() should work:
-    await db.execute_write("create table foo (t text)", block=True)
+    await db.execute_write("create table foo (t text)")
     assert await db.table_names() == ["foo"]

From 63537dd3decfd59636f4a42b336785ef49f0cec0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 12:34:55 -0800
Subject: [PATCH 1286/2303] Allow 'explain query plan' with more whitespace,
 closes #1588

---
 datasette/utils/__init__.py | 8 ++++----
 tests/test_utils.py         | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index c339113c..bc3155a5 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -162,11 +162,11 @@ class InvalidSql(Exception):
 
 allowed_sql_res = [
     re.compile(r"^select\b"),
-    re.compile(r"^explain select\b"),
-    re.compile(r"^explain query plan select\b"),
+    re.compile(r"^explain\s+select\b"),
+    re.compile(r"^explain\s+query\s+plan\s+select\b"),
     re.compile(r"^with\b"),
-    re.compile(r"^explain with\b"),
-    re.compile(r"^explain query plan with\b"),
+    re.compile(r"^explain\s+with\b"),
+    re.compile(r"^explain\s+query\s+plan\s+with\b"),
 ]
 allowed_pragmas = (
     "database_list",
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e1b61072..e7d67045 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -157,7 +157,9 @@ def test_validate_sql_select_bad(bad_sql):
         "select '# Hello there\n\n* This is a list\n* of items\n--\n[And a link](https://github.com/simonw/datasette-render-markdown).'\nas demo_markdown",
         "select 1 + 1",
         "explain select 1 + 1",
+        "explain\nselect 1 + 1",
         "explain query plan select 1 + 1",
+        "explain  query  plan\nselect 1 + 1",
         "SELECT\nblah FROM foo",
         "WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;",
         "explain WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;",

From 4b23f01f3e668c8f2a2f1a294be49f49b4073969 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 13:35:54 -0800
Subject: [PATCH 1287/2303] CLI reference docs, maintained by cog - refs #1594

---
 .github/workflows/test.yml |   3 +
 docs/cli-reference.rst     | 355 +++++++++++++++++++++++++++++++++++++
 docs/index.rst             |   1 +
 setup.py                   |   1 +
 4 files changed, 360 insertions(+)
 create mode 100644 docs/cli-reference.rst

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0b3635fe..704931a6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,3 +28,6 @@ jobs:
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
+    - name: Check if cog needs to be run
+      run: |
+        cog --check docs/*.rst
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
new file mode 100644
index 00000000..adf89633
--- /dev/null
+++ b/docs/cli-reference.rst
@@ -0,0 +1,355 @@
+.. _cli_reference:
+
+===============
+ CLI reference
+===============
+
+This page lists the ``--help`` for every ``datasette`` CLI command.
+
+.. [[[cog
+    from datasette import cli
+    from click.testing import CliRunner
+    import textwrap
+    commands = [
+        ["--help"],
+        ["serve", "--help"],
+        ["serve", "--help-settings"],
+        ["plugins", "--help"],
+        ["publish", "--help"],
+        ["publish", "cloudrun", "--help"],
+        ["publish", "heroku", "--help"],
+        ["package", "--help"],
+        ["inspect", "--help"],
+        ["install", "--help"],
+        ["uninstall", "--help"],
+    ]
+    for command in commands:
+        title = "datasette " + " ".join(command)
+        cog.out(title + "\n")
+        cog.out(("=" * len(title)) + "\n\n")
+        cog.out("::\n\n")
+        result = CliRunner().invoke(cli.cli, command)
+        output = result.output.replace("Usage: cli ", "Usage: datasette ")
+        cog.out(textwrap.indent(output, '    '))
+        cog.out("\n\n")
+.. ]]]
+datasette --help
+================
+
+::
+
+    Usage: datasette [OPTIONS] COMMAND [ARGS]...
+
+      Datasette is an open source multi-tool for exploring and publishing data
+
+      About Datasette: https://datasette.io/
+      Full documentation: https://docs.datasette.io/
+
+    Options:
+      --version  Show the version and exit.
+      --help     Show this message and exit.
+
+    Commands:
+      serve*     Serve up specified SQLite database files with a web UI
+      inspect
+      install    Install Python packages - e.g.
+      package    Package specified SQLite files into a new datasette Docker...
+      plugins    List currently available plugins
+      publish    Publish specified SQLite database files to the internet along...
+      uninstall  Uninstall Python packages (e.g.
+
+
+datasette serve --help
+======================
+
+::
+
+    Usage: datasette serve [OPTIONS] [FILES]...
+
+      Serve up specified SQLite database files with a web UI
+
+    Options:
+      -i, --immutable PATH      Database files to open in immutable mode
+      -h, --host TEXT           Host for server. Defaults to 127.0.0.1 which means
+                                only connections from the local machine will be
+                                allowed. Use 0.0.0.0 to listen to all IPs and allow
+                                access from other machines.
+      -p, --port INTEGER RANGE  Port for server, defaults to 8001. Use -p 0 to
+                                automatically assign an available port.
+                                [0<=x<=65535]
+      --uds TEXT                Bind to a Unix domain socket
+      --reload                  Automatically reload if code or metadata change
+                                detected - useful for development
+      --cors                    Enable CORS by serving Access-Control-Allow-Origin:
+                                *
+      --load-extension TEXT     Path to a SQLite extension to load
+      --inspect-file TEXT       Path to JSON file created using "datasette inspect"
+      -m, --metadata FILENAME   Path to JSON/YAML file containing license/source
+                                metadata
+      --template-dir DIRECTORY  Path to directory containing custom templates
+      --plugins-dir DIRECTORY   Path to directory containing custom plugins
+      --static MOUNT:DIRECTORY  Serve static files from this directory at /MOUNT/...
+      --memory                  Make /_memory database available
+      --config CONFIG           Deprecated: set config option using
+                                configname:value. Use --setting instead.
+      --setting SETTING...      Setting, see docs.datasette.io/en/stable/config.html
+      --secret TEXT             Secret used for signing secure values, such as
+                                signed cookies
+      --root                    Output URL that sets a cookie authenticating the
+                                root user
+      --get TEXT                Run an HTTP GET request against this path, print
+                                results and exit
+      --version-note TEXT       Additional note to show on /-/versions
+      --help-settings           Show available settings
+      --pdb                     Launch debugger on any errors
+      -o, --open                Open Datasette in your web browser
+      --create                  Create database files if they do not exist
+      --crossdb                 Enable cross-database joins using the /_memory
+                                database
+      --ssl-keyfile TEXT        SSL key file
+      --ssl-certfile TEXT       SSL certificate file
+      --help                    Show this message and exit.
+
+
+datasette serve --help-settings
+===============================
+
+::
+
+    Settings:
+      default_page_size            Default page size for the table view
+                                   (default=100)
+      max_returned_rows            Maximum rows that can be returned from a table or
+                                   custom query (default=1000)
+      num_sql_threads              Number of threads in the thread pool for
+                                   executing SQLite queries (default=3)
+      sql_time_limit_ms            Time limit for a SQL query in milliseconds
+                                   (default=1000)
+      default_facet_size           Number of values to return for requested facets
+                                   (default=30)
+      facet_time_limit_ms          Time limit for calculating a requested facet
+                                   (default=200)
+      facet_suggest_time_limit_ms  Time limit for calculating a suggested facet
+                                   (default=50)
+      hash_urls                    Include DB file contents hash in URLs, for far-
+                                   future caching (default=False)
+      allow_facet                  Allow users to specify columns to facet using
+                                   ?_facet= parameter (default=True)
+      allow_download               Allow users to download the original SQLite
+                                   database files (default=True)
+      suggest_facets               Calculate and display suggested facets
+                                   (default=True)
+      default_cache_ttl            Default HTTP cache TTL (used in Cache-Control:
+                                   max-age= header) (default=5)
+      default_cache_ttl_hashed     Default HTTP cache TTL for hashed URL pages
+                                   (default=31536000)
+      cache_size_kb                SQLite cache size in KB (0 == use SQLite default)
+                                   (default=0)
+      allow_csv_stream             Allow .csv?_stream=1 to download all rows
+                                   (ignoring max_returned_rows) (default=True)
+      max_csv_mb                   Maximum size allowed for CSV export in MB - set 0
+                                   to disable this limit (default=100)
+      truncate_cells_html          Truncate cells longer than this in HTML table
+                                   view - set 0 to disable (default=2048)
+      force_https_urls             Force URLs in API output to always use https://
+                                   protocol (default=False)
+      template_debug               Allow display of template debug information with
+                                   ?_context=1 (default=False)
+      trace_debug                  Allow display of SQL trace debug information with
+                                   ?_trace=1 (default=False)
+      base_url                     Datasette URLs should use this base path
+                                   (default=/)
+
+
+
+datasette plugins --help
+========================
+
+::
+
+    Usage: datasette plugins [OPTIONS]
+
+      List currently available plugins
+
+    Options:
+      --all                    Include built-in default plugins
+      --plugins-dir DIRECTORY  Path to directory containing custom plugins
+      --help                   Show this message and exit.
+
+
+datasette publish --help
+========================
+
+::
+
+    Usage: datasette publish [OPTIONS] COMMAND [ARGS]...
+
+      Publish specified SQLite database files to the internet along with a
+      Datasette-powered interface and API
+
+    Options:
+      --help  Show this message and exit.
+
+    Commands:
+      cloudrun
+      heroku
+
+
+datasette publish cloudrun --help
+=================================
+
+::
+
+    Usage: datasette publish cloudrun [OPTIONS] [FILES]...
+
+    Options:
+      -m, --metadata FILENAME         Path to JSON/YAML file containing metadata to
+                                      publish
+      --extra-options TEXT            Extra options to pass to datasette serve
+      --branch TEXT                   Install datasette from a GitHub branch e.g.
+                                      main
+      --template-dir DIRECTORY        Path to directory containing custom templates
+      --plugins-dir DIRECTORY         Path to directory containing custom plugins
+      --static MOUNT:DIRECTORY        Serve static files from this directory at
+                                      /MOUNT/...
+      --install TEXT                  Additional packages (e.g. plugins) to install
+      --plugin-secret <TEXT TEXT TEXT>...
+                                      Secrets to pass to plugins, e.g. --plugin-
+                                      secret datasette-auth-github client_id xxx
+      --version-note TEXT             Additional note to show on /-/versions
+      --secret TEXT                   Secret used for signing secure values, such as
+                                      signed cookies
+      --title TEXT                    Title for metadata
+      --license TEXT                  License label for metadata
+      --license_url TEXT              License URL for metadata
+      --source TEXT                   Source label for metadata
+      --source_url TEXT               Source URL for metadata
+      --about TEXT                    About label for metadata
+      --about_url TEXT                About URL for metadata
+      -n, --name TEXT                 Application name to use when building
+      --service TEXT                  Cloud Run service to deploy (or over-write)
+      --spatialite                    Enable SpatialLite extension
+      --show-files                    Output the generated Dockerfile and
+                                      metadata.json
+      --memory TEXT                   Memory to allocate in Cloud Run, e.g. 1Gi
+      --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
+      --apt-get-install TEXT          Additional packages to apt-get install
+      --help                          Show this message and exit.
+
+
+datasette publish heroku --help
+===============================
+
+::
+
+    Usage: datasette publish heroku [OPTIONS] [FILES]...
+
+    Options:
+      -m, --metadata FILENAME         Path to JSON/YAML file containing metadata to
+                                      publish
+      --extra-options TEXT            Extra options to pass to datasette serve
+      --branch TEXT                   Install datasette from a GitHub branch e.g.
+                                      main
+      --template-dir DIRECTORY        Path to directory containing custom templates
+      --plugins-dir DIRECTORY         Path to directory containing custom plugins
+      --static MOUNT:DIRECTORY        Serve static files from this directory at
+                                      /MOUNT/...
+      --install TEXT                  Additional packages (e.g. plugins) to install
+      --plugin-secret <TEXT TEXT TEXT>...
+                                      Secrets to pass to plugins, e.g. --plugin-
+                                      secret datasette-auth-github client_id xxx
+      --version-note TEXT             Additional note to show on /-/versions
+      --secret TEXT                   Secret used for signing secure values, such as
+                                      signed cookies
+      --title TEXT                    Title for metadata
+      --license TEXT                  License label for metadata
+      --license_url TEXT              License URL for metadata
+      --source TEXT                   Source label for metadata
+      --source_url TEXT               Source URL for metadata
+      --about TEXT                    About label for metadata
+      --about_url TEXT                About URL for metadata
+      -n, --name TEXT                 Application name to use when deploying
+      --tar TEXT                      --tar option to pass to Heroku, e.g.
+                                      --tar=/usr/local/bin/gtar
+      --help                          Show this message and exit.
+
+
+datasette package --help
+========================
+
+::
+
+    Usage: datasette package [OPTIONS] FILES...
+
+      Package specified SQLite files into a new datasette Docker container
+
+    Options:
+      -t, --tag TEXT            Name for the resulting Docker container, can
+                                optionally use name:tag format
+      -m, --metadata FILENAME   Path to JSON/YAML file containing metadata to
+                                publish
+      --extra-options TEXT      Extra options to pass to datasette serve
+      --branch TEXT             Install datasette from a GitHub branch e.g. main
+      --template-dir DIRECTORY  Path to directory containing custom templates
+      --plugins-dir DIRECTORY   Path to directory containing custom plugins
+      --static MOUNT:DIRECTORY  Serve static files from this directory at /MOUNT/...
+      --install TEXT            Additional packages (e.g. plugins) to install
+      --spatialite              Enable SpatialLite extension
+      --version-note TEXT       Additional note to show on /-/versions
+      --secret TEXT             Secret used for signing secure values, such as
+                                signed cookies
+      -p, --port INTEGER RANGE  Port to run the server on, defaults to 8001
+                                [1<=x<=65535]
+      --title TEXT              Title for metadata
+      --license TEXT            License label for metadata
+      --license_url TEXT        License URL for metadata
+      --source TEXT             Source label for metadata
+      --source_url TEXT         Source URL for metadata
+      --about TEXT              About label for metadata
+      --about_url TEXT          About URL for metadata
+      --help                    Show this message and exit.
+
+
+datasette inspect --help
+========================
+
+::
+
+    Usage: datasette inspect [OPTIONS] [FILES]...
+
+    Options:
+      --inspect-file TEXT
+      --load-extension TEXT  Path to a SQLite extension to load
+      --help                 Show this message and exit.
+
+
+datasette install --help
+========================
+
+::
+
+    Usage: datasette install [OPTIONS] PACKAGES...
+
+      Install Python packages - e.g. Datasette plugins - into the same environment
+      as Datasette
+
+    Options:
+      -U, --upgrade  Upgrade packages to latest version
+      --help         Show this message and exit.
+
+
+datasette uninstall --help
+==========================
+
+::
+
+    Usage: datasette uninstall [OPTIONS] PACKAGES...
+
+      Uninstall Python packages (e.g. plugins) from the Datasette environment
+
+    Options:
+      -y, --yes  Don't ask for confirmation
+      --help     Show this message and exit.
+
+
+.. [[[end]]]
diff --git a/docs/index.rst b/docs/index.rst
index eafc5bdb..36e42848 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -60,4 +60,5 @@ Contents
    testing_plugins
    internals
    contributing
+   cli-reference
    changelog
diff --git a/setup.py b/setup.py
index 1f8855cf..e9ef082a 100644
--- a/setup.py
+++ b/setup.py
@@ -74,6 +74,7 @@ setup(
             "black==21.12b0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
+            "cogapp>=3.3.0",
         ],
         "rich": ["rich"],
     },

From 5698e2af0182677c0f1f7f5b3bc61415bb6c93f8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 13:55:13 -0800
Subject: [PATCH 1288/2303] Promote Datasette Desktop in installation docs,
 closes #1466

---
 docs/installation.rst | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 723f1e3f..ac3dcca2 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -7,8 +7,7 @@
 .. note::
     If you just want to try Datasette out you don't need to install anything: see :ref:`getting_started_glitch`
 
-There are two main options for installing Datasette. You can install it directly
-on to your machine, or you can install it using Docker.
+There are two main options for installing Datasette. You can install it directly on to your machine, or you can install it using Docker.
 
 If you want to start making contributions to the Datasette project by installing a copy that lets you directly modify the code, take a look at our guide to :ref:`devenvironment`.
 
@@ -20,6 +19,13 @@ If you want to start making contributions to the Datasette project by installing
 Basic installation
 ==================
 
+.. _installation_datasette_desktop:
+
+Datasette Desktop for Mac
+-------------------------
+
+`Datasette Desktop <https://datasette.io/desktop>`__ is a packaged Mac application which bundles Datasette together with Python and allows you to install and run Datasette directly on your laptop. This is the best option for local installation if you are not comfortable using the command line.
+
 .. _installation_homebrew:
 
 Using Homebrew

From 3658e57ac2de0bec0ea5de36e3ddd09784ecf65e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 14:20:07 -0800
Subject: [PATCH 1289/2303] Fixed bug with table title element, closes #1560

---
 datasette/templates/table.html |  3 +--
 tests/test_table_html.py       | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index f3749b57..e3c6f38d 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -1,7 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %}
-    {% if human_description_en %}where {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{ super() }}
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 50d679a0..f68e05a5 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1038,3 +1038,22 @@ def test_sort_rowid_with_next(app_client):
 
 def assert_querystring_equal(expected, actual):
     assert sorted(expected.split("&")) == sorted(actual.split("&"))
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    (
+        (
+            "/fixtures/facetable",
+            "fixtures: facetable: 15 rows",
+        ),
+        (
+            "/fixtures/facetable?on_earth__exact=1",
+            "fixtures: facetable: 14 rows where on_earth = 1",
+        ),
+    ),
+)
+def test_table_page_title(app_client, path, expected):
+    response = app_client.get(path)
+    title = Soup(response.text, "html.parser").find("title").text
+    assert title == expected

From 88bc2ceae1151ec859f477d527b40f7e36012017 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:07:30 -0800
Subject: [PATCH 1290/2303] --help summary for 'datasette inspect', closes
 #1597

---
 datasette/cli.py       | 6 ++++++
 docs/cli-reference.rst | 7 ++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 22e2338a..12d3d728 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -136,6 +136,12 @@ def cli():
 @click.option("--inspect-file", default="-")
 @sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
+    """
+    Generate JSON summary of provided database files
+
+    This can then be passed to "datasette --inspect-file" to speed up count
+    operations against immutable database files.
+    """
     app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
     loop = asyncio.get_event_loop()
     inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index adf89633..7ac6debe 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -51,7 +51,7 @@ datasette --help
 
     Commands:
       serve*     Serve up specified SQLite database files with a web UI
-      inspect
+      inspect    Generate JSON summary of provided database files
       install    Install Python packages - e.g.
       package    Package specified SQLite files into a new datasette Docker...
       plugins    List currently available plugins
@@ -317,6 +317,11 @@ datasette inspect --help
 
     Usage: datasette inspect [OPTIONS] [FILES]...
 
+      Generate JSON summary of provided database files
+
+      This can then be passed to "datasette --inspect-file" to speed up count
+      operations against immutable database files.
+
     Options:
       --inspect-file TEXT
       --load-extension TEXT  Path to a SQLite extension to load

From 8f5c44a1669427019b288f5b5debec67a90f908b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:09:38 -0800
Subject: [PATCH 1291/2303] Better --help summaries for install and uninstall

---
 datasette/cli.py       | 4 ++--
 docs/cli-reference.rst | 9 ++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 12d3d728..18054448 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -307,7 +307,7 @@ def package(
     "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version"
 )
 def install(packages, upgrade):
-    """Install Python packages - e.g. Datasette plugins - into the same environment as Datasette"""
+    """Install plugins and packages from PyPI into the same environment as Datasette"""
     args = ["pip", "install"]
     if upgrade:
         args += ["--upgrade"]
@@ -320,7 +320,7 @@ def install(packages, upgrade):
 @click.argument("packages", nargs=-1, required=True)
 @click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation")
 def uninstall(packages, yes):
-    """Uninstall Python packages (e.g. plugins) from the Datasette environment"""
+    """Uninstall plugins and Python packages from the Datasette environment"""
     sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else [])
     run_module("pip", run_name="__main__")
 
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 7ac6debe..37a30606 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -52,11 +52,11 @@ datasette --help
     Commands:
       serve*     Serve up specified SQLite database files with a web UI
       inspect    Generate JSON summary of provided database files
-      install    Install Python packages - e.g.
+      install    Install plugins and packages from PyPI into the same...
       package    Package specified SQLite files into a new datasette Docker...
       plugins    List currently available plugins
       publish    Publish specified SQLite database files to the internet along...
-      uninstall  Uninstall Python packages (e.g.
+      uninstall  Uninstall plugins and Python packages from the Datasette...
 
 
 datasette serve --help
@@ -335,8 +335,7 @@ datasette install --help
 
     Usage: datasette install [OPTIONS] PACKAGES...
 
-      Install Python packages - e.g. Datasette plugins - into the same environment
-      as Datasette
+      Install plugins and packages from PyPI into the same environment as Datasette
 
     Options:
       -U, --upgrade  Upgrade packages to latest version
@@ -350,7 +349,7 @@ datasette uninstall --help
 
     Usage: datasette uninstall [OPTIONS] PACKAGES...
 
-      Uninstall Python packages (e.g. plugins) from the Datasette environment
+      Uninstall plugins and Python packages from the Datasette environment
 
     Options:
       -y, --yes  Don't ask for confirmation

From 8cf4b77a92f5170c33e0079f2bab48a4f36b6934 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:10:52 -0800
Subject: [PATCH 1292/2303] Better copy for 'datasette plugins --help'

---
 datasette/cli.py       | 2 +-
 docs/cli-reference.rst | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 18054448..af09453f 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -190,7 +190,7 @@ pm.hook.publish_subcommand(publish=publish)
     help="Path to directory containing custom plugins",
 )
 def plugins(all, plugins_dir):
-    """List currently available plugins"""
+    """List currently installed plugins"""
     app = Datasette([], plugins_dir=plugins_dir)
     click.echo(json.dumps(app._plugins(all=all), indent=4))
 
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 37a30606..69a7cdd9 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -54,7 +54,7 @@ datasette --help
       inspect    Generate JSON summary of provided database files
       install    Install plugins and packages from PyPI into the same...
       package    Package specified SQLite files into a new datasette Docker...
-      plugins    List currently available plugins
+      plugins    List currently installed plugins
       publish    Publish specified SQLite database files to the internet along...
       uninstall  Uninstall plugins and Python packages from the Datasette...
 
@@ -169,7 +169,7 @@ datasette plugins --help
 
     Usage: datasette plugins [OPTIONS]
 
-      List currently available plugins
+      List currently installed plugins
 
     Options:
       --all                    Include built-in default plugins

From 515f8d38ebae203efc15ca79a8b42848276b35e5 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:12:54 -0800
Subject: [PATCH 1293/2303] Help summaries for publish cloudrun/heroku

---
 datasette/publish/cloudrun.py | 1 +
 datasette/publish/heroku.py   | 1 +
 docs/cli-reference.rst        | 8 ++++++--
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py
index 1fabcafd..a1e2f580 100644
--- a/datasette/publish/cloudrun.py
+++ b/datasette/publish/cloudrun.py
@@ -74,6 +74,7 @@ def publish_subcommand(publish):
         cpu,
         apt_get_extras,
     ):
+        "Publish databases to Datasette running on Cloud Run"
         fail_if_publish_binary_not_installed(
             "gcloud", "Google Cloud", "https://cloud.google.com/sdk/"
         )
diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py
index 2ebbd4bd..171252ce 100644
--- a/datasette/publish/heroku.py
+++ b/datasette/publish/heroku.py
@@ -50,6 +50,7 @@ def publish_subcommand(publish):
         name,
         tar,
     ):
+        "Publish databases to Datasette running on Heroku"
         fail_if_publish_binary_not_installed(
             "heroku", "Heroku", "https://cli.heroku.com"
         )
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 69a7cdd9..f3279f6d 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -191,8 +191,8 @@ datasette publish --help
       --help  Show this message and exit.
 
     Commands:
-      cloudrun
-      heroku
+      cloudrun  Publish databases to Datasette running on Cloud Run
+      heroku    Publish databases to Datasette running on Heroku
 
 
 datasette publish cloudrun --help
@@ -202,6 +202,8 @@ datasette publish cloudrun --help
 
     Usage: datasette publish cloudrun [OPTIONS] [FILES]...
 
+      Publish databases to Datasette running on Cloud Run
+
     Options:
       -m, --metadata FILENAME         Path to JSON/YAML file containing metadata to
                                       publish
@@ -244,6 +246,8 @@ datasette publish heroku --help
 
     Usage: datasette publish heroku [OPTIONS] [FILES]...
 
+      Publish databases to Datasette running on Heroku
+
     Options:
       -m, --metadata FILENAME         Path to JSON/YAML file containing metadata to
                                       publish

From 3a0f7d64889cd79d5d00d3251e8ab77ff52de60d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:27:21 -0800
Subject: [PATCH 1294/2303] Fixed hidden form fields bug #1527

---
 datasette/views/table.py |  2 +-
 tests/test_table_html.py | 22 +++++++++++++++++-----
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 9808fd24..77fb2850 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -815,7 +815,7 @@ class TableView(RowTableShared):
                 if (
                     key.startswith("_")
                     and key not in ("_sort", "_search", "_next")
-                    and not key.endswith("__exact")
+                    and "__" not in key
                 ):
                     for value in request.args.getlist(key):
                         form_hidden_args.append((key, value))
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index f68e05a5..021268c3 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -143,17 +143,29 @@ def test_existing_filter_redirects(app_client):
     assert "?" not in response.headers["Location"]
 
 
-def test_exact_parameter_results_in_correct_hidden_fields(app_client):
+@pytest.mark.parametrize(
+    "qs,expected_hidden",
+    (
+        # Things that should be reflected in hidden form fields:
+        ("_facet=_neighborhood", {"_facet": "_neighborhood"}),
+        ("_where=1+=+1&_col=_city_id", {"_where": "1 = 1", "_col": "_city_id"}),
+        # Things that should NOT be reflected in hidden form fields:
+        (
+            "_facet=_neighborhood&_neighborhood__exact=Downtown",
+            {"_facet": "_neighborhood"},
+        ),
+        ("_facet=_neighborhood&_city_id__gt=1", {"_facet": "_neighborhood"}),
+    ),
+)
+def test_reflected_hidden_form_fields(app_client, qs, expected_hidden):
     # https://github.com/simonw/datasette/issues/1527
-    response = app_client.get(
-        "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown"
-    )
+    response = app_client.get("/fixtures/facetable?{}".format(qs))
     # In this case we should NOT have a hidden _neighborhood__exact=Downtown field
     form = Soup(response.body, "html.parser").find("form")
     hidden_inputs = {
         input["name"]: input["value"] for input in form.select("input[type=hidden]")
     }
-    assert hidden_inputs == {"_facet": "_neighborhood"}
+    assert hidden_inputs == expected_hidden
 
 
 def test_empty_search_parameter_gets_removed(app_client):

From 76d66d5b2bf10249c0beaac0999b93ac8d757f48 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:30:00 -0800
Subject: [PATCH 1295/2303] Tweak order of documentation contents

---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index 36e42848..acca943f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -59,6 +59,6 @@ Contents
    plugin_hooks
    testing_plugins
    internals
-   contributing
    cli-reference
+   contributing
    changelog

From 714b4df1b1b2aeab8cde3a309627c42355439dda Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:36:28 -0800
Subject: [PATCH 1296/2303] Fixed reStructuredText warning, refs #1594

---
 docs/cli-reference.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index f3279f6d..f529782d 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -23,6 +23,7 @@ This page lists the ``--help`` for every ``datasette`` CLI command.
         ["install", "--help"],
         ["uninstall", "--help"],
     ]
+    cog.out("\n")
     for command in commands:
         title = "datasette " + " ".join(command)
         cog.out(title + "\n")
@@ -33,6 +34,7 @@ This page lists the ``--help`` for every ``datasette`` CLI command.
         cog.out(textwrap.indent(output, '    '))
         cog.out("\n\n")
 .. ]]]
+
 datasette --help
 ================
 

From ab7d6a7179e9939c2764989e508bfa8eba31f3b1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:38:16 -0800
Subject: [PATCH 1297/2303] Updated settings help URL to avoid redirect

---
 datasette/cli.py        | 2 +-
 datasette/views/base.py | 2 +-
 docs/cli-reference.rst  | 3 ++-
 tests/test_html.py      | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index af09453f..9d1b5ee5 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -400,7 +400,7 @@ def uninstall(packages, yes):
     "--setting",
     "settings",
     type=Setting(),
-    help="Setting, see docs.datasette.io/en/stable/config.html",
+    help="Setting, see docs.datasette.io/en/stable/settings.html",
     multiple=True,
 )
 @click.option(
diff --git a/datasette/views/base.py b/datasette/views/base.py
index a9953dfd..b1cacb3f 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -493,7 +493,7 @@ class DataView(BaseView):
             raise DatasetteError(
                 """
                 SQL query took too long. The time limit is controlled by the
-                <a href="https://docs.datasette.io/en/stable/config.html#sql-time-limit-ms">sql_time_limit_ms</a>
+                <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
                 configuration option.
             """,
                 title="SQL Interrupted",
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index f529782d..74adb92d 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -94,7 +94,8 @@ datasette serve --help
       --memory                  Make /_memory database available
       --config CONFIG           Deprecated: set config option using
                                 configname:value. Use --setting instead.
-      --setting SETTING...      Setting, see docs.datasette.io/en/stable/config.html
+      --setting SETTING...      Setting, see
+                                docs.datasette.io/en/stable/settings.html
       --secret TEXT             Secret used for signing secure values, such as
                                 signed cookies
       --root                    Output URL that sets a cookie authenticating the
diff --git a/tests/test_html.py b/tests/test_html.py
index bfe5c8f9..3f0a88a9 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -155,7 +155,7 @@ def test_sql_time_limit(app_client_shorter_time_limit):
     response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)")
     assert 400 == response.status
     expected_html_fragment = """
-        <a href="https://docs.datasette.io/en/stable/config.html#sql-time-limit-ms">sql_time_limit_ms</a>
+        <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
     """.strip()
     assert expected_html_fragment in response.text
 

From 10659c3f1f82458adfa65c61f4dcc8d9af5467ed Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:38:53 -0800
Subject: [PATCH 1298/2303] datasette-debug-asgi plugin to help investigate
 #1590

---
 demos/apache-proxy/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index ab7b9d16..6c921963 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -27,7 +27,7 @@ ARG DATASETTE_REF
 
 RUN pip install \
     https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip \
-    datasette-redirect-to-https
+    datasette-redirect-to-https datasette-debug-asgi
 
 ADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf
 

From 3664ddd400062123e99500d28b160c7944408c1a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 16:47:53 -0800
Subject: [PATCH 1299/2303] Replace update-docs-help.py with cog, closes #1598

---
 docs/cli-reference.rst                   | 24 ++++++++++++++
 docs/datasette-package-help.txt          | 29 ----------------
 docs/datasette-publish-cloudrun-help.txt | 33 -------------------
 docs/datasette-publish-heroku-help.txt   | 29 ----------------
 docs/datasette-serve-help.txt            | 42 ------------------------
 docs/getting_started.rst                 |  9 +----
 docs/publish.rst                         |  6 ++--
 tests/test_docs.py                       | 20 -----------
 update-docs-help.py                      | 25 --------------
 9 files changed, 28 insertions(+), 189 deletions(-)
 delete mode 100644 docs/datasette-package-help.txt
 delete mode 100644 docs/datasette-publish-cloudrun-help.txt
 delete mode 100644 docs/datasette-publish-heroku-help.txt
 delete mode 100644 docs/datasette-serve-help.txt
 delete mode 100644 update-docs-help.py

diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 74adb92d..155a005d 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -26,6 +26,8 @@ This page lists the ``--help`` for every ``datasette`` CLI command.
     cog.out("\n")
     for command in commands:
         title = "datasette " + " ".join(command)
+        ref = "_cli_help_" + ("_".join(command).replace("-", "_"))
+        cog.out(".. {}:\n\n".format(ref))
         cog.out(title + "\n")
         cog.out(("=" * len(title)) + "\n\n")
         cog.out("::\n\n")
@@ -35,6 +37,8 @@ This page lists the ``--help`` for every ``datasette`` CLI command.
         cog.out("\n\n")
 .. ]]]
 
+.. _cli_help___help:
+
 datasette --help
 ================
 
@@ -61,6 +65,8 @@ datasette --help
       uninstall  Uninstall plugins and Python packages from the Datasette...
 
 
+.. _cli_help_serve___help:
+
 datasette serve --help
 ======================
 
@@ -114,6 +120,8 @@ datasette serve --help
       --help                    Show this message and exit.
 
 
+.. _cli_help_serve___help_settings:
+
 datasette serve --help-settings
 ===============================
 
@@ -165,6 +173,8 @@ datasette serve --help-settings
 
 
 
+.. _cli_help_plugins___help:
+
 datasette plugins --help
 ========================
 
@@ -180,6 +190,8 @@ datasette plugins --help
       --help                   Show this message and exit.
 
 
+.. _cli_help_publish___help:
+
 datasette publish --help
 ========================
 
@@ -198,6 +210,8 @@ datasette publish --help
       heroku    Publish databases to Datasette running on Heroku
 
 
+.. _cli_help_publish_cloudrun___help:
+
 datasette publish cloudrun --help
 =================================
 
@@ -242,6 +256,8 @@ datasette publish cloudrun --help
       --help                          Show this message and exit.
 
 
+.. _cli_help_publish_heroku___help:
+
 datasette publish heroku --help
 ===============================
 
@@ -281,6 +297,8 @@ datasette publish heroku --help
       --help                          Show this message and exit.
 
 
+.. _cli_help_package___help:
+
 datasette package --help
 ========================
 
@@ -317,6 +335,8 @@ datasette package --help
       --help                    Show this message and exit.
 
 
+.. _cli_help_inspect___help:
+
 datasette inspect --help
 ========================
 
@@ -335,6 +355,8 @@ datasette inspect --help
       --help                 Show this message and exit.
 
 
+.. _cli_help_install___help:
+
 datasette install --help
 ========================
 
@@ -349,6 +371,8 @@ datasette install --help
       --help         Show this message and exit.
 
 
+.. _cli_help_uninstall___help:
+
 datasette uninstall --help
 ==========================
 
diff --git a/docs/datasette-package-help.txt b/docs/datasette-package-help.txt
deleted file mode 100644
index 7cfac1b1..00000000
--- a/docs/datasette-package-help.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-$ datasette package --help
-
-Usage: datasette package [OPTIONS] FILES...
-
-  Package specified SQLite files into a new datasette Docker container
-
-Options:
-  -t, --tag TEXT            Name for the resulting Docker container, can optionally use
-                            name:tag format
-  -m, --metadata FILENAME   Path to JSON/YAML file containing metadata to publish
-  --extra-options TEXT      Extra options to pass to datasette serve
-  --branch TEXT             Install datasette from a GitHub branch e.g. main
-  --template-dir DIRECTORY  Path to directory containing custom templates
-  --plugins-dir DIRECTORY   Path to directory containing custom plugins
-  --static MOUNT:DIRECTORY  Serve static files from this directory at /MOUNT/...
-  --install TEXT            Additional packages (e.g. plugins) to install
-  --spatialite              Enable SpatialLite extension
-  --version-note TEXT       Additional note to show on /-/versions
-  --secret TEXT             Secret used for signing secure values, such as signed
-                            cookies
-  -p, --port INTEGER RANGE  Port to run the server on, defaults to 8001  [1<=x<=65535]
-  --title TEXT              Title for metadata
-  --license TEXT            License label for metadata
-  --license_url TEXT        License URL for metadata
-  --source TEXT             Source label for metadata
-  --source_url TEXT         Source URL for metadata
-  --about TEXT              About label for metadata
-  --about_url TEXT          About URL for metadata
-  --help                    Show this message and exit.
diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt
deleted file mode 100644
index 34481b40..00000000
--- a/docs/datasette-publish-cloudrun-help.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-$ datasette publish cloudrun --help
-
-Usage: datasette publish cloudrun [OPTIONS] [FILES]...
-
-Options:
-  -m, --metadata FILENAME         Path to JSON/YAML file containing metadata to publish
-  --extra-options TEXT            Extra options to pass to datasette serve
-  --branch TEXT                   Install datasette from a GitHub branch e.g. main
-  --template-dir DIRECTORY        Path to directory containing custom templates
-  --plugins-dir DIRECTORY         Path to directory containing custom plugins
-  --static MOUNT:DIRECTORY        Serve static files from this directory at /MOUNT/...
-  --install TEXT                  Additional packages (e.g. plugins) to install
-  --plugin-secret <TEXT TEXT TEXT>...
-                                  Secrets to pass to plugins, e.g. --plugin-secret
-                                  datasette-auth-github client_id xxx
-  --version-note TEXT             Additional note to show on /-/versions
-  --secret TEXT                   Secret used for signing secure values, such as signed
-                                  cookies
-  --title TEXT                    Title for metadata
-  --license TEXT                  License label for metadata
-  --license_url TEXT              License URL for metadata
-  --source TEXT                   Source label for metadata
-  --source_url TEXT               Source URL for metadata
-  --about TEXT                    About label for metadata
-  --about_url TEXT                About URL for metadata
-  -n, --name TEXT                 Application name to use when building
-  --service TEXT                  Cloud Run service to deploy (or over-write)
-  --spatialite                    Enable SpatialLite extension
-  --show-files                    Output the generated Dockerfile and metadata.json
-  --memory TEXT                   Memory to allocate in Cloud Run, e.g. 1Gi
-  --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
-  --apt-get-install TEXT          Additional packages to apt-get install
-  --help                          Show this message and exit.
diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt
deleted file mode 100644
index 9d633e95..00000000
--- a/docs/datasette-publish-heroku-help.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-$ datasette publish heroku --help
-
-Usage: datasette publish heroku [OPTIONS] [FILES]...
-
-Options:
-  -m, --metadata FILENAME         Path to JSON/YAML file containing metadata to publish
-  --extra-options TEXT            Extra options to pass to datasette serve
-  --branch TEXT                   Install datasette from a GitHub branch e.g. main
-  --template-dir DIRECTORY        Path to directory containing custom templates
-  --plugins-dir DIRECTORY         Path to directory containing custom plugins
-  --static MOUNT:DIRECTORY        Serve static files from this directory at /MOUNT/...
-  --install TEXT                  Additional packages (e.g. plugins) to install
-  --plugin-secret <TEXT TEXT TEXT>...
-                                  Secrets to pass to plugins, e.g. --plugin-secret
-                                  datasette-auth-github client_id xxx
-  --version-note TEXT             Additional note to show on /-/versions
-  --secret TEXT                   Secret used for signing secure values, such as signed
-                                  cookies
-  --title TEXT                    Title for metadata
-  --license TEXT                  License label for metadata
-  --license_url TEXT              License URL for metadata
-  --source TEXT                   Source label for metadata
-  --source_url TEXT               Source URL for metadata
-  --about TEXT                    About label for metadata
-  --about_url TEXT                About URL for metadata
-  -n, --name TEXT                 Application name to use when deploying
-  --tar TEXT                      --tar option to pass to Heroku, e.g.
-                                  --tar=/usr/local/bin/gtar
-  --help                          Show this message and exit.
diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt
deleted file mode 100644
index 2911977a..00000000
--- a/docs/datasette-serve-help.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-$ datasette serve --help
-
-Usage: datasette serve [OPTIONS] [FILES]...
-
-  Serve up specified SQLite database files with a web UI
-
-Options:
-  -i, --immutable PATH      Database files to open in immutable mode
-  -h, --host TEXT           Host for server. Defaults to 127.0.0.1 which means only
-                            connections from the local machine will be allowed. Use
-                            0.0.0.0 to listen to all IPs and allow access from other
-                            machines.
-  -p, --port INTEGER RANGE  Port for server, defaults to 8001. Use -p 0 to automatically
-                            assign an available port.  [0<=x<=65535]
-  --uds TEXT                Bind to a Unix domain socket
-  --reload                  Automatically reload if code or metadata change detected -
-                            useful for development
-  --cors                    Enable CORS by serving Access-Control-Allow-Origin: *
-  --load-extension TEXT     Path to a SQLite extension to load
-  --inspect-file TEXT       Path to JSON file created using "datasette inspect"
-  -m, --metadata FILENAME   Path to JSON/YAML file containing license/source metadata
-  --template-dir DIRECTORY  Path to directory containing custom templates
-  --plugins-dir DIRECTORY   Path to directory containing custom plugins
-  --static MOUNT:DIRECTORY  Serve static files from this directory at /MOUNT/...
-  --memory                  Make /_memory database available
-  --config CONFIG           Deprecated: set config option using configname:value. Use
-                            --setting instead.
-  --setting SETTING...      Setting, see docs.datasette.io/en/stable/config.html
-  --secret TEXT             Secret used for signing secure values, such as signed
-                            cookies
-  --root                    Output URL that sets a cookie authenticating the root user
-  --get TEXT                Run an HTTP GET request against this path, print results and
-                            exit
-  --version-note TEXT       Additional note to show on /-/versions
-  --help-settings           Show available settings
-  --pdb                     Launch debugger on any errors
-  -o, --open                Open Datasette in your web browser
-  --create                  Create database files if they do not exist
-  --crossdb                 Enable cross-database joins using the /_memory database
-  --ssl-keyfile TEXT        SSL key file
-  --ssl-certfile TEXT       SSL certificate file
-  --help                    Show this message and exit.
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 52434fdc..3e357afb 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -161,11 +161,4 @@ The ``--get`` option can specify the path to a page within Datasette and cause D
 
 The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This means you can use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions.
 
-.. _getting_started_serve_help:
-
-datasette serve --help
-----------------------
-
-Running ``datasette downloads.db`` executes the default ``serve`` sub-command, and is equivalent to running ``datasette serve downloads.db``. The full list of options to that command is shown below.
-
-.. literalinclude:: datasette-serve-help.txt
+Running ``datasette`` without specifying a command runs the default command, ``datasette serve``.  See :ref:`cli_help_serve___help` for the full list of options for that command.
diff --git a/docs/publish.rst b/docs/publish.rst
index f6895f53..1d9664e7 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -47,7 +47,7 @@ Once it has finished it will output a URL like this one::
 
 Cloud Run provides a URL on the ``.run.app`` domain, but you can also point your own domain or subdomain at your Cloud Run service - see `mapping custom domains <https://cloud.google.com/run/docs/mapping-custom-domains>`__ in the Cloud Run documentation for details.
 
-.. literalinclude:: datasette-publish-cloudrun-help.txt
+See :ref:`cli_help_publish_cloudrun___help` for the full list of options for this command.
 
 Publishing to Heroku
 --------------------
@@ -64,7 +64,7 @@ This will output some details about the new deployment, including a URL like thi
 
 You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app.
 
-.. literalinclude:: datasette-publish-heroku-help.txt
+See :ref:`cli_help_publish_heroku___help` for the full list of options for this command.
 
 .. _publish_vercel:
 
@@ -171,4 +171,4 @@ You can customize the port that is exposed by the container using the ``--port``
 
 A full list of options can be seen by running ``datasette package --help``:
 
-.. literalinclude:: datasette-package-help.txt
+See :ref:`cli_help_package___help` for the full list of options for this command.
\ No newline at end of file
diff --git a/tests/test_docs.py b/tests/test_docs.py
index d0cb036d..0d17b8e3 100644
--- a/tests/test_docs.py
+++ b/tests/test_docs.py
@@ -33,26 +33,6 @@ def test_settings_are_documented(settings_headings, setting):
     assert setting.name in settings_headings
 
 
-@pytest.mark.parametrize(
-    "name,filename",
-    (
-        ("serve", "datasette-serve-help.txt"),
-        ("package", "datasette-package-help.txt"),
-        ("publish heroku", "datasette-publish-heroku-help.txt"),
-        ("publish cloudrun", "datasette-publish-cloudrun-help.txt"),
-    ),
-)
-def test_help_includes(name, filename):
-    expected = (docs_path / filename).read_text()
-    runner = CliRunner()
-    result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88)
-    actual = f"$ datasette {name} --help\n\n{result.output}"
-    # actual has "Usage: cli package [OPTIONS] FILES"
-    # because it doesn't know that cli will be aliased to datasette
-    expected = expected.replace("Usage: datasette", "Usage: cli")
-    assert expected == actual, "Run python update-docs-help.py to fix this"
-
-
 @pytest.fixture(scope="session")
 def plugin_hooks_content():
     return (docs_path / "plugin_hooks.rst").read_text()
diff --git a/update-docs-help.py b/update-docs-help.py
deleted file mode 100644
index 292d1dcd..00000000
--- a/update-docs-help.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from click.testing import CliRunner
-from datasette.cli import cli
-from pathlib import Path
-
-docs_path = Path(__file__).parent / "docs"
-
-includes = (
-    ("serve", "datasette-serve-help.txt"),
-    ("package", "datasette-package-help.txt"),
-    ("publish heroku", "datasette-publish-heroku-help.txt"),
-    ("publish cloudrun", "datasette-publish-cloudrun-help.txt"),
-)
-
-
-def update_help_includes():
-    for name, filename in includes:
-        runner = CliRunner()
-        result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88)
-        actual = f"$ datasette {name} --help\n\n{result.output}"
-        actual = actual.replace("Usage: cli ", "Usage: datasette ")
-        (docs_path / filename).write_text(actual)
-
-
-if __name__ == "__main__":
-    update_help_includes()

From cb29119db9115b1f40de2fb45263ed77e3bfbb3e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jan 2022 17:36:51 -0800
Subject: [PATCH 1300/2303] Release 0.60

Refs #473, #625, #1527, #1544, #1547, #1551, #1552, #1555, #1556, #1557,
#1563, #1564, #1568, #1570, #1575, #1579, #1588, #1594
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 37 ++++++++++++++++++++++++-------------
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index 290fbcf3..a4e340b3 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.60a1"
+__version__ = "0.60"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 99d3315e..d7e2af39 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,30 +4,41 @@
 Changelog
 =========
 
-.. _v0_60a1:
+.. _v0_60:
 
-0.60a1 (2021-12-19)
--------------------
+0.60 (2022-01-13)
+-----------------
 
+Plugins and internals
+~~~~~~~~~~~~~~~~~~~~~
+
+- New plugin hook: :ref:`plugin_hook_filters_from_request`, which runs on the table page and can be used to support new custom query string parameters that modify the SQL query. (:issue:`473`)
+- Added two additional methods for writing to the database: :ref:`database_execute_write_script` and :ref:`database_execute_write_many`. (:issue:`1570`)
+- The :ref:`db.execute_write() <database_execute_write>` internal method now defaults to blocking until the write operation has completed. Previously it defaulted to queuing the write and then continuing to run code while the write was in the queue. (:issue:`1579`)
 - Database write connections now execute the :ref:`plugin_hook_prepare_connection` plugin hook. (:issue:`1564`)
 - The ``Datasette()`` constructor no longer requires the ``files=`` argument, and is now documented at :ref:`internals_datasette`. (:issue:`1563`)
 - The tracing feature now traces write queries, not just read queries. (:issue:`1568`)
-- Added two methods for writing to the database: :ref:`database_execute_write_script` and :ref:`database_execute_write_many`. (:issue:`1570`)
-- Made several performance improvements to the database schema introspection code that runs when Datasette first starts up. (:issue:`1555`)
-- Fixed bug where writable canned queries could not be used with custom templates.  (:issue:`1547`)
+- The query string variables exposed by ``request.args`` will now include blank strings for arguments such as ``foo`` in ``?foo=&bar=1`` rather than ignoring those parameters entirely. (:issue:`1551`)
 
-.. _v0_60a0:
+Faceting
+~~~~~~~~
 
-0.60a0 (2021-12-17)
--------------------
-
-- New plugin hook: :ref:`plugin_hook_filters_from_request`, which runs on the table page and can be used to support new custom query string parameters that modify the SQL query. (:issue:`473`)
 - The number of unique values in a facet is now always displayed. Previously it was only displayed if the user specified ``?_facet_size=max``. (:issue:`1556`)
-- Fixed bug where ``?_facet_array=tags&_facet=tags`` would only display one of the two selected facets. (:issue:`625`)
 - Facets of type ``date`` or ``array`` can now be configured in ``metadata.json``, see :ref:`facets_metadata`. Thanks, David Larlet. (:issue:`1552`)
 - New ``?_nosuggest=1`` parameter for table views, which disables facet suggestion. (:issue:`1557`)
+- Fixed bug where ``?_facet_array=tags&_facet=tags`` would only display one of the two selected facets. (:issue:`625`)
+
+Other small fixes
+~~~~~~~~~~~~~~~~~
+
+- Made several performance improvements to the database schema introspection code that runs when Datasette first starts up. (:issue:`1555`)
 - Label columns detected for foreign keys are now case-insensitive, so ``Name`` or ``TITLE`` will be detected in the same way as ``name`` or ``title``. (:issue:`1544`)
-- The query string variables exposed by ``request.args`` will now include blank strings for arguments such as ``foo`` in ``?foo=&bar=1`` rather than ignoring those parameters entirely. (:issue:`1551`)
+- Upgraded Pluggy dependency to 1.0. (:issue:`1575`)
+- Now using `Plausible analytics <https://plausible.io/>`__ for the Datasette documentation.
+- ``explain query plan`` is now allowed with varying amounts of whitespace in the query. (:issue:`1588`)
+- New :ref:`cli_reference` page showing the output of ``--help`` for each of the ``datasette`` sub-commands. This lead to several small improvements to the help copy. (:issue:`1594`)
+- Fixed bug where writable canned queries could not be used with custom templates.  (:issue:`1547`)
+- Improved fix for a bug where columns with a underscore prefix could result in unnecessary hidden form fields. (:issue:`1527`)
 
 .. _v0_59_4:
 

From 58652dd925bb7509b43905423ec00083bd374dc1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 20:12:46 -0800
Subject: [PATCH 1301/2303] Hidden tables sqlite1/2/3/4, closes #1587

---
 datasette/database.py |  4 +++-
 tests/test_api.py     | 12 ++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/datasette/database.py b/datasette/database.py
index e908d1ea..06dc8da5 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -345,7 +345,9 @@ class Database:
                     """
                 select name from sqlite_master
                 where rootpage = 0
-                and sql like '%VIRTUAL TABLE%USING FTS%'
+                and (
+                    sql like '%VIRTUAL TABLE%USING FTS%'
+                ) or name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
             """
                 )
             ).rows
diff --git a/tests/test_api.py b/tests/test_api.py
index 574ebb41..47ec3a8c 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1030,3 +1030,15 @@ async def test_db_path(app_client):
 
     # Previously this broke if path was a pathlib.Path:
     await datasette.refresh_schemas()
+
+
+@pytest.mark.asyncio
+async def test_hidden_sqlite_stat1_table():
+    ds = Datasette()
+    db = ds.add_memory_database("db")
+    await db.execute_write("create table normal (id integer primary key, name text)")
+    await db.execute_write("create index idx on normal (name)")
+    await db.execute_write("analyze")
+    data = (await ds.client.get("/db.json?_show_hidden=1")).json()
+    tables = [(t["name"], t["hidden"]) for t in data["tables"]]
+    assert tables == [("normal", False), ("sqlite_stat1", True)]

From fae3983c51f4a3aca8335f3e01ff85ef27076fbf Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 20:31:22 -0800
Subject: [PATCH 1302/2303] Drop support for Python 3.6, closes #1577

Refs #1606
---
 .github/workflows/publish.yml | 6 +++---
 .github/workflows/test.yml    | 2 +-
 README.md                     | 2 +-
 docs/contributing.rst         | 2 +-
 docs/installation.rst         | 2 +-
 docs/introspection.rst        | 8 ++++----
 setup.py                      | 3 +--
 7 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 17c6ae9b..3cfc67da 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -38,7 +38,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.9'
+        python-version: '3.10'
     - uses: actions/cache@v2
       name: Configure pip caching
       with:
@@ -66,7 +66,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.9'
+        python-version: '3.10'
     - uses: actions/cache@v2
       name: Configure pip caching
       with:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 704931a6..78c289bb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/README.md b/README.md
index ce15ccf4..107d81da 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ You can also install it using `pip` or `pipx`:
 
     pip install datasette
 
-Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
+Datasette requires Python 3.7 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
 
 ## Basic usage
 
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 07f2a0e4..b74f2f36 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -19,7 +19,7 @@ General guidelines
 Setting up a development environment
 ------------------------------------
 
-If you have Python 3.6 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew <https://docs.python-guide.org/starting/install3/osx/>`__) you can install an editable copy of Datasette using the following steps.
+If you have Python 3.7 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew <https://docs.python-guide.org/starting/install3/osx/>`__) you can install an editable copy of Datasette using the following steps.
 
 If you want to use GitHub to publish your changes, first `create a fork of datasette <https://github.com/simonw/datasette/fork>`__ under your own GitHub account.
 
diff --git a/docs/installation.rst b/docs/installation.rst
index ac3dcca2..e8bef9cd 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -56,7 +56,7 @@ If the latest packaged release of Datasette has not yet been made available thro
 Using pip
 ---------
 
-Datasette requires Python 3.6 or higher. Visit `InstallPython3.com <https://installpython3.com/>`__ for step-by-step installation guides for your operating system.
+Datasette requires Python 3.7 or higher. Visit `InstallPython3.com <https://installpython3.com/>`__ for step-by-step installation guides for your operating system.
 
 You can install Datasette and its dependencies using ``pip``::
 
diff --git a/docs/introspection.rst b/docs/introspection.rst
index d1a0a854..e08ca911 100644
--- a/docs/introspection.rst
+++ b/docs/introspection.rst
@@ -38,11 +38,11 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la
 
     {
         "datasette": {
-            "version": "0.21"
+            "version": "0.60"
         },
         "python": {
-            "full": "3.6.5 (default, May  5 2018, 03:07:21) \n[GCC 6.3.0 20170516]",
-            "version": "3.6.5"
+            "full": "3.8.12 (default, Dec 21 2021, 10:45:09) \n[GCC 10.2.1 20210110]",
+            "version": "3.8.12"
         },
         "sqlite": {
             "extensions": {
@@ -62,7 +62,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la
                 "ENABLE_RTREE",
                 "THREADSAFE=1"
             ],
-            "version": "3.16.2"
+            "version": "3.37.0"
         }
     }
 
diff --git a/setup.py b/setup.py
index e9ef082a..dade0a88 100644
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,7 @@ setup(
     packages=find_packages(exclude=("tests",)),
     package_data={"datasette": ["templates/*.html"]},
     include_package_data=True,
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     install_requires=[
         "asgiref>=3.2.10,<3.5.0",
         "click>=7.1.1,<8.1.0",
@@ -91,6 +91,5 @@ setup(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.6",
     ],
 )

From 14e320329f756b7d8e298c4e2251d8a0b194c9c4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 20:38:49 -0800
Subject: [PATCH 1303/2303] Hidden tables data_licenses, KNN, KNN2 for
 SpatiaLite, closes #1601

---
 datasette/database.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/datasette/database.py b/datasette/database.py
index 06dc8da5..6ce87215 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -365,6 +365,9 @@ class Database:
                 "sqlite_sequence",
                 "views_geometry_columns",
                 "virts_geometry_columns",
+                "data_licenses",
+                "KNN",
+                "KNN2",
             ] + [
                 r[0]
                 for r in (

From 43c30ce0236ebbc7e9cec98a3822265eb2691430 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 21:04:09 -0800
Subject: [PATCH 1304/2303] Use cog to maintain default plugin list in
 plugins.rst, closes #1600

Also fixed a bug I spotted where datasette.filters showed the same hook three times.
---
 datasette/app.py |  2 +-
 docs/plugins.rst | 97 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 91 insertions(+), 8 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index bd663509..0a89a9f3 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -770,7 +770,7 @@ class Datasette:
                 "static": p["static_path"] is not None,
                 "templates": p["templates_path"] is not None,
                 "version": p.get("version"),
-                "hooks": p["hooks"],
+                "hooks": list(set(p["hooks"])),
             }
             for p in ps
         ]
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 020030f1..4a2c0194 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -91,36 +91,119 @@ You can also use the ``datasette plugins`` command::
         }
     ]
 
+.. [[[cog
+    from datasette import cli
+    from click.testing import CliRunner
+    import textwrap, json
+    cog.out("\n")
+    result = CliRunner().invoke(cli.cli, ["plugins", "--all"])
+    # cog.out() with text containing newlines was unindenting for some reason
+    cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::\n")
+    plugins = [p for p in json.loads(result.output) if p["name"].startswith("datasette.")]
+    indented = textwrap.indent(json.dumps(plugins, indent=4), "    ")
+    for line in indented.split("\n"):
+        cog.outl(line)
+    cog.out("\n\n")
+.. ]]]
+
 If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::
 
-    $ datasette plugins --all
     [
+        {
+            "name": "datasette.publish.heroku",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "publish_subcommand"
+            ]
+        },
         {
             "name": "datasette.sql_functions",
             "static": false,
             "templates": false,
-            "version": null
+            "version": null,
+            "hooks": [
+                "prepare_connection"
+            ]
         },
         {
-            "name": "datasette.publish.cloudrun",
+            "name": "datasette.actor_auth_cookie",
             "static": false,
             "templates": false,
-            "version": null
+            "version": null,
+            "hooks": [
+                "actor_from_request"
+            ]
+        },
+        {
+            "name": "datasette.blob_renderer",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "register_output_renderer"
+            ]
         },
         {
             "name": "datasette.facets",
             "static": false,
             "templates": false,
-            "version": null
+            "version": null,
+            "hooks": [
+                "register_facet_classes"
+            ]
         },
         {
-            "name": "datasette.publish.heroku",
+            "name": "datasette.default_magic_parameters",
             "static": false,
             "templates": false,
-            "version": null
+            "version": null,
+            "hooks": [
+                "register_magic_parameters"
+            ]
+        },
+        {
+            "name": "datasette.default_permissions",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "permission_allowed"
+            ]
+        },
+        {
+            "name": "datasette.default_menu_links",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "menu_links"
+            ]
+        },
+        {
+            "name": "datasette.filters",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "filters_from_request"
+            ]
+        },
+        {
+            "name": "datasette.publish.cloudrun",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "publish_subcommand"
+            ]
         }
     ]
 
+
+.. [[[end]]]
+
 You can add the ``--plugins-dir=`` option to include any plugins found in that directory.
 
 .. _plugins_configuration:

From e1770766ce3ae6669305662ba618be610367af77 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 21:14:04 -0800
Subject: [PATCH 1305/2303] Return plugins and hooks in predictable order

---
 datasette/app.py  |  3 ++-
 docs/plugins.rst  | 58 +++++++++++++++++++++++------------------------
 tests/test_cli.py |  4 +---
 3 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 0a89a9f3..49858a4a 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -764,13 +764,14 @@ class Datasette:
             should_show_all = all
         if not should_show_all:
             ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS]
+        ps.sort(key=lambda p: p["name"])
         return [
             {
                 "name": p["name"],
                 "static": p["static_path"] is not None,
                 "templates": p["templates_path"] is not None,
                 "version": p.get("version"),
-                "hooks": list(set(p["hooks"])),
+                "hooks": list(sorted(set(p["hooks"]))),
             }
             for p in ps
         ]
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 4a2c0194..f2ed02f7 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -109,24 +109,6 @@ You can also use the ``datasette plugins`` command::
 If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::
 
     [
-        {
-            "name": "datasette.publish.heroku",
-            "static": false,
-            "templates": false,
-            "version": null,
-            "hooks": [
-                "publish_subcommand"
-            ]
-        },
-        {
-            "name": "datasette.sql_functions",
-            "static": false,
-            "templates": false,
-            "version": null,
-            "hooks": [
-                "prepare_connection"
-            ]
-        },
         {
             "name": "datasette.actor_auth_cookie",
             "static": false,
@@ -145,15 +127,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "register_output_renderer"
             ]
         },
-        {
-            "name": "datasette.facets",
-            "static": false,
-            "templates": false,
-            "version": null,
-            "hooks": [
-                "register_facet_classes"
-            ]
-        },
         {
             "name": "datasette.default_magic_parameters",
             "static": false,
@@ -163,6 +136,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "register_magic_parameters"
             ]
         },
+        {
+            "name": "datasette.default_menu_links",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "menu_links"
+            ]
+        },
         {
             "name": "datasette.default_permissions",
             "static": false,
@@ -173,12 +155,12 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
             ]
         },
         {
-            "name": "datasette.default_menu_links",
+            "name": "datasette.facets",
             "static": false,
             "templates": false,
             "version": null,
             "hooks": [
-                "menu_links"
+                "register_facet_classes"
             ]
         },
         {
@@ -198,6 +180,24 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
             "hooks": [
                 "publish_subcommand"
             ]
+        },
+        {
+            "name": "datasette.publish.heroku",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "publish_subcommand"
+            ]
+        },
+        {
+            "name": "datasette.sql_functions",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "prepare_connection"
+            ]
         }
     ]
 
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 763fe2e7..bbc5df30 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -106,9 +106,7 @@ def test_spatialite_error_if_cannot_find_load_extension_spatialite():
 def test_plugins_cli(app_client):
     runner = CliRunner()
     result1 = runner.invoke(cli, ["plugins"])
-    assert sorted(EXPECTED_PLUGINS, key=lambda p: p["name"]) == sorted(
-        json.loads(result1.output), key=lambda p: p["name"]
-    )
+    assert json.loads(result1.output) == EXPECTED_PLUGINS
     # Try with --all
     result2 = runner.invoke(cli, ["plugins", "--all"])
     names = [p["name"] for p in json.loads(result2.output)]

From 0467723ee57c2cbc0f02daa47cef632dd4651df0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 21:46:03 -0800
Subject: [PATCH 1306/2303] New, improved favicon - refs #1603

---
 datasette/app.py             |  11 ++++++++++-
 datasette/static/favicon.png | Bin 0 -> 1207 bytes
 tests/test_html.py           |   4 +++-
 3 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 datasette/static/favicon.png

diff --git a/datasette/app.py b/datasette/app.py
index 49858a4a..b2942cd9 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -70,6 +70,7 @@ from .utils.asgi import (
     Response,
     asgi_static,
     asgi_send,
+    asgi_send_file,
     asgi_send_html,
     asgi_send_json,
     asgi_send_redirect,
@@ -178,9 +179,17 @@ SETTINGS = (
 
 DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS}
 
+FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png"
+
 
 async def favicon(request, send):
-    await asgi_send(send, "", 200)
+    await asgi_send_file(
+        send,
+        str(FAVICON_PATH),
+        content_type="image/png",
+        chunk_size=4096,
+        headers={"Cache-Control": "max-age=3600, immutable, public"},
+    )
 
 
 class Datasette:
diff --git a/datasette/static/favicon.png b/datasette/static/favicon.png
new file mode 100644
index 0000000000000000000000000000000000000000..36d6334fd7714da87aad44bba0d9c2a869cd0b86
GIT binary patch
literal 1207
zcmeAS@N?(olHy`uVBq!ia0vp^86eET1|(%=wk`xxoCO|{#S9GG!XV7ZFl&wkP>``W
z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eBu8*gSV@L(#+gXPB%B~{E@6V2KU$RE|a=`mo
z=TfEn3fnR!zH@Oseo)=wmh*$NvI}i5UR0HpZD-+iO=*9nWijR7qiH4)<$WST)$;i@
z7Pn&6W#%ORmVfzw^0n8MkLM=U{Z2c!d+{f!Nh;ZFoU_%gg<K8^2~y1O^E`j0N^4WS
z)t0-HC$IUQZ-4!?)*=bf+}{^PjLiQWdq0o&?c~!gYhy%W{N?P*4DbJ%$Td@YV}E$l
zz53T~-v6xLzVW{<o?)`-SjGzf8K$QX&1;>t_1E7u_MKO{uKo<L)=N)*Wi*l5{#(e?
z19Rm#E0fQ>nKnPKDs9jE<8O=B=Jm9FI<kL@+QiR_=i0Y!`~P6ESF!uD#ORRy!B?Hv
z+lv_q-|MgM`yEtze)7W$`G3k|LyF?`XZ<R;{^8-f=i#Z#H_lpGsVg%rded{MCyw*j
zy=H2E>zh^cE<R6m@y;h}4=TKFi;%clvb^Zx#Lm3nb4nMKr+zqK{IWmjO8AAxpIdJV
z=iJt@5#8jGxBBMwqYh^lO}PFg&d?%GAmE%)2lvM2B!wSYRuu)CCBArO`k$C|At=ek
zJn_$uWe$4ZVonqVR7dG$Nv%46EkygXV5Gv-|1;jHaWu9y9atI4yzk2IKn|WW3I!S*
zGF(jQS$tK7KY~)-UVU|!7dfpTU-xnHnG>PgJ8#X~AG#rFrX&j=YooK{g8%^w5yH&A
zu=@`Lb!Nbnmw#>8dOX(lIu939d((l1P%H5<cVx22q=BsR^F6U?rJvzaTg5vjOxNVh
zdt;2j?!~7WqCo#m+JXCb=j^xMe5!j}(p{Z`%N!Omk3hCtu>Z9eGn54i6jk3bDLT6V
z(?LnOy0g1j|L5P)W(y179CBjQ4UMUHd=0k78I~oh-Vxm-($q3(!3(t?#dH69edwFe
z8oYk{U(GqSF+Hy4isG-QHQ(}?v_>-}(Im|H|CQYv-*<n!yzp%sFs`FqgEnlqoRPw-
zSbY4q<-EKsIXUsi@0Ju7P5OGszIE-TP_0b6D}3TSpH)3?ef&}uclmwteDT-k+P-Dl
z^s^q-De}EPMgKZ)yg|vUuj?(;V-@QzpZ;}Ad8K-^-O&tfd+z<m_UZjUwJqt}k^JZH
zo8x$HegC`YyVv6d(V?g7KU9S4ym@By>+cVXYPIVVUa9r1zW3x~{B3L5XDjDF@~uA}
zp6fgF*VMny^fq~}43T|4*J6vWv}(7)Icsgx-y80q|26qW>$K~W*PdB@%So*DsOB-_
z=bN*RF3o)Fy}u;n^4wRK`eQW3!Y|a>)NJLO@oekkJdvlXd#cw}1p8iomBlP^=5O7<
pA8K}rANu~Q{$t<kbEaiK<ImRe6uW@kXMyDmgQu&X%Q~loCIC6tDUtvH

literal 0
HcmV?d00001

diff --git a/tests/test_html.py b/tests/test_html.py
index 3f0a88a9..735d12ff 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -63,7 +63,9 @@ def test_homepage_options(app_client):
 def test_favicon(app_client):
     response = app_client.get("/favicon.ico")
     assert response.status == 200
-    assert "" == response.text
+    assert response.headers["cache-control"] == "max-age=3600, immutable, public"
+    assert response.headers["content-length"] == "1207"
+    assert response.headers["content-type"] == "image/png"
 
 
 def test_static(app_client):

From b2eebf5ebf222b61a21625527851b77347d3d662 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 21:52:00 -0800
Subject: [PATCH 1307/2303] No need to send this, it's got a default, refs
 #1603

---
 datasette/app.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index b2942cd9..09d7d034 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -187,7 +187,6 @@ async def favicon(request, send):
         send,
         str(FAVICON_PATH),
         content_type="image/png",
-        chunk_size=4096,
         headers={"Cache-Control": "max-age=3600, immutable, public"},
     )
 

From b01c9b68d151e1656fc180815c0d8480e35fc961 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 21:54:41 -0800
Subject: [PATCH 1308/2303] Oops I pushed the wrong favicon, refs #1603

---
 datasette/static/favicon.png | Bin 1207 -> 1358 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/datasette/static/favicon.png b/datasette/static/favicon.png
index 36d6334fd7714da87aad44bba0d9c2a869cd0b86..a27139ae6ecb09660f96d262d3779de29f7b5d4f 100644
GIT binary patch
literal 1358
zcmV-U1+n^xP)<h;3K|Lk000e1NJLTq004jh004jp0{{R3^x%>C0000;P)t-s0001%
zmb`Fqr$j`OPfwIOI*uG1hYt>g|NsB|{QdCp_1@p^(bD6=!P>XE+_AIRsjA42k-T+v
zrB+v&Dk+Hr0*3$qfB*n>9JfE)00008bW%=J07QSN(*FMbUcFRk000E7Nkl<Zc-rk-
zje4sf486525v_s>@%`WQlKxzNVr%CWzi#uW>voplSwP4@{_OZL9ExLB6vg4y?TeGK
zxnWMl)7zgdqJ`^`=cF%a;XEnfZ~_DX0hkjY04$p0#Qg^g4mhGP`9S3ZfVHOsyFUU7
z#G%tFCv-O;3Iy9H0!|1B<DlF8=+1O68`nZ%>HhuALbX|-?j<*B7==YsgcaaKiLv)L
zR~e4ruNx2mhbRC<c#EtdAW$ucAO!R?YO?W`YeNO!G#vuildl268^F5pk3*oYm4N2p
zYrq0_83Vw}>_!pP!O;Q=uhM_!kwFpBR0sbAoGjp31uE6rmB2!vqppAi3=e@S_*n0v
zGty(YfGe8&)i!jlbNTZ8k6gf6ZwrwM*c>(yU^fP)5dafI5T+Q;E%LO90Kjouuqqgu
z7@dQT7){iyRiobWOrHWV_GJXwv`t$_RNXqRjB#<?CGl(>SH)J`Q=U;o0<f1M%8L4n
zx(vCRdXy@nWaF$ZW$dZi#Iw9Ot0pO*3t)c-6%*nrK*nY_0TC+2JZ}O*W>$dY0_27Q
z$RHIUH;sTSKQ96sw}B@SP)h{JpA><;4XoA%KC&m=ek@?4VT8>`!{HJ`ce-oqfT)#Y
zG_zt{LtImb)Rd<|@j4?y#CNGBW;`PT5d#3;r52f4oE0EAMGWT_dBSW`iX%=j3pmE`
zD&UQ<=^Ru<DWcLw=qNqm+0I&-$;VLAHS-vHBU#G(V<>&E_v$f}|CN-MwPV)mi1yKN
z&k6T7@C`De4~@XjZUetOC*%{AUGI*Bany_z!BPFZm06OIaB4as!7nEye4bD&o+lK`
z<_Xc_#$W<qTUbnU!YV?ol+u@ox@gB$F<Py3lXz<8**GheSWP~LlJ|r%?T9=@pN+Gk
zoG|Ues(+lkbwoZVeBKe2RJgpIkDidoJNFRqps2h@N93)1M|3~@VC2i+bHe|~2Y!77
z+Fzd$?QI~d067%_`Fg!TJ{KUpBNrgQK^u5?rnKb(5>G!Ey%GRu`WS=5JJd8cZ<Q`y
zTjVo)lhgt;ob<GBtR26(PQ1v|@(2XSU3SEICgGNNj>sZlW@~p?wG9Bmwok!=u6XmF
za9=zA4x{0o59~SN-Uc%Fgx*9x%?T?{h5sFgo{D*{mJ_}|7*$_P$@mOOwek#!nw%k-
z&xkn1IkU*qe%uR<%OW5Mi#%ceeL~O7P>ama&b-vxJEU{P4q7v=TjVJa)|f4CwtJt@
zPtyg|m(~l)4}YI<kkdJ#XVw*9*n&m1?PwUymLDwu)}_p!5BTwE6c&nyfC8);n^JBT
z0a@zN1UoDa2B{Hjm=jh80nfoUCkui?kUs)G0MS;bf&@`3$lV5f3jj2Sc&C@32SLj0
z5pbV1B2WO*tyXBRWj$LtUF9W~t7=36I8XpQ3PB3Su9lfsPUWQ^5qt$i3P6n5cPe;C
zzD?heS51hvBaoGc=AIql{>xz;1prQLx-0;}3cxWlgs^^}S-R&EEeN*$DIOO*C~-R<
zX5L0jdwPaSLA1l+>8@9c&tY)|*l^D)M{zuxv&jcn`mvbX`kWVkc6=QE0tPO=y`~{k
QD*ylh07*qoM6N<$g1B&Oe*gdg

literal 1207
zcmeAS@N?(olHy`uVBq!ia0vp^86eET1|(%=wk`xxoCO|{#S9GG!XV7ZFl&wkP>``W
z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eBu8*gSV@L(#+gXPB%B~{E@6V2KU$RE|a=`mo
z=TfEn3fnR!zH@Oseo)=wmh*$NvI}i5UR0HpZD-+iO=*9nWijR7qiH4)<$WST)$;i@
z7Pn&6W#%ORmVfzw^0n8MkLM=U{Z2c!d+{f!Nh;ZFoU_%gg<K8^2~y1O^E`j0N^4WS
z)t0-HC$IUQZ-4!?)*=bf+}{^PjLiQWdq0o&?c~!gYhy%W{N?P*4DbJ%$Td@YV}E$l
zz53T~-v6xLzVW{<o?)`-SjGzf8K$QX&1;>t_1E7u_MKO{uKo<L)=N)*Wi*l5{#(e?
z19Rm#E0fQ>nKnPKDs9jE<8O=B=Jm9FI<kL@+QiR_=i0Y!`~P6ESF!uD#ORRy!B?Hv
z+lv_q-|MgM`yEtze)7W$`G3k|LyF?`XZ<R;{^8-f=i#Z#H_lpGsVg%rded{MCyw*j
zy=H2E>zh^cE<R6m@y;h}4=TKFi;%clvb^Zx#Lm3nb4nMKr+zqK{IWmjO8AAxpIdJV
z=iJt@5#8jGxBBMwqYh^lO}PFg&d?%GAmE%)2lvM2B!wSYRuu)CCBArO`k$C|At=ek
zJn_$uWe$4ZVonqVR7dG$Nv%46EkygXV5Gv-|1;jHaWu9y9atI4yzk2IKn|WW3I!S*
zGF(jQS$tK7KY~)-UVU|!7dfpTU-xnHnG>PgJ8#X~AG#rFrX&j=YooK{g8%^w5yH&A
zu=@`Lb!Nbnmw#>8dOX(lIu939d((l1P%H5<cVx22q=BsR^F6U?rJvzaTg5vjOxNVh
zdt;2j?!~7WqCo#m+JXCb=j^xMe5!j}(p{Z`%N!Omk3hCtu>Z9eGn54i6jk3bDLT6V
z(?LnOy0g1j|L5P)W(y179CBjQ4UMUHd=0k78I~oh-Vxm-($q3(!3(t?#dH69edwFe
z8oYk{U(GqSF+Hy4isG-QHQ(}?v_>-}(Im|H|CQYv-*<n!yzp%sFs`FqgEnlqoRPw-
zSbY4q<-EKsIXUsi@0Ju7P5OGszIE-TP_0b6D}3TSpH)3?ef&}uclmwteDT-k+P-Dl
z^s^q-De}EPMgKZ)yg|vUuj?(;V-@QzpZ;}Ad8K-^-O&tfd+z<m_UZjUwJqt}k^JZH
zo8x$HegC`YyVv6d(V?g7KU9S4ym@By>+cVXYPIVVUa9r1zW3x~{B3L5XDjDF@~uA}
zp6fgF*VMny^fq~}43T|4*J6vWv}(7)Icsgx-y80q|26qW>$K~W*PdB@%So*DsOB-_
z=bN*RF3o)Fy}u;n^4wRK`eQW3!Y|a>)NJLO@oekkJdvlXd#cw}1p8iomBlP^=5O7<
pA8K}rANu~Q{$t<kbEaiK<ImRe6uW@kXMyDmgQu&X%Q~loCIC6tDUtvH


From 7c67483f5e61f7c46410a433a55280d62280327f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 19 Jan 2022 21:57:14 -0800
Subject: [PATCH 1309/2303] Make test_favicon flexible to changing icon sizes,
 refs #1603

---
 tests/test_html.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_html.py b/tests/test_html.py
index 735d12ff..aa718857 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -64,7 +64,7 @@ def test_favicon(app_client):
     response = app_client.get("/favicon.ico")
     assert response.status == 200
     assert response.headers["cache-control"] == "max-age=3600, immutable, public"
-    assert response.headers["content-length"] == "1207"
+    assert int(response.headers["content-length"]) > 100
     assert response.headers["content-type"] == "image/png"
 
 

From 150967d98ef6c5b6064587e7ed30cbdd9b992b8e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 20 Jan 2022 10:43:15 -0800
Subject: [PATCH 1310/2303] Hand-edited pixel favicon, refs #1603

---
 datasette/static/favicon.png | Bin 1358 -> 208 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/datasette/static/favicon.png b/datasette/static/favicon.png
index a27139ae6ecb09660f96d262d3779de29f7b5d4f..4993163f703c425c6512dec380dafe2ca52051a7 100644
GIT binary patch
literal 208
zcmeAS@N?(olHy`uVBq!ia0vp^3LwnF3?v&v(vJfvi2$DvS0Mc#47TsQvvJdnm8(v&
zGBmL;l)e{q>Icd(l?3?(GYEgu?Rf^|rFyzJhE&{2KERTb>7mfr*l2iDf^!C=sKnw1
z79FNc4F-(`OP(xAS+ay@QOJ}j3QP)SnoLihzI2EP<5Qdv5^(o|!kVa_4lxb=X6adN
yPl8f;beC`>@CQ#ili=ZWZBd)t&CVH}j0{Q7(vQX;{-qAIkipZ{&t;ucLK6Tps7T!a

literal 1358
zcmV-U1+n^xP)<h;3K|Lk000e1NJLTq004jh004jp0{{R3^x%>C0000;P)t-s0001%
zmb`Fqr$j`OPfwIOI*uG1hYt>g|NsB|{QdCp_1@p^(bD6=!P>XE+_AIRsjA42k-T+v
zrB+v&Dk+Hr0*3$qfB*n>9JfE)00008bW%=J07QSN(*FMbUcFRk000E7Nkl<Zc-rk-
zje4sf486525v_s>@%`WQlKxzNVr%CWzi#uW>voplSwP4@{_OZL9ExLB6vg4y?TeGK
zxnWMl)7zgdqJ`^`=cF%a;XEnfZ~_DX0hkjY04$p0#Qg^g4mhGP`9S3ZfVHOsyFUU7
z#G%tFCv-O;3Iy9H0!|1B<DlF8=+1O68`nZ%>HhuALbX|-?j<*B7==YsgcaaKiLv)L
zR~e4ruNx2mhbRC<c#EtdAW$ucAO!R?YO?W`YeNO!G#vuildl268^F5pk3*oYm4N2p
zYrq0_83Vw}>_!pP!O;Q=uhM_!kwFpBR0sbAoGjp31uE6rmB2!vqppAi3=e@S_*n0v
zGty(YfGe8&)i!jlbNTZ8k6gf6ZwrwM*c>(yU^fP)5dafI5T+Q;E%LO90Kjouuqqgu
z7@dQT7){iyRiobWOrHWV_GJXwv`t$_RNXqRjB#<?CGl(>SH)J`Q=U;o0<f1M%8L4n
zx(vCRdXy@nWaF$ZW$dZi#Iw9Ot0pO*3t)c-6%*nrK*nY_0TC+2JZ}O*W>$dY0_27Q
z$RHIUH;sTSKQ96sw}B@SP)h{JpA><;4XoA%KC&m=ek@?4VT8>`!{HJ`ce-oqfT)#Y
zG_zt{LtImb)Rd<|@j4?y#CNGBW;`PT5d#3;r52f4oE0EAMGWT_dBSW`iX%=j3pmE`
zD&UQ<=^Ru<DWcLw=qNqm+0I&-$;VLAHS-vHBU#G(V<>&E_v$f}|CN-MwPV)mi1yKN
z&k6T7@C`De4~@XjZUetOC*%{AUGI*Bany_z!BPFZm06OIaB4as!7nEye4bD&o+lK`
z<_Xc_#$W<qTUbnU!YV?ol+u@ox@gB$F<Py3lXz<8**GheSWP~LlJ|r%?T9=@pN+Gk
zoG|Ues(+lkbwoZVeBKe2RJgpIkDidoJNFRqps2h@N93)1M|3~@VC2i+bHe|~2Y!77
z+Fzd$?QI~d067%_`Fg!TJ{KUpBNrgQK^u5?rnKb(5>G!Ey%GRu`WS=5JJd8cZ<Q`y
zTjVo)lhgt;ob<GBtR26(PQ1v|@(2XSU3SEICgGNNj>sZlW@~p?wG9Bmwok!=u6XmF
za9=zA4x{0o59~SN-Uc%Fgx*9x%?T?{h5sFgo{D*{mJ_}|7*$_P$@mOOwek#!nw%k-
z&xkn1IkU*qe%uR<%OW5Mi#%ceeL~O7P>ama&b-vxJEU{P4q7v=TjVJa)|f4CwtJt@
zPtyg|m(~l)4}YI<kkdJ#XVw*9*n&m1?PwUymLDwu)}_p!5BTwE6c&nyfC8);n^JBT
z0a@zN1UoDa2B{Hjm=jh80nfoUCkui?kUs)G0MS;bf&@`3$lV5f3jj2Sc&C@32SLj0
z5pbV1B2WO*tyXBRWj$LtUF9W~t7=36I8XpQ3PB3Su9lfsPUWQ^5qt$i3P6n5cPe;C
zzD?heS51hvBaoGc=AIql{>xz;1prQLx-0;}3cxWlgs^^}S-R&EEeN*$DIOO*C~-R<
zX5L0jdwPaSLA1l+>8@9c&tY)|*l^D)M{zuxv&jcn`mvbX`kWVkc6=QE0tPO=y`~{k
QD*ylh07*qoM6N<$g1B&Oe*gdg


From ffca55dfd7cc9b53522c2e5a2fa1ff67c9beadf2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 20 Jan 2022 14:40:44 -0800
Subject: [PATCH 1311/2303] Show link to /stable/ on /latest/ pages, refs #1608

---
 docs/_templates/layout.html | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index db16b428..785cdc7c 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -26,3 +26,36 @@
 {% include "searchbox.html" %}
 
 {% endblock %}
+
+{% block footer %}
+{{ super() }}
+<script>
+jQuery(function ($) {
+  // Show banner linking to /stable/ if this is a /latest/ page
+  if (!/\/latest\//.test(location.pathname)) {
+    return;
+  }
+  var stableUrl = location.pathname.replace("/latest/", "/stable/");
+  // Check it's not a 404
+  fetch(stableUrl, { method: "HEAD" }).then((response) => {
+    if (response.status == 200) {
+      var warning = $(
+        `<div class="admonition warning">
+           <p class="first admonition-title">Note</p>
+           <p class="last">
+             This documentation covers the <strong>development version</strong> of Datasette.</p>
+             <p>See <a href="${stableUrl}">this page</a> for the current stable release.
+           </p>
+        </div>`
+      );
+      warning.find("a").attr("href", stableUrl);
+      var body = $("div.body");
+      if (!body.length) {
+        body = $("div.document");
+      }
+      body.prepend(warning);
+    }
+  });
+});
+</script>
+{% endblock %}

From d194db4204b732af57138e1fb0924ec77354dd58 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 20 Jan 2022 18:01:47 -0800
Subject: [PATCH 1312/2303] Output pip freeze to show installed packages, refs
 #1609

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 78c289bb..2caf9447 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,6 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install -e '.[test]'
+        pip freeze
     - name: Run tests
       run: |
         pytest -n auto -m "not serial"

From 68cc1e2dbb0b841af7a7691ea6b4e7d31b09cc5e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 25 Jan 2022 10:28:05 -0800
Subject: [PATCH 1313/2303] Move queries to top of database page, refs #1612

---
 datasette/templates/database.html | 24 ++++++++++++++----------
 tests/test_html.py                | 30 +++++++++++++++++++++++++-----
 2 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index 2d182d1b..c1e39bd1 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -67,10 +67,23 @@
     </div>
 {% endif %}
 
+{% if queries %}
+    <h2 id="queries">Queries</h2>
+    <ul class="bullets">
+        {% for query in queries %}
+            <li><a href="{{ urls.query(database, query.name) }}{% if query.fragment %}#{{ query.fragment }}{% endif %}" title="{{ query.description or query.sql }}">{{ query.title or query.name }}</a>{% if query.private %} 🔒{% endif %}</li>
+        {% endfor %}
+    </ul>
+{% endif %}
+
+{% if tables %}
+<h2 id="tables">Tables</h2>
+{% endif %}
+
 {% for table in tables %}
 {% if show_hidden or not table.hidden %}
 <div class="db-table">
-    <h2><a href="{{ urls.table(database, table.name) }}">{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if table.hidden %}<em> (hidden)</em>{% endif %}</h2>
+    <h3><a href="{{ urls.table(database, table.name) }}">{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if table.hidden %}<em> (hidden)</em>{% endif %}</h3>
     <p><em>{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}</em></p>
     <p>{% if table.count is none %}Many rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}</p>
 </div>
@@ -90,15 +103,6 @@
     </ul>
 {% endif %}
 
-{% if queries %}
-    <h2 id="queries">Queries</h2>
-    <ul class="bullets">
-        {% for query in queries %}
-            <li><a href="{{ urls.query(database, query.name) }}{% if query.fragment %}#{{ query.fragment }}{% endif %}" title="{{ query.description or query.sql }}">{{ query.title or query.name }}</a>{% if query.private %} 🔒{% endif %}</li>
-        {% endfor %}
-    </ul>
-{% endif %}
-
 {% if allow_download %}
     <p class="download-sqlite">Download SQLite DB: <a href="{{ urls.database(database) }}.db">{{ database }}.db</a> <em>{{ format_bytes(size) }}</em></p>
 {% endif %}
diff --git a/tests/test_html.py b/tests/test_html.py
index aa718857..1bbf335c 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -110,12 +110,32 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash):
 
 def test_database_page(app_client):
     response = app_client.get("/fixtures")
-    assert (
-        b"<p><em>pk, foreign_key_with_label, foreign_key_with_blank_label, "
-        b"foreign_key_with_no_label, foreign_key_compound_pk1, "
-        b"foreign_key_compound_pk2</em></p>"
-    ) in response.body
     soup = Soup(response.body, "html.parser")
+    # Should have a <textarea> for executing SQL
+    assert "<textarea" in response.text
+
+    # And a list of tables
+    for fragment in (
+        '<h2 id="tables">Tables</h2>',
+        '<h3><a href="/fixtures/sortable">sortable</a></h3>',
+        "<p><em>pk, foreign_key_with_label, foreign_key_with_blank_label, ",
+    ):
+        assert fragment in response.text
+
+    # And views
+    views_ul = soup.find("h2", text="Views").find_next_sibling("ul")
+    assert views_ul is not None
+    assert [
+        ("/fixtures/paginated_view", "paginated_view"),
+        ("/fixtures/searchable_view", "searchable_view"),
+        (
+            "/fixtures/searchable_view_configured_by_metadata",
+            "searchable_view_configured_by_metadata",
+        ),
+        ("/fixtures/simple_view", "simple_view"),
+    ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")])
+
+    # And a list of canned queries
     queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul")
     assert queries_ul is not None
     assert [

From 84391763a8d5911c387c9965c86c8d45f39b31fb Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 25 Jan 2022 10:39:03 -0800
Subject: [PATCH 1314/2303] Clarify that magic parameters don't work for custom
 SQL

---
 docs/sql_queries.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index f9a36490..010e3205 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -275,6 +275,8 @@ Magic parameters
 
 Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or query string.
 
+These magic parameters are only supported for canned queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query.
+
 Available magic parameters are:
 
 ``_actor_*`` - e.g. ``_actor_id``, ``_actor_name``

From 2aa686c6554bf6b8230eb5b3019574df6cc99225 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 26 Jan 2022 10:21:05 -0800
Subject: [PATCH 1315/2303] It's not a weekly newsletter

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 107d81da..557d9290 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 * Live demo of current main: https://latest.datasette.io/
 * Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions)
 
-Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem.
+Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem.
 
 ## Installation
 

From 3ef47a0896c7e63404a34e465b7160c80eaa571d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 27 Nov 2021 12:08:42 -0800
Subject: [PATCH 1316/2303] Link rel=alternate header for tables and rows

Also added Access-Control-Expose-Headers: Link to --cors mode.

Closes #1533

Refs https://github.com/simonw/datasette-notebook/issues/2

LL#	metadata.json.1
---
 datasette/templates/base.html  |  2 +-
 datasette/templates/row.html   |  3 ++-
 datasette/templates/table.html |  3 ++-
 datasette/utils/__init__.py    |  1 +
 datasette/views/base.py        | 12 ++++++++++--
 datasette/views/table.py       | 22 +++++++++++++++++++++-
 docs/json_api.rst              | 20 ++++++++++++++++++++
 tests/test_api.py              |  1 +
 tests/test_table_html.py       | 28 ++++++++++++++++++++++++++++
 9 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/datasette/templates/base.html b/datasette/templates/base.html
index c9aa7e31..836b7bb7 100644
--- a/datasette/templates/base.html
+++ b/datasette/templates/base.html
@@ -10,7 +10,7 @@
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.sri %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>
 {% endfor %}
-{% block extra_head %}{% endblock %}
+{%- block extra_head %}{% endblock -%}
 </head>
 <body class="{% block body_class %}{% endblock %}">
 <div class="not-footer">
diff --git a/datasette/templates/row.html b/datasette/templates/row.html
index c86e979d..1ac16268 100644
--- a/datasette/templates/row.html
+++ b/datasette/templates/row.html
@@ -3,7 +3,8 @@
 {% block title %}{{ database }}: {{ table }}{% endblock %}
 
 {% block extra_head %}
-{{ super() }}
+{{- super() -}}
+<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 <style>
 @media only screen and (max-width: 576px) {
 {% for column in columns %}
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index e3c6f38d..403e1d5b 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -3,7 +3,8 @@
 {% block title %}{{ database }}: {{ table }}: {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
-{{ super() }}
+{{- super() -}}
+<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 <script src="{{ urls.static('table.js') }}" defer></script>
 <style>
 @media only screen and (max-width: 576px) {
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index bc3155a5..dc4e1c99 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1094,3 +1094,4 @@ async def derive_named_parameters(db, sql):
 def add_cors_headers(headers):
     headers["Access-Control-Allow-Origin"] = "*"
     headers["Access-Control-Allow-Headers"] = "Authorization"
+    headers["Access-Control-Expose-Headers"] = "Link"
diff --git a/datasette/views/base.py b/datasette/views/base.py
index b1cacb3f..a414892a 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -137,10 +137,18 @@ class BaseView:
                 ],
             },
         }
+        # Hacky cheat to add extra headers
+        headers = {}
+        if "_extra_headers" in context:
+            headers.update(context["_extra_headers"])
         return Response.html(
             await self.ds.render_template(
-                template, template_context, request=request, view_name=self.name
-            )
+                template,
+                template_context,
+                request=request,
+                view_name=self.name,
+            ),
+            headers=headers,
         )
 
     @classmethod
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 77fb2850..6bbee352 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -17,6 +17,7 @@ from datasette.utils import (
     is_url,
     path_from_row_pks,
     path_with_added_args,
+    path_with_format,
     path_with_removed_args,
     path_with_replaced_args,
     to_css_class,
@@ -850,7 +851,12 @@ class TableView(RowTableShared):
                 for table_column in table_columns
                 if table_column not in columns
             ]
+            alternate_url_json = self.ds.absolute_url(
+                request,
+                self.ds.urls.path(path_with_format(request=request, format="json")),
+            )
             d = {
+                "alternate_url_json": alternate_url_json,
                 "table_actions": table_actions,
                 "use_rowid": use_rowid,
                 "filters": filters,
@@ -881,6 +887,11 @@ class TableView(RowTableShared):
                 "metadata": metadata,
                 "view_definition": await db.get_view_definition(table),
                 "table_definition": await db.get_table_definition(table),
+                "_extra_headers": {
+                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
+                        alternate_url_json
+                    )
+                },
             }
             d.update(extra_context_from_filters)
             return d
@@ -964,8 +975,12 @@ class RowView(RowTableShared):
             )
             for column in display_columns:
                 column["sortable"] = False
-
+            alternate_url_json = self.ds.absolute_url(
+                request,
+                self.ds.urls.path(path_with_format(request=request, format="json")),
+            )
             return {
+                "alternate_url_json": alternate_url_json,
                 "foreign_key_tables": await self.foreign_key_tables(
                     database, table, pk_values
                 ),
@@ -980,6 +995,11 @@ class RowView(RowTableShared):
                 .get(database, {})
                 .get("tables", {})
                 .get(table, {}),
+                "_extra_headers": {
+                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
+                        alternate_url_json
+                    )
+                },
             }
 
         data = {
diff --git a/docs/json_api.rst b/docs/json_api.rst
index bd55c163..b5a6744b 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -14,6 +14,7 @@ served with the following additional HTTP headers::
 
     Access-Control-Allow-Origin: *
     Access-Control-Allow-Headers: Authorization
+    Access-Control-Expose-Headers: Link
 
 This means JavaScript running on any domain will be able to make cross-origin
 requests to fetch the data.
@@ -435,3 +436,22 @@ looks like::
 
 The column in the foreign key table that is used for the label can be specified
 in ``metadata.json`` - see :ref:`label_columns`.
+
+.. _json_api_discover_alternate:
+
+Discovering the JSON for a page
+-------------------------------
+
+The :ref:`table <TableView>` and :ref:`row <RowView>` HTML pages both provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism.
+
+You can find this near the top of those pages, looking like this:
+
+.. code-block:: python
+
+    <link rel="alternate"
+      type="application/json+datasette"
+      href="https://latest.datasette.io/fixtures/sortable.json">
+
+The JSON URL is also made available in a ``Link`` HTTP header for the page::
+
+    Link: https://latest.datasette.io/fixtures/sortable.json; rel="alternate"; type="application/json+datasette"
diff --git a/tests/test_api.py b/tests/test_api.py
index 47ec3a8c..9741ffc5 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -977,6 +977,7 @@ def test_cors(app_client_with_cors, path, status_code):
     assert response.status == status_code
     assert response.headers["Access-Control-Allow-Origin"] == "*"
     assert response.headers["Access-Control-Allow-Headers"] == "Authorization"
+    assert response.headers["Access-Control-Expose-Headers"] == "Link"
 
 
 @pytest.mark.parametrize(
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 021268c3..7d08d230 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1069,3 +1069,31 @@ def test_table_page_title(app_client, path, expected):
     response = app_client.get(path)
     title = Soup(response.text, "html.parser").find("title").text
     assert title == expected
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    (
+        (
+            "/fixtures/table%2Fwith%2Fslashes.csv",
+            "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json",
+        ),
+        ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
+        (
+            "/fixtures/no_primary_key/1",
+            "http://localhost/fixtures/no_primary_key/1.json",
+        ),
+    ),
+)
+def test_alternate_url_json(app_client, path, expected):
+    response = app_client.get(path)
+    link = response.headers["link"]
+    assert link == '{}; rel="alternate"; type="application/json+datasette"'.format(
+        expected
+    )
+    assert (
+        '<link rel="alternate" type="application/json+datasette" href="{}">'.format(
+            expected
+        )
+        in response.text
+    )

From b72b2423c79dea4600b2337949db98269d0b6215 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 2 Feb 2022 13:21:11 -0800
Subject: [PATCH 1317/2303] rel=alternate JSON for queries and database pages,
 closes #1620

---
 datasette/templates/database.html |  3 ++-
 datasette/templates/query.html    |  3 ++-
 datasette/views/database.py       | 32 ++++++++++++++++++++++
 docs/json_api.rst                 |  2 +-
 tests/test_canned_queries.py      |  9 +++++++
 tests/test_html.py                | 45 +++++++++++++++++++++++++++++++
 tests/test_table_html.py          | 28 -------------------
 7 files changed, 91 insertions(+), 31 deletions(-)

diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index c1e39bd1..8f0c65d7 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -3,7 +3,8 @@
 {% block title %}{{ database }}{% endblock %}
 
 {% block extra_head %}
-{{ super() }}
+{{- super() -}}
+<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 {% include "_codemirror.html" %}
 {% endblock %}
 
diff --git a/datasette/templates/query.html b/datasette/templates/query.html
index 75f7f1b1..d0121976 100644
--- a/datasette/templates/query.html
+++ b/datasette/templates/query.html
@@ -3,7 +3,8 @@
 {% block title %}{{ database }}{% if query and query.sql %}: {{ query.sql }}{% endif %}{% endblock %}
 
 {% block extra_head %}
-{{ super() }}
+{{- super() -}}
+<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 {% if columns %}
 <style>
 @media only screen and (max-width: 576px) {
diff --git a/datasette/views/database.py b/datasette/views/database.py
index e26706e7..f3641dc5 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -123,6 +123,10 @@ class DatabaseView(DataView):
 
         attached_databases = [d.name for d in await db.attached_databases()]
 
+        alternate_url_json = self.ds.absolute_url(
+            request,
+            self.ds.urls.path(path_with_format(request=request, format="json")),
+        )
         return (
             {
                 "database": database,
@@ -140,6 +144,7 @@ class DatabaseView(DataView):
                 ),
             },
             {
+                "alternate_url_json": alternate_url_json,
                 "database_actions": database_actions,
                 "show_hidden": request.args.get("_show_hidden"),
                 "editable": True,
@@ -148,6 +153,11 @@ class DatabaseView(DataView):
                 and not db.is_mutable
                 and not db.is_memory,
                 "attached_databases": attached_databases,
+                "_extra_headers": {
+                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
+                        alternate_url_json
+                    )
+                },
             },
             (f"database-{to_css_class(database)}.html", "database.html"),
         )
@@ -308,7 +318,14 @@ class QueryView(DataView):
             else:
 
                 async def extra_template():
+                    alternate_url_json = self.ds.absolute_url(
+                        request,
+                        self.ds.urls.path(
+                            path_with_format(request=request, format="json")
+                        ),
+                    )
                     return {
+                        "alternate_url_json": alternate_url_json,
                         "request": request,
                         "path_with_added_args": path_with_added_args,
                         "path_with_removed_args": path_with_removed_args,
@@ -316,6 +333,11 @@ class QueryView(DataView):
                         "canned_query": canned_query,
                         "success_message": request.args.get("_success") or "",
                         "canned_write": True,
+                        "_extra_headers": {
+                            "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
+                                alternate_url_json
+                            )
+                        },
                     }
 
                 return (
@@ -448,7 +470,12 @@ class QueryView(DataView):
                     show_hide_link = path_with_added_args(request, {"_hide_sql": 1})
                     show_hide_text = "hide"
             hide_sql = show_hide_text == "show"
+            alternate_url_json = self.ds.absolute_url(
+                request,
+                self.ds.urls.path(path_with_format(request=request, format="json")),
+            )
             return {
+                "alternate_url_json": alternate_url_json,
                 "display_rows": display_rows,
                 "custom_sql": True,
                 "named_parameter_values": named_parameter_values,
@@ -462,6 +489,11 @@ class QueryView(DataView):
                 "show_hide_text": show_hide_text,
                 "show_hide_hidden": markupsafe.Markup(show_hide_hidden),
                 "hide_sql": hide_sql,
+                "_extra_headers": {
+                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
+                        alternate_url_json
+                    )
+                },
             }
 
         return (
diff --git a/docs/json_api.rst b/docs/json_api.rst
index b5a6744b..4f9eaddb 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -442,7 +442,7 @@ in ``metadata.json`` - see :ref:`label_columns`.
 Discovering the JSON for a page
 -------------------------------
 
-The :ref:`table <TableView>` and :ref:`row <RowView>` HTML pages both provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism.
+The :ref:`database <DatabaseView>`, :ref:`table <TableView>`, :ref:`custom/canned query <sql>` and :ref:`row <RowView>` HTML pages all provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism.
 
 You can find this near the top of those pages, looking like this:
 
diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index c5ccaf5c..5abffdcc 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -364,3 +364,12 @@ def test_canned_write_custom_template(canned_write_client):
         in response.text
     )
     assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text
+    # And test for link rel=alternate while we're here:
+    assert (
+        '<link rel="alternate" type="application/json+datasette" href="http://localhost/data/update_name.json">'
+        in response.text
+    )
+    assert (
+        response.headers["link"]
+        == 'http://localhost/data/update_name.json; rel="alternate"; type="application/json+datasette"'
+    )
diff --git a/tests/test_html.py b/tests/test_html.py
index 1bbf335c..273e4914 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -870,3 +870,48 @@ def test_trace_correctly_escaped(app_client):
     response = app_client.get("/fixtures?sql=select+'<h1>Hello'&_trace=1")
     assert "select '<h1>Hello" not in response.text
     assert "select '<h1>Hello" in response.text
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    (
+        # Table page
+        (
+            "/fixtures/table%2Fwith%2Fslashes.csv",
+            "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json",
+        ),
+        ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
+        # Row page
+        (
+            "/fixtures/no_primary_key/1",
+            "http://localhost/fixtures/no_primary_key/1.json",
+        ),
+        # Database index page
+        (
+            "/fixtures",
+            "http://localhost/fixtures.json",
+        ),
+        # Custom query page
+        (
+            "/fixtures?sql=select+*+from+facetable",
+            "http://localhost/fixtures.json?sql=select+*+from+facetable",
+        ),
+        # Canned query page
+        (
+            "/fixtures/neighborhood_search?text=town",
+            "http://localhost/fixtures/neighborhood_search.json?text=town",
+        ),
+    ),
+)
+def test_alternate_url_json(app_client, path, expected):
+    response = app_client.get(path)
+    link = response.headers["link"]
+    assert link == '{}; rel="alternate"; type="application/json+datasette"'.format(
+        expected
+    )
+    assert (
+        '<link rel="alternate" type="application/json+datasette" href="{}">'.format(
+            expected
+        )
+        in response.text
+    )
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 7d08d230..021268c3 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1069,31 +1069,3 @@ def test_table_page_title(app_client, path, expected):
     response = app_client.get(path)
     title = Soup(response.text, "html.parser").find("title").text
     assert title == expected
-
-
-@pytest.mark.parametrize(
-    "path,expected",
-    (
-        (
-            "/fixtures/table%2Fwith%2Fslashes.csv",
-            "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json",
-        ),
-        ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
-        (
-            "/fixtures/no_primary_key/1",
-            "http://localhost/fixtures/no_primary_key/1.json",
-        ),
-    ),
-)
-def test_alternate_url_json(app_client, path, expected):
-    response = app_client.get(path)
-    link = response.headers["link"]
-    assert link == '{}; rel="alternate"; type="application/json+datasette"'.format(
-        expected
-    )
-    assert (
-        '<link rel="alternate" type="application/json+datasette" href="{}">'.format(
-            expected
-        )
-        in response.text
-    )

From 8d5779acf0041cfd0db7f68f468419f9008b86ec Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 2 Feb 2022 13:32:47 -0800
Subject: [PATCH 1318/2303] Refactored alternate_url_json mechanism, refs
 #1620, #1533

---
 datasette/templates/base.html     |  3 +++
 datasette/templates/database.html |  1 -
 datasette/templates/query.html    |  1 -
 datasette/templates/row.html      |  1 -
 datasette/templates/table.html    |  1 -
 datasette/views/base.py           | 17 +++++++++++++---
 datasette/views/database.py       | 32 -------------------------------
 datasette/views/table.py          | 20 -------------------
 docs/json_api.rst                 |  4 ++--
 tests/test_html.py                |  9 ++++++++-
 10 files changed, 27 insertions(+), 62 deletions(-)

diff --git a/datasette/templates/base.html b/datasette/templates/base.html
index 836b7bb7..c3a71acb 100644
--- a/datasette/templates/base.html
+++ b/datasette/templates/base.html
@@ -10,6 +10,9 @@
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.sri %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>
 {% endfor %}
+{%- if alternate_url_json -%}
+    <link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
+{%- endif -%}
 {%- block extra_head %}{% endblock -%}
 </head>
 <body class="{% block body_class %}{% endblock %}">
diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index 8f0c65d7..e76bc49e 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -4,7 +4,6 @@
 
 {% block extra_head %}
 {{- super() -}}
-<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 {% include "_codemirror.html" %}
 {% endblock %}
 
diff --git a/datasette/templates/query.html b/datasette/templates/query.html
index d0121976..8c920527 100644
--- a/datasette/templates/query.html
+++ b/datasette/templates/query.html
@@ -4,7 +4,6 @@
 
 {% block extra_head %}
 {{- super() -}}
-<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 {% if columns %}
 <style>
 @media only screen and (max-width: 576px) {
diff --git a/datasette/templates/row.html b/datasette/templates/row.html
index 1ac16268..10770ce9 100644
--- a/datasette/templates/row.html
+++ b/datasette/templates/row.html
@@ -4,7 +4,6 @@
 
 {% block extra_head %}
 {{- super() -}}
-<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 <style>
 @media only screen and (max-width: 576px) {
 {% for column in columns %}
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index 403e1d5b..81bd044a 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -4,7 +4,6 @@
 
 {% block extra_head %}
 {{- super() -}}
-<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}">
 <script src="{{ urls.static('table.js') }}" defer></script>
 <style>
 @media only screen and (max-width: 576px) {
diff --git a/datasette/views/base.py b/datasette/views/base.py
index a414892a..c74d6141 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -55,6 +55,7 @@ class DatasetteError(Exception):
 
 class BaseView:
     ds = None
+    has_json_alternate = True
 
     def __init__(self, datasette):
         self.ds = datasette
@@ -137,10 +138,20 @@ class BaseView:
                 ],
             },
         }
-        # Hacky cheat to add extra headers
         headers = {}
-        if "_extra_headers" in context:
-            headers.update(context["_extra_headers"])
+        if self.has_json_alternate:
+            alternate_url_json = self.ds.absolute_url(
+                request,
+                self.ds.urls.path(path_with_format(request=request, format="json")),
+            )
+            template_context["alternate_url_json"] = alternate_url_json
+            headers.update(
+                {
+                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
+                        alternate_url_json
+                    )
+                }
+            )
         return Response.html(
             await self.ds.render_template(
                 template,
diff --git a/datasette/views/database.py b/datasette/views/database.py
index f3641dc5..e26706e7 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -123,10 +123,6 @@ class DatabaseView(DataView):
 
         attached_databases = [d.name for d in await db.attached_databases()]
 
-        alternate_url_json = self.ds.absolute_url(
-            request,
-            self.ds.urls.path(path_with_format(request=request, format="json")),
-        )
         return (
             {
                 "database": database,
@@ -144,7 +140,6 @@ class DatabaseView(DataView):
                 ),
             },
             {
-                "alternate_url_json": alternate_url_json,
                 "database_actions": database_actions,
                 "show_hidden": request.args.get("_show_hidden"),
                 "editable": True,
@@ -153,11 +148,6 @@ class DatabaseView(DataView):
                 and not db.is_mutable
                 and not db.is_memory,
                 "attached_databases": attached_databases,
-                "_extra_headers": {
-                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
-                        alternate_url_json
-                    )
-                },
             },
             (f"database-{to_css_class(database)}.html", "database.html"),
         )
@@ -318,14 +308,7 @@ class QueryView(DataView):
             else:
 
                 async def extra_template():
-                    alternate_url_json = self.ds.absolute_url(
-                        request,
-                        self.ds.urls.path(
-                            path_with_format(request=request, format="json")
-                        ),
-                    )
                     return {
-                        "alternate_url_json": alternate_url_json,
                         "request": request,
                         "path_with_added_args": path_with_added_args,
                         "path_with_removed_args": path_with_removed_args,
@@ -333,11 +316,6 @@ class QueryView(DataView):
                         "canned_query": canned_query,
                         "success_message": request.args.get("_success") or "",
                         "canned_write": True,
-                        "_extra_headers": {
-                            "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
-                                alternate_url_json
-                            )
-                        },
                     }
 
                 return (
@@ -470,12 +448,7 @@ class QueryView(DataView):
                     show_hide_link = path_with_added_args(request, {"_hide_sql": 1})
                     show_hide_text = "hide"
             hide_sql = show_hide_text == "show"
-            alternate_url_json = self.ds.absolute_url(
-                request,
-                self.ds.urls.path(path_with_format(request=request, format="json")),
-            )
             return {
-                "alternate_url_json": alternate_url_json,
                 "display_rows": display_rows,
                 "custom_sql": True,
                 "named_parameter_values": named_parameter_values,
@@ -489,11 +462,6 @@ class QueryView(DataView):
                 "show_hide_text": show_hide_text,
                 "show_hide_hidden": markupsafe.Markup(show_hide_hidden),
                 "hide_sql": hide_sql,
-                "_extra_headers": {
-                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
-                        alternate_url_json
-                    )
-                },
             }
 
         return (
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 6bbee352..be9e9c3b 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -851,12 +851,7 @@ class TableView(RowTableShared):
                 for table_column in table_columns
                 if table_column not in columns
             ]
-            alternate_url_json = self.ds.absolute_url(
-                request,
-                self.ds.urls.path(path_with_format(request=request, format="json")),
-            )
             d = {
-                "alternate_url_json": alternate_url_json,
                 "table_actions": table_actions,
                 "use_rowid": use_rowid,
                 "filters": filters,
@@ -887,11 +882,6 @@ class TableView(RowTableShared):
                 "metadata": metadata,
                 "view_definition": await db.get_view_definition(table),
                 "table_definition": await db.get_table_definition(table),
-                "_extra_headers": {
-                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
-                        alternate_url_json
-                    )
-                },
             }
             d.update(extra_context_from_filters)
             return d
@@ -975,12 +965,7 @@ class RowView(RowTableShared):
             )
             for column in display_columns:
                 column["sortable"] = False
-            alternate_url_json = self.ds.absolute_url(
-                request,
-                self.ds.urls.path(path_with_format(request=request, format="json")),
-            )
             return {
-                "alternate_url_json": alternate_url_json,
                 "foreign_key_tables": await self.foreign_key_tables(
                     database, table, pk_values
                 ),
@@ -995,11 +980,6 @@ class RowView(RowTableShared):
                 .get(database, {})
                 .get("tables", {})
                 .get(table, {}),
-                "_extra_headers": {
-                    "Link": '{}; rel="alternate"; type="application/json+datasette"'.format(
-                        alternate_url_json
-                    )
-                },
             }
 
         data = {
diff --git a/docs/json_api.rst b/docs/json_api.rst
index 4f9eaddb..aa6fcdaa 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -442,9 +442,9 @@ in ``metadata.json`` - see :ref:`label_columns`.
 Discovering the JSON for a page
 -------------------------------
 
-The :ref:`database <DatabaseView>`, :ref:`table <TableView>`, :ref:`custom/canned query <sql>` and :ref:`row <RowView>` HTML pages all provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism.
+Most of the HTML pages served by Datasette provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism.
 
-You can find this near the top of those pages, looking like this:
+You can find this near the top of the source code of those pages, looking like this:
 
 .. code-block:: python
 
diff --git a/tests/test_html.py b/tests/test_html.py
index 273e4914..4b6cbd13 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -875,12 +875,14 @@ def test_trace_correctly_escaped(app_client):
 @pytest.mark.parametrize(
     "path,expected",
     (
+        # Instance index page
+        ("/", "http://localhost/.json"),
         # Table page
+        ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
         (
             "/fixtures/table%2Fwith%2Fslashes.csv",
             "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json",
         ),
-        ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
         # Row page
         (
             "/fixtures/no_primary_key/1",
@@ -901,6 +903,11 @@ def test_trace_correctly_escaped(app_client):
             "/fixtures/neighborhood_search?text=town",
             "http://localhost/fixtures/neighborhood_search.json?text=town",
         ),
+        # /-/ pages
+        (
+            "/-/plugins",
+            "http://localhost/-/plugins.json",
+        ),
     ),
 )
 def test_alternate_url_json(app_client, path, expected):

From 23a09b0f6af33c52acf8c1d9002fe475b42fee10 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 2 Feb 2022 13:48:52 -0800
Subject: [PATCH 1319/2303] Remove JSON rel=alternate from some pages, closes
 #1623

---
 datasette/views/special.py |  6 ++++++
 tests/test_html.py         | 12 ++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/datasette/views/special.py b/datasette/views/special.py
index 3cb626a5..cdd530f0 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -43,6 +43,7 @@ class JsonDataView(BaseView):
 
 class PatternPortfolioView(BaseView):
     name = "patterns"
+    has_json_alternate = False
 
     async def get(self, request):
         await self.check_permission(request, "view-instance")
@@ -51,6 +52,7 @@ class PatternPortfolioView(BaseView):
 
 class AuthTokenView(BaseView):
     name = "auth_token"
+    has_json_alternate = False
 
     async def get(self, request):
         token = request.args.get("token") or ""
@@ -69,6 +71,7 @@ class AuthTokenView(BaseView):
 
 class LogoutView(BaseView):
     name = "logout"
+    has_json_alternate = False
 
     async def get(self, request):
         if not request.actor:
@@ -88,6 +91,7 @@ class LogoutView(BaseView):
 
 class PermissionsDebugView(BaseView):
     name = "permissions_debug"
+    has_json_alternate = False
 
     async def get(self, request):
         await self.check_permission(request, "view-instance")
@@ -103,6 +107,7 @@ class PermissionsDebugView(BaseView):
 
 class AllowDebugView(BaseView):
     name = "allow_debug"
+    has_json_alternate = False
 
     async def get(self, request):
         errors = []
@@ -137,6 +142,7 @@ class AllowDebugView(BaseView):
 
 class MessagesDebugView(BaseView):
     name = "messages_debug"
+    has_json_alternate = False
 
     async def get(self, request):
         await self.check_permission(request, "view-instance")
diff --git a/tests/test_html.py b/tests/test_html.py
index 4b6cbd13..d5f4250d 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -922,3 +922,15 @@ def test_alternate_url_json(app_client, path, expected):
         )
         in response.text
     )
+
+
+@pytest.mark.parametrize(
+    "path",
+    ("/-/patterns", "/-/messages", "/-/allow-debug", "/fixtures.db"),
+)
+def test_no_alternate_url_json(app_client, path):
+    response = app_client.get(path)
+    assert "link" not in response.headers
+    assert (
+        '<link rel="alternate" type="application/json+datasette"' not in response.text
+    )

From a9d8824617268c4d214dd3be2174ac452044f737 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 2 Feb 2022 13:58:52 -0800
Subject: [PATCH 1320/2303] Test against Python 3.11-dev

Closes #1621
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2caf9447..478e1f34 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

From b5e6b1a9e1332fca3effe45d55dd06ee4249f163 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 2 Feb 2022 14:23:51 -0800
Subject: [PATCH 1321/2303] Bump black from 21.12b0 to 22.1.0 (#1616)

Bumps [black](https://github.com/psf/black) from 21.12b0 to 22.1.0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits/22.1.0)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index dade0a88..6accaa30 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.10,<0.17",
             "beautifulsoup4>=4.8.1,<4.11.0",
-            "black==21.12b0",
+            "black==22.1.0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
             "cogapp>=3.3.0",

From 1af1041f91a9b91b321078d354132d1df5204660 Mon Sep 17 00:00:00 2001
From: Robert Christie <robc@pobox.com>
Date: Thu, 3 Feb 2022 01:58:35 +0000
Subject: [PATCH 1322/2303] Jinja template_name should use "/" even on Windows
 (#1617)

Closes #1545. Thanks, Robert Christie
---
 datasette/app.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 09d7d034..7bdf076c 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1212,9 +1212,10 @@ class DatasetteRouter:
         else:
             # Is there a pages/* template matching this path?
             route_path = request.scope.get("route_path", request.scope["path"])
-            template_path = os.path.join("pages", *route_path.split("/")) + ".html"
+            # Jinja requires template names to use "/" even on Windows
+            template_name = "pages" + route_path + ".html"
             try:
-                template = self.ds.jinja_env.select_template([template_path])
+                template = self.ds.jinja_env.select_template([template_name])
             except TemplateNotFound:
                 template = None
             if template is None:

From ac239d34ab2de6987afac43f5d38b576b26e9457 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 4 Feb 2022 20:45:13 -0800
Subject: [PATCH 1323/2303] Refactor test_trace into separate test module, refs
 #1576

---
 tests/test_api.py    | 51 ------------------------------------------
 tests/test_tracer.py | 53 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 51 deletions(-)
 create mode 100644 tests/test_tracer.py

diff --git a/tests/test_api.py b/tests/test_api.py
index 9741ffc5..57471af2 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -911,57 +911,6 @@ def test_config_force_https_urls():
         assert client.ds._last_request.scheme == "https"
 
 
-@pytest.mark.parametrize("trace_debug", (True, False))
-def test_trace(trace_debug):
-    with make_app_client(settings={"trace_debug": trace_debug}) as client:
-        response = client.get("/fixtures/simple_primary_key.json?_trace=1")
-        assert response.status == 200
-
-    data = response.json
-    if not trace_debug:
-        assert "_trace" not in data
-        return
-
-    assert "_trace" in data
-    trace_info = data["_trace"]
-    assert isinstance(trace_info["request_duration_ms"], float)
-    assert isinstance(trace_info["sum_trace_duration_ms"], float)
-    assert isinstance(trace_info["num_traces"], int)
-    assert isinstance(trace_info["traces"], list)
-    traces = trace_info["traces"]
-    assert len(traces) == trace_info["num_traces"]
-    for trace in traces:
-        assert isinstance(trace["type"], str)
-        assert isinstance(trace["start"], float)
-        assert isinstance(trace["end"], float)
-        assert trace["duration_ms"] == (trace["end"] - trace["start"]) * 1000
-        assert isinstance(trace["traceback"], list)
-        assert isinstance(trace["database"], str)
-        assert isinstance(trace["sql"], str)
-        assert isinstance(trace.get("params"), (list, dict, None.__class__))
-
-    sqls = [trace["sql"] for trace in traces if "sql" in trace]
-    # There should be a mix of different types of SQL statement
-    expected = (
-        "CREATE TABLE ",
-        "PRAGMA ",
-        "INSERT OR REPLACE INTO ",
-        "INSERT INTO",
-        "select ",
-    )
-    for prefix in expected:
-        assert any(
-            sql.startswith(prefix) for sql in sqls
-        ), "No trace beginning with: {}".format(prefix)
-
-    # Should be at least one executescript
-    assert any(trace for trace in traces if trace.get("executescript"))
-    # And at least one executemany
-    execute_manys = [trace for trace in traces if trace.get("executemany")]
-    assert execute_manys
-    assert all(isinstance(trace["count"], int) for trace in execute_manys)
-
-
 @pytest.mark.parametrize(
     "path,status_code",
     [
diff --git a/tests/test_tracer.py b/tests/test_tracer.py
new file mode 100644
index 00000000..20a4427e
--- /dev/null
+++ b/tests/test_tracer.py
@@ -0,0 +1,53 @@
+import pytest
+from .fixtures import make_app_client
+
+
+@pytest.mark.parametrize("trace_debug", (True, False))
+def test_trace(trace_debug):
+    with make_app_client(settings={"trace_debug": trace_debug}) as client:
+        response = client.get("/fixtures/simple_primary_key.json?_trace=1")
+        assert response.status == 200
+
+    data = response.json
+    if not trace_debug:
+        assert "_trace" not in data
+        return
+
+    assert "_trace" in data
+    trace_info = data["_trace"]
+    assert isinstance(trace_info["request_duration_ms"], float)
+    assert isinstance(trace_info["sum_trace_duration_ms"], float)
+    assert isinstance(trace_info["num_traces"], int)
+    assert isinstance(trace_info["traces"], list)
+    traces = trace_info["traces"]
+    assert len(traces) == trace_info["num_traces"]
+    for trace in traces:
+        assert isinstance(trace["type"], str)
+        assert isinstance(trace["start"], float)
+        assert isinstance(trace["end"], float)
+        assert trace["duration_ms"] == (trace["end"] - trace["start"]) * 1000
+        assert isinstance(trace["traceback"], list)
+        assert isinstance(trace["database"], str)
+        assert isinstance(trace["sql"], str)
+        assert isinstance(trace.get("params"), (list, dict, None.__class__))
+
+    sqls = [trace["sql"] for trace in traces if "sql" in trace]
+    # There should be a mix of different types of SQL statement
+    expected = (
+        "CREATE TABLE ",
+        "PRAGMA ",
+        "INSERT OR REPLACE INTO ",
+        "INSERT INTO",
+        "select ",
+    )
+    for prefix in expected:
+        assert any(
+            sql.startswith(prefix) for sql in sqls
+        ), "No trace beginning with: {}".format(prefix)
+
+    # Should be at least one executescript
+    assert any(trace for trace in traces if trace.get("executescript"))
+    # And at least one executemany
+    execute_manys = [trace for trace in traces if trace.get("executemany")]
+    assert execute_manys
+    assert all(isinstance(trace["count"], int) for trace in execute_manys)

From da53e0360da4771ffb56a8e3eb3f7476f3168299 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 4 Feb 2022 21:19:49 -0800
Subject: [PATCH 1324/2303] tracer.trace_child_tasks() for asyncio.gather
 tracing

Also added documentation for datasette.tracer module.

Closes #1576
---
 datasette/tracer.py        | 20 +++++++----
 docs/internals.rst         | 71 ++++++++++++++++++++++++++++++++++++++
 tests/plugins/my_plugin.py | 12 +++++++
 tests/test_tracer.py       | 15 ++++++++
 4 files changed, 111 insertions(+), 7 deletions(-)

diff --git a/datasette/tracer.py b/datasette/tracer.py
index 6703f060..fc7338b0 100644
--- a/datasette/tracer.py
+++ b/datasette/tracer.py
@@ -1,5 +1,6 @@
 import asyncio
 from contextlib import contextmanager
+from contextvars import ContextVar
 from markupsafe import escape
 import time
 import json
@@ -9,20 +10,25 @@ tracers = {}
 
 TRACE_RESERVED_KEYS = {"type", "start", "end", "duration_ms", "traceback"}
 
-
-# asyncio.current_task was introduced in Python 3.7:
-for obj in (asyncio, asyncio.Task):
-    current_task = getattr(obj, "current_task", None)
-    if current_task is not None:
-        break
+trace_task_id = ContextVar("trace_task_id", default=None)
 
 
 def get_task_id():
+    current = trace_task_id.get(None)
+    if current is not None:
+        return current
     try:
         loop = asyncio.get_event_loop()
     except RuntimeError:
         return None
-    return id(current_task(loop=loop))
+    return id(asyncio.current_task(loop=loop))
+
+
+@contextmanager
+def trace_child_tasks():
+    token = trace_task_id.set(get_task_id())
+    yield
+    trace_task_id.reset(token)
 
 
 @contextmanager
diff --git a/docs/internals.rst b/docs/internals.rst
index 6a5666fd..a5dbdfb4 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -864,3 +864,74 @@ parse_metadata(content)
 This function accepts a string containing either JSON or YAML, expected to be of the format described in :ref:`metadata`. It returns a nested Python dictionary representing the parsed data from that string.
 
 If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception.
+
+.. _internals_tracer
+
+datasette.tracer
+================
+
+Running Datasette with ``--setting trace_debug 1`` enables trace debug output, which can then be viewed by adding ``?_trace=1`` to the query string for any page.
+
+You can see an example of this at the bottom of `latest.datasette.io/fixtures/facetable?_trace=1 <https://latest.datasette.io/fixtures/facetable?_trace=1>`__. The JSON output shows full details of every SQL query that was executed to generate the page.
+
+The `datasette-pretty-traces <https://datasette.io/plugins/datasette-pretty-traces>`__ plugin can be installed to provide a more readable display of this information. You can see `a demo of that here <https://latest-with-plugins.datasette.io/github/commits?_trace=1>`__.
+
+You can add your own custom traces to the JSON output using the ``trace()`` context manager. This takes a string that identifies the type of trace being recorded, and records any keyword arguments as additional JSON keys on the resulting trace object.
+
+The start and end time, duration and a traceback of where the trace was executed will be automatically attached to the JSON object.
+
+This example uses trace to record the start, end and duration of any HTTP GET requests made using the function:
+
+.. code-block:: python
+
+    from datasette.tracer import trace
+    import httpx
+
+    async def fetch_url(url):
+        with trace("fetch-url", url=url):
+            async with httpx.AsyncClient() as client:
+                return await client.get(url)
+
+.. _internals_tracer_trace_child_tasks
+
+Tracing child tasks
+-------------------
+
+If your code uses a mechanism such as ``asyncio.gather()`` to execute code in additional tasks you may find that some of the traces are missing from the display.
+
+You can use the ``trace_child_tasks()`` context manager to ensure these child tasks are correctly handled.
+
+.. code-block:: python
+
+    from datasette import tracer
+
+    with tracer.trace_child_tasks():
+        results = await asyncio.gather(
+            # ... async tasks here
+        )
+
+This example uses the :ref:`register_routes() <plugin_register_routes>` plugin hook to add a page at ``/parallel-queries`` which executes two SQL queries in parallel using ``asyncio.gather()`` and returns their results.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette import tracer
+
+    @hookimpl
+    def register_routes():
+
+        async def parallel_queries(datasette):
+            db = datasette.get_database()
+            with tracer.trace_child_tasks():
+                one, two = await asyncio.gather(
+                    db.execute("select 1"),
+                    db.execute("select 2"),
+                )
+            return Response.json({"one": one.single_value(), "two": two.single_value()})
+
+        return [
+            (r"/parallel-queries$", parallel_queries),
+        ]
+
+
+Adding ``?_trace=1`` will show that the trace covers both of those child tasks.
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 75c76ea8..610cea17 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -1,5 +1,7 @@
+import asyncio
 from datasette import hookimpl
 from datasette.facets import Facet
+from datasette import tracer
 from datasette.utils import path_with_added_args
 from datasette.utils.asgi import asgi_send_json, Response
 import base64
@@ -270,6 +272,15 @@ def register_routes():
     def asgi_scope(scope):
         return Response.json(scope, default=repr)
 
+    async def parallel_queries(datasette):
+        db = datasette.get_database()
+        with tracer.trace_child_tasks():
+            one, two = await asyncio.gather(
+                db.execute("select coalesce(sleep(0.1), 1)"),
+                db.execute("select coalesce(sleep(0.1), 2)"),
+            )
+        return Response.json({"one": one.single_value(), "two": two.single_value()})
+
     return [
         (r"/one/$", one),
         (r"/two/(?P<name>.*)$", two),
@@ -281,6 +292,7 @@ def register_routes():
         (r"/add-message/$", add_message),
         (r"/render-message/$", render_message),
         (r"/asgi-scope$", asgi_scope),
+        (r"/parallel-queries$", parallel_queries),
     ]
 
 
diff --git a/tests/test_tracer.py b/tests/test_tracer.py
index 20a4427e..ceadee50 100644
--- a/tests/test_tracer.py
+++ b/tests/test_tracer.py
@@ -51,3 +51,18 @@ def test_trace(trace_debug):
     execute_manys = [trace for trace in traces if trace.get("executemany")]
     assert execute_manys
     assert all(isinstance(trace["count"], int) for trace in execute_manys)
+
+
+def test_trace_parallel_queries():
+    with make_app_client(settings={"trace_debug": True}) as client:
+        response = client.get("/parallel-queries?_trace=1")
+        assert response.status == 200
+
+    data = response.json
+    assert data["one"] == 1
+    assert data["two"] == 2
+    trace_info = data["_trace"]
+    traces = [trace for trace in trace_info["traces"] if "sql" in trace]
+    one, two = traces
+    # "two" should have started before "one" ended
+    assert two["start"] < one["end"]

From 1c6b297e3ec288cf1f838796df499a9c21c31664 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 4 Feb 2022 21:28:35 -0800
Subject: [PATCH 1325/2303] Link to datasette.tracer from trace_debug docs,
 refs #1576

---
 docs/settings.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/settings.rst b/docs/settings.rst
index 7cc4bae0..da06d6a0 100644
--- a/docs/settings.rst
+++ b/docs/settings.rst
@@ -302,6 +302,8 @@ Some examples:
 * https://latest.datasette.io/?_trace=1
 * https://latest.datasette.io/fixtures/roadside_attractions?_trace=1
 
+See :ref:`internals_tracer` for details on how to hook into this mechanism as a plugin author.
+
 .. _setting_base_url:
 
 base_url

From d25b55ab5e4d7368d374ea752b2232755869d40d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 5 Feb 2022 22:32:23 -0800
Subject: [PATCH 1326/2303] Fixed rST warnings

---
 docs/internals.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index a5dbdfb4..0b010295 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -865,7 +865,7 @@ This function accepts a string containing either JSON or YAML, expected to be of
 
 If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception.
 
-.. _internals_tracer
+.. _internals_tracer:
 
 datasette.tracer
 ================
@@ -892,7 +892,7 @@ This example uses trace to record the start, end and duration of any HTTP GET re
             async with httpx.AsyncClient() as client:
                 return await client.get(url)
 
-.. _internals_tracer_trace_child_tasks
+.. _internals_tracer_trace_child_tasks:
 
 Tracing child tasks
 -------------------

From 8a25ea9bcae7ae4c9a4bd99f90c955828ff5676d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 5 Feb 2022 22:34:33 -0800
Subject: [PATCH 1327/2303] Implemented import shortcuts, closes #957

---
 datasette/__init__.py      |  2 ++
 docs/internals.rst         | 15 +++++++++++++++
 docs/plugin_hooks.rst      |  4 +++-
 tests/plugins/my_plugin.py |  9 +++++++++
 4 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/datasette/__init__.py b/datasette/__init__.py
index 0e59760a..faa36051 100644
--- a/datasette/__init__.py
+++ b/datasette/__init__.py
@@ -1,3 +1,5 @@
 from datasette.version import __version_info__, __version__  # noqa
+from datasette.utils.asgi import Forbidden, NotFound, Response  # noqa
+from datasette.utils import actor_matches_allow  # noqa
 from .hookspecs import hookimpl  # noqa
 from .hookspecs import hookspec  # noqa
diff --git a/docs/internals.rst b/docs/internals.rst
index 0b010295..632f7d7a 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -935,3 +935,18 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h
 
 
 Adding ``?_trace=1`` will show that the trace covers both of those child tasks.
+
+.. _internals_shortcuts:
+
+Import shortcuts
+================
+
+The following commonly used symbols can be imported directly from the ``datasette`` module:
+
+.. code-block:: python
+
+    from datasette import Response
+    from datasette import Forbidden
+    from datasette import NotFound
+    from datasette import hookimpl
+    from datasette import actor_matches_allow
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 88e1def0..1308b704 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -542,7 +542,7 @@ Return a list of ``(regex, view_function)`` pairs, something like this:
 
 .. code-block:: python
 
-    from datasette.utils.asgi import Response
+    from datasette import Response
     import html
 
 
@@ -582,6 +582,8 @@ The view function can be a regular function or an ``async def`` function, depend
 
 The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only).
 
+It can also rase the ``datasette.NotFound`` exception to return a 404 not found error, or the ``datasette.Forbidden`` exception for a 403 forbidden.
+
 See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes used by your plugin.
 
 Examples: `datasette-auth-github <https://datasette.io/plugins/datasette-auth-github>`__, `datasette-psutil <https://datasette.io/plugins/datasette-psutil>`__
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 610cea17..1c9b0575 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -300,6 +300,15 @@ def register_routes():
 def startup(datasette):
     datasette._startup_hook_fired = True
 
+    # And test some import shortcuts too
+    from datasette import Response
+    from datasette import Forbidden
+    from datasette import NotFound
+    from datasette import hookimpl
+    from datasette import actor_matches_allow
+
+    _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow)
+
 
 @hookimpl
 def canned_queries(datasette, database, actor):

From 9b83ff2ee4d3cb5bfc5cb09a3ec99819ac214434 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 5 Feb 2022 22:46:33 -0800
Subject: [PATCH 1328/2303] Fixed spelling of "raise"

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 1308b704..a63d441e 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -582,7 +582,7 @@ The view function can be a regular function or an ``async def`` function, depend
 
 The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only).
 
-It can also rase the ``datasette.NotFound`` exception to return a 404 not found error, or the ``datasette.Forbidden`` exception for a 403 forbidden.
+It can also raise the ``datasette.NotFound`` exception to return a 404 not found error, or the ``datasette.Forbidden`` exception for a 403 forbidden.
 
 See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes used by your plugin.
 

From d9b508ffaa91f9f1840b366f5d282712d445f16b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 6 Feb 2022 22:30:00 -0800
Subject: [PATCH 1329/2303] @documented decorator plus unit test plus
 sphinx.ext.autodoc

New mechanism for marking datasette.utils functions that should be covered by the
documentation, then testing that they have indeed been documented.

Also enabled sphinx.ext.autodoc which can now be used to embed the documented
versions of those functions.

Refs #1176
---
 datasette/utils/__init__.py | 16 ++++++++++++++--
 docs/conf.py                |  2 +-
 docs/internals.rst          | 11 +++++++++++
 tests/test_docs.py          | 18 +++++++++++++++++-
 4 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index dc4e1c99..610e916f 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -12,6 +12,7 @@ import os
 import re
 import shlex
 import tempfile
+import typing
 import time
 import types
 import shutil
@@ -59,8 +60,17 @@ Column = namedtuple(
     "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden")
 )
 
+functions_marked_as_documented = []
 
-async def await_me_maybe(value):
+
+def documented(fn):
+    functions_marked_as_documented.append(fn)
+    return fn
+
+
+@documented
+async def await_me_maybe(value: typing.Any) -> typing.Any:
+    "If value is callable, call it. If awaitable, await it. Otherwise return it."
     if callable(value):
         value = value()
     if asyncio.iscoroutine(value):
@@ -915,7 +925,9 @@ class BadMetadataError(Exception):
     pass
 
 
-def parse_metadata(content):
+@documented
+def parse_metadata(content: str) -> dict:
+    "Detects if content is JSON or YAML and parses it appropriately."
     # content can be JSON or YAML
     try:
         return json.loads(content)
diff --git a/docs/conf.py b/docs/conf.py
index 89009ea9..d114bc52 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -31,7 +31,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx.ext.extlinks"]
+extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"]
 
 extlinks = {
     "issue": ("https://github.com/simonw/datasette/issues/%s", "#"),
diff --git a/docs/internals.rst b/docs/internals.rst
index 632f7d7a..12ef5c54 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -865,6 +865,17 @@ This function accepts a string containing either JSON or YAML, expected to be of
 
 If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception.
 
+.. autofunction:: datasette.utils.parse_metadata
+
+.. _internals_utils_await_me_maybe:
+
+await_me_maybe(value)
+---------------------
+
+Utility function for calling ``await`` on a return value if it is awaitable, otherwise returning the value. This is used by Datasette to support plugin hooks that can optionally return awaitable functions. Read more about this function in `The “await me maybe” pattern for Python asyncio <https://simonwillison.net/2020/Sep/2/await-me-maybe/>`__.
+
+.. autofunction:: datasette.utils.await_me_maybe
+
 .. _internals_tracer:
 
 datasette.tracer
diff --git a/tests/test_docs.py b/tests/test_docs.py
index 0d17b8e3..cd5a6c13 100644
--- a/tests/test_docs.py
+++ b/tests/test_docs.py
@@ -2,7 +2,7 @@
 Tests to ensure certain things are documented.
 """
 from click.testing import CliRunner
-from datasette import app
+from datasette import app, utils
 from datasette.cli import cli
 from datasette.filters import Filters
 from pathlib import Path
@@ -86,3 +86,19 @@ def documented_table_filters():
 @pytest.mark.parametrize("filter", [f.key for f in Filters._filters])
 def test_table_filters_are_documented(documented_table_filters, filter):
     assert filter in documented_table_filters
+
+
+@pytest.fixture(scope="session")
+def documented_fns():
+    internals_rst = (docs_path / "internals.rst").read_text()
+    # Any line that starts .. _internals_utils_X
+    lines = internals_rst.split("\n")
+    prefix = ".. _internals_utils_"
+    return {
+        line.split(prefix)[1].split(":")[0] for line in lines if line.startswith(prefix)
+    }
+
+
+@pytest.mark.parametrize("fn", utils.functions_marked_as_documented)
+def test_functions_marked_with_documented_are_documented(documented_fns, fn):
+    assert fn.__name__ in documented_fns

From fdce6f29e19c3c6b477b72f86e187abee9627b92 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 6 Feb 2022 22:38:27 -0800
Subject: [PATCH 1330/2303] Reconfigure ReadTheDocs, refs #1176

---
 .readthedocs.yaml                 | 8 ++++++--
 docs/readthedocs-requirements.txt | 1 -
 2 files changed, 6 insertions(+), 3 deletions(-)
 delete mode 100644 docs/readthedocs-requirements.txt

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 70db5313..60b73b30 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -9,5 +9,9 @@ sphinx:
    configuration: docs/conf.py
 
 python:
-   install:
-   - requirements: docs/readthedocs-requirements.txt
+  version: "3.9"
+  install:
+  - method: pip
+    path: .
+    extra_requirements:
+    - docs
diff --git a/docs/readthedocs-requirements.txt b/docs/readthedocs-requirements.txt
deleted file mode 100644
index db1851ad..00000000
--- a/docs/readthedocs-requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-docutils<0.19

From 03305ea183b1534bc4cef3a721fe5f3700273b84 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 6 Feb 2022 22:40:47 -0800
Subject: [PATCH 1331/2303] Remove python.version, refs #1176

---
 .readthedocs.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 60b73b30..e157fb9c 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -9,7 +9,6 @@ sphinx:
    configuration: docs/conf.py
 
 python:
-  version: "3.9"
   install:
   - method: pip
     path: .

From 0cd982fc6af45b60e0c9306516dd412ae948c89b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Feb 2022 15:28:46 -0800
Subject: [PATCH 1332/2303] De-duplicate 'datasette db.db db.db', closes #1632

Refs https://github.com/simonw/datasette-publish-fly/pull/12
---
 datasette/cli.py  |  3 +++
 tests/test_cli.py | 12 ++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/datasette/cli.py b/datasette/cli.py
index 9d1b5ee5..61e7ce91 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -549,6 +549,9 @@ def serve(
                     )
                 )
 
+    # De-duplicate files so 'datasette db.db db.db' only attaches one /db
+    files = list(dict.fromkeys(files))
+
     try:
         ds = Datasette(files, **kwargs)
     except SpatialiteNotFound:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index bbc5df30..3fbfdee2 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -257,6 +257,7 @@ def test_serve_create(ensure_eventloop, tmpdir):
 
 
 def test_serve_duplicate_database_names(ensure_eventloop, tmpdir):
+    "'datasette db.db nested/db.db' should attach two databases, /db and /db_2"
     runner = CliRunner()
     db_1_path = str(tmpdir / "db.db")
     nested = tmpdir / "nested"
@@ -270,6 +271,17 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir):
     assert {db["name"] for db in databases} == {"db", "db_2"}
 
 
+def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir):
+    "'datasette db.db db.db' should only attach one database, /db"
+    runner = CliRunner()
+    db_path = str(tmpdir / "db.db")
+    sqlite3.connect(db_path).execute("vacuum")
+    result = runner.invoke(cli, [db_path, db_path, "--get", "/-/databases.json"])
+    assert result.exit_code == 0, result.output
+    databases = json.loads(result.output)
+    assert {db["name"] for db in databases} == {"db"}
+
+
 @pytest.mark.parametrize(
     "filename", ["test-database (1).sqlite", "database (1).sqlite"]
 )

From fa5fc327adbbf70656ac533912f3fc0526a3873d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Feb 2022 15:32:54 -0800
Subject: [PATCH 1333/2303] Release 0.60.2

Refs #1632
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index a4e340b3..91224615 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.60"
+__version__ = "0.60.2"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index d7e2af39..c58c8444 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,20 @@
 Changelog
 =========
 
+.. _v0_60.2:
+
+0.60.2 (2022-02-07)
+-------------------
+
+- Fixed a bug where Datasette would open the same file twice with two different database names if you ran ``datasette file.db file.db``. (:issue:`1632`)
+
+.. _v0_60.1:
+
+0.60.1 (2022-01-20)
+-------------------
+
+- Fixed a bug where installation on Python 3.6 stopped working due to a change to an underlying dependency. This release can now be installed on Python 3.6, but is the last release of Datasette that will support anything less than Python 3.7. (:issue:`1609`)
+
 .. _v0_60:
 
 0.60 (2022-01-13)

From 5bfd001b55357106dba090c83a1c88912a004665 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Feb 2022 15:42:37 -0800
Subject: [PATCH 1334/2303] Use de-dupe idiom that works with Python 3.6, refs
 #1632

---
 datasette/cli.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 61e7ce91..a8da0741 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -550,7 +550,9 @@ def serve(
                 )
 
     # De-duplicate files so 'datasette db.db db.db' only attaches one /db
-    files = list(dict.fromkeys(files))
+    files_seen = set()
+    deduped_files = [f for f in files if f not in files_seen and not files_seen.add(f)]
+    files = deduped_files
 
     try:
         ds = Datasette(files, **kwargs)

From 1b2f0ab6bbc9274dac1ba5fe126b1d6b8587ea96 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Feb 2022 15:43:20 -0800
Subject: [PATCH 1335/2303] Revert "Use de-dupe idiom that works with Python
 3.6, refs #1632"

This reverts commit 5bfd001b55357106dba090c83a1c88912a004665.

No need for this on the main branch because it doesn't support Python 3.6 any more.
---
 datasette/cli.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index a8da0741..61e7ce91 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -550,9 +550,7 @@ def serve(
                 )
 
     # De-duplicate files so 'datasette db.db db.db' only attaches one /db
-    files_seen = set()
-    deduped_files = [f for f in files if f not in files_seen and not files_seen.add(f)]
-    files = deduped_files
+    files = list(dict.fromkeys(files))
 
     try:
         ds = Datasette(files, **kwargs)

From 458f03ad3a454d271f47a643f4530bd8b60ddb76 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 8 Feb 2022 22:32:19 -0800
Subject: [PATCH 1336/2303] More SpatiaLite details on /-/versions, closes
 #1607

---
 datasette/app.py            | 12 ++++++++++++
 datasette/utils/__init__.py | 32 ++++++++++++++++++++++++++++++++
 tests/test_spatialite.py    | 21 +++++++++++++++++++++
 3 files changed, 65 insertions(+)
 create mode 100644 tests/test_spatialite.py

diff --git a/datasette/app.py b/datasette/app.py
index 7bdf076c..8c5480cf 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -46,6 +46,7 @@ from .database import Database, QueryInterrupted
 
 from .utils import (
     PrefixedUrlString,
+    SPATIALITE_FUNCTIONS,
     StartupError,
     add_cors_headers,
     async_call_with_supported_arguments,
@@ -724,6 +725,17 @@ class Datasette:
                     sqlite_extensions[extension] = None
             except Exception:
                 pass
+        # More details on SpatiaLite
+        if "spatialite" in sqlite_extensions:
+            spatialite_details = {}
+            for fn in SPATIALITE_FUNCTIONS:
+                try:
+                    result = conn.execute("select {}()".format(fn))
+                    spatialite_details[fn] = result.fetchone()[0]
+                except Exception as e:
+                    spatialite_details[fn] = {"error": str(e)}
+            sqlite_extensions["spatialite"] = spatialite_details
+
         # Figure out supported FTS versions
         fts_versions = []
         for fts in ("FTS5", "FTS4", "FTS3"):
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 610e916f..e17b4d7f 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -52,9 +52,41 @@ SPATIALITE_PATHS = (
     "/usr/local/lib/mod_spatialite.dylib",
     "/usr/local/lib/mod_spatialite.so",
 )
+# Used to display /-/versions.json SpatiaLite information
+SPATIALITE_FUNCTIONS = (
+    "spatialite_version",
+    "spatialite_target_cpu",
+    "check_strict_sql_quoting",
+    "freexl_version",
+    "proj_version",
+    "geos_version",
+    "rttopo_version",
+    "libxml2_version",
+    "HasIconv",
+    "HasMathSQL",
+    "HasGeoCallbacks",
+    "HasProj",
+    "HasProj6",
+    "HasGeos",
+    "HasGeosAdvanced",
+    "HasGeosTrunk",
+    "HasGeosReentrant",
+    "HasGeosOnlyReentrant",
+    "HasMiniZip",
+    "HasRtTopo",
+    "HasLibXML2",
+    "HasEpsg",
+    "HasFreeXL",
+    "HasGeoPackage",
+    "HasGCP",
+    "HasTopology",
+    "HasKNN",
+    "HasRouting",
+)
 # Length of hash subset used in hashed URLs:
 HASH_LENGTH = 7
 
+
 # Can replace this with Column from sqlite_utils when I add that dependency
 Column = namedtuple(
     "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden")
diff --git a/tests/test_spatialite.py b/tests/test_spatialite.py
new file mode 100644
index 00000000..8b98c5d6
--- /dev/null
+++ b/tests/test_spatialite.py
@@ -0,0 +1,21 @@
+from datasette.app import Datasette
+from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS
+import pytest
+
+
+def has_spatialite():
+    try:
+        find_spatialite()
+        return True
+    except SpatialiteNotFound:
+        return False
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_spatialite(), reason="Requires SpatiaLite")
+async def test_spatialite_version_info():
+    ds = Datasette(sqlite_extensions=["spatialite"])
+    response = await ds.client.get("/-/versions.json")
+    assert response.status_code == 200
+    spatialite = response.json()["sqlite"]["extensions"]["spatialite"]
+    assert set(SPATIALITE_FUNCTIONS) == set(spatialite)

From 7d24fd405f3c60e4c852c5d746c91aa2ba23cf5b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 9 Feb 2022 09:47:54 -0800
Subject: [PATCH 1337/2303] datasette-auth-passwords is now an example of
 register_commands

Refs https://github.com/simonw/datasette-auth-passwords/issues/19
---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index a63d441e..92cf662f 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -633,7 +633,7 @@ Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-d
 
     pip install -e path/to/my/datasette-plugin
 
-Example: `datasette-verify <https://datasette.io/plugins/datasette-verify>`_
+Examples: `datasette-auth-passwords <https://datasette.io/plugins/datasette-auth-passwords>`__, `datasette-verify <https://datasette.io/plugins/datasette-verify>`__
 
 .. _plugin_register_facet_classes:
 

From dd94157f8958bdfe9f45575add934ccf1aba6d63 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 27 Feb 2022 10:04:03 -0800
Subject: [PATCH 1338/2303] Link to tutorials from documentation index page

---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index acca943f..a2888822 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -25,7 +25,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 
 `Explore a demo <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__, watch `a presentation about the project <https://static.simonwillison.net/static/2018/pybay-datasette/>`__ or :ref:`getting_started_glitch`.
 
-More examples: https://datasette.io/examples
+Interested in learning Datasette? Start with `the official tutorials <https://datasette.io/tutorials>`__.
 
 Support questions, feedback? Join our `GitHub Discussions forum <https://github.com/simonw/datasette/discussions>`__.
 

From 5010d1359b9e9db90a5a69a3ca22d12862893e00 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 5 Mar 2022 11:45:04 -0800
Subject: [PATCH 1339/2303] Fix for test failure caused by SQLite 3.37.0+,
 closes #1647

---
 datasette/templates/_table.html  |  2 +-
 tests/test_internals_database.py | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html
index d91a1a57..5332f831 100644
--- a/datasette/templates/_table.html
+++ b/datasette/templates/_table.html
@@ -4,7 +4,7 @@
         <thead>
             <tr>
                 {% for column in display_columns %}
-                    <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}">
+                    <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type.lower() }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}">
                         {% if not column.sortable %}
                             {{ column.name }}
                         {% else %}
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index bcecb486..31538a24 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -279,7 +279,15 @@ async def test_table_columns(db, table, expected):
 @pytest.mark.asyncio
 async def test_table_column_details(db, table, expected):
     columns = await db.table_column_details(table)
-    assert columns == expected
+    # Convert "type" to lowercase before comparison
+    # https://github.com/simonw/datasette/issues/1647
+    compare_columns = [
+        Column(
+            c.cid, c.name, c.type.lower(), c.notnull, c.default_value, c.is_pk, c.hidden
+        )
+        for c in columns
+    ]
+    assert compare_columns == expected
 
 
 @pytest.mark.asyncio

From a22ec96c3ac555337eb49121450723a273fb52d1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 5 Mar 2022 17:29:53 -0800
Subject: [PATCH 1340/2303] Update pytest-asyncio requirement from <0.17,>=0.10
 to >=0.10,<0.19 (#1631)

Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.10.0...v0.18.0)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6accaa30..6a097e0f 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,7 @@ setup(
         "test": [
             "pytest>=5.2.2,<6.3.0",
             "pytest-xdist>=2.2.1,<2.6",
-            "pytest-asyncio>=0.10,<0.17",
+            "pytest-asyncio>=0.10,<0.19",
             "beautifulsoup4>=4.8.1,<4.11.0",
             "black==22.1.0",
             "pytest-timeout>=1.4.2,<2.1",

From b21839dd1a005f6269c4e9a9f763195fe7aa9c86 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 5 Mar 2022 17:30:05 -0800
Subject: [PATCH 1341/2303] Update pytest requirement from <6.3.0,>=5.2.2 to
 >=5.2.2,<7.1.0 (#1629)

Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...7.0.0)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6a097e0f..b9db0700 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@ setup(
     extras_require={
         "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
         "test": [
-            "pytest>=5.2.2,<6.3.0",
+            "pytest>=5.2.2,<7.1.0",
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.10,<0.19",
             "beautifulsoup4>=4.8.1,<4.11.0",

From 73f2d25f70d741c6b53f7312674c91f0aec83e17 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 5 Mar 2022 17:30:27 -0800
Subject: [PATCH 1342/2303] Update asgiref requirement from <3.5.0,>=3.2.10 to
 >=3.2.10,<3.6.0 (#1610)

Updates the requirements on [asgiref](https://github.com/django/asgiref) to permit the latest version.
- [Release notes](https://github.com/django/asgiref/releases)
- [Changelog](https://github.com/django/asgiref/blob/main/CHANGELOG.txt)
- [Commits](https://github.com/django/asgiref/compare/3.2.10...3.5.0)

---
updated-dependencies:
- dependency-name: asgiref
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b9db0700..b13f7496 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.7",
     install_requires=[
-        "asgiref>=3.2.10,<3.5.0",
+        "asgiref>=3.2.10,<3.6.0",
         "click>=7.1.1,<8.1.0",
         "click-default-group~=1.2.2",
         "Jinja2>=2.10.3,<3.1.0",

From 7b78279b93b6e7a5fce6b53e5a85ca421a801496 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 5 Mar 2022 17:41:49 -0800
Subject: [PATCH 1343/2303] Update pytest-timeout requirement from <2.1,>=1.4.2
 to >=1.4.2,<2.2 (#1602)

Updates the requirements on [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-timeout/releases)
- [Commits](https://github.com/pytest-dev/pytest-timeout/compare/1.4.2...2.1.0)

---
updated-dependencies:
- dependency-name: pytest-timeout
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b13f7496..8e69c2f5 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,7 @@ setup(
             "pytest-asyncio>=0.10,<0.19",
             "beautifulsoup4>=4.8.1,<4.11.0",
             "black==22.1.0",
-            "pytest-timeout>=1.4.2,<2.1",
+            "pytest-timeout>=1.4.2,<2.2",
             "trustme>=0.7,<0.10",
             "cogapp>=3.3.0",
         ],

From 0499f174c063283aa9b589d475a32077aaf7adc5 Mon Sep 17 00:00:00 2001
From: David Larlet <3556+davidbgk@users.noreply.github.com>
Date: Sat, 5 Mar 2022 20:58:31 -0500
Subject: [PATCH 1344/2303] Typo in docs about default redirect status code
 (#1589)

---
 docs/custom_templates.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst
index 3e4eb633..97dea2af 100644
--- a/docs/custom_templates.rst
+++ b/docs/custom_templates.rst
@@ -428,7 +428,7 @@ You can use the ``custom_redirect(location)`` function to redirect users to anot
 
 Now requests to ``http://localhost:8001/datasette`` will result in a redirect.
 
-These redirects are served with a ``301 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function:
+These redirects are served with a ``302 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function:
 
 .. code-block:: jinja
 

From de810f49cc57a4f88e4a1553d26c579253ce4531 Mon Sep 17 00:00:00 2001
From: Dan Peterson <danp@danp.net>
Date: Sun, 6 Mar 2022 15:39:15 -0400
Subject: [PATCH 1345/2303] Add /opt/homebrew to where spatialite extension can
 be found (#1649)

Helps homebrew on Apple Silicon setups find spatialite without needing
a full path.

Similar to #1114

Thanks, @danp
---
 datasette/utils/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index e17b4d7f..133b9bc7 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -51,6 +51,7 @@ SPATIALITE_PATHS = (
     "/usr/lib/x86_64-linux-gnu/mod_spatialite.so",
     "/usr/local/lib/mod_spatialite.dylib",
     "/usr/local/lib/mod_spatialite.so",
+    "/opt/homebrew/lib/mod_spatialite.dylib",
 )
 # Used to display /-/versions.json SpatiaLite information
 SPATIALITE_FUNCTIONS = (

From 1baa030eca375f839f3471237547ab403523e643 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 07:38:29 -0800
Subject: [PATCH 1346/2303] Switch to dash encoding for table/database/row-pk
 in paths

* Dash encoding functions, tests and docs, refs #1439
* dash encoding is now like percent encoding but with dashes
* Use dash-encoding for row PKs and ?_next=, refs #1439
* Use dash encoding for table names, refs #1439
* Use dash encoding for database names, too, refs #1439

See also https://simonwillison.net/2022/Mar/5/dash-encoding/
---
 datasette/url_builder.py     | 10 ++++----
 datasette/utils/__init__.py  | 41 ++++++++++++++++++++++++++---
 datasette/views/base.py      | 24 ++++++++---------
 datasette/views/table.py     |  9 ++++---
 docs/internals.rst           | 26 +++++++++++++++++++
 tests/fixtures.py            |  1 +
 tests/test_api.py            | 19 +++++++++++---
 tests/test_cli.py            |  5 ++--
 tests/test_html.py           | 50 ++++++++++++++++++++++++------------
 tests/test_internals_urls.py |  2 +-
 tests/test_table_api.py      |  7 +++--
 tests/test_table_html.py     | 12 ++++++---
 tests/test_utils.py          | 20 ++++++++++++++-
 13 files changed, 173 insertions(+), 53 deletions(-)

diff --git a/datasette/url_builder.py b/datasette/url_builder.py
index 2bcda869..eebfe31e 100644
--- a/datasette/url_builder.py
+++ b/datasette/url_builder.py
@@ -1,4 +1,4 @@
-from .utils import path_with_format, HASH_LENGTH, PrefixedUrlString
+from .utils import dash_encode, path_with_format, HASH_LENGTH, PrefixedUrlString
 import urllib
 
 
@@ -31,20 +31,20 @@ class Urls:
         db = self.ds.databases[database]
         if self.ds.setting("hash_urls") and db.hash:
             path = self.path(
-                f"{urllib.parse.quote(database)}-{db.hash[:HASH_LENGTH]}", format=format
+                f"{dash_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format
             )
         else:
-            path = self.path(urllib.parse.quote(database), format=format)
+            path = self.path(dash_encode(database), format=format)
         return path
 
     def table(self, database, table, format=None):
-        path = f"{self.database(database)}/{urllib.parse.quote_plus(table)}"
+        path = f"{self.database(database)}/{dash_encode(table)}"
         if format is not None:
             path = path_with_format(path=path, format=format)
         return PrefixedUrlString(path)
 
     def query(self, database, query, format=None):
-        path = f"{self.database(database)}/{urllib.parse.quote_plus(query)}"
+        path = f"{self.database(database)}/{dash_encode(query)}"
         if format is not None:
             path = path_with_format(path=path, format=format)
         return PrefixedUrlString(path)
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 133b9bc7..79feeef6 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -112,12 +112,12 @@ async def await_me_maybe(value: typing.Any) -> typing.Any:
 
 
 def urlsafe_components(token):
-    """Splits token on commas and URL decodes each component"""
-    return [urllib.parse.unquote_plus(b) for b in token.split(",")]
+    """Splits token on commas and dash-decodes each component"""
+    return [dash_decode(b) for b in token.split(",")]
 
 
 def path_from_row_pks(row, pks, use_rowid, quote=True):
-    """Generate an optionally URL-quoted unique identifier
+    """Generate an optionally dash-quoted unique identifier
     for a row from its primary keys."""
     if use_rowid:
         bits = [row["rowid"]]
@@ -126,7 +126,7 @@ def path_from_row_pks(row, pks, use_rowid, quote=True):
             row[pk]["value"] if isinstance(row[pk], dict) else row[pk] for pk in pks
         ]
     if quote:
-        bits = [urllib.parse.quote_plus(str(bit)) for bit in bits]
+        bits = [dash_encode(str(bit)) for bit in bits]
     else:
         bits = [str(bit) for bit in bits]
 
@@ -1140,3 +1140,36 @@ def add_cors_headers(headers):
     headers["Access-Control-Allow-Origin"] = "*"
     headers["Access-Control-Allow-Headers"] = "Authorization"
     headers["Access-Control-Expose-Headers"] = "Link"
+
+
+_DASH_ENCODING_SAFE = frozenset(
+    b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    b"abcdefghijklmnopqrstuvwxyz"
+    b"0123456789_"
+    # This is the same as Python percent-encoding but I removed
+    # '.' and '-' and '~'
+)
+
+
+class DashEncoder(dict):
+    # Keeps a cache internally, via __missing__
+    def __missing__(self, b):
+        # Handle a cache miss, store encoded string in cache and return.
+        res = chr(b) if b in _DASH_ENCODING_SAFE else "-{:02X}".format(b)
+        self[b] = res
+        return res
+
+
+_dash_encoder = DashEncoder().__getitem__
+
+
+@documented
+def dash_encode(s: str) -> str:
+    "Returns dash-encoded string - for example ``/foo/bar`` -> ``-2Ffoo-2Fbar``"
+    return "".join(_dash_encoder(char) for char in s.encode("utf-8"))
+
+
+@documented
+def dash_decode(s: str) -> str:
+    "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``"
+    return urllib.parse.unquote(s.replace("-", "%"))
diff --git a/datasette/views/base.py b/datasette/views/base.py
index c74d6141..7cd385b7 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -17,6 +17,8 @@ from datasette.utils import (
     InvalidSql,
     LimitedWriter,
     call_with_supported_arguments,
+    dash_decode,
+    dash_encode,
     path_from_row_pks,
     path_with_added_args,
     path_with_removed_args,
@@ -203,17 +205,17 @@ class DataView(BaseView):
     async def resolve_db_name(self, request, db_name, **kwargs):
         hash = None
         name = None
-        db_name = urllib.parse.unquote_plus(db_name)
-        if db_name not in self.ds.databases and "-" in db_name:
+        decoded_name = dash_decode(db_name)
+        if decoded_name not in self.ds.databases and "-" in db_name:
             # No matching DB found, maybe it's a name-hash?
             name_bit, hash_bit = db_name.rsplit("-", 1)
-            if name_bit not in self.ds.databases:
+            if dash_decode(name_bit) not in self.ds.databases:
                 raise NotFound(f"Database not found: {name}")
             else:
-                name = name_bit
+                name = dash_decode(name_bit)
                 hash = hash_bit
         else:
-            name = db_name
+            name = decoded_name
 
         try:
             db = self.ds.databases[name]
@@ -233,9 +235,7 @@ class DataView(BaseView):
                     return await db.table_exists(t)
 
                 table, _format = await resolve_table_and_format(
-                    table_and_format=urllib.parse.unquote_plus(
-                        kwargs["table_and_format"]
-                    ),
+                    table_and_format=dash_decode(kwargs["table_and_format"]),
                     table_exists=async_table_exists,
                     allowed_formats=self.ds.renderers.keys(),
                 )
@@ -243,11 +243,11 @@ class DataView(BaseView):
                 if _format:
                     kwargs["as_format"] = f".{_format}"
             elif kwargs.get("table"):
-                kwargs["table"] = urllib.parse.unquote_plus(kwargs["table"])
+                kwargs["table"] = dash_decode(kwargs["table"])
 
             should_redirect = self.ds.urls.path(f"{name}-{expected}")
             if kwargs.get("table"):
-                should_redirect += "/" + urllib.parse.quote_plus(kwargs["table"])
+                should_redirect += "/" + dash_encode(kwargs["table"])
             if kwargs.get("pk_path"):
                 should_redirect += "/" + kwargs["pk_path"]
             if kwargs.get("as_format"):
@@ -467,7 +467,7 @@ class DataView(BaseView):
                 return await db.table_exists(t)
 
             table, _ext_format = await resolve_table_and_format(
-                table_and_format=urllib.parse.unquote_plus(args["table_and_format"]),
+                table_and_format=dash_decode(args["table_and_format"]),
                 table_exists=async_table_exists,
                 allowed_formats=self.ds.renderers.keys(),
             )
@@ -475,7 +475,7 @@ class DataView(BaseView):
             args["table"] = table
             del args["table_and_format"]
         elif "table" in args:
-            args["table"] = urllib.parse.unquote_plus(args["table"])
+            args["table"] = dash_decode(args["table"])
         return _format, args
 
     async def view_get(self, request, database, hash, correct_hash_provided, **kwargs):
diff --git a/datasette/views/table.py b/datasette/views/table.py
index be9e9c3b..1d81755e 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -12,6 +12,7 @@ from datasette.utils import (
     MultiParams,
     append_querystring,
     compound_keys_after_sql,
+    dash_encode,
     escape_sqlite,
     filters_should_redirect,
     is_url,
@@ -142,7 +143,7 @@ class RowTableShared(DataView):
                             '<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format(
                                 base_url=base_url,
                                 database=database,
-                                table=urllib.parse.quote_plus(table),
+                                table=dash_encode(table),
                                 flat_pks=str(markupsafe.escape(pk_path)),
                                 flat_pks_quoted=path_from_row_pks(row, pks, not pks),
                             )
@@ -199,8 +200,8 @@ class RowTableShared(DataView):
                         link_template.format(
                             database=database,
                             base_url=base_url,
-                            table=urllib.parse.quote_plus(other_table),
-                            link_id=urllib.parse.quote_plus(str(value)),
+                            table=dash_encode(other_table),
+                            link_id=dash_encode(str(value)),
                             id=str(markupsafe.escape(value)),
                             label=str(markupsafe.escape(label)) or "-",
                         )
@@ -765,7 +766,7 @@ class TableView(RowTableShared):
                 if prefix is None:
                     prefix = "$null"
                 else:
-                    prefix = urllib.parse.quote_plus(str(prefix))
+                    prefix = dash_encode(str(prefix))
                 next_value = f"{prefix},{next_value}"
                 added_args = {"_next": next_value}
                 if sort:
diff --git a/docs/internals.rst b/docs/internals.rst
index 12ef5c54..d035e1f1 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -876,6 +876,32 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth
 
 .. autofunction:: datasette.utils.await_me_maybe
 
+.. _internals_dash_encoding:
+
+Dash encoding
+-------------
+
+Datasette uses a custom encoding scheme in some places, called **dash encoding**. This is primarily used for table names and row primary keys, to avoid any confusion between ``/`` characters in those values and the Datasette URLs that reference them.
+
+Dash encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``-`` hyphen character used in place of ``%``.
+
+Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_`` will be replaced by the numeric equivalent preceded by a hyphen. For example:
+
+- ``/`` becomes ``-2F``
+- ``.`` becomes ``-2E``
+- ``%`` becomes ``-25``
+- ``-`` becomes ``-2D``
+- Space character becomes ``-20``
+- ``polls/2022.primary`` becomes ``polls-2F2022-2Eprimary``
+
+.. _internals_utils_dash_encode:
+
+.. autofunction:: datasette.utils.dash_encode
+
+.. _internals_utils_dash_decode:
+
+.. autofunction:: datasette.utils.dash_decode
+
 .. _internals_tracer:
 
 datasette.tracer
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 26f0cf7b..11f09c41 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -406,6 +406,7 @@ CREATE TABLE compound_primary_key (
 );
 
 INSERT INTO compound_primary_key VALUES ('a', 'b', 'c');
+INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c');
 
 CREATE TABLE compound_three_primary_keys (
   pk1 varchar(30),
diff --git a/tests/test_api.py b/tests/test_api.py
index 57471af2..dd916cf0 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -143,7 +143,7 @@ def test_database_page(app_client):
             "name": "compound_primary_key",
             "columns": ["pk1", "pk2", "content"],
             "primary_keys": ["pk1", "pk2"],
-            "count": 1,
+            "count": 2,
             "hidden": False,
             "fts_table": None,
             "foreign_keys": {"incoming": [], "outgoing": []},
@@ -942,7 +942,7 @@ def test_cors(app_client_with_cors, path, status_code):
 )
 def test_database_with_space_in_name(app_client_two_attached_databases, path):
     response = app_client_two_attached_databases.get(
-        "/extra database" + path, follow_redirects=True
+        "/extra-20database" + path, follow_redirects=True
     )
     assert response.status == 200
 
@@ -953,7 +953,7 @@ def test_common_prefix_database_names(app_client_conflicting_database_names):
         d["name"]
         for d in app_client_conflicting_database_names.get("/-/databases.json").json
     ]
-    for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")):
+    for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-2Dbar.json")):
         data = app_client_conflicting_database_names.get(path).json
         assert db_name == data["database"]
 
@@ -992,3 +992,16 @@ async def test_hidden_sqlite_stat1_table():
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
     assert tables == [("normal", False), ("sqlite_stat1", True)]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("db_name", ("foo", r"fo%o", "f~/c.d"))
+async def test_dash_encoded_database_names(db_name):
+    ds = Datasette()
+    ds.add_memory_database(db_name)
+    response = await ds.client.get("/.json")
+    assert db_name in response.json().keys()
+    path = response.json()[db_name]["path"]
+    # And the JSON for that database
+    response2 = await ds.client.get(path + ".json")
+    assert response2.status_code == 200
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 3fbfdee2..e30c2ad3 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -9,6 +9,7 @@ from datasette.app import SETTINGS
 from datasette.plugins import DEFAULT_PLUGINS
 from datasette.cli import cli, serve
 from datasette.version import __version__
+from datasette.utils import dash_encode
 from datasette.utils.sqlite import sqlite3
 from click.testing import CliRunner
 import io
@@ -294,12 +295,12 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename):
     assert result1.exit_code == 0, result1.output
     filename_no_stem = filename.rsplit(".", 1)[0]
     expected_link = '<a href="/{}">{}</a>'.format(
-        urllib.parse.quote(filename_no_stem), filename_no_stem
+        dash_encode(filename_no_stem), filename_no_stem
     )
     assert expected_link in result1.output
     # Now try hitting that database page
     result2 = runner.invoke(
-        cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))]
+        cli, [db_path, "--get", "/{}".format(dash_encode(filename_no_stem))]
     )
     assert result2.exit_code == 0, result2.output
 
diff --git a/tests/test_html.py b/tests/test_html.py
index d5f4250d..b4a12b8a 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -29,7 +29,7 @@ def test_homepage(app_client_two_attached_databases):
     )
     # Should be two attached databases
     assert [
-        {"href": r"/extra%20database", "text": "extra database"},
+        {"href": r"/extra-20database", "text": "extra database"},
         {"href": "/fixtures", "text": "fixtures"},
     ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")]
     # Database should show count text and attached tables
@@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases):
         {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a")
     ]
     assert [
-        {"href": r"/extra%20database/searchable", "text": "searchable"},
-        {"href": r"/extra%20database/searchable_view", "text": "searchable_view"},
+        {"href": r"/extra-20database/searchable", "text": "searchable"},
+        {"href": r"/extra-20database/searchable_view", "text": "searchable_view"},
     ] == table_links
 
 
@@ -140,7 +140,7 @@ def test_database_page(app_client):
     assert queries_ul is not None
     assert [
         (
-            "/fixtures/%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC",
+            "/fixtures/-F0-9D-90-9C-F0-9D-90-A2-F0-9D-90-AD-F0-9D-90-A2-F0-9D-90-9E-F0-9D-90-AC",
             "𝐜𝐢𝐭𝐢𝐞𝐬",
         ),
         ("/fixtures/from_async_hook", "from_async_hook"),
@@ -193,11 +193,11 @@ def test_row_redirects_with_url_hash(app_client_with_hash):
 
 
 def test_row_strange_table_name_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures/table%2Fwith%2Fslashes.csv/3")
+    response = app_client_with_hash.get("/fixtures/table-2Fwith-2Fslashes-2Ecsv/3")
     assert response.status == 302
-    assert response.headers["Location"].endswith("/table%2Fwith%2Fslashes.csv/3")
+    assert response.headers["Location"].endswith("/table-2Fwith-2Fslashes-2Ecsv/3")
     response = app_client_with_hash.get(
-        "/fixtures/table%2Fwith%2Fslashes.csv/3", follow_redirects=True
+        "/fixtures/table-2Fwith-2Fslashes-2Ecsv/3", follow_redirects=True
     )
     assert response.status == 200
 
@@ -345,20 +345,38 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l
     assert link == expected_link
 
 
-def test_row_html_compound_primary_key(app_client):
-    response = app_client.get("/fixtures/compound_primary_key/a,b")
+@pytest.mark.parametrize(
+    "path,expected",
+    (
+        (
+            "/fixtures/compound_primary_key/a,b",
+            [
+                [
+                    '<td class="col-pk1 type-str">a</td>',
+                    '<td class="col-pk2 type-str">b</td>',
+                    '<td class="col-content type-str">c</td>',
+                ]
+            ],
+        ),
+        (
+            "/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd",
+            [
+                [
+                    '<td class="col-pk1 type-str">a/b</td>',
+                    '<td class="col-pk2 type-str">.c-d</td>',
+                    '<td class="col-content type-str">c</td>',
+                ]
+            ],
+        ),
+    ),
+)
+def test_row_html_compound_primary_key(app_client, path, expected):
+    response = app_client.get(path)
     assert response.status == 200
     table = Soup(response.body, "html.parser").find("table")
     assert ["pk1", "pk2", "content"] == [
         th.string.strip() for th in table.select("thead th")
     ]
-    expected = [
-        [
-            '<td class="col-pk1 type-str">a</td>',
-            '<td class="col-pk2 type-str">b</td>',
-            '<td class="col-content type-str">c</td>',
-        ]
-    ]
     assert expected == [
         [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
     ]
diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py
index e486e4c9..16515ad6 100644
--- a/tests/test_internals_urls.py
+++ b/tests/test_internals_urls.py
@@ -121,7 +121,7 @@ def test_database(ds, base_url, format, expected):
         ("/", "name", None, "/_memory/name"),
         ("/prefix/", "name", None, "/prefix/_memory/name"),
         ("/", "name", "json", "/_memory/name.json"),
-        ("/", "name.json", "json", "/_memory/name.json?_format=json"),
+        ("/", "name.json", "json", "/_memory/name-2Ejson.json"),
     ],
 )
 def test_table_and_query(ds, base_url, name, format, expected):
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 6a6daed5..cc38d392 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -136,7 +136,10 @@ def test_table_shape_object(app_client):
 
 def test_table_shape_object_compound_primary_key(app_client):
     response = app_client.get("/fixtures/compound_primary_key.json?_shape=object")
-    assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json
+    assert response.json == {
+        "a,b": {"pk1": "a", "pk2": "b", "content": "c"},
+        "a-2Fb,-2Ec-2Dd": {"pk1": "a/b", "pk2": ".c-d", "content": "c"},
+    }
 
 
 def test_table_with_slashes_in_name(app_client):
@@ -308,7 +311,7 @@ def test_sortable(app_client, query_string, sort_key, human_description_en):
         path = response.json["next_url"]
         if path:
             path = path.replace("http://localhost", "")
-    assert 5 == page
+    assert page == 5
     expected = list(generate_sortable_rows(201))
     expected.sort(key=sort_key)
     assert [r["content"] for r in expected] == [r["content"] for r in fetched]
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 021268c3..77d97d80 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -563,11 +563,17 @@ def test_table_html_compound_primary_key(app_client):
             '<td class="col-pk1 type-str">a</td>',
             '<td class="col-pk2 type-str">b</td>',
             '<td class="col-content type-str">c</td>',
-        ]
+        ],
+        [
+            '<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd">a/b,.c-d</a></td>',
+            '<td class="col-pk1 type-str">a/b</td>',
+            '<td class="col-pk2 type-str">.c-d</td>',
+            '<td class="col-content type-str">c</td>',
+        ],
     ]
-    assert expected == [
+    assert [
         [str(td) for td in tr.select("td")] for tr in table.select("tbody tr")
-    ]
+    ] == expected
 
 
 def test_table_html_foreign_key_links(app_client):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e7d67045..1c3ab495 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -93,7 +93,7 @@ def test_path_with_replaced_args(path, args, expected):
     "row,pks,expected_path",
     [
         ({"A": "foo", "B": "bar"}, ["A", "B"], "foo,bar"),
-        ({"A": "f,o", "B": "bar"}, ["A", "B"], "f%2Co,bar"),
+        ({"A": "f,o", "B": "bar"}, ["A", "B"], "f-2Co,bar"),
         ({"A": 123}, ["A"], "123"),
         (
             utils.CustomRow(
@@ -646,3 +646,21 @@ async def test_derive_named_parameters(sql, expected):
     db = ds.get_database("_memory")
     params = await utils.derive_named_parameters(db, sql)
     assert params == expected
+
+
+@pytest.mark.parametrize(
+    "original,expected",
+    (
+        ("abc", "abc"),
+        ("/foo/bar", "-2Ffoo-2Fbar"),
+        ("/-/bar", "-2F-2D-2Fbar"),
+        ("-/db-/table.csv", "-2D-2Fdb-2D-2Ftable-2Ecsv"),
+        (r"%~-/", "-25-7E-2D-2F"),
+        ("-25-7E-2D-2F", "-2D25-2D7E-2D2D-2D2F"),
+    ),
+)
+def test_dash_encoding(original, expected):
+    actual = utils.dash_encode(original)
+    assert actual == expected
+    # And test round-trip
+    assert original == utils.dash_decode(actual)

From 644d25d1de78a36b105cca479e7b3e4375a6eadc Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 08:01:03 -0800
Subject: [PATCH 1347/2303] Redirect old % URLs to new - encoded URLs, closes
 #1650

Refs #1439
---
 datasette/app.py   | 7 +++++++
 tests/test_html.py | 6 ++++++
 2 files changed, 13 insertions(+)

diff --git a/datasette/app.py b/datasette/app.py
index 8c5480cf..2907d90e 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1211,6 +1211,13 @@ class DatasetteRouter:
         return await self.handle_404(request, send)
 
     async def handle_404(self, request, send, exception=None):
+        # If path contains % encoding, redirect to dash encoding
+        if "%" in request.path:
+            # Try the same path but with "%" replaced by "-"
+            # and "-" replaced with "-2D"
+            new_path = request.path.replace("-", "-2D").replace("%", "-")
+            await asgi_send_redirect(send, new_path)
+            return
         # If URL has a trailing slash, redirect to URL without it
         path = request.scope.get(
             "raw_path", request.scope["path"].encode("utf8")
diff --git a/tests/test_html.py b/tests/test_html.py
index b4a12b8a..3e24009e 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -952,3 +952,9 @@ def test_no_alternate_url_json(app_client, path):
     assert (
         '<link rel="alternate" type="application/json+datasette"' not in response.text
     )
+
+
+def test_redirect_percent_encoding_to_dash_encoding(app_client):
+    response = app_client.get("/fivethirtyeight/twitter-ratio%2Fsenators")
+    assert response.status == 302
+    assert response.headers["location"] == "/fivethirtyeight/twitter-2Dratio-2Fsenators"

From d714c67d656c46e012b24ccca53b59409440334f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 08:09:15 -0800
Subject: [PATCH 1348/2303] asyncio_mode = strict to avoid pytest warnings

---
 pytest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytest.ini b/pytest.ini
index d702ce5f..559e518c 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,3 +8,4 @@ filterwarnings=
     ignore:.*current_task.*:PendingDeprecationWarning
 markers =
     serial: tests to avoid using with pytest-xdist
+asyncio_mode = strict

From 020effe47bf89f35182960a9645f2383a42ebd54 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 08:18:07 -0800
Subject: [PATCH 1349/2303] Preserve query string in % to - redirects, refs
 #1650

---
 datasette/app.py   |  2 ++
 tests/test_html.py | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 2907d90e..7abccc05 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1216,6 +1216,8 @@ class DatasetteRouter:
             # Try the same path but with "%" replaced by "-"
             # and "-" replaced with "-2D"
             new_path = request.path.replace("-", "-2D").replace("%", "-")
+            if request.query_string:
+                new_path += "?{}".format(request.query_string)
             await asgi_send_redirect(send, new_path)
             return
         # If URL has a trailing slash, redirect to URL without it
diff --git a/tests/test_html.py b/tests/test_html.py
index 3e24009e..de703284 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -954,7 +954,18 @@ def test_no_alternate_url_json(app_client, path):
     )
 
 
-def test_redirect_percent_encoding_to_dash_encoding(app_client):
-    response = app_client.get("/fivethirtyeight/twitter-ratio%2Fsenators")
+@pytest.mark.parametrize(
+    "path,expected",
+    (
+        (
+            "/fivethirtyeight/twitter-ratio%2Fsenators",
+            "/fivethirtyeight/twitter-2Dratio-2Fsenators",
+        ),
+        # query string should be preserved
+        ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"),
+    ),
+)
+def test_redirect_percent_encoding_to_dash_encoding(app_client, path, expected):
+    response = app_client.get(path)
     assert response.status == 302
-    assert response.headers["location"] == "/fivethirtyeight/twitter-2Dratio-2Fsenators"
+    assert response.headers["location"] == expected

From c85d669de387b40e667fd6942c6cc1c15b4f5964 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 11:26:08 -0800
Subject: [PATCH 1350/2303] Fix bug with percentage redirects, close #1650

---
 datasette/utils/__init__.py | 7 ++++++-
 tests/test_html.py          | 4 ++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 79feeef6..e7c9fb1c 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -10,6 +10,7 @@ import markupsafe
 import mergedeep
 import os
 import re
+import secrets
 import shlex
 import tempfile
 import typing
@@ -1172,4 +1173,8 @@ def dash_encode(s: str) -> str:
 @documented
 def dash_decode(s: str) -> str:
     "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``"
-    return urllib.parse.unquote(s.replace("-", "%"))
+    # Avoid accidentally decoding a %2f style sequence
+    temp = secrets.token_hex(16)
+    s = s.replace("%", temp)
+    decoded = urllib.parse.unquote(s.replace("-", "%"))
+    return decoded.replace(temp, "%")
diff --git a/tests/test_html.py b/tests/test_html.py
index de703284..55d78c05 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -961,6 +961,10 @@ def test_no_alternate_url_json(app_client, path):
             "/fivethirtyeight/twitter-ratio%2Fsenators",
             "/fivethirtyeight/twitter-2Dratio-2Fsenators",
         ),
+        (
+            "/fixtures/table%2Fwith%2Fslashes",
+            "/fixtures/table-2Fwith-2Fslashes",
+        ),
         # query string should be preserved
         ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"),
     ),

From bb499942c15c4e2cfa4b6afab8f8debe5948c009 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 11:33:31 -0800
Subject: [PATCH 1351/2303] Fixed tests for urlsafe_components, refs #1650

---
 tests/test_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1c3ab495..ff4f649a 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -19,8 +19,8 @@ from unittest.mock import patch
         ("foo", ["foo"]),
         ("foo,bar", ["foo", "bar"]),
         ("123,433,112", ["123", "433", "112"]),
-        ("123%2C433,112", ["123,433", "112"]),
-        ("123%2F433%2F112", ["123/433/112"]),
+        ("123-2C433,112", ["123,433", "112"]),
+        ("123-2F433-2F112", ["123/433/112"]),
     ],
 )
 def test_urlsafe_components(path, expected):

From c5791156d92615f25696ba93dae5bb2dcc192c98 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 14:04:10 -0800
Subject: [PATCH 1352/2303] Code of conduct, refs #1654

---
 CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 CODE_OF_CONDUCT.md

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 00000000..14d4c567
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+`swillison+datasette-code-of-conduct@gmail.com`.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

From 239aed182053903ed69108776b6864d42bfe1eb4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 15 Mar 2022 08:36:35 -0700
Subject: [PATCH 1353/2303] Revert "Code of conduct, refs #1654"

This reverts commit c5791156d92615f25696ba93dae5bb2dcc192c98.

Refs #1658
---
 CODE_OF_CONDUCT.md | 128 ---------------------------------------------
 1 file changed, 128 deletions(-)
 delete mode 100644 CODE_OF_CONDUCT.md

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
deleted file mode 100644
index 14d4c567..00000000
--- a/CODE_OF_CONDUCT.md
+++ /dev/null
@@ -1,128 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, religion, or sexual identity
-and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
-  overall community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or
-  advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email
-  address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-`swillison+datasette-code-of-conduct@gmail.com`.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
-of actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or
-permanent ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
-the community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.0, available at
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
-
-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see the FAQ at
-https://www.contributor-covenant.org/faq. Translations are available at
-https://www.contributor-covenant.org/translations.

From 5a353a32b9c4d75acbe3193fd72f735a8e78516a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 15 Mar 2022 08:37:14 -0700
Subject: [PATCH 1354/2303] Revert "Fixed tests for urlsafe_components, refs
 #1650"

This reverts commit bb499942c15c4e2cfa4b6afab8f8debe5948c009.

Refs #1658
---
 tests/test_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils.py b/tests/test_utils.py
index ff4f649a..1c3ab495 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -19,8 +19,8 @@ from unittest.mock import patch
         ("foo", ["foo"]),
         ("foo,bar", ["foo", "bar"]),
         ("123,433,112", ["123", "433", "112"]),
-        ("123-2C433,112", ["123,433", "112"]),
-        ("123-2F433-2F112", ["123/433/112"]),
+        ("123%2C433,112", ["123,433", "112"]),
+        ("123%2F433%2F112", ["123/433/112"]),
     ],
 )
 def test_urlsafe_components(path, expected):

From 77e718c3ffb30473759a8b1ed347f73cb2ff5cfe Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 15 Mar 2022 08:37:31 -0700
Subject: [PATCH 1355/2303] Revert "Fix bug with percentage redirects, close
 #1650"

This reverts commit c85d669de387b40e667fd6942c6cc1c15b4f5964.

Refs #1658
---
 datasette/utils/__init__.py | 7 +------
 tests/test_html.py          | 4 ----
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index e7c9fb1c..79feeef6 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -10,7 +10,6 @@ import markupsafe
 import mergedeep
 import os
 import re
-import secrets
 import shlex
 import tempfile
 import typing
@@ -1173,8 +1172,4 @@ def dash_encode(s: str) -> str:
 @documented
 def dash_decode(s: str) -> str:
     "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``"
-    # Avoid accidentally decoding a %2f style sequence
-    temp = secrets.token_hex(16)
-    s = s.replace("%", temp)
-    decoded = urllib.parse.unquote(s.replace("-", "%"))
-    return decoded.replace(temp, "%")
+    return urllib.parse.unquote(s.replace("-", "%"))
diff --git a/tests/test_html.py b/tests/test_html.py
index 55d78c05..de703284 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -961,10 +961,6 @@ def test_no_alternate_url_json(app_client, path):
             "/fivethirtyeight/twitter-ratio%2Fsenators",
             "/fivethirtyeight/twitter-2Dratio-2Fsenators",
         ),
-        (
-            "/fixtures/table%2Fwith%2Fslashes",
-            "/fixtures/table-2Fwith-2Fslashes",
-        ),
         # query string should be preserved
         ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"),
     ),

From 645381a5ed23c016281e8c6c7d141518f91b67e5 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 15 Mar 2022 08:36:35 -0700
Subject: [PATCH 1356/2303] Add code of conduct again

Refs #1658
---
 CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 CODE_OF_CONDUCT.md

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 00000000..14d4c567
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+`swillison+datasette-code-of-conduct@gmail.com`.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

From c10cd48baf106659bf3f129ad7bfb2226be73821 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 7 Mar 2022 11:56:59 -0800
Subject: [PATCH 1357/2303] Min pytest-asyncio of 0.17

So that the asyncio_mode in pytest.ini does not produce
a warning on older versions of that library.
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 8e69c2f5..e70839d6 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,7 @@ setup(
         "test": [
             "pytest>=5.2.2,<7.1.0",
             "pytest-xdist>=2.2.1,<2.6",
-            "pytest-asyncio>=0.10,<0.19",
+            "pytest-asyncio>=0.17,<0.19",
             "beautifulsoup4>=4.8.1,<4.11.0",
             "black==22.1.0",
             "pytest-timeout>=1.4.2,<2.2",

From a35393b29cfb5b8abdc6a94e577af1c9a5c13652 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 15 Mar 2022 11:01:57 -0700
Subject: [PATCH 1358/2303] Tilde encoding (#1659)

Closes #1657

Refs #1439
---
 datasette/app.py             | 11 +++++----
 datasette/url_builder.py     | 10 ++++----
 datasette/utils/__init__.py  | 37 ++++++++++++++++-------------
 datasette/views/base.py      | 25 +++++++++++---------
 datasette/views/table.py     | 14 +++++++----
 docs/csv_export.rst          | 18 ---------------
 docs/internals.rst           | 34 +++++++++++++--------------
 tests/test_api.py            | 17 ++++----------
 tests/test_cli.py            |  6 ++---
 tests/test_html.py           | 45 +++++++++++++++++++++---------------
 tests/test_internals_urls.py |  2 +-
 tests/test_table_api.py      |  9 +++++---
 tests/test_table_html.py     |  2 +-
 tests/test_utils.py          | 36 +++++++++--------------------
 14 files changed, 125 insertions(+), 141 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 7abccc05..b39ef7cd 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1211,11 +1211,14 @@ class DatasetteRouter:
         return await self.handle_404(request, send)
 
     async def handle_404(self, request, send, exception=None):
-        # If path contains % encoding, redirect to dash encoding
+        # If path contains % encoding, redirect to tilde encoding
         if "%" in request.path:
-            # Try the same path but with "%" replaced by "-"
-            # and "-" replaced with "-2D"
-            new_path = request.path.replace("-", "-2D").replace("%", "-")
+            # Try the same path but with "%" replaced by "~"
+            # and "~" replaced with "~7E"
+            # and "." replaced with "~2E"
+            new_path = (
+                request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E")
+            )
             if request.query_string:
                 new_path += "?{}".format(request.query_string)
             await asgi_send_redirect(send, new_path)
diff --git a/datasette/url_builder.py b/datasette/url_builder.py
index eebfe31e..9f072462 100644
--- a/datasette/url_builder.py
+++ b/datasette/url_builder.py
@@ -1,4 +1,4 @@
-from .utils import dash_encode, path_with_format, HASH_LENGTH, PrefixedUrlString
+from .utils import tilde_encode, path_with_format, HASH_LENGTH, PrefixedUrlString
 import urllib
 
 
@@ -31,20 +31,20 @@ class Urls:
         db = self.ds.databases[database]
         if self.ds.setting("hash_urls") and db.hash:
             path = self.path(
-                f"{dash_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format
+                f"{tilde_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format
             )
         else:
-            path = self.path(dash_encode(database), format=format)
+            path = self.path(tilde_encode(database), format=format)
         return path
 
     def table(self, database, table, format=None):
-        path = f"{self.database(database)}/{dash_encode(table)}"
+        path = f"{self.database(database)}/{tilde_encode(table)}"
         if format is not None:
             path = path_with_format(path=path, format=format)
         return PrefixedUrlString(path)
 
     def query(self, database, query, format=None):
-        path = f"{self.database(database)}/{dash_encode(query)}"
+        path = f"{self.database(database)}/{tilde_encode(query)}"
         if format is not None:
             path = path_with_format(path=path, format=format)
         return PrefixedUrlString(path)
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 79feeef6..bd591459 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -15,6 +15,7 @@ import tempfile
 import typing
 import time
 import types
+import secrets
 import shutil
 import urllib
 import yaml
@@ -112,12 +113,12 @@ async def await_me_maybe(value: typing.Any) -> typing.Any:
 
 
 def urlsafe_components(token):
-    """Splits token on commas and dash-decodes each component"""
-    return [dash_decode(b) for b in token.split(",")]
+    """Splits token on commas and tilde-decodes each component"""
+    return [tilde_decode(b) for b in token.split(",")]
 
 
 def path_from_row_pks(row, pks, use_rowid, quote=True):
-    """Generate an optionally dash-quoted unique identifier
+    """Generate an optionally tilde-encoded unique identifier
     for a row from its primary keys."""
     if use_rowid:
         bits = [row["rowid"]]
@@ -126,7 +127,7 @@ def path_from_row_pks(row, pks, use_rowid, quote=True):
             row[pk]["value"] if isinstance(row[pk], dict) else row[pk] for pk in pks
         ]
     if quote:
-        bits = [dash_encode(str(bit)) for bit in bits]
+        bits = [tilde_encode(str(bit)) for bit in bits]
     else:
         bits = [str(bit) for bit in bits]
 
@@ -1142,34 +1143,38 @@ def add_cors_headers(headers):
     headers["Access-Control-Expose-Headers"] = "Link"
 
 
-_DASH_ENCODING_SAFE = frozenset(
+_TILDE_ENCODING_SAFE = frozenset(
     b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
     b"abcdefghijklmnopqrstuvwxyz"
-    b"0123456789_"
+    b"0123456789_-"
     # This is the same as Python percent-encoding but I removed
-    # '.' and '-' and '~'
+    # '.' and '~'
 )
 
 
-class DashEncoder(dict):
+class TildeEncoder(dict):
     # Keeps a cache internally, via __missing__
     def __missing__(self, b):
         # Handle a cache miss, store encoded string in cache and return.
-        res = chr(b) if b in _DASH_ENCODING_SAFE else "-{:02X}".format(b)
+        res = chr(b) if b in _TILDE_ENCODING_SAFE else "~{:02X}".format(b)
         self[b] = res
         return res
 
 
-_dash_encoder = DashEncoder().__getitem__
+_tilde_encoder = TildeEncoder().__getitem__
 
 
 @documented
-def dash_encode(s: str) -> str:
-    "Returns dash-encoded string - for example ``/foo/bar`` -> ``-2Ffoo-2Fbar``"
-    return "".join(_dash_encoder(char) for char in s.encode("utf-8"))
+def tilde_encode(s: str) -> str:
+    "Returns tilde-encoded string - for example ``/foo/bar`` -> ``~2Ffoo~2Fbar``"
+    return "".join(_tilde_encoder(char) for char in s.encode("utf-8"))
 
 
 @documented
-def dash_decode(s: str) -> str:
-    "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``"
-    return urllib.parse.unquote(s.replace("-", "%"))
+def tilde_decode(s: str) -> str:
+    "Decodes a tilde-encoded string, so ``~2Ffoo~2Fbar`` -> ``/foo/bar``"
+    # Avoid accidentally decoding a %2f style sequence
+    temp = secrets.token_hex(16)
+    s = s.replace("%", temp)
+    decoded = urllib.parse.unquote(s.replace("~", "%"))
+    return decoded.replace(temp, "%")
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 7cd385b7..1c0c3f9b 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -10,6 +10,7 @@ import pint
 
 from datasette import __version__
 from datasette.database import QueryInterrupted
+from datasette.utils.asgi import Request
 from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
@@ -17,8 +18,8 @@ from datasette.utils import (
     InvalidSql,
     LimitedWriter,
     call_with_supported_arguments,
-    dash_decode,
-    dash_encode,
+    tilde_decode,
+    tilde_encode,
     path_from_row_pks,
     path_with_added_args,
     path_with_removed_args,
@@ -205,14 +206,14 @@ class DataView(BaseView):
     async def resolve_db_name(self, request, db_name, **kwargs):
         hash = None
         name = None
-        decoded_name = dash_decode(db_name)
+        decoded_name = tilde_decode(db_name)
         if decoded_name not in self.ds.databases and "-" in db_name:
             # No matching DB found, maybe it's a name-hash?
             name_bit, hash_bit = db_name.rsplit("-", 1)
-            if dash_decode(name_bit) not in self.ds.databases:
+            if tilde_decode(name_bit) not in self.ds.databases:
                 raise NotFound(f"Database not found: {name}")
             else:
-                name = dash_decode(name_bit)
+                name = tilde_decode(name_bit)
                 hash = hash_bit
         else:
             name = decoded_name
@@ -235,7 +236,7 @@ class DataView(BaseView):
                     return await db.table_exists(t)
 
                 table, _format = await resolve_table_and_format(
-                    table_and_format=dash_decode(kwargs["table_and_format"]),
+                    table_and_format=tilde_decode(kwargs["table_and_format"]),
                     table_exists=async_table_exists,
                     allowed_formats=self.ds.renderers.keys(),
                 )
@@ -243,11 +244,11 @@ class DataView(BaseView):
                 if _format:
                     kwargs["as_format"] = f".{_format}"
             elif kwargs.get("table"):
-                kwargs["table"] = dash_decode(kwargs["table"])
+                kwargs["table"] = tilde_decode(kwargs["table"])
 
             should_redirect = self.ds.urls.path(f"{name}-{expected}")
             if kwargs.get("table"):
-                should_redirect += "/" + dash_encode(kwargs["table"])
+                should_redirect += "/" + tilde_encode(kwargs["table"])
             if kwargs.get("pk_path"):
                 should_redirect += "/" + kwargs["pk_path"]
             if kwargs.get("as_format"):
@@ -291,6 +292,7 @@ class DataView(BaseView):
             if not request.args.get(key)
         ]
         if extra_parameters:
+            # Replace request object with a new one with modified scope
             if not request.query_string:
                 new_query_string = "&".join(extra_parameters)
             else:
@@ -300,7 +302,8 @@ class DataView(BaseView):
             new_scope = dict(
                 request.scope, query_string=new_query_string.encode("latin-1")
             )
-            request.scope = new_scope
+            receive = request.receive
+            request = Request(new_scope, receive)
         if stream:
             # Some quick soundness checks
             if not self.ds.setting("allow_csv_stream"):
@@ -467,7 +470,7 @@ class DataView(BaseView):
                 return await db.table_exists(t)
 
             table, _ext_format = await resolve_table_and_format(
-                table_and_format=dash_decode(args["table_and_format"]),
+                table_and_format=tilde_decode(args["table_and_format"]),
                 table_exists=async_table_exists,
                 allowed_formats=self.ds.renderers.keys(),
             )
@@ -475,7 +478,7 @@ class DataView(BaseView):
             args["table"] = table
             del args["table_and_format"]
         elif "table" in args:
-            args["table"] = dash_decode(args["table"])
+            args["table"] = tilde_decode(args["table"])
         return _format, args
 
     async def view_get(self, request, database, hash, correct_hash_provided, **kwargs):
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 1d81755e..72b8e9a4 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -12,7 +12,8 @@ from datasette.utils import (
     MultiParams,
     append_querystring,
     compound_keys_after_sql,
-    dash_encode,
+    tilde_decode,
+    tilde_encode,
     escape_sqlite,
     filters_should_redirect,
     is_url,
@@ -143,7 +144,7 @@ class RowTableShared(DataView):
                             '<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format(
                                 base_url=base_url,
                                 database=database,
-                                table=dash_encode(table),
+                                table=tilde_encode(table),
                                 flat_pks=str(markupsafe.escape(pk_path)),
                                 flat_pks_quoted=path_from_row_pks(row, pks, not pks),
                             )
@@ -200,8 +201,8 @@ class RowTableShared(DataView):
                         link_template.format(
                             database=database,
                             base_url=base_url,
-                            table=dash_encode(other_table),
-                            link_id=dash_encode(str(value)),
+                            table=tilde_encode(other_table),
+                            link_id=tilde_encode(str(value)),
                             id=str(markupsafe.escape(value)),
                             label=str(markupsafe.escape(label)) or "-",
                         )
@@ -346,6 +347,8 @@ class TableView(RowTableShared):
                 write=bool(canned_query.get("write")),
             )
 
+        table = tilde_decode(table)
+
         db = self.ds.databases[database]
         is_view = bool(await db.get_view_definition(table))
         table_exists = bool(await db.table_exists(table))
@@ -766,7 +769,7 @@ class TableView(RowTableShared):
                 if prefix is None:
                     prefix = "$null"
                 else:
-                    prefix = dash_encode(str(prefix))
+                    prefix = tilde_encode(str(prefix))
                 next_value = f"{prefix},{next_value}"
                 added_args = {"_next": next_value}
                 if sort:
@@ -938,6 +941,7 @@ class RowView(RowTableShared):
     name = "row"
 
     async def data(self, request, database, hash, table, pk_path, default_labels=False):
+        table = tilde_decode(table)
         await self.check_permissions(
             request,
             [
diff --git a/docs/csv_export.rst b/docs/csv_export.rst
index b1cc673c..023fa05e 100644
--- a/docs/csv_export.rst
+++ b/docs/csv_export.rst
@@ -59,21 +59,3 @@ truncation error message.
 You can increase or remove this limit using the :ref:`setting_max_csv_mb` config
 setting. You can also disable the CSV export feature entirely using
 :ref:`setting_allow_csv_stream`.
-
-A note on URLs
---------------
-
-The default URL for the CSV representation of a table is that table with
-``.csv`` appended to it:
-
-* https://latest.datasette.io/fixtures/facetable - HTML interface
-* https://latest.datasette.io/fixtures/facetable.csv - CSV export
-* https://latest.datasette.io/fixtures/facetable.json - JSON API
-
-This pattern doesn't work for tables with names that already end in ``.csv`` or
-``.json``. For those tables, you can instead use the ``_format=`` query string
-parameter:
-
-* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv - HTML interface
-* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv?_format=csv - CSV export
-* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv?_format=json - JSON API
diff --git a/docs/internals.rst b/docs/internals.rst
index d035e1f1..3d223603 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -545,7 +545,7 @@ These functions can be accessed via the ``{{ urls }}`` object in Datasette templ
     <a href="{{ urls.table("fixtures", "facetable") }}">facetable table</a>
     <a href="{{ urls.query("fixtures", "pragma_cache_size") }}">pragma_cache_size query</a>
 
-Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) arguments to get back URLs to the JSON representation. This is usually the path with ``.json`` added on the end, but it may use ``?_format=json`` in cases where the path already includes ``.json``, for example a URL to a table named ``table.json``.
+Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) arguments to get back URLs to the JSON representation. This is the path with ``.json`` added on the end.
 
 These methods each return a ``datasette.utils.PrefixedUrlString`` object, which is a subclass of the Python ``str`` type. This allows the logic that considers the ``base_url`` setting to detect if that prefix has already been applied to the path.
 
@@ -876,31 +876,31 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth
 
 .. autofunction:: datasette.utils.await_me_maybe
 
-.. _internals_dash_encoding:
+.. _internals_tilde_encoding:
 
-Dash encoding
--------------
+Tilde encoding
+--------------
 
-Datasette uses a custom encoding scheme in some places, called **dash encoding**. This is primarily used for table names and row primary keys, to avoid any confusion between ``/`` characters in those values and the Datasette URLs that reference them.
+Datasette uses a custom encoding scheme in some places, called **tilde encoding**. This is primarily used for table names and row primary keys, to avoid any confusion between ``/`` characters in those values and the Datasette URLs that reference them.
 
-Dash encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``-`` hyphen character used in place of ``%``.
+Tilde encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``~`` tilde character used in place of ``%``.
 
-Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_`` will be replaced by the numeric equivalent preceded by a hyphen. For example:
+Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_-`` will be replaced by the numeric equivalent preceded by a tilde. For example:
 
-- ``/`` becomes ``-2F``
-- ``.`` becomes ``-2E``
-- ``%`` becomes ``-25``
-- ``-`` becomes ``-2D``
-- Space character becomes ``-20``
-- ``polls/2022.primary`` becomes ``polls-2F2022-2Eprimary``
+- ``/`` becomes ``~2F``
+- ``.`` becomes ``~2E``
+- ``%`` becomes ``~25``
+- ``~`` becomes ``~7E``
+- Space character becomes ``~20``
+- ``polls/2022.primary`` becomes ``polls~2F2022~2Eprimary``
 
-.. _internals_utils_dash_encode:
+.. _internals_utils_tilde_encode:
 
-.. autofunction:: datasette.utils.dash_encode
+.. autofunction:: datasette.utils.tilde_encode
 
-.. _internals_utils_dash_decode:
+.. _internals_utils_tilde_decode:
 
-.. autofunction:: datasette.utils.dash_decode
+.. autofunction:: datasette.utils.tilde_decode
 
 .. _internals_tracer:
 
diff --git a/tests/test_api.py b/tests/test_api.py
index dd916cf0..87d91e56 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -679,18 +679,9 @@ def test_row(app_client):
     assert [{"id": "1", "content": "hello"}] == response.json["rows"]
 
 
-def test_row_format_in_querystring(app_client):
-    # regression test for https://github.com/simonw/datasette/issues/563
-    response = app_client.get(
-        "/fixtures/simple_primary_key/1?_format=json&_shape=objects"
-    )
-    assert response.status == 200
-    assert [{"id": "1", "content": "hello"}] == response.json["rows"]
-
-
 def test_row_strange_table_name(app_client):
     response = app_client.get(
-        "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects"
+        "/fixtures/table~2Fwith~2Fslashes~2Ecsv/3.json?_shape=objects"
     )
     assert response.status == 200
     assert [{"pk": "3", "content": "hey"}] == response.json["rows"]
@@ -942,7 +933,7 @@ def test_cors(app_client_with_cors, path, status_code):
 )
 def test_database_with_space_in_name(app_client_two_attached_databases, path):
     response = app_client_two_attached_databases.get(
-        "/extra-20database" + path, follow_redirects=True
+        "/extra~20database" + path, follow_redirects=True
     )
     assert response.status == 200
 
@@ -953,7 +944,7 @@ def test_common_prefix_database_names(app_client_conflicting_database_names):
         d["name"]
         for d in app_client_conflicting_database_names.get("/-/databases.json").json
     ]
-    for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-2Dbar.json")):
+    for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")):
         data = app_client_conflicting_database_names.get(path).json
         assert db_name == data["database"]
 
@@ -996,7 +987,7 @@ async def test_hidden_sqlite_stat1_table():
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("db_name", ("foo", r"fo%o", "f~/c.d"))
-async def test_dash_encoded_database_names(db_name):
+async def test_tilde_encoded_database_names(db_name):
     ds = Datasette()
     ds.add_memory_database(db_name)
     response = await ds.client.get("/.json")
diff --git a/tests/test_cli.py b/tests/test_cli.py
index e30c2ad3..5afe72c1 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -9,7 +9,7 @@ from datasette.app import SETTINGS
 from datasette.plugins import DEFAULT_PLUGINS
 from datasette.cli import cli, serve
 from datasette.version import __version__
-from datasette.utils import dash_encode
+from datasette.utils import tilde_encode
 from datasette.utils.sqlite import sqlite3
 from click.testing import CliRunner
 import io
@@ -295,12 +295,12 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename):
     assert result1.exit_code == 0, result1.output
     filename_no_stem = filename.rsplit(".", 1)[0]
     expected_link = '<a href="/{}">{}</a>'.format(
-        dash_encode(filename_no_stem), filename_no_stem
+        tilde_encode(filename_no_stem), filename_no_stem
     )
     assert expected_link in result1.output
     # Now try hitting that database page
     result2 = runner.invoke(
-        cli, [db_path, "--get", "/{}".format(dash_encode(filename_no_stem))]
+        cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))]
     )
     assert result2.exit_code == 0, result2.output
 
diff --git a/tests/test_html.py b/tests/test_html.py
index de703284..76a8423a 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -29,7 +29,7 @@ def test_homepage(app_client_two_attached_databases):
     )
     # Should be two attached databases
     assert [
-        {"href": r"/extra-20database", "text": "extra database"},
+        {"href": "/extra~20database", "text": "extra database"},
         {"href": "/fixtures", "text": "fixtures"},
     ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")]
     # Database should show count text and attached tables
@@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases):
         {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a")
     ]
     assert [
-        {"href": r"/extra-20database/searchable", "text": "searchable"},
-        {"href": r"/extra-20database/searchable_view", "text": "searchable_view"},
+        {"href": r"/extra~20database/searchable", "text": "searchable"},
+        {"href": r"/extra~20database/searchable_view", "text": "searchable_view"},
     ] == table_links
 
 
@@ -139,15 +139,15 @@ def test_database_page(app_client):
     queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul")
     assert queries_ul is not None
     assert [
-        (
-            "/fixtures/-F0-9D-90-9C-F0-9D-90-A2-F0-9D-90-AD-F0-9D-90-A2-F0-9D-90-9E-F0-9D-90-AC",
-            "𝐜𝐢𝐭𝐢𝐞𝐬",
-        ),
         ("/fixtures/from_async_hook", "from_async_hook"),
         ("/fixtures/from_hook", "from_hook"),
         ("/fixtures/magic_parameters", "magic_parameters"),
         ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"),
         ("/fixtures/pragma_cache_size", "pragma_cache_size"),
+        (
+            "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC",
+            "𝐜𝐢𝐭𝐢𝐞𝐬",
+        ),
     ] == sorted(
         [(a["href"], a.text) for a in queries_ul.find_all("a")], key=lambda p: p[0]
     )
@@ -193,11 +193,11 @@ def test_row_redirects_with_url_hash(app_client_with_hash):
 
 
 def test_row_strange_table_name_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures/table-2Fwith-2Fslashes-2Ecsv/3")
+    response = app_client_with_hash.get("/fixtures/table~2Fwith~2Fslashes~2Ecsv/3")
     assert response.status == 302
-    assert response.headers["Location"].endswith("/table-2Fwith-2Fslashes-2Ecsv/3")
+    assert response.headers["Location"].endswith("/table~2Fwith~2Fslashes~2Ecsv/3")
     response = app_client_with_hash.get(
-        "/fixtures/table-2Fwith-2Fslashes-2Ecsv/3", follow_redirects=True
+        "/fixtures/table~2Fwith~2Fslashes~2Ecsv/3", follow_redirects=True
     )
     assert response.status == 200
 
@@ -229,7 +229,7 @@ def test_row_page_does_not_truncate():
             ["query", "db-fixtures", "query-neighborhood_search"],
         ),
         (
-            "/fixtures/table%2Fwith%2Fslashes.csv",
+            "/fixtures/table~2Fwith~2Fslashes~2Ecsv",
             ["table", "db-fixtures", "table-tablewithslashescsv-fa7563"],
         ),
         (
@@ -255,7 +255,7 @@ def test_css_classes_on_body(app_client, path, expected_classes):
             "table-fixtures-simple_primary_key.html, *table.html",
         ),
         (
-            "/fixtures/table%2Fwith%2Fslashes.csv",
+            "/fixtures/table~2Fwith~2Fslashes~2Ecsv",
             "table-fixtures-tablewithslashescsv-fa7563.html, *table.html",
         ),
         (
@@ -359,7 +359,7 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l
             ],
         ),
         (
-            "/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd",
+            "/fixtures/compound_primary_key/a~2Fb,~2Ec~2Dd",
             [
                 [
                     '<td class="col-pk1 type-str">a/b</td>',
@@ -816,7 +816,8 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
         ),
         ("/fixtures/pragma_cache_size", None),
         (
-            "/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬",
+            # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬
+            "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC",
             "/fixtures?sql=select+id%2C+name+from+facet_cities+order+by+id+limit+1%3B",
         ),
         ("/fixtures/magic_parameters", None),
@@ -824,6 +825,7 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
 )
 def test_edit_sql_link_on_canned_queries(app_client, path, expected):
     response = app_client.get(path)
+    assert response.status == 200
     expected_link = f'<a href="{expected}" class="canned-query-edit-sql">Edit SQL</a>'
     if expected:
         assert expected_link in response.text
@@ -898,8 +900,8 @@ def test_trace_correctly_escaped(app_client):
         # Table page
         ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
         (
-            "/fixtures/table%2Fwith%2Fslashes.csv",
-            "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json",
+            "/fixtures/table~2Fwith~2Fslashes~2Ecsv",
+            "http://localhost/fixtures/table~2Fwith~2Fslashes~2Ecsv.json",
         ),
         # Row page
         (
@@ -930,6 +932,7 @@ def test_trace_correctly_escaped(app_client):
 )
 def test_alternate_url_json(app_client, path, expected):
     response = app_client.get(path)
+    assert response.status == 200
     link = response.headers["link"]
     assert link == '{}; rel="alternate"; type="application/json+datasette"'.format(
         expected
@@ -959,13 +962,17 @@ def test_no_alternate_url_json(app_client, path):
     (
         (
             "/fivethirtyeight/twitter-ratio%2Fsenators",
-            "/fivethirtyeight/twitter-2Dratio-2Fsenators",
+            "/fivethirtyeight/twitter-ratio~2Fsenators",
+        ),
+        (
+            "/fixtures/table%2Fwith%2Fslashes.csv",
+            "/fixtures/table~2Fwith~2Fslashes~2Ecsv",
         ),
         # query string should be preserved
-        ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"),
+        ("/foo/bar%2Fbaz?id=5", "/foo/bar~2Fbaz?id=5"),
     ),
 )
-def test_redirect_percent_encoding_to_dash_encoding(app_client, path, expected):
+def test_redirect_percent_encoding_to_tilde_encoding(app_client, path, expected):
     response = app_client.get(path)
     assert response.status == 302
     assert response.headers["location"] == expected
diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py
index 16515ad6..4307789c 100644
--- a/tests/test_internals_urls.py
+++ b/tests/test_internals_urls.py
@@ -121,7 +121,7 @@ def test_database(ds, base_url, format, expected):
         ("/", "name", None, "/_memory/name"),
         ("/prefix/", "name", None, "/prefix/_memory/name"),
         ("/", "name", "json", "/_memory/name.json"),
-        ("/", "name.json", "json", "/_memory/name-2Ejson.json"),
+        ("/", "name.json", "json", "/_memory/name~2Ejson.json"),
     ],
 )
 def test_table_and_query(ds, base_url, name, format, expected):
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index cc38d392..3ab369b3 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -138,13 +138,13 @@ def test_table_shape_object_compound_primary_key(app_client):
     response = app_client.get("/fixtures/compound_primary_key.json?_shape=object")
     assert response.json == {
         "a,b": {"pk1": "a", "pk2": "b", "content": "c"},
-        "a-2Fb,-2Ec-2Dd": {"pk1": "a/b", "pk2": ".c-d", "content": "c"},
+        "a~2Fb,~2Ec-d": {"pk1": "a/b", "pk2": ".c-d", "content": "c"},
     }
 
 
 def test_table_with_slashes_in_name(app_client):
     response = app_client.get(
-        "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json"
+        "/fixtures/table~2Fwith~2Fslashes~2Ecsv.json?_shape=objects"
     )
     assert response.status == 200
     data = response.json
@@ -1032,7 +1032,10 @@ def test_infinity_returned_as_invalid_json_if_requested(app_client):
 
 
 def test_custom_query_with_unicode_characters(app_client):
-    response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array")
+    # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json
+    response = app_client.get(
+        "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC.json?_shape=array"
+    )
     assert [{"id": 1, "name": "San Francisco"}] == response.json
 
 
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 77d97d80..d40f017a 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -565,7 +565,7 @@ def test_table_html_compound_primary_key(app_client):
             '<td class="col-content type-str">c</td>',
         ],
         [
-            '<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd">a/b,.c-d</a></td>',
+            '<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a~2Fb,~2Ec-d">a/b,.c-d</a></td>',
             '<td class="col-pk1 type-str">a/b</td>',
             '<td class="col-pk2 type-str">.c-d</td>',
             '<td class="col-content type-str">c</td>',
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1c3ab495..790aadc7 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -19,8 +19,8 @@ from unittest.mock import patch
         ("foo", ["foo"]),
         ("foo,bar", ["foo", "bar"]),
         ("123,433,112", ["123", "433", "112"]),
-        ("123%2C433,112", ["123,433", "112"]),
-        ("123%2F433%2F112", ["123/433/112"]),
+        ("123~2C433,112", ["123,433", "112"]),
+        ("123~2F433~2F112", ["123/433/112"]),
     ],
 )
 def test_urlsafe_components(path, expected):
@@ -93,7 +93,7 @@ def test_path_with_replaced_args(path, args, expected):
     "row,pks,expected_path",
     [
         ({"A": "foo", "B": "bar"}, ["A", "B"], "foo,bar"),
-        ({"A": "f,o", "B": "bar"}, ["A", "B"], "f-2Co,bar"),
+        ({"A": "f,o", "B": "bar"}, ["A", "B"], "f~2Co,bar"),
         ({"A": 123}, ["A"], "123"),
         (
             utils.CustomRow(
@@ -393,9 +393,7 @@ def test_table_columns():
         ("/foo?sql=select+1", "json", {}, "/foo.json?sql=select+1"),
         ("/foo/bar", "json", {}, "/foo/bar.json"),
         ("/foo/bar", "csv", {}, "/foo/bar.csv"),
-        ("/foo/bar.csv", "json", {}, "/foo/bar.csv?_format=json"),
         ("/foo/bar", "csv", {"_dl": 1}, "/foo/bar.csv?_dl=1"),
-        ("/foo/b.csv", "json", {"_dl": 1}, "/foo/b.csv?_dl=1&_format=json"),
         (
             "/sf-trees/Street_Tree_List?_search=cherry&_size=1000",
             "csv",
@@ -410,18 +408,6 @@ def test_path_with_format(path, format, extra_qs, expected):
     assert expected == actual
 
 
-def test_path_with_format_replace_format():
-    request = Request.fake("/foo/bar.csv")
-    assert (
-        utils.path_with_format(request=request, format="blob")
-        == "/foo/bar.csv?_format=blob"
-    )
-    assert (
-        utils.path_with_format(request=request, format="blob", replace_format="csv")
-        == "/foo/bar.blob"
-    )
-
-
 @pytest.mark.parametrize(
     "bytes,expected",
     [
@@ -652,15 +638,15 @@ async def test_derive_named_parameters(sql, expected):
     "original,expected",
     (
         ("abc", "abc"),
-        ("/foo/bar", "-2Ffoo-2Fbar"),
-        ("/-/bar", "-2F-2D-2Fbar"),
-        ("-/db-/table.csv", "-2D-2Fdb-2D-2Ftable-2Ecsv"),
-        (r"%~-/", "-25-7E-2D-2F"),
-        ("-25-7E-2D-2F", "-2D25-2D7E-2D2D-2D2F"),
+        ("/foo/bar", "~2Ffoo~2Fbar"),
+        ("/-/bar", "~2F-~2Fbar"),
+        ("-/db-/table.csv", "-~2Fdb-~2Ftable~2Ecsv"),
+        (r"%~-/", "~25~7E-~2F"),
+        ("~25~7E~2D~2F", "~7E25~7E7E~7E2D~7E2F"),
     ),
 )
-def test_dash_encoding(original, expected):
-    actual = utils.dash_encode(original)
+def test_tilde_encoding(original, expected):
+    actual = utils.tilde_encode(original)
     assert actual == expected
     # And test round-trip
-    assert original == utils.dash_decode(actual)
+    assert original == utils.tilde_decode(actual)

From 77a904fea14f743560af9cc668146339bdbbd0a9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Mar 2022 11:03:01 -0700
Subject: [PATCH 1359/2303] Update pytest requirement from <7.1.0,>=5.2.2 to
 >=5.2.2,<7.2.0 (#1656)

Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...7.1.0)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index e70839d6..4b58b8c4 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@ setup(
     extras_require={
         "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
         "test": [
-            "pytest>=5.2.2,<7.1.0",
+            "pytest>=5.2.2,<7.2.0",
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.17,<0.19",
             "beautifulsoup4>=4.8.1,<4.11.0",

From 30e5f0e67c38054a8087a2a4eae3fc4d1779af90 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 17 Mar 2022 14:30:02 -0700
Subject: [PATCH 1360/2303] Documented internals used by datasette-hashed-urls

Closes #1663
---
 docs/internals.rst | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index 3d223603..117cb95c 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -217,12 +217,18 @@ You can create your own instance of this - for example to help write tests for a
         }
     })
 
+Constructor parameters include:
+
+- ``files=[...]`` - a list of database files to open
+- ``immutables=[...]`` - a list of database files to open in immutable mode
+- ``metadata={...}`` - a dictionary of :ref:`metadata`
+
 .. _datasette_databases:
 
 .databases
 ----------
 
-Property exposing an ordered dictionary of databases currently connected to Datasette.
+Property exposing a ``collections.OrderedDict`` of databases currently connected to Datasette.
 
 The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances.
 
@@ -582,6 +588,13 @@ The arguments are as follows:
 
 The first argument is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments.
 
+.. _database_hash:
+
+db.hash
+-------
+
+If the database was opened in immutable mode, this property returns the 64 character SHA-256 hash of the database contents as a string. Otherwise it returns ``None``.
+
 .. _database_execute:
 
 await db.execute(sql, ...)

From d4f60c2388c01ddce1b16f95c16d310e037c9912 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 18 Mar 2022 17:12:03 -0700
Subject: [PATCH 1361/2303] Remove hashed URL mode

Also simplified how view class routing works.

Refs #1661
---
 datasette/app.py             |   2 +-
 datasette/views/base.py      | 153 ++++++-----------------------------
 datasette/views/database.py  |  19 +++--
 datasette/views/index.py     |   3 +-
 datasette/views/special.py   |   3 +-
 datasette/views/table.py     |  34 ++++----
 tests/fixtures.py            |   6 --
 tests/test_api.py            |  29 -------
 tests/test_custom_pages.py   |  42 +++++-----
 tests/test_html.py           |  28 -------
 tests/test_internals_urls.py |  18 -----
 tests/test_table_api.py      |   8 --
 12 files changed, 79 insertions(+), 266 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index b39ef7cd..3099ada7 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1097,7 +1097,7 @@ class Datasette:
         )
         add_route(
             TableView.as_view(self),
-            r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)",
+            r"/(?P<db_name>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$",
         )
         add_route(
             RowView.as_view(self),
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 1c0c3f9b..e31beb19 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -122,11 +122,11 @@ class BaseView:
     async def delete(self, request, *args, **kwargs):
         return Response.text("Method not allowed", status=405)
 
-    async def dispatch_request(self, request, *args, **kwargs):
+    async def dispatch_request(self, request):
         if self.ds:
             await self.ds.refresh_schemas()
         handler = getattr(self, request.method.lower(), None)
-        return await handler(request, *args, **kwargs)
+        return await handler(request)
 
     async def render(self, templates, request, context=None):
         context = context or {}
@@ -169,9 +169,7 @@ class BaseView:
     def as_view(cls, *class_args, **class_kwargs):
         async def view(request, send):
             self = view.view_class(*class_args, **class_kwargs)
-            return await self.dispatch_request(
-                request, **request.scope["url_route"]["kwargs"]
-            )
+            return await self.dispatch_request(request)
 
         view.view_class = cls
         view.__doc__ = cls.__doc__
@@ -200,90 +198,14 @@ class DataView(BaseView):
             add_cors_headers(r.headers)
         return r
 
-    async def data(self, request, database, hash, **kwargs):
+    async def data(self, request):
         raise NotImplementedError
 
-    async def resolve_db_name(self, request, db_name, **kwargs):
-        hash = None
-        name = None
-        decoded_name = tilde_decode(db_name)
-        if decoded_name not in self.ds.databases and "-" in db_name:
-            # No matching DB found, maybe it's a name-hash?
-            name_bit, hash_bit = db_name.rsplit("-", 1)
-            if tilde_decode(name_bit) not in self.ds.databases:
-                raise NotFound(f"Database not found: {name}")
-            else:
-                name = tilde_decode(name_bit)
-                hash = hash_bit
-        else:
-            name = decoded_name
-
-        try:
-            db = self.ds.databases[name]
-        except KeyError:
-            raise NotFound(f"Database not found: {name}")
-
-        # Verify the hash
-        expected = "000"
-        if db.hash is not None:
-            expected = db.hash[:HASH_LENGTH]
-        correct_hash_provided = expected == hash
-
-        if not correct_hash_provided:
-            if "table_and_format" in kwargs:
-
-                async def async_table_exists(t):
-                    return await db.table_exists(t)
-
-                table, _format = await resolve_table_and_format(
-                    table_and_format=tilde_decode(kwargs["table_and_format"]),
-                    table_exists=async_table_exists,
-                    allowed_formats=self.ds.renderers.keys(),
-                )
-                kwargs["table"] = table
-                if _format:
-                    kwargs["as_format"] = f".{_format}"
-            elif kwargs.get("table"):
-                kwargs["table"] = tilde_decode(kwargs["table"])
-
-            should_redirect = self.ds.urls.path(f"{name}-{expected}")
-            if kwargs.get("table"):
-                should_redirect += "/" + tilde_encode(kwargs["table"])
-            if kwargs.get("pk_path"):
-                should_redirect += "/" + kwargs["pk_path"]
-            if kwargs.get("as_format"):
-                should_redirect += kwargs["as_format"]
-            if kwargs.get("as_db"):
-                should_redirect += kwargs["as_db"]
-
-            if (
-                (self.ds.setting("hash_urls") or "_hash" in request.args)
-                and
-                # Redirect only if database is immutable
-                not self.ds.databases[name].is_mutable
-            ):
-                return name, expected, correct_hash_provided, should_redirect
-
-        return name, expected, correct_hash_provided, None
-
     def get_templates(self, database, table=None):
         assert NotImplemented
 
-    async def get(self, request, db_name, **kwargs):
-        (
-            database,
-            hash,
-            correct_hash_provided,
-            should_redirect,
-        ) = await self.resolve_db_name(request, db_name, **kwargs)
-        if should_redirect:
-            return self.redirect(request, should_redirect, remove_args={"_hash"})
-
-        return await self.view_get(
-            request, database, hash, correct_hash_provided, **kwargs
-        )
-
-    async def as_csv(self, request, database, hash, **kwargs):
+    async def as_csv(self, request, database):
+        kwargs = {}
         stream = request.args.get("_stream")
         # Do not calculate facets or counts:
         extra_parameters = [
@@ -313,9 +235,7 @@ class DataView(BaseView):
             kwargs["_size"] = "max"
         # Fetch the first page
         try:
-            response_or_template_contexts = await self.data(
-                request, database, hash, **kwargs
-            )
+            response_or_template_contexts = await self.data(request)
             if isinstance(response_or_template_contexts, Response):
                 return response_or_template_contexts
             elif len(response_or_template_contexts) == 4:
@@ -367,10 +287,11 @@ class DataView(BaseView):
             next = None
             while first or (next and stream):
                 try:
+                    kwargs = {}
                     if next:
                         kwargs["_next"] = next
                     if not first:
-                        data, _, _ = await self.data(request, database, hash, **kwargs)
+                        data, _, _ = await self.data(request, **kwargs)
                     if first:
                         if request.args.get("_header") != "off":
                             await writer.writerow(headings)
@@ -445,60 +366,39 @@ class DataView(BaseView):
             if not trace:
                 content_type = "text/csv; charset=utf-8"
             disposition = 'attachment; filename="{}.csv"'.format(
-                kwargs.get("table", database)
+                request.url_vars.get("table", database)
             )
             headers["content-disposition"] = disposition
 
         return AsgiStream(stream_fn, headers=headers, content_type=content_type)
 
-    async def get_format(self, request, database, args):
-        """Determine the format of the response from the request, from URL
-        parameters or from a file extension.
-
-        `args` is a dict of the path components parsed from the URL by the router.
-        """
-        # If ?_format= is provided, use that as the format
-        _format = request.args.get("_format", None)
-        if not _format:
-            _format = (args.pop("as_format", None) or "").lstrip(".")
+    def get_format(self, request):
+        # Format is the bit from the path following the ., if one exists
+        last_path_component = request.path.split("/")[-1]
+        if "." in last_path_component:
+            return last_path_component.split(".")[-1]
         else:
-            args.pop("as_format", None)
-        if "table_and_format" in args:
-            db = self.ds.databases[database]
+            return None
 
-            async def async_table_exists(t):
-                return await db.table_exists(t)
-
-            table, _ext_format = await resolve_table_and_format(
-                table_and_format=tilde_decode(args["table_and_format"]),
-                table_exists=async_table_exists,
-                allowed_formats=self.ds.renderers.keys(),
-            )
-            _format = _format or _ext_format
-            args["table"] = table
-            del args["table_and_format"]
-        elif "table" in args:
-            args["table"] = tilde_decode(args["table"])
-        return _format, args
-
-    async def view_get(self, request, database, hash, correct_hash_provided, **kwargs):
-        _format, kwargs = await self.get_format(request, database, kwargs)
+    async def get(self, request):
+        db_name = request.url_vars["db_name"]
+        database = tilde_decode(db_name)
+        _format = self.get_format(request)
+        data_kwargs = {}
 
         if _format == "csv":
-            return await self.as_csv(request, database, hash, **kwargs)
+            return await self.as_csv(request, database)
 
         if _format is None:
             # HTML views default to expanding all foreign key labels
-            kwargs["default_labels"] = True
+            data_kwargs["default_labels"] = True
 
         extra_template_data = {}
         start = time.perf_counter()
         status_code = None
         templates = []
         try:
-            response_or_template_contexts = await self.data(
-                request, database, hash, **kwargs
-            )
+            response_or_template_contexts = await self.data(request, **data_kwargs)
             if isinstance(response_or_template_contexts, Response):
                 return response_or_template_contexts
             # If it has four items, it includes an HTTP status code
@@ -650,10 +550,7 @@ class DataView(BaseView):
 
         ttl = request.args.get("_ttl", None)
         if ttl is None or not ttl.isdigit():
-            if correct_hash_provided:
-                ttl = self.ds.setting("default_cache_ttl_hashed")
-            else:
-                ttl = self.ds.setting("default_cache_ttl")
+            ttl = self.ds.setting("default_cache_ttl")
 
         return self.set_response_headers(r, ttl)
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index e26706e7..48635e01 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -12,6 +12,7 @@ from datasette.utils import (
     await_me_maybe,
     check_visibility,
     derive_named_parameters,
+    tilde_decode,
     to_css_class,
     validate_sql_select,
     is_url,
@@ -21,7 +22,7 @@ from datasette.utils import (
     sqlite3,
     InvalidSql,
 )
-from datasette.utils.asgi import AsgiFileDownload, Response, Forbidden
+from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden
 from datasette.plugins import pm
 
 from .base import DatasetteError, DataView
@@ -30,7 +31,8 @@ from .base import DatasetteError, DataView
 class DatabaseView(DataView):
     name = "database"
 
-    async def data(self, request, database, hash, default_labels=False, _size=None):
+    async def data(self, request, default_labels=False, _size=None):
+        database = tilde_decode(request.url_vars["db_name"])
         await self.check_permissions(
             request,
             [
@@ -45,10 +47,13 @@ class DatabaseView(DataView):
             sql = request.args.get("sql")
             validate_sql_select(sql)
             return await QueryView(self.ds).data(
-                request, database, hash, sql, _size=_size, metadata=metadata
+                request, sql, _size=_size, metadata=metadata
             )
 
-        db = self.ds.databases[database]
+        try:
+            db = self.ds.databases[database]
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database))
 
         table_counts = await db.table_counts(5)
         hidden_table_names = set(await db.hidden_table_names())
@@ -156,7 +161,8 @@ class DatabaseView(DataView):
 class DatabaseDownload(DataView):
     name = "database_download"
 
-    async def view_get(self, request, database, hash, correct_hash_present, **kwargs):
+    async def get(self, request):
+        database = tilde_decode(request.url_vars["db_name"])
         await self.check_permissions(
             request,
             [
@@ -191,8 +197,6 @@ class QueryView(DataView):
     async def data(
         self,
         request,
-        database,
-        hash,
         sql,
         editable=True,
         canned_query=None,
@@ -201,6 +205,7 @@ class QueryView(DataView):
         named_parameters=None,
         write=False,
     ):
+        database = tilde_decode(request.url_vars["db_name"])
         params = {key: request.args.get(key) for key in request.args}
         if "sql" in params:
             params.pop("sql")
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 18454759..311a49db 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -18,7 +18,8 @@ COUNT_DB_SIZE_LIMIT = 100 * 1024 * 1024
 class IndexView(BaseView):
     name = "index"
 
-    async def get(self, request, as_format):
+    async def get(self, request):
+        as_format = request.url_vars["as_format"]
         await self.check_permission(request, "view-instance")
         databases = []
         for name, db in self.ds.databases.items():
diff --git a/datasette/views/special.py b/datasette/views/special.py
index cdd530f0..c7b5061f 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -14,7 +14,8 @@ class JsonDataView(BaseView):
         self.data_callback = data_callback
         self.needs_request = needs_request
 
-    async def get(self, request, as_format):
+    async def get(self, request):
+        as_format = request.url_vars["as_format"]
         await self.check_permission(request, "view-instance")
         if self.needs_request:
             data = self.data_callback(request)
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 72b8e9a4..8bdc7417 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -271,20 +271,18 @@ class RowTableShared(DataView):
 class TableView(RowTableShared):
     name = "table"
 
-    async def post(self, request, db_name, table_and_format):
+    async def post(self, request):
+        db_name = tilde_decode(request.url_vars["db_name"])
+        table = tilde_decode(request.url_vars["table"])
         # Handle POST to a canned query
-        canned_query = await self.ds.get_canned_query(
-            db_name, table_and_format, request.actor
-        )
+        canned_query = await self.ds.get_canned_query(db_name, table, request.actor)
         assert canned_query, "You may only POST to a canned query"
         return await QueryView(self.ds).data(
             request,
-            db_name,
-            None,
             canned_query["sql"],
             metadata=canned_query,
             editable=False,
-            canned_query=table_and_format,
+            canned_query=table,
             named_parameters=canned_query.get("params"),
             write=bool(canned_query.get("write")),
         )
@@ -325,20 +323,22 @@ class TableView(RowTableShared):
     async def data(
         self,
         request,
-        database,
-        hash,
-        table,
         default_labels=False,
         _next=None,
         _size=None,
     ):
+        database = tilde_decode(request.url_vars["db_name"])
+        table = tilde_decode(request.url_vars["table"])
+        try:
+            db = self.ds.databases[database]
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database))
+
         # If this is a canned query, not a table, then dispatch to QueryView instead
         canned_query = await self.ds.get_canned_query(database, table, request.actor)
         if canned_query:
             return await QueryView(self.ds).data(
                 request,
-                database,
-                hash,
                 canned_query["sql"],
                 metadata=canned_query,
                 editable=False,
@@ -347,9 +347,6 @@ class TableView(RowTableShared):
                 write=bool(canned_query.get("write")),
             )
 
-        table = tilde_decode(table)
-
-        db = self.ds.databases[database]
         is_view = bool(await db.get_view_definition(table))
         table_exists = bool(await db.table_exists(table))
 
@@ -940,8 +937,9 @@ async def _sql_params_pks(db, table, pk_values):
 class RowView(RowTableShared):
     name = "row"
 
-    async def data(self, request, database, hash, table, pk_path, default_labels=False):
-        table = tilde_decode(table)
+    async def data(self, request, default_labels=False):
+        database = tilde_decode(request.url_vars["db_name"])
+        table = tilde_decode(request.url_vars["table"])
         await self.check_permissions(
             request,
             [
@@ -950,7 +948,7 @@ class RowView(RowTableShared):
                 "view-instance",
             ],
         )
-        pk_values = urlsafe_components(pk_path)
+        pk_values = urlsafe_components(request.url_vars["pk_path"])
         db = self.ds.databases[database]
         sql, params, pks = await _sql_params_pks(db, table, pk_values)
         results = await db.execute(sql, params, truncate=True)
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 11f09c41..342a3020 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -214,12 +214,6 @@ def app_client_two_attached_databases_one_immutable():
         yield client
 
 
-@pytest.fixture(scope="session")
-def app_client_with_hash():
-    with make_app_client(settings={"hash_urls": True}, is_immutable=True) as client:
-        yield client
-
-
 @pytest.fixture(scope="session")
 def app_client_with_trace():
     with make_app_client(settings={"trace_debug": True}, is_immutable=True) as client:
diff --git a/tests/test_api.py b/tests/test_api.py
index 87d91e56..46e41afb 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -825,35 +825,6 @@ def test_config_redirects_to_settings(app_client, path, expected_redirect):
     assert response.headers["Location"] == expected_redirect
 
 
-@pytest.mark.parametrize(
-    "path,expected_redirect",
-    [
-        ("/fixtures/facetable.json?_hash=1", "/fixtures-HASH/facetable.json"),
-        (
-            "/fixtures/facetable.json?city_id=1&_hash=1",
-            "/fixtures-HASH/facetable.json?city_id=1",
-        ),
-    ],
-)
-def test_hash_parameter(
-    app_client_two_attached_databases_one_immutable, path, expected_redirect
-):
-    # First get the current hash for the fixtures database
-    current_hash = app_client_two_attached_databases_one_immutable.ds.databases[
-        "fixtures"
-    ].hash[:7]
-    response = app_client_two_attached_databases_one_immutable.get(path)
-    assert response.status == 302
-    location = response.headers["Location"]
-    assert expected_redirect.replace("HASH", current_hash) == location
-
-
-def test_hash_parameter_ignored_for_mutable_databases(app_client):
-    path = "/fixtures/facetable.json?_hash=1"
-    response = app_client.get(path)
-    assert response.status == 200
-
-
 test_json_columns_default_expected = [
     {"intval": 1, "strval": "s", "floatval": 0.5, "jsonval": '{"foo": "bar"}'}
 ]
diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py
index 66b7437a..f2cfe394 100644
--- a/tests/test_custom_pages.py
+++ b/tests/test_custom_pages.py
@@ -21,61 +21,61 @@ def custom_pages_client_with_base_url():
 
 def test_custom_pages_view_name(custom_pages_client):
     response = custom_pages_client.get("/about")
-    assert 200 == response.status
-    assert "ABOUT! view_name:page" == response.text
+    assert response.status == 200
+    assert response.text == "ABOUT! view_name:page"
 
 
 def test_request_is_available(custom_pages_client):
     response = custom_pages_client.get("/request")
-    assert 200 == response.status
-    assert "path:/request" == response.text
+    assert response.status == 200
+    assert response.text == "path:/request"
 
 
 def test_custom_pages_with_base_url(custom_pages_client_with_base_url):
     response = custom_pages_client_with_base_url.get("/prefix/request")
-    assert 200 == response.status
-    assert "path:/prefix/request" == response.text
+    assert response.status == 200
+    assert response.text == "path:/prefix/request"
 
 
 def test_custom_pages_nested(custom_pages_client):
     response = custom_pages_client.get("/nested/nest")
-    assert 200 == response.status
-    assert "Nest!" == response.text
+    assert response.status == 200
+    assert response.text == "Nest!"
     response = custom_pages_client.get("/nested/nest2")
-    assert 404 == response.status
+    assert response.status == 404
 
 
 def test_custom_status(custom_pages_client):
     response = custom_pages_client.get("/202")
-    assert 202 == response.status
-    assert "202!" == response.text
+    assert response.status == 202
+    assert response.text == "202!"
 
 
 def test_custom_headers(custom_pages_client):
     response = custom_pages_client.get("/headers")
-    assert 200 == response.status
-    assert "foo" == response.headers["x-this-is-foo"]
-    assert "bar" == response.headers["x-this-is-bar"]
-    assert "FOOBAR" == response.text
+    assert response.status == 200
+    assert response.headers["x-this-is-foo"] == "foo"
+    assert response.headers["x-this-is-bar"] == "bar"
+    assert response.text == "FOOBAR"
 
 
 def test_custom_content_type(custom_pages_client):
     response = custom_pages_client.get("/atom")
-    assert 200 == response.status
+    assert response.status == 200
     assert response.headers["content-type"] == "application/xml"
-    assert "<?xml ...>" == response.text
+    assert response.text == "<?xml ...>"
 
 
 def test_redirect(custom_pages_client):
     response = custom_pages_client.get("/redirect")
-    assert 302 == response.status
-    assert "/example" == response.headers["Location"]
+    assert response.status == 302
+    assert response.headers["Location"] == "/example"
 
 
 def test_redirect2(custom_pages_client):
     response = custom_pages_client.get("/redirect2")
-    assert 301 == response.status
-    assert "/example" == response.headers["Location"]
+    assert response.status == 301
+    assert response.headers["Location"] == "/example"
 
 
 @pytest.mark.parametrize(
diff --git a/tests/test_html.py b/tests/test_html.py
index 76a8423a..6e4c22b1 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -5,7 +5,6 @@ from .fixtures import (  # noqa
     app_client_base_url_prefix,
     app_client_shorter_time_limit,
     app_client_two_attached_databases,
-    app_client_with_hash,
     make_app_client,
     METADATA,
 )
@@ -101,13 +100,6 @@ def test_not_allowed_methods():
             assert response.status == 405
 
 
-def test_database_page_redirects_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures")
-    assert response.status == 302
-    response = app_client_with_hash.get("/fixtures", follow_redirects=True)
-    assert "fixtures" in response.text
-
-
 def test_database_page(app_client):
     response = app_client.get("/fixtures")
     soup = Soup(response.body, "html.parser")
@@ -182,26 +174,6 @@ def test_sql_time_limit(app_client_shorter_time_limit):
     assert expected_html_fragment in response.text
 
 
-def test_row_redirects_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures/simple_primary_key/1")
-    assert response.status == 302
-    assert response.headers["Location"].endswith("/1")
-    response = app_client_with_hash.get(
-        "/fixtures/simple_primary_key/1", follow_redirects=True
-    )
-    assert response.status == 200
-
-
-def test_row_strange_table_name_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures/table~2Fwith~2Fslashes~2Ecsv/3")
-    assert response.status == 302
-    assert response.headers["Location"].endswith("/table~2Fwith~2Fslashes~2Ecsv/3")
-    response = app_client_with_hash.get(
-        "/fixtures/table~2Fwith~2Fslashes~2Ecsv/3", follow_redirects=True
-    )
-    assert response.status == 200
-
-
 def test_row_page_does_not_truncate():
     with make_app_client(settings={"truncate_cells_html": 5}) as client:
         response = client.get("/fixtures/facetable/1")
diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py
index 4307789c..d60aafcf 100644
--- a/tests/test_internals_urls.py
+++ b/tests/test_internals_urls.py
@@ -1,6 +1,5 @@
 from datasette.app import Datasette
 from datasette.utils import PrefixedUrlString
-from .fixtures import app_client_with_hash
 import pytest
 
 
@@ -147,20 +146,3 @@ def test_row(ds, base_url, format, expected):
     actual = ds.urls.row("_memory", "facetable", "1", format=format)
     assert actual == expected
     assert isinstance(actual, PrefixedUrlString)
-
-
-@pytest.mark.parametrize("base_url", ["/", "/prefix/"])
-def test_database_hashed(app_client_with_hash, base_url):
-    ds = app_client_with_hash.ds
-    original_base_url = ds._settings["base_url"]
-    try:
-        ds._settings["base_url"] = base_url
-        db_hash = ds.get_database("fixtures").hash
-        assert len(db_hash) == 64
-        expected = f"{base_url}fixtures-{db_hash[:7]}"
-        assert ds.urls.database("fixtures") == expected
-        assert ds.urls.table("fixtures", "name") == expected + "/name"
-        assert ds.urls.query("fixtures", "name") == expected + "/name"
-    finally:
-        # Reset this since fixture is shared with other tests
-        ds._settings["base_url"] = original_base_url
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 3ab369b3..3d0a7fbd 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -2,7 +2,6 @@ from datasette.utils import detect_json1
 from datasette.utils.sqlite import sqlite_version
 from .fixtures import (  # noqa
     app_client,
-    app_client_with_hash,
     app_client_with_trace,
     app_client_returned_rows_matches_page_size,
     generate_compound_rows,
@@ -41,13 +40,6 @@ def test_table_not_exists_json(app_client):
     } == app_client.get("/fixtures/blah.json").json
 
 
-def test_jsono_redirects_to_shape_objects(app_client_with_hash):
-    response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono")
-    response = app_client_with_hash.get(response_1.headers["Location"])
-    assert response.status == 302
-    assert response.headers["Location"].endswith("?_shape=objects")
-
-
 def test_table_shape_arrays(app_client):
     response = app_client.get("/fixtures/simple_primary_key.json?_shape=arrays")
     assert [

From 8658c66438ec71edc7e9adc495f4692b937a0f57 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 18 Mar 2022 17:19:31 -0700
Subject: [PATCH 1362/2303] Show error if --setting hash_urls 1 used, refs
 #1661

---
 datasette/app.py  | 15 ++++++++++-----
 datasette/cli.py  | 21 ++++++++++++++++++---
 tests/test_cli.py |  7 +++++++
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 3099ada7..c1c0663d 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -118,11 +118,6 @@ SETTINGS = (
         50,
         "Time limit for calculating a suggested facet",
     ),
-    Setting(
-        "hash_urls",
-        False,
-        "Include DB file contents hash in URLs, for far-future caching",
-    ),
     Setting(
         "allow_facet",
         True,
@@ -177,6 +172,16 @@ SETTINGS = (
     ),
     Setting("base_url", "/", "Datasette URLs should use this base path"),
 )
+OBSOLETE_SETTINGS = {
+    option.name: option
+    for option in (
+        Setting(
+            "hash_urls",
+            False,
+            "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead",
+        ),
+    )
+}
 
 DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS}
 
diff --git a/datasette/cli.py b/datasette/cli.py
index 61e7ce91..b94ac192 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -12,7 +12,14 @@ from subprocess import call
 import sys
 from runpy import run_module
 import webbrowser
-from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm
+from .app import (
+    OBSOLETE_SETTINGS,
+    Datasette,
+    DEFAULT_SETTINGS,
+    SETTINGS,
+    SQLITE_LIMIT_ATTACHED,
+    pm,
+)
 from .utils import (
     StartupError,
     check_connection,
@@ -50,8 +57,12 @@ class Config(click.ParamType):
             return
         name, value = config.split(":", 1)
         if name not in DEFAULT_SETTINGS:
+            if name in OBSOLETE_SETTINGS:
+                msg = OBSOLETE_SETTINGS[name].help
+            else:
+                msg = f"{name} is not a valid option (--help-settings to see all)"
             self.fail(
-                f"{name} is not a valid option (--help-settings to see all)",
+                msg,
                 param,
                 ctx,
             )
@@ -83,8 +94,12 @@ class Setting(CompositeParamType):
     def convert(self, config, param, ctx):
         name, value = config
         if name not in DEFAULT_SETTINGS:
+            if name in OBSOLETE_SETTINGS:
+                msg = OBSOLETE_SETTINGS[name].help
+            else:
+                msg = f"{name} is not a valid option (--help-settings to see all)"
             self.fail(
-                f"{name} is not a valid option (--help-settings to see all)",
+                msg,
                 param,
                 ctx,
             )
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 5afe72c1..89e8d044 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -310,3 +310,10 @@ def test_help_settings():
     result = runner.invoke(cli, ["--help-settings"])
     for setting in SETTINGS:
         assert setting.name in result.output
+
+
+def test_help_error_on_hash_urls_setting():
+    runner = CliRunner()
+    result = runner.invoke(cli, ["--setting", "hash_urls", 1])
+    assert result.exit_code == 2
+    assert 'The hash_urls setting has been removed' in result.output

From 9979dcd07f9921ac30c4c0b5ea60d09cd1e10556 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 18 Mar 2022 17:25:14 -0700
Subject: [PATCH 1363/2303] Also remove default_cache_ttl_hashed setting, refs
 #1661

---
 datasette/app.py         | 17 +++--------------
 datasette/cli.py         | 16 ++++++++--------
 datasette/url_builder.py |  9 +--------
 tests/test_api.py        |  2 --
 tests/test_cli.py        |  7 ++++---
 5 files changed, 16 insertions(+), 35 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index c1c0663d..f52e3283 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -134,11 +134,6 @@ SETTINGS = (
         5,
         "Default HTTP cache TTL (used in Cache-Control: max-age= header)",
     ),
-    Setting(
-        "default_cache_ttl_hashed",
-        365 * 24 * 60 * 60,
-        "Default HTTP cache TTL for hashed URL pages",
-    ),
     Setting("cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)"),
     Setting(
         "allow_csv_stream",
@@ -172,17 +167,11 @@ SETTINGS = (
     ),
     Setting("base_url", "/", "Datasette URLs should use this base path"),
 )
+_HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead"
 OBSOLETE_SETTINGS = {
-    option.name: option
-    for option in (
-        Setting(
-            "hash_urls",
-            False,
-            "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead",
-        ),
-    )
+    "hash_urls": _HASH_URLS_REMOVED,
+    "default_cache_ttl_hashed": _HASH_URLS_REMOVED,
 }
-
 DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS}
 
 FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png"
diff --git a/datasette/cli.py b/datasette/cli.py
index b94ac192..3c6e1b2c 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -57,10 +57,10 @@ class Config(click.ParamType):
             return
         name, value = config.split(":", 1)
         if name not in DEFAULT_SETTINGS:
-            if name in OBSOLETE_SETTINGS:
-                msg = OBSOLETE_SETTINGS[name].help
-            else:
-                msg = f"{name} is not a valid option (--help-settings to see all)"
+            msg = (
+                OBSOLETE_SETTINGS.get(name)
+                or f"{name} is not a valid option (--help-settings to see all)"
+            )
             self.fail(
                 msg,
                 param,
@@ -94,10 +94,10 @@ class Setting(CompositeParamType):
     def convert(self, config, param, ctx):
         name, value = config
         if name not in DEFAULT_SETTINGS:
-            if name in OBSOLETE_SETTINGS:
-                msg = OBSOLETE_SETTINGS[name].help
-            else:
-                msg = f"{name} is not a valid option (--help-settings to see all)"
+            msg = (
+                OBSOLETE_SETTINGS.get(name)
+                or f"{name} is not a valid option (--help-settings to see all)"
+            )
             self.fail(
                 msg,
                 param,
diff --git a/datasette/url_builder.py b/datasette/url_builder.py
index 9f072462..498ec85d 100644
--- a/datasette/url_builder.py
+++ b/datasette/url_builder.py
@@ -28,14 +28,7 @@ class Urls:
         return self.path("-/logout")
 
     def database(self, database, format=None):
-        db = self.ds.databases[database]
-        if self.ds.setting("hash_urls") and db.hash:
-            path = self.path(
-                f"{tilde_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format
-            )
-        else:
-            path = self.path(tilde_encode(database), format=format)
-        return path
+        return self.path(tilde_encode(database), format=format)
 
     def table(self, database, table, format=None):
         path = f"{self.database(database)}/{tilde_encode(table)}"
diff --git a/tests/test_api.py b/tests/test_api.py
index 46e41afb..d3c94023 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -798,14 +798,12 @@ def test_settings_json(app_client):
         "allow_facet": True,
         "suggest_facets": True,
         "default_cache_ttl": 5,
-        "default_cache_ttl_hashed": 365 * 24 * 60 * 60,
         "num_sql_threads": 1,
         "cache_size_kb": 0,
         "allow_csv_stream": True,
         "max_csv_mb": 100,
         "truncate_cells_html": 2048,
         "force_https_urls": False,
-        "hash_urls": False,
         "template_debug": False,
         "trace_debug": False,
         "base_url": "/",
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 89e8d044..dca65f26 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -312,8 +312,9 @@ def test_help_settings():
         assert setting.name in result.output
 
 
-def test_help_error_on_hash_urls_setting():
+@pytest.mark.parametrize("setting", ("hash_urls", "default_cache_ttl_hashed"))
+def test_help_error_on_hash_urls_setting(setting):
     runner = CliRunner()
-    result = runner.invoke(cli, ["--setting", "hash_urls", 1])
+    result = runner.invoke(cli, ["--setting", setting, 1])
     assert result.exit_code == 2
-    assert 'The hash_urls setting has been removed' in result.output
+    assert "The hash_urls setting has been removed" in result.output

From 32963018e7edfab1233de7c7076c428d0e5c7813 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 18 Mar 2022 17:33:06 -0700
Subject: [PATCH 1364/2303] Updated documentation to remove hash_urls, refs
 #1661

---
 docs/cli-reference.rst |  4 ----
 docs/performance.rst   | 18 +++++++++++-------
 docs/settings.rst      | 27 ---------------------------
 3 files changed, 11 insertions(+), 38 deletions(-)

diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 155a005d..69670d8a 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -142,8 +142,6 @@ datasette serve --help-settings
                                    (default=200)
       facet_suggest_time_limit_ms  Time limit for calculating a suggested facet
                                    (default=50)
-      hash_urls                    Include DB file contents hash in URLs, for far-
-                                   future caching (default=False)
       allow_facet                  Allow users to specify columns to facet using
                                    ?_facet= parameter (default=True)
       allow_download               Allow users to download the original SQLite
@@ -152,8 +150,6 @@ datasette serve --help-settings
                                    (default=True)
       default_cache_ttl            Default HTTP cache TTL (used in Cache-Control:
                                    max-age= header) (default=5)
-      default_cache_ttl_hashed     Default HTTP cache TTL for hashed URL pages
-                                   (default=31536000)
       cache_size_kb                SQLite cache size in KB (0 == use SQLite default)
                                    (default=0)
       allow_csv_stream             Allow .csv?_stream=1 to download all rows
diff --git a/docs/performance.rst b/docs/performance.rst
index bcf3208e..d37f1804 100644
--- a/docs/performance.rst
+++ b/docs/performance.rst
@@ -60,18 +60,22 @@ The :ref:`setting_default_cache_ttl` setting sets the default HTTP cache TTL for
 
 You can also change the cache timeout on a per-request basis using the ``?_ttl=10`` query string parameter. This can be useful when you are working with the Datasette JSON API - you may decide that a specific query can be cached for a longer time, or maybe you need to set ``?_ttl=0`` for some requests for example if you are running a SQL ``order by random()`` query.
 
-Hashed URL mode
----------------
+datasette-hashed-urls
+---------------------
 
-When you open a database file in immutable mode using the ``-i`` option, Datasette calculates a SHA-256 hash of the contents of that file on startup. This content hash can then optionally be used to create URLs that are guaranteed to change if the contents of the file changes in the future. This results in URLs that can then be cached indefinitely by both browsers and caching proxies - an enormous potential performance optimization.
+If you open a database file in immutable mode using the ``-i`` option, you can be assured that the content of that database will not change for the lifetime of the Datasette server.
 
-You can enable these hashed URLs in two ways: using the :ref:`setting_hash_urls` configuration setting (which affects all requests to Datasette) or via the ``?_hash=1`` query string parameter (which only applies to the current request).
+The `datasette-hashed-urls plugin <https://datasette.io/plugins/datasette-hashed-urls>`__ implements an optimization where your database is served with part of the SHA-256 hash of the database contents baked into the URL.
 
-With hashed URLs enabled, any request to e.g. ``/mydatabase/mytable`` will 302 redirect to ``mydatabase-455fe3a/mytable``. The URL containing the hash will be served with a very long cache expire header - configured using :ref:`setting_default_cache_ttl_hashed` which defaults to 365 days.
+A database at ``/fixtures`` will instead be served at ``/fixtures-aa7318b``, and a year-long cache expiry header will be returned with those pages.
 
-Since these responses are cached for a long time, you may wish to build API clients against the non-hashed version of these URLs. These 302 redirects are served extremely quickly, so this should still be a performant way to work against the Datasette API.
+This will then be cached by both browsers and caching proxies such as Cloudflare or Fastly, providing a potentially significant performance boost.
 
-If you run Datasette behind an `HTTP/2 server push <https://en.wikipedia.org/wiki/HTTP/2_Server_Push>`__ aware proxy such as Cloudflare Datasette will serve the 302 redirects in such a way that the redirected page will be efficiently "pushed" to the browser as part of the response, without the browser needing to make a second HTTP request to fetch the redirected resource.
+To install the plugin, run the following::
+
+    datasette install datasette-hashed-urls
 
 .. note::
+    Prior to Datasette 0.61 hashed URL mode was a core Datasette feature, enabled using the ``hash_urls`` setting. This implementation has now been removed in favor of the ``datasette-hashed-urls`` plugin.
+
     Prior to Datasette 0.28 hashed URL mode was the default behaviour for Datasette, since all database files were assumed to be immutable and unchanging. From 0.28 onwards the default has been to treat database files as mutable unless explicitly configured otherwise.
diff --git a/docs/settings.rst b/docs/settings.rst
index da06d6a0..60c4b36d 100644
--- a/docs/settings.rst
+++ b/docs/settings.rst
@@ -178,17 +178,6 @@ Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-ag
 
     datasette mydatabase.db --setting default_cache_ttl 60
 
-.. _setting_default_cache_ttl_hashed:
-
-default_cache_ttl_hashed
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Default HTTP caching max-age for responses served using using the :ref:`hashed-urls mechanism <setting_hash_urls>`. Defaults to 365 days (31536000 seconds).
-
-::
-
-    datasette mydatabase.db --setting default_cache_ttl_hashed 10000
-
 .. _setting_cache_size_kb:
 
 cache_size_kb
@@ -251,22 +240,6 @@ HTTP but is served to the outside world via a proxy that enables HTTPS.
 
     datasette mydatabase.db --setting force_https_urls 1
 
-.. _setting_hash_urls:
-
-hash_urls
-~~~~~~~~~
-
-When enabled, this setting causes Datasette to append a content hash of the
-database file to the URL path for every table and query within that database.
-
-When combined with far-future expire headers this ensures that queries can be
-cached forever, safe in the knowledge that any modifications to the database
-itself will result in new, uncached URL paths.
-
-::
-
-    datasette mydatabase.db --setting hash_urls 1
-
 .. _setting_template_debug:
 
 template_debug

From 4e47a2d894b96854348343374c8e97c9d7055cf6 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 18 Mar 2022 18:37:54 -0700
Subject: [PATCH 1365/2303] Fixed bug where tables with a column called n
 caused 500 errors

Closes #1228
---
 datasette/facets.py              |  6 +++---
 tests/fixtures.py                | 33 ++++++++++++++++----------------
 tests/test_api.py                |  1 +
 tests/test_csv.py                | 32 +++++++++++++++----------------
 tests/test_internals_database.py | 10 ++++++++++
 tests/test_plugins.py            |  6 ++++--
 tests/test_table_api.py          |  8 ++++++++
 7 files changed, 59 insertions(+), 37 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index a1bb4a5f..b15a758c 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -151,10 +151,10 @@ class ColumnFacet(Facet):
             if column in already_enabled:
                 continue
             suggested_facet_sql = """
-                select {column}, count(*) as n from (
+                select {column} as value, count(*) as n from (
                     {sql}
-                ) where {column} is not null
-                group by {column}
+                ) where value is not null
+                group by value
                 limit {limit}
             """.format(
                 column=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 342a3020..e0e4ec7b 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -564,26 +564,27 @@ CREATE TABLE facetable (
     tags text,
     complex_array text,
     distinct_some_null,
+    n text,
     FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
 );
 INSERT INTO facetable
-    (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null)
+    (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n)
 VALUES
-    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'),
-    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'),
-    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null),
-    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null),
-    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null),
-    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null),
-    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null),
-    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null),
-    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null),
-    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null),
-    ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null),
-    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null),
-    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null),
-    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null),
-    ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null)
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null)
 ;
 
 CREATE TABLE binary_data (
diff --git a/tests/test_api.py b/tests/test_api.py
index d3c94023..421bb1fe 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -210,6 +210,7 @@ def test_database_page(app_client):
                 "tags",
                 "complex_array",
                 "distinct_some_null",
+                "n",
             ],
             "primary_keys": ["pk"],
             "count": 15,
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 8749cd8b..7fc25a09 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -24,22 +24,22 @@ world
 )
 
 EXPECTED_TABLE_WITH_LABELS_CSV = """
-pk,created,planet_int,on_earth,state,_city_id,_city_id_label,_neighborhood,tags,complex_array,distinct_some_null
-1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one
-2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two
-3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[],
-4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[],
-5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[],
-6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[],
-7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[],
-8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[],
-9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[],
-10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[],
-11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[],
-12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[],
-13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[],
-14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[],
-15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[],
+pk,created,planet_int,on_earth,state,_city_id,_city_id_label,_neighborhood,tags,complex_array,distinct_some_null,n
+1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one,n1
+2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two,n2
+3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[],,
+4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[],,
+5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[],,
+6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[],,
+7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[],,
+8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[],,
+9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[],,
+10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[],,
+11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[],,
+12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[],,
+13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[],,
+14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[],,
+15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[],,
 """.lstrip().replace(
     "\n", "\r\n"
 )
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 31538a24..551f67e1 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -86,6 +86,7 @@ async def test_table_exists(db, tables, exists):
                 "tags",
                 "complex_array",
                 "distinct_some_null",
+                "n",
             ],
         ),
         (
@@ -204,6 +205,15 @@ async def test_table_columns(db, table, expected):
                     is_pk=0,
                     hidden=0,
                 ),
+                Column(
+                    cid=10,
+                    name="n",
+                    type="text",
+                    notnull=0,
+                    default_value=None,
+                    is_pk=0,
+                    hidden=0,
+                ),
             ],
         ),
         (
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 656f39e4..15bde962 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -442,6 +442,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "tags",
             "complex_array",
             "distinct_some_null",
+            "n",
         ],
         "rows": [
             "<sqlite3.Row object at 0xXXX>",
@@ -460,7 +461,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "<sqlite3.Row object at 0xXXX>",
             "<sqlite3.Row object at 0xXXX>",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
@@ -531,8 +532,9 @@ def test_hook_register_output_renderer_can_render(app_client):
             "tags",
             "complex_array",
             "distinct_some_null",
+            "n",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 3d0a7fbd..9db383c3 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -532,6 +532,7 @@ def test_table_filter_json_arraycontains(app_client):
             '["tag1", "tag2"]',
             '[{"foo": "bar"}]',
             "one",
+            "n1",
         ],
         [
             2,
@@ -544,6 +545,7 @@ def test_table_filter_json_arraycontains(app_client):
             '["tag1", "tag3"]',
             "[]",
             "two",
+            "n2",
         ],
     ]
 
@@ -565,6 +567,7 @@ def test_table_filter_json_arraynotcontains(app_client):
             '["tag1", "tag2"]',
             '[{"foo": "bar"}]',
             "one",
+            "n1",
         ]
     ]
 
@@ -585,6 +588,7 @@ def test_table_filter_extra_where(app_client):
             '["tag1", "tag3"]',
             "[]",
             "two",
+            "n2",
         ]
     ] == response.json["rows"]
 
@@ -958,6 +962,7 @@ def test_expand_labels(app_client):
             "tags": '["tag1", "tag3"]',
             "complex_array": "[]",
             "distinct_some_null": "two",
+            "n": "n2",
         },
         "13": {
             "pk": 13,
@@ -970,6 +975,7 @@ def test_expand_labels(app_client):
             "tags": "[]",
             "complex_array": "[]",
             "distinct_some_null": None,
+            "n": None,
         },
     } == response.json
 
@@ -1161,6 +1167,7 @@ def test_generated_columns_are_visible_in_datasette():
                 "tags",
                 "complex_array",
                 "distinct_some_null",
+                "n",
             ],
         ),
         (
@@ -1188,6 +1195,7 @@ def test_generated_columns_are_visible_in_datasette():
                 "tags",
                 "complex_array",
                 "distinct_some_null",
+                "n",
             ],
         ),
         (

From 711767bcd3c1e76a0861fe7f24069ff1c8efc97a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 18 Mar 2022 21:03:08 -0700
Subject: [PATCH 1366/2303] Refactored URL routing to add tests, closes #1666

Refs #1660
---
 datasette/app.py            | 54 ++++++++++++++++++++-----------------
 datasette/utils/__init__.py |  8 ++++++
 tests/test_routes.py        | 34 +++++++++++++++++++++++
 3 files changed, 72 insertions(+), 24 deletions(-)
 create mode 100644 tests/test_routes.py

diff --git a/datasette/app.py b/datasette/app.py
index f52e3283..8987112c 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -60,6 +60,7 @@ from .utils import (
     module_from_path,
     parse_metadata,
     resolve_env_secrets,
+    resolve_routes,
     to_css_class,
 )
 from .utils.asgi import (
@@ -974,8 +975,7 @@ class Datasette:
             output.append(script)
         return output
 
-    def app(self):
-        """Returns an ASGI app function that serves the whole of Datasette"""
+    def _routes(self):
         routes = []
 
         for routes_to_add in pm.hook.register_routes(datasette=self):
@@ -1099,6 +1099,15 @@ class Datasette:
             + renderer_regex
             + r")?$",
         )
+        return [
+            # Compile any strings to regular expressions
+            ((re.compile(pattern) if isinstance(pattern, str) else pattern), view)
+            for pattern, view in routes
+        ]
+
+    def app(self):
+        """Returns an ASGI app function that serves the whole of Datasette"""
+        routes = self._routes()
         self._register_custom_units()
 
         async def setup_db():
@@ -1129,12 +1138,7 @@ class Datasette:
 class DatasetteRouter:
     def __init__(self, datasette, routes):
         self.ds = datasette
-        routes = routes or []
-        self.routes = [
-            # Compile any strings to regular expressions
-            ((re.compile(pattern) if isinstance(pattern, str) else pattern), view)
-            for pattern, view in routes
-        ]
+        self.routes = routes or []
         # Build a list of pages/blah/{name}.html matching expressions
         pattern_templates = [
             filepath
@@ -1187,22 +1191,24 @@ class DatasetteRouter:
                 break
         scope_modifications["actor"] = actor or default_actor
         scope = dict(scope, **scope_modifications)
-        for regex, view in self.routes:
-            match = regex.match(path)
-            if match is not None:
-                new_scope = dict(scope, url_route={"kwargs": match.groupdict()})
-                request.scope = new_scope
-                try:
-                    response = await view(request, send)
-                    if response:
-                        self.ds._write_messages_to_response(request, response)
-                        await response.asgi_send(send)
-                    return
-                except NotFound as exception:
-                    return await self.handle_404(request, send, exception)
-                except Exception as exception:
-                    return await self.handle_500(request, send, exception)
-        return await self.handle_404(request, send)
+
+        match, view = resolve_routes(self.routes, path)
+
+        if match is None:
+            return await self.handle_404(request, send)
+
+        new_scope = dict(scope, url_route={"kwargs": match.groupdict()})
+        request.scope = new_scope
+        try:
+            response = await view(request, send)
+            if response:
+                self.ds._write_messages_to_response(request, response)
+                await response.asgi_send(send)
+            return
+        except NotFound as exception:
+            return await self.handle_404(request, send, exception)
+        except Exception as exception:
+            return await self.handle_500(request, send, exception)
 
     async def handle_404(self, request, send, exception=None):
         # If path contains % encoding, redirect to tilde encoding
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index bd591459..ccdf8ad4 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1178,3 +1178,11 @@ def tilde_decode(s: str) -> str:
     s = s.replace("%", temp)
     decoded = urllib.parse.unquote(s.replace("~", "%"))
     return decoded.replace(temp, "%")
+
+
+def resolve_routes(routes, path):
+    for regex, view in routes:
+        match = regex.match(path)
+        if match is not None:
+            return match, view
+    return None, None
diff --git a/tests/test_routes.py b/tests/test_routes.py
new file mode 100644
index 00000000..a1960f14
--- /dev/null
+++ b/tests/test_routes.py
@@ -0,0 +1,34 @@
+from datasette.app import Datasette
+from datasette.utils import resolve_routes
+import pytest
+
+
+@pytest.fixture(scope="session")
+def routes():
+    ds = Datasette()
+    return ds._routes()
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    (
+        ("/", "IndexView"),
+        ("/foo", "DatabaseView"),
+        ("/foo.csv", "DatabaseView"),
+        ("/foo.json", "DatabaseView"),
+        ("/foo.humbug", "DatabaseView"),
+        ("/foo/humbug", "TableView"),
+        ("/foo/humbug.json", "TableView"),
+        ("/foo/humbug.blah", "TableView"),
+        ("/foo/humbug/1", "RowView"),
+        ("/foo/humbug/1.json", "RowView"),
+        ("/-/metadata.json", "JsonDataView"),
+        ("/-/metadata", "JsonDataView"),
+    ),
+)
+def test_routes(routes, path, expected):
+    match, view = resolve_routes(routes, path)
+    if expected is None:
+        assert match is None
+    else:
+        assert view.view_class.__name__ == expected

From 764738dfcb16cd98b0987d443f59d5baa9d3c332 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 09:30:22 -0700
Subject: [PATCH 1367/2303] test_routes also now asserts matches, refs #1666

---
 tests/test_routes.py | 41 +++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/tests/test_routes.py b/tests/test_routes.py
index a1960f14..6718c232 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -10,25 +10,34 @@ def routes():
 
 
 @pytest.mark.parametrize(
-    "path,expected",
+    "path,expected_class,expected_matches",
     (
-        ("/", "IndexView"),
-        ("/foo", "DatabaseView"),
-        ("/foo.csv", "DatabaseView"),
-        ("/foo.json", "DatabaseView"),
-        ("/foo.humbug", "DatabaseView"),
-        ("/foo/humbug", "TableView"),
-        ("/foo/humbug.json", "TableView"),
-        ("/foo/humbug.blah", "TableView"),
-        ("/foo/humbug/1", "RowView"),
-        ("/foo/humbug/1.json", "RowView"),
-        ("/-/metadata.json", "JsonDataView"),
-        ("/-/metadata", "JsonDataView"),
+        ("/", "IndexView", {"as_format": ""}),
+        ("/foo", "DatabaseView", {"as_format": None, "db_name": "foo"}),
+        ("/foo.csv", "DatabaseView", {"as_format": ".csv", "db_name": "foo"}),
+        ("/foo.json", "DatabaseView", {"as_format": ".json", "db_name": "foo"}),
+        ("/foo.humbug", "DatabaseView", {"as_format": None, "db_name": "foo.humbug"}),
+        ("/foo/humbug", "TableView", {"db_name": "foo", "table": "humbug"}),
+        ("/foo/humbug.json", "TableView", {"db_name": "foo", "table": "humbug"}),
+        ("/foo/humbug.blah", "TableView", {"db_name": "foo", "table": "humbug"}),
+        (
+            "/foo/humbug/1",
+            "RowView",
+            {"as_format": None, "db_name": "foo", "pk_path": "1", "table": "humbug"},
+        ),
+        (
+            "/foo/humbug/1.json",
+            "RowView",
+            {"as_format": ".json", "db_name": "foo", "pk_path": "1", "table": "humbug"},
+        ),
+        ("/-/metadata.json", "JsonDataView", {"as_format": ".json"}),
+        ("/-/metadata", "JsonDataView", {"as_format": ""}),
     ),
 )
-def test_routes(routes, path, expected):
+def test_routes(routes, path, expected_class, expected_matches):
     match, view = resolve_routes(routes, path)
-    if expected is None:
+    if expected_class is None:
         assert match is None
     else:
-        assert view.view_class.__name__ == expected
+        assert view.view_class.__name__ == expected_class
+        assert match.groupdict() == expected_matches

From 61419388c134001118aaf7dfb913562d467d7913 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 09:52:08 -0700
Subject: [PATCH 1368/2303] Rename route match groups for consistency, refs
 #1667, #1660

---
 datasette/app.py            | 28 ++++++++++++----------------
 datasette/blob_renderer.py  |  4 ++--
 datasette/views/base.py     |  2 +-
 datasette/views/database.py |  6 +++---
 datasette/views/index.py    |  2 +-
 datasette/views/special.py  |  2 +-
 datasette/views/table.py    |  8 ++++----
 tests/test_routes.py        | 24 ++++++++++++------------
 8 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 8987112c..5259c50c 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -988,7 +988,7 @@ class Datasette:
         # Generate a regex snippet to match all registered renderer file extensions
         renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
 
-        add_route(IndexView.as_view(self), r"/(?P<as_format>(\.jsono?)?$)")
+        add_route(IndexView.as_view(self), r"/(?P<format>(\.jsono?)?$)")
         # TODO: /favicon.ico and /-/static/ deserve far-future cache expires
         add_route(favicon, "/favicon.ico")
 
@@ -1020,21 +1020,21 @@ class Datasette:
         )
         add_route(
             JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()),
-            r"/-/metadata(?P<as_format>(\.json)?)$",
+            r"/-/metadata(?P<format>(\.json)?)$",
         )
         add_route(
             JsonDataView.as_view(self, "versions.json", self._versions),
-            r"/-/versions(?P<as_format>(\.json)?)$",
+            r"/-/versions(?P<format>(\.json)?)$",
         )
         add_route(
             JsonDataView.as_view(
                 self, "plugins.json", self._plugins, needs_request=True
             ),
-            r"/-/plugins(?P<as_format>(\.json)?)$",
+            r"/-/plugins(?P<format>(\.json)?)$",
         )
         add_route(
             JsonDataView.as_view(self, "settings.json", lambda: self._settings),
-            r"/-/settings(?P<as_format>(\.json)?)$",
+            r"/-/settings(?P<format>(\.json)?)$",
         )
         add_route(
             permanent_redirect("/-/settings.json"),
@@ -1046,15 +1046,15 @@ class Datasette:
         )
         add_route(
             JsonDataView.as_view(self, "threads.json", self._threads),
-            r"/-/threads(?P<as_format>(\.json)?)$",
+            r"/-/threads(?P<format>(\.json)?)$",
         )
         add_route(
             JsonDataView.as_view(self, "databases.json", self._connected_databases),
-            r"/-/databases(?P<as_format>(\.json)?)$",
+            r"/-/databases(?P<format>(\.json)?)$",
         )
         add_route(
             JsonDataView.as_view(self, "actor.json", self._actor, needs_request=True),
-            r"/-/actor(?P<as_format>(\.json)?)$",
+            r"/-/actor(?P<format>(\.json)?)$",
         )
         add_route(
             AuthTokenView.as_view(self),
@@ -1080,22 +1080,18 @@ class Datasette:
             PatternPortfolioView.as_view(self),
             r"/-/patterns$",
         )
-        add_route(
-            DatabaseDownload.as_view(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$"
-        )
+        add_route(DatabaseDownload.as_view(self), r"/(?P<database>[^/]+?)\.db$")
         add_route(
             DatabaseView.as_view(self),
-            r"/(?P<db_name>[^/]+?)(?P<as_format>"
-            + renderer_regex
-            + r"|.jsono|\.csv)?$",
+            r"/(?P<database>[^/]+?)(?P<format>" + renderer_regex + r"|.jsono|\.csv)?$",
         )
         add_route(
             TableView.as_view(self),
-            r"/(?P<db_name>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$",
+            r"/(?P<database>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$",
         )
         add_route(
             RowView.as_view(self),
-            r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>"
+            r"/(?P<database>[^/]+)/(?P<table>[^/]+?)/(?P<pks>[^/]+?)(?P<format>"
             + renderer_regex
             + r")?$",
         )
diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py
index 217b3638..4d8c6bea 100644
--- a/datasette/blob_renderer.py
+++ b/datasette/blob_renderer.py
@@ -34,8 +34,8 @@ async def render_blob(datasette, database, rows, columns, request, table, view_n
     filename_bits = []
     if table:
         filename_bits.append(to_css_class(table))
-    if "pk_path" in request.url_vars:
-        filename_bits.append(request.url_vars["pk_path"])
+    if "pks" in request.url_vars:
+        filename_bits.append(request.url_vars["pks"])
     filename_bits.append(to_css_class(blob_column))
     if blob_hash:
         filename_bits.append(blob_hash[:6])
diff --git a/datasette/views/base.py b/datasette/views/base.py
index e31beb19..0bbf98bb 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -381,7 +381,7 @@ class DataView(BaseView):
             return None
 
     async def get(self, request):
-        db_name = request.url_vars["db_name"]
+        db_name = request.url_vars["database"]
         database = tilde_decode(db_name)
         _format = self.get_format(request)
         data_kwargs = {}
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 48635e01..93bd1011 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -32,7 +32,7 @@ class DatabaseView(DataView):
     name = "database"
 
     async def data(self, request, default_labels=False, _size=None):
-        database = tilde_decode(request.url_vars["db_name"])
+        database = tilde_decode(request.url_vars["database"])
         await self.check_permissions(
             request,
             [
@@ -162,7 +162,7 @@ class DatabaseDownload(DataView):
     name = "database_download"
 
     async def get(self, request):
-        database = tilde_decode(request.url_vars["db_name"])
+        database = tilde_decode(request.url_vars["database"])
         await self.check_permissions(
             request,
             [
@@ -205,7 +205,7 @@ class QueryView(DataView):
         named_parameters=None,
         write=False,
     ):
-        database = tilde_decode(request.url_vars["db_name"])
+        database = tilde_decode(request.url_vars["database"])
         params = {key: request.args.get(key) for key in request.args}
         if "sql" in params:
             params.pop("sql")
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 311a49db..f5e31181 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -19,7 +19,7 @@ class IndexView(BaseView):
     name = "index"
 
     async def get(self, request):
-        as_format = request.url_vars["as_format"]
+        as_format = request.url_vars["format"]
         await self.check_permission(request, "view-instance")
         databases = []
         for name, db in self.ds.databases.items():
diff --git a/datasette/views/special.py b/datasette/views/special.py
index c7b5061f..395ee587 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -15,7 +15,7 @@ class JsonDataView(BaseView):
         self.needs_request = needs_request
 
     async def get(self, request):
-        as_format = request.url_vars["as_format"]
+        as_format = request.url_vars["format"]
         await self.check_permission(request, "view-instance")
         if self.needs_request:
             data = self.data_callback(request)
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 8bdc7417..ea4f24b7 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -272,7 +272,7 @@ class TableView(RowTableShared):
     name = "table"
 
     async def post(self, request):
-        db_name = tilde_decode(request.url_vars["db_name"])
+        db_name = tilde_decode(request.url_vars["database"])
         table = tilde_decode(request.url_vars["table"])
         # Handle POST to a canned query
         canned_query = await self.ds.get_canned_query(db_name, table, request.actor)
@@ -327,7 +327,7 @@ class TableView(RowTableShared):
         _next=None,
         _size=None,
     ):
-        database = tilde_decode(request.url_vars["db_name"])
+        database = tilde_decode(request.url_vars["database"])
         table = tilde_decode(request.url_vars["table"])
         try:
             db = self.ds.databases[database]
@@ -938,7 +938,7 @@ class RowView(RowTableShared):
     name = "row"
 
     async def data(self, request, default_labels=False):
-        database = tilde_decode(request.url_vars["db_name"])
+        database = tilde_decode(request.url_vars["database"])
         table = tilde_decode(request.url_vars["table"])
         await self.check_permissions(
             request,
@@ -948,7 +948,7 @@ class RowView(RowTableShared):
                 "view-instance",
             ],
         )
-        pk_values = urlsafe_components(request.url_vars["pk_path"])
+        pk_values = urlsafe_components(request.url_vars["pks"])
         db = self.ds.databases[database]
         sql, params, pks = await _sql_params_pks(db, table, pk_values)
         results = await db.execute(sql, params, truncate=True)
diff --git a/tests/test_routes.py b/tests/test_routes.py
index 6718c232..349ac302 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -12,26 +12,26 @@ def routes():
 @pytest.mark.parametrize(
     "path,expected_class,expected_matches",
     (
-        ("/", "IndexView", {"as_format": ""}),
-        ("/foo", "DatabaseView", {"as_format": None, "db_name": "foo"}),
-        ("/foo.csv", "DatabaseView", {"as_format": ".csv", "db_name": "foo"}),
-        ("/foo.json", "DatabaseView", {"as_format": ".json", "db_name": "foo"}),
-        ("/foo.humbug", "DatabaseView", {"as_format": None, "db_name": "foo.humbug"}),
-        ("/foo/humbug", "TableView", {"db_name": "foo", "table": "humbug"}),
-        ("/foo/humbug.json", "TableView", {"db_name": "foo", "table": "humbug"}),
-        ("/foo/humbug.blah", "TableView", {"db_name": "foo", "table": "humbug"}),
+        ("/", "IndexView", {"format": ""}),
+        ("/foo", "DatabaseView", {"format": None, "database": "foo"}),
+        ("/foo.csv", "DatabaseView", {"format": ".csv", "database": "foo"}),
+        ("/foo.json", "DatabaseView", {"format": ".json", "database": "foo"}),
+        ("/foo.humbug", "DatabaseView", {"format": None, "database": "foo.humbug"}),
+        ("/foo/humbug", "TableView", {"database": "foo", "table": "humbug"}),
+        ("/foo/humbug.json", "TableView", {"database": "foo", "table": "humbug"}),
+        ("/foo/humbug.blah", "TableView", {"database": "foo", "table": "humbug"}),
         (
             "/foo/humbug/1",
             "RowView",
-            {"as_format": None, "db_name": "foo", "pk_path": "1", "table": "humbug"},
+            {"format": None, "database": "foo", "pks": "1", "table": "humbug"},
         ),
         (
             "/foo/humbug/1.json",
             "RowView",
-            {"as_format": ".json", "db_name": "foo", "pk_path": "1", "table": "humbug"},
+            {"format": ".json", "database": "foo", "pks": "1", "table": "humbug"},
         ),
-        ("/-/metadata.json", "JsonDataView", {"as_format": ".json"}),
-        ("/-/metadata", "JsonDataView", {"as_format": ""}),
+        ("/-/metadata.json", "JsonDataView", {"format": ".json"}),
+        ("/-/metadata", "JsonDataView", {"format": ""}),
     ),
 )
 def test_routes(routes, path, expected_class, expected_matches):

From b9c2b1cfc8692b9700416db98721fa3ec982f6be Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 13:29:10 -0700
Subject: [PATCH 1369/2303] Consistent treatment of format in route capturing,
 refs #1667

Also refs #1660
---
 datasette/app.py     | 30 ++++++++++++------------------
 tests/test_api.py    |  4 ++--
 tests/test_routes.py | 32 ++++++++++++++++++++++----------
 3 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 5259c50c..edef34e9 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -985,10 +985,7 @@ class Datasette:
         def add_route(view, regex):
             routes.append((regex, view))
 
-        # Generate a regex snippet to match all registered renderer file extensions
-        renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
-
-        add_route(IndexView.as_view(self), r"/(?P<format>(\.jsono?)?$)")
+        add_route(IndexView.as_view(self), r"/(\.(?P<format>jsono?))?$")
         # TODO: /favicon.ico and /-/static/ deserve far-future cache expires
         add_route(favicon, "/favicon.ico")
 
@@ -1020,21 +1017,21 @@ class Datasette:
         )
         add_route(
             JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()),
-            r"/-/metadata(?P<format>(\.json)?)$",
+            r"/-/metadata(\.(?P<format>json))?$",
         )
         add_route(
             JsonDataView.as_view(self, "versions.json", self._versions),
-            r"/-/versions(?P<format>(\.json)?)$",
+            r"/-/versions(\.(?P<format>json))?$",
         )
         add_route(
             JsonDataView.as_view(
                 self, "plugins.json", self._plugins, needs_request=True
             ),
-            r"/-/plugins(?P<format>(\.json)?)$",
+            r"/-/plugins(\.(?P<format>json))?$",
         )
         add_route(
             JsonDataView.as_view(self, "settings.json", lambda: self._settings),
-            r"/-/settings(?P<format>(\.json)?)$",
+            r"/-/settings(\.(?P<format>json))?$",
         )
         add_route(
             permanent_redirect("/-/settings.json"),
@@ -1046,15 +1043,15 @@ class Datasette:
         )
         add_route(
             JsonDataView.as_view(self, "threads.json", self._threads),
-            r"/-/threads(?P<format>(\.json)?)$",
+            r"/-/threads(\.(?P<format>json))?$",
         )
         add_route(
             JsonDataView.as_view(self, "databases.json", self._connected_databases),
-            r"/-/databases(?P<format>(\.json)?)$",
+            r"/-/databases(\.(?P<format>json))?$",
         )
         add_route(
             JsonDataView.as_view(self, "actor.json", self._actor, needs_request=True),
-            r"/-/actor(?P<format>(\.json)?)$",
+            r"/-/actor(\.(?P<format>json))?$",
         )
         add_route(
             AuthTokenView.as_view(self),
@@ -1080,20 +1077,17 @@ class Datasette:
             PatternPortfolioView.as_view(self),
             r"/-/patterns$",
         )
-        add_route(DatabaseDownload.as_view(self), r"/(?P<database>[^/]+?)\.db$")
+        add_route(DatabaseDownload.as_view(self), r"/(?P<database>[^\/\.]+)\.db$")
         add_route(
-            DatabaseView.as_view(self),
-            r"/(?P<database>[^/]+?)(?P<format>" + renderer_regex + r"|.jsono|\.csv)?$",
+            DatabaseView.as_view(self), r"/(?P<database>[^\/\.]+)(\.(?P<format>\w+))?$"
         )
         add_route(
             TableView.as_view(self),
-            r"/(?P<database>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$",
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$",
         )
         add_route(
             RowView.as_view(self),
-            r"/(?P<database>[^/]+)/(?P<table>[^/]+?)/(?P<pks>[^/]+?)(?P<format>"
-            + renderer_regex
-            + r")?$",
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^/]+?)/(?P<pks>[^/]+?)(\.(?P<format>\w+))?$",
         )
         return [
             # Compile any strings to regular expressions
diff --git a/tests/test_api.py b/tests/test_api.py
index 421bb1fe..253c1718 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -629,8 +629,8 @@ def test_old_memory_urls_redirect(app_client_no_files, path, expected_redirect):
 
 
 def test_database_page_for_database_with_dot_in_name(app_client_with_dot):
-    response = app_client_with_dot.get("/fixtures.dot.json")
-    assert 200 == response.status
+    response = app_client_with_dot.get("/fixtures~2Edot.json")
+    assert response.status == 200
 
 
 def test_custom_sql(app_client):
diff --git a/tests/test_routes.py b/tests/test_routes.py
index 349ac302..1fa55018 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -12,14 +12,26 @@ def routes():
 @pytest.mark.parametrize(
     "path,expected_class,expected_matches",
     (
-        ("/", "IndexView", {"format": ""}),
+        ("/", "IndexView", {"format": None}),
         ("/foo", "DatabaseView", {"format": None, "database": "foo"}),
-        ("/foo.csv", "DatabaseView", {"format": ".csv", "database": "foo"}),
-        ("/foo.json", "DatabaseView", {"format": ".json", "database": "foo"}),
-        ("/foo.humbug", "DatabaseView", {"format": None, "database": "foo.humbug"}),
-        ("/foo/humbug", "TableView", {"database": "foo", "table": "humbug"}),
-        ("/foo/humbug.json", "TableView", {"database": "foo", "table": "humbug"}),
-        ("/foo/humbug.blah", "TableView", {"database": "foo", "table": "humbug"}),
+        ("/foo.csv", "DatabaseView", {"format": "csv", "database": "foo"}),
+        ("/foo.json", "DatabaseView", {"format": "json", "database": "foo"}),
+        ("/foo.humbug", "DatabaseView", {"format": "humbug", "database": "foo"}),
+        (
+            "/foo/humbug",
+            "TableView",
+            {"database": "foo", "table": "humbug", "format": None},
+        ),
+        (
+            "/foo/humbug.json",
+            "TableView",
+            {"database": "foo", "table": "humbug", "format": "json"},
+        ),
+        (
+            "/foo/humbug.blah",
+            "TableView",
+            {"database": "foo", "table": "humbug", "format": "blah"},
+        ),
         (
             "/foo/humbug/1",
             "RowView",
@@ -28,10 +40,10 @@ def routes():
         (
             "/foo/humbug/1.json",
             "RowView",
-            {"format": ".json", "database": "foo", "pks": "1", "table": "humbug"},
+            {"format": "json", "database": "foo", "pks": "1", "table": "humbug"},
         ),
-        ("/-/metadata.json", "JsonDataView", {"format": ".json"}),
-        ("/-/metadata", "JsonDataView", {"format": ""}),
+        ("/-/metadata.json", "JsonDataView", {"format": "json"}),
+        ("/-/metadata", "JsonDataView", {"format": None}),
     ),
 )
 def test_routes(routes, path, expected_class, expected_matches):

From 798f075ef9b98819fdb564f9f79c78975a0f71e8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 13:32:29 -0700
Subject: [PATCH 1370/2303] Read format from route captures, closes #1667

Refs #1660
---
 datasette/utils/__init__.py | 20 --------------------
 datasette/views/base.py     | 12 +-----------
 tests/test_utils.py         | 25 -------------------------
 3 files changed, 1 insertion(+), 56 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index ccdf8ad4..c89b9d23 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -731,26 +731,6 @@ def module_from_path(path, name):
     return mod
 
 
-async def resolve_table_and_format(
-    table_and_format, table_exists, allowed_formats=None
-):
-    if allowed_formats is None:
-        allowed_formats = []
-    if "." in table_and_format:
-        # Check if a table exists with this exact name
-        it_exists = await table_exists(table_and_format)
-        if it_exists:
-            return table_and_format, None
-
-    # Check if table ends with a known format
-    formats = list(allowed_formats) + ["csv", "jsono"]
-    for _format in formats:
-        if table_and_format.endswith(f".{_format}"):
-            table = table_and_format[: -(len(_format) + 1)]
-            return table, _format
-    return table_and_format, None
-
-
 def path_with_format(
     *, request=None, path=None, format=None, extra_qs=None, replace_format=None
 ):
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 0bbf98bb..24e97d95 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -19,12 +19,10 @@ from datasette.utils import (
     LimitedWriter,
     call_with_supported_arguments,
     tilde_decode,
-    tilde_encode,
     path_from_row_pks,
     path_with_added_args,
     path_with_removed_args,
     path_with_format,
-    resolve_table_and_format,
     sqlite3,
     HASH_LENGTH,
 )
@@ -372,18 +370,10 @@ class DataView(BaseView):
 
         return AsgiStream(stream_fn, headers=headers, content_type=content_type)
 
-    def get_format(self, request):
-        # Format is the bit from the path following the ., if one exists
-        last_path_component = request.path.split("/")[-1]
-        if "." in last_path_component:
-            return last_path_component.split(".")[-1]
-        else:
-            return None
-
     async def get(self, request):
         db_name = request.url_vars["database"]
         database = tilde_decode(db_name)
-        _format = self.get_format(request)
+        _format = request.url_vars["format"]
         data_kwargs = {}
 
         if _format == "csv":
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 790aadc7..7b41a87f 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -351,31 +351,6 @@ def test_compound_keys_after_sql():
     )
 
 
-async def table_exists(table):
-    return table == "exists.csv"
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "table_and_format,expected_table,expected_format",
-    [
-        ("blah", "blah", None),
-        ("blah.csv", "blah", "csv"),
-        ("blah.json", "blah", "json"),
-        ("blah.baz", "blah.baz", None),
-        ("exists.csv", "exists.csv", None),
-    ],
-)
-async def test_resolve_table_and_format(
-    table_and_format, expected_table, expected_format
-):
-    actual_table, actual_format = await utils.resolve_table_and_format(
-        table_and_format, table_exists, ["json"]
-    )
-    assert expected_table == actual_table
-    assert expected_format == actual_format
-
-
 def test_table_columns():
     conn = sqlite3.connect(":memory:")
     conn.executescript(

From 7a6654a253dee243518dc542ce4c06dbb0d0801d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 17:11:17 -0700
Subject: [PATCH 1371/2303] Databases can now have a .route separate from their
 .name, refs #1668

---
 datasette/app.py                  | 13 ++++++--
 datasette/database.py             |  1 +
 datasette/views/base.py           | 12 +++++--
 datasette/views/database.py       | 18 ++++++-----
 datasette/views/table.py          | 29 ++++++++++++-----
 docs/internals.rst                | 11 ++++---
 tests/test_internals_datasette.py |  1 +
 tests/test_routes.py              | 52 ++++++++++++++++++++++++++++++-
 8 files changed, 111 insertions(+), 26 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index edef34e9..5c8101a3 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -388,13 +388,18 @@ class Datasette:
     def unsign(self, signed, namespace="default"):
         return URLSafeSerializer(self._secret, namespace).loads(signed)
 
-    def get_database(self, name=None):
+    def get_database(self, name=None, route=None):
+        if route is not None:
+            matches = [db for db in self.databases.values() if db.route == route]
+            if not matches:
+                raise KeyError
+            return matches[0]
         if name is None:
-            # Return first no-_schemas database
+            # Return first database that isn't "_internal"
             name = [key for key in self.databases.keys() if key != "_internal"][0]
         return self.databases[name]
 
-    def add_database(self, db, name=None):
+    def add_database(self, db, name=None, route=None):
         new_databases = self.databases.copy()
         if name is None:
             # Pick a unique name for this database
@@ -407,6 +412,7 @@ class Datasette:
             name = "{}_{}".format(suggestion, i)
             i += 1
         db.name = name
+        db.route = route or name
         new_databases[name] = db
         # don't mutate! that causes race conditions with live import
         self.databases = new_databases
@@ -693,6 +699,7 @@ class Datasette:
         return [
             {
                 "name": d.name,
+                "route": d.route,
                 "path": d.path,
                 "size": d.size,
                 "is_mutable": d.is_mutable,
diff --git a/datasette/database.py b/datasette/database.py
index 6ce87215..ba594a8c 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -31,6 +31,7 @@ class Database:
         self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None
     ):
         self.name = None
+        self.route = None
         self.ds = ds
         self.path = path
         self.is_mutable = is_mutable
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 24e97d95..afa9eaa6 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -371,13 +371,19 @@ class DataView(BaseView):
         return AsgiStream(stream_fn, headers=headers, content_type=content_type)
 
     async def get(self, request):
-        db_name = request.url_vars["database"]
-        database = tilde_decode(db_name)
+        database_route = tilde_decode(request.url_vars["database"])
+
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
+
         _format = request.url_vars["format"]
         data_kwargs = {}
 
         if _format == "csv":
-            return await self.as_csv(request, database)
+            return await self.as_csv(request, database_route)
 
         if _format is None:
             # HTML views default to expanding all foreign key labels
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 93bd1011..2563c5b2 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -32,7 +32,13 @@ class DatabaseView(DataView):
     name = "database"
 
     async def data(self, request, default_labels=False, _size=None):
-        database = tilde_decode(request.url_vars["database"])
+        database_route = tilde_decode(request.url_vars["database"])
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
+
         await self.check_permissions(
             request,
             [
@@ -50,11 +56,6 @@ class DatabaseView(DataView):
                 request, sql, _size=_size, metadata=metadata
             )
 
-        try:
-            db = self.ds.databases[database]
-        except KeyError:
-            raise NotFound("Database not found: {}".format(database))
-
         table_counts = await db.table_counts(5)
         hidden_table_names = set(await db.hidden_table_names())
         all_foreign_keys = await db.get_all_foreign_keys()
@@ -171,9 +172,10 @@ class DatabaseDownload(DataView):
                 "view-instance",
             ],
         )
-        if database not in self.ds.databases:
+        try:
+            db = self.ds.get_database(route=database)
+        except KeyError:
             raise DatasetteError("Invalid database", status=404)
-        db = self.ds.databases[database]
         if db.is_memory:
             raise DatasetteError("Cannot download in-memory databases", status=404)
         if not self.ds.setting("allow_download") or db.is_mutable:
diff --git a/datasette/views/table.py b/datasette/views/table.py
index ea4f24b7..7fa1da3a 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -272,10 +272,15 @@ class TableView(RowTableShared):
     name = "table"
 
     async def post(self, request):
-        db_name = tilde_decode(request.url_vars["database"])
+        database_route = tilde_decode(request.url_vars["database"])
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
         table = tilde_decode(request.url_vars["table"])
         # Handle POST to a canned query
-        canned_query = await self.ds.get_canned_query(db_name, table, request.actor)
+        canned_query = await self.ds.get_canned_query(database, table, request.actor)
         assert canned_query, "You may only POST to a canned query"
         return await QueryView(self.ds).data(
             request,
@@ -327,12 +332,13 @@ class TableView(RowTableShared):
         _next=None,
         _size=None,
     ):
-        database = tilde_decode(request.url_vars["database"])
+        database_route = tilde_decode(request.url_vars["database"])
         table = tilde_decode(request.url_vars["table"])
         try:
-            db = self.ds.databases[database]
+            db = self.ds.get_database(route=database_route)
         except KeyError:
-            raise NotFound("Database not found: {}".format(database))
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
 
         # If this is a canned query, not a table, then dispatch to QueryView instead
         canned_query = await self.ds.get_canned_query(database, table, request.actor)
@@ -938,8 +944,13 @@ class RowView(RowTableShared):
     name = "row"
 
     async def data(self, request, default_labels=False):
-        database = tilde_decode(request.url_vars["database"])
+        database_route = tilde_decode(request.url_vars["database"])
         table = tilde_decode(request.url_vars["table"])
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
         await self.check_permissions(
             request,
             [
@@ -949,7 +960,11 @@ class RowView(RowTableShared):
             ],
         )
         pk_values = urlsafe_components(request.url_vars["pks"])
-        db = self.ds.databases[database]
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
         sql, params, pks = await _sql_params_pks(db, table, pk_values)
         results = await db.execute(sql, params, truncate=True)
         columns = [r[0] for r in results.description]
diff --git a/docs/internals.rst b/docs/internals.rst
index 117cb95c..323256c7 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -307,14 +307,17 @@ Returns the specified database object. Raises a ``KeyError`` if the database doe
 
 .. _datasette_add_database:
 
-.add_database(db, name=None)
-----------------------------
+.add_database(db, name=None, route=None)
+----------------------------------------
 
 ``db`` - datasette.database.Database instance
     The database to be attached.
 
 ``name`` - string, optional
-    The name to be used for this database - this will be used in the URL path, e.g. ``/dbname``. If not specified Datasette will pick one based on the filename or memory name.
+    The name to be used for this database . If not specified Datasette will pick one based on the filename or memory name.
+
+``route`` - string, optional
+    This will be used in the URL path. If not specified, it will default to the same thing as the ``name``.
 
 The ``datasette.add_database(db)`` method lets you add a new database to the current Datasette instance.
 
@@ -371,7 +374,7 @@ Using either of these pattern will result in the in-memory database being served
 ``name`` - string
     The name of the database to be removed.
 
-This removes a database that has been previously added. ``name=`` is the unique name of that database, used in its URL path.
+This removes a database that has been previously added. ``name=`` is the unique name of that database.
 
 .. _datasette_sign:
 
diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py
index adf84be9..cc200a2d 100644
--- a/tests/test_internals_datasette.py
+++ b/tests/test_internals_datasette.py
@@ -55,6 +55,7 @@ async def test_datasette_constructor():
     assert databases == [
         {
             "name": "_memory",
+            "route": "_memory",
             "path": None,
             "size": 0,
             "is_mutable": False,
diff --git a/tests/test_routes.py b/tests/test_routes.py
index 1fa55018..dd3bc644 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -1,6 +1,7 @@
-from datasette.app import Datasette
+from datasette.app import Datasette, Database
 from datasette.utils import resolve_routes
 import pytest
+import pytest_asyncio
 
 
 @pytest.fixture(scope="session")
@@ -53,3 +54,52 @@ def test_routes(routes, path, expected_class, expected_matches):
     else:
         assert view.view_class.__name__ == expected_class
         assert match.groupdict() == expected_matches
+
+
+@pytest_asyncio.fixture
+async def ds_with_route():
+    ds = Datasette()
+    ds.remove_database("_memory")
+    db = Database(ds, is_memory=True, memory_name="route-name-db")
+    ds.add_database(db, name="name", route="route-name")
+    await db.execute_write_script(
+        """
+        create table if not exists t (id integer primary key);
+        insert or replace into t (id) values (1);
+    """
+    )
+    return ds
+
+
+@pytest.mark.asyncio
+async def test_db_with_route_databases(ds_with_route):
+    response = await ds_with_route.client.get("/-/databases.json")
+    assert response.json()[0] == {
+        "name": "name",
+        "route": "route-name",
+        "path": None,
+        "size": 0,
+        "is_mutable": True,
+        "is_memory": True,
+        "hash": None,
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "path,expected_status",
+    (
+        ("/", 200),
+        ("/name", 404),
+        ("/name/t", 404),
+        ("/name/t/1", 404),
+        ("/route-name", 200),
+        ("/route-name/t", 200),
+        ("/route-name/t/1", 200),
+    ),
+)
+async def test_db_with_route_that_does_not_match_name(
+    ds_with_route, path, expected_status
+):
+    response = await ds_with_route.client.get(path)
+    assert response.status_code == expected_status

From e10da9af3595c0a4e09c6f370103571aa4ea106e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 17:21:56 -0700
Subject: [PATCH 1372/2303] alternative-route demo, refs #1668

---
 .github/workflows/deploy-latest.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index 1ae96e89..92aa1c6b 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -42,6 +42,17 @@ jobs:
         sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
+    - name: Set up the alternate-route demo
+      run: |
+        echo '
+        from datasette import hookimpl
+
+        @hookimpl
+        def startup(datasette):
+            db = datasette.get_database("fixtures2")
+            db.route = "alternative-route"
+        ' > plugins/alternative_route.py
+        cp fixtures.db fixtures2.db
     - name: Set up Cloud Run
       uses: google-github-actions/setup-gcloud@master
       with:
@@ -54,7 +65,7 @@ jobs:
         gcloud config set project datasette-222320
         export SUFFIX="-${GITHUB_REF#refs/heads/}"
         export SUFFIX=${SUFFIX#-main}
-        datasette publish cloudrun fixtures.db extra_database.db \
+        datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \
             -m fixtures.json \
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \

From cdbae2b93f441653616dd889644c63e4150ceec1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 17:31:23 -0700
Subject: [PATCH 1373/2303] Fixed internal links to respect db.route, refs
 #1668

---
 datasette/url_builder.py |  3 ++-
 datasette/views/table.py |  5 ++---
 tests/test_routes.py     | 22 +++++++++++++---------
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/datasette/url_builder.py b/datasette/url_builder.py
index 498ec85d..574bf3c1 100644
--- a/datasette/url_builder.py
+++ b/datasette/url_builder.py
@@ -28,7 +28,8 @@ class Urls:
         return self.path("-/logout")
 
     def database(self, database, format=None):
-        return self.path(tilde_encode(database), format=format)
+        db = self.ds.get_database(database)
+        return self.path(tilde_encode(db.route), format=format)
 
     def table(self, database, table, format=None):
         path = f"{self.database(database)}/{tilde_encode(table)}"
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 7fa1da3a..8745c28a 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -141,10 +141,9 @@ class RowTableShared(DataView):
                         "is_special_link_column": is_special_link_column,
                         "raw": pk_path,
                         "value": markupsafe.Markup(
-                            '<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format(
+                            '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format(
                                 base_url=base_url,
-                                database=database,
-                                table=tilde_encode(table),
+                                table_path=self.ds.urls.table(database, table),
                                 flat_pks=str(markupsafe.escape(pk_path)),
                                 flat_pks_quoted=path_from_row_pks(row, pks, not pks),
                             )
diff --git a/tests/test_routes.py b/tests/test_routes.py
index dd3bc644..211b77b5 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -61,7 +61,7 @@ async def ds_with_route():
     ds = Datasette()
     ds.remove_database("_memory")
     db = Database(ds, is_memory=True, memory_name="route-name-db")
-    ds.add_database(db, name="name", route="route-name")
+    ds.add_database(db, name="original-name", route="custom-route-name")
     await db.execute_write_script(
         """
         create table if not exists t (id integer primary key);
@@ -75,8 +75,8 @@ async def ds_with_route():
 async def test_db_with_route_databases(ds_with_route):
     response = await ds_with_route.client.get("/-/databases.json")
     assert response.json()[0] == {
-        "name": "name",
-        "route": "route-name",
+        "name": "original-name",
+        "route": "custom-route-name",
         "path": None,
         "size": 0,
         "is_mutable": True,
@@ -90,12 +90,12 @@ async def test_db_with_route_databases(ds_with_route):
     "path,expected_status",
     (
         ("/", 200),
-        ("/name", 404),
-        ("/name/t", 404),
-        ("/name/t/1", 404),
-        ("/route-name", 200),
-        ("/route-name/t", 200),
-        ("/route-name/t/1", 200),
+        ("/original-name", 404),
+        ("/original-name/t", 404),
+        ("/original-name/t/1", 404),
+        ("/custom-route-name", 200),
+        ("/custom-route-name/t", 200),
+        ("/custom-route-name/t/1", 200),
     ),
 )
 async def test_db_with_route_that_does_not_match_name(
@@ -103,3 +103,7 @@ async def test_db_with_route_that_does_not_match_name(
 ):
     response = await ds_with_route.client.get(path)
     assert response.status_code == expected_status
+    # There should be links to custom-route-name but none to original-name
+    if response.status_code == 200:
+        assert "/custom-route-name" in response.text
+        assert "/original-name" not in response.text

From 5471e3c4914837de957e206d8fb80c9ec383bc2e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 18:14:40 -0700
Subject: [PATCH 1374/2303] Release 0.61a0

Refs #957, #1533, #1545, #1576, #1577, #1587, #1601, #1603, #1607, #1612, #1621, #1649, #1654, #1657, #1661, #1668
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 29 +++++++++++++++++++++++++++--
 docs/performance.rst |  2 ++
 3 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index 91224615..ccc1e04b 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.60.2"
+__version__ = "0.61a0"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index c58c8444..0f3d3aff 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,14 +4,39 @@
 Changelog
 =========
 
-.. _v0_60.2:
+.. _v0_61_a0:
+
+0.61a0 (2022-03-19)
+-------------------
+
+- Removed hashed URL mode from Datasette. The new ``datasette-hashed-urls`` plugin can be used to achieve the same result, see :ref:`performance_hashed_urls` for details. (:issue:`1661`)
+- Databases can now have a custom path within the Datasette instance that is indpendent of the database name, using the ``db.route`` property. (:issue:`1668`)
+- URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`)
+- Table and row HTML pages now include a ``<link rel="alternate" type="application/json+datasette" href="...">`` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`)
+- ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header.
+- Canned queries are now shown at the top of the database page, directly below the SQL editor. Previously they were shown at the bottom, below the list of tables. (:issue:`1612`)
+- Datasette now has a default favicon. (:issue:`1603`)
+- ``sqlite_stat`` tables are now hidden by default. (:issue:`1587`)
+- SpatiaLite tables ``data_licenses``, ``KNN`` and ``KNN2`` are now hidden by default. (:issue:`1601`)
+- Python 3.6 is no longer supported. (:issue:`1577`)
+- Tests now run against Python 3.11-dev. (:issue:`1621`)
+- Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`)
+- SQL query tracing mechanism now works for queries executed in ``asyncio`` sub-tasks, such as those created by ``asyncio.gather()``. (:issue:`1576`)
+- :ref:`internals_tracer` mechanism is now documented.
+- Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`)
+- ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`)
+- Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__.
+- Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__)
+- Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`)
+
+.. _v0_60_2:
 
 0.60.2 (2022-02-07)
 -------------------
 
 - Fixed a bug where Datasette would open the same file twice with two different database names if you ran ``datasette file.db file.db``. (:issue:`1632`)
 
-.. _v0_60.1:
+.. _v0_60_1:
 
 0.60.1 (2022-01-20)
 -------------------
diff --git a/docs/performance.rst b/docs/performance.rst
index d37f1804..89bbf5ae 100644
--- a/docs/performance.rst
+++ b/docs/performance.rst
@@ -60,6 +60,8 @@ The :ref:`setting_default_cache_ttl` setting sets the default HTTP cache TTL for
 
 You can also change the cache timeout on a per-request basis using the ``?_ttl=10`` query string parameter. This can be useful when you are working with the Datasette JSON API - you may decide that a specific query can be cached for a longer time, or maybe you need to set ``?_ttl=0`` for some requests for example if you are running a SQL ``order by random()`` query.
 
+.. _performance_hashed_urls:
+
 datasette-hashed-urls
 ---------------------
 

From cb4854a435cc1418665edec2a73664ad74a32017 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 18:17:58 -0700
Subject: [PATCH 1375/2303] Fixed typo

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 0f3d3aff..9f5a143c 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -10,7 +10,7 @@ Changelog
 -------------------
 
 - Removed hashed URL mode from Datasette. The new ``datasette-hashed-urls`` plugin can be used to achieve the same result, see :ref:`performance_hashed_urls` for details. (:issue:`1661`)
-- Databases can now have a custom path within the Datasette instance that is indpendent of the database name, using the ``db.route`` property. (:issue:`1668`)
+- Databases can now have a custom path within the Datasette instance that is independent of the database name, using the ``db.route`` property. (:issue:`1668`)
 - URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`)
 - Table and row HTML pages now include a ``<link rel="alternate" type="application/json+datasette" href="...">`` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`)
 - ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header.

From 4a4164b81191dec35e423486a208b05a9edc65e4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 19 Mar 2022 18:23:03 -0700
Subject: [PATCH 1376/2303] Added another note to the 0.61a0 release notes,
 refs #1228

---
 docs/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9f5a143c..05ad85f2 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -28,6 +28,7 @@ Changelog
 - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__.
 - Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__)
 - Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`)
+- Fixed error caused when a table had a column named ``n``. (:issue:`1228`)
 
 .. _v0_60_2:
 

From e627510b760198ccedba9e5af47a771e847785c9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 21 Mar 2022 10:13:16 -0700
Subject: [PATCH 1377/2303] BaseView.check_permissions is now
 datasette.ensure_permissions, closes #1675

Refs #1660
---
 datasette/app.py            | 35 +++++++++++++++++++++++++++++++++++
 datasette/views/base.py     | 26 --------------------------
 datasette/views/database.py | 12 ++++++------
 datasette/views/table.py    |  8 ++++----
 docs/internals.rst          | 26 ++++++++++++++++++++++++++
 5 files changed, 71 insertions(+), 36 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 5c8101a3..9e509e96 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1,4 +1,5 @@
 import asyncio
+from typing import Sequence, Union, Tuple
 import asgi_csrf
 import collections
 import datetime
@@ -628,6 +629,40 @@ class Datasette:
         )
         return result
 
+    async def ensure_permissions(
+        self,
+        actor: dict,
+        permissions: Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]],
+    ):
+        """
+        permissions is a list of (action, resource) tuples or 'action' strings
+
+        Raises datasette.Forbidden() if any of the checks fail
+        """
+        for permission in permissions:
+            if isinstance(permission, str):
+                action = permission
+                resource = None
+            elif isinstance(permission, (tuple, list)) and len(permission) == 2:
+                action, resource = permission
+            else:
+                assert (
+                    False
+                ), "permission should be string or tuple of two items: {}".format(
+                    repr(permission)
+                )
+            ok = await self.permission_allowed(
+                actor,
+                action,
+                resource=resource,
+                default=None,
+            )
+            if ok is not None:
+                if ok:
+                    return
+                else:
+                    raise Forbidden(action)
+
     async def execute(
         self,
         db_name,
diff --git a/datasette/views/base.py b/datasette/views/base.py
index afa9eaa6..d1e684a2 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -76,32 +76,6 @@ class BaseView:
         if not ok:
             raise Forbidden(action)
 
-    async def check_permissions(self, request, permissions):
-        """permissions is a list of (action, resource) tuples or 'action' strings"""
-        for permission in permissions:
-            if isinstance(permission, str):
-                action = permission
-                resource = None
-            elif isinstance(permission, (tuple, list)) and len(permission) == 2:
-                action, resource = permission
-            else:
-                assert (
-                    False
-                ), "permission should be string or tuple of two items: {}".format(
-                    repr(permission)
-                )
-            ok = await self.ds.permission_allowed(
-                request.actor,
-                action,
-                resource=resource,
-                default=None,
-            )
-            if ok is not None:
-                if ok:
-                    return
-                else:
-                    raise Forbidden(action)
-
     def database_color(self, database):
         return "ff0000"
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 2563c5b2..69ed1233 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -39,8 +39,8 @@ class DatabaseView(DataView):
             raise NotFound("Database not found: {}".format(database_route))
         database = db.name
 
-        await self.check_permissions(
-            request,
+        await self.ds.ensure_permissions(
+            request.actor,
             [
                 ("view-database", database),
                 "view-instance",
@@ -164,8 +164,8 @@ class DatabaseDownload(DataView):
 
     async def get(self, request):
         database = tilde_decode(request.url_vars["database"])
-        await self.check_permissions(
-            request,
+        await self.ds.ensure_permissions(
+            request.actor,
             [
                 ("view-database-download", database),
                 ("view-database", database),
@@ -217,8 +217,8 @@ class QueryView(DataView):
         private = False
         if canned_query:
             # Respect canned query permissions
-            await self.check_permissions(
-                request,
+            await self.ds.ensure_permissions(
+                request.actor,
                 [
                     ("view-query", (database, canned_query)),
                     ("view-database", database),
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 8745c28a..84169820 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -360,8 +360,8 @@ class TableView(RowTableShared):
             raise NotFound(f"Table not found: {table}")
 
         # Ensure user has permission to view this table
-        await self.check_permissions(
-            request,
+        await self.ds.ensure_permissions(
+            request.actor,
             [
                 ("view-table", (database, table)),
                 ("view-database", database),
@@ -950,8 +950,8 @@ class RowView(RowTableShared):
         except KeyError:
             raise NotFound("Database not found: {}".format(database_route))
         database = db.name
-        await self.check_permissions(
-            request,
+        await self.ds.ensure_permissions(
+            request.actor,
             [
                 ("view-table", (database, table)),
                 ("view-database", database),
diff --git a/docs/internals.rst b/docs/internals.rst
index 323256c7..12adde00 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -295,6 +295,32 @@ If neither ``metadata.json`` nor any of the plugins provide an answer to the per
 
 See :ref:`permissions` for a full list of permission actions included in Datasette core.
 
+.. _datasette_permission_allowed:
+
+await .ensure_permissions(actor, permissions)
+---------------------------------------------
+
+``actor`` - dictionary
+    The authenticated actor. This is usually ``request.actor``.
+
+``permissions`` - list
+    A list of permissions to check. Each permission in that list can be a string ``action`` name or a 2-tuple of ``(action, resource)``.
+
+This method allows multiple permissions to be checked at onced. It raises a ``datasette.Forbidden`` exception if any of the checks are denied before one of them is explicitly granted.
+
+This is useful when you need to check multiple permissions at once. For example, an actor should be able to view a table if either one of the following checks returns ``True`` or not a single one of them returns ``False``:
+
+.. code-block:: python
+
+    await self.ds.ensure_permissions(
+        request.actor,
+        [
+            ("view-table", (database, table)),
+            ("view-database", database),
+            "view-instance",
+        ]
+    )
+
 .. _datasette_get_database:
 
 .get_database(name)

From dfafce6d962d615d98a7080e546c7b3662ae7d34 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 21 Mar 2022 11:37:27 -0700
Subject: [PATCH 1378/2303] Display no-opinion permission checks on
 /-/permissions

---
 datasette/templates/permissions_debug.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html
index d898ea8c..db709c14 100644
--- a/datasette/templates/permissions_debug.html
+++ b/datasette/templates/permissions_debug.html
@@ -10,6 +10,9 @@
 .check-result-false {
     color: red;
 }
+.check-result-no-opinion {
+    color: #aaa;
+}
 .check h2 {
     font-size: 1em
 }
@@ -38,6 +41,8 @@
             <span class="check-when">{{ check.when }}</span>
             {% if check.result %}
                 <span class="check-result check-result-true">✓</span>
+            {% elif check.result is none %}
+                <span class="check-result check-result-no-opinion">none</span>
             {% else %}
                 <span class="check-result check-result-false">✗</span>
             {% endif %}

From 194e4f6c3fffde69eb196f8535ca45386b40ec2d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 21 Mar 2022 11:41:56 -0700
Subject: [PATCH 1379/2303] Removed check_permission() from BaseView, closes
 #1677

Refs #1660
---
 datasette/app.py            |  1 +
 datasette/views/base.py     | 10 ----------
 datasette/views/database.py |  2 +-
 datasette/views/index.py    |  2 +-
 datasette/views/special.py  | 10 +++++-----
 tests/test_permissions.py   | 13 ++++++++-----
 6 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 9e509e96..22ae211f 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -639,6 +639,7 @@ class Datasette:
 
         Raises datasette.Forbidden() if any of the checks fail
         """
+        assert actor is None or isinstance(actor, dict)
         for permission in permissions:
             if isinstance(permission, str):
                 action = permission
diff --git a/datasette/views/base.py b/datasette/views/base.py
index d1e684a2..221e1882 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -66,16 +66,6 @@ class BaseView:
         response.body = b""
         return response
 
-    async def check_permission(self, request, action, resource=None):
-        ok = await self.ds.permission_allowed(
-            request.actor,
-            action,
-            resource=resource,
-            default=True,
-        )
-        if not ok:
-            raise Forbidden(action)
-
     def database_color(self, database):
         return "ff0000"
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 69ed1233..31a1839f 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -229,7 +229,7 @@ class QueryView(DataView):
                 None, "view-query", (database, canned_query), default=True
             )
         else:
-            await self.check_permission(request, "execute-sql", database)
+            await self.ds.ensure_permissions(request.actor, [("execute-sql", database)])
 
         # Extract any :named parameters
         named_parameters = named_parameters or await derive_named_parameters(
diff --git a/datasette/views/index.py b/datasette/views/index.py
index f5e31181..1c391e26 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -20,7 +20,7 @@ class IndexView(BaseView):
 
     async def get(self, request):
         as_format = request.url_vars["format"]
-        await self.check_permission(request, "view-instance")
+        await self.ds.ensure_permissions(request.actor, ["view-instance"])
         databases = []
         for name, db in self.ds.databases.items():
             visible, database_private = await check_visibility(
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 395ee587..dd834528 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -16,7 +16,7 @@ class JsonDataView(BaseView):
 
     async def get(self, request):
         as_format = request.url_vars["format"]
-        await self.check_permission(request, "view-instance")
+        await self.ds.ensure_permissions(request.actor, ["view-instance"])
         if self.needs_request:
             data = self.data_callback(request)
         else:
@@ -47,7 +47,7 @@ class PatternPortfolioView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
-        await self.check_permission(request, "view-instance")
+        await self.ds.ensure_permissions(request.actor, ["view-instance"])
         return await self.render(["patterns.html"], request=request)
 
 
@@ -95,7 +95,7 @@ class PermissionsDebugView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
-        await self.check_permission(request, "view-instance")
+        await self.ds.ensure_permissions(request.actor, ["view-instance"])
         if not await self.ds.permission_allowed(request.actor, "permissions-debug"):
             raise Forbidden("Permission denied")
         return await self.render(
@@ -146,11 +146,11 @@ class MessagesDebugView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
-        await self.check_permission(request, "view-instance")
+        await self.ds.ensure_permissions(request.actor, ["view-instance"])
         return await self.render(["messages_debug.html"], request)
 
     async def post(self, request):
-        await self.check_permission(request, "view-instance")
+        await self.ds.ensure_permissions(request.actor, ["view-instance"])
         post = await request.post_vars()
         message = post.get("message", "")
         message_type = post.get("message_type") or "INFO"
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 788523b0..f4169dbe 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -321,17 +321,20 @@ def test_permissions_debug(app_client):
     checks = [
         {
             "action": div.select_one(".check-action").text,
-            "result": bool(div.select(".check-result-true")),
+            # True = green tick, False = red cross, None = gray None
+            "result": None
+            if div.select(".check-result-no-opinion")
+            else bool(div.select(".check-result-true")),
             "used_default": bool(div.select(".check-used-default")),
         }
         for div in check_divs
     ]
-    assert [
+    assert checks == [
         {"action": "permissions-debug", "result": True, "used_default": False},
-        {"action": "view-instance", "result": True, "used_default": True},
+        {"action": "view-instance", "result": None, "used_default": True},
         {"action": "permissions-debug", "result": False, "used_default": True},
-        {"action": "view-instance", "result": True, "used_default": True},
-    ] == checks
+        {"action": "view-instance", "result": None, "used_default": True},
+    ]
 
 
 @pytest.mark.parametrize(

From 1a7750eb29fd15dd2eea3b9f6e33028ce441b143 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 21 Mar 2022 12:01:37 -0700
Subject: [PATCH 1380/2303] Documented datasette.check_visibility() method,
 closes #1678

---
 datasette/app.py            | 18 ++++++++++++++++++
 datasette/utils/__init__.py | 19 -------------------
 datasette/views/database.py | 10 +++-------
 datasette/views/index.py    | 11 ++++-------
 docs/internals.rst          | 28 +++++++++++++++++++++++++++-
 5 files changed, 52 insertions(+), 34 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 22ae211f..c9eede26 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -664,6 +664,24 @@ class Datasette:
                 else:
                     raise Forbidden(action)
 
+    async def check_visibility(self, actor, action, resource):
+        """Returns (visible, private) - visible = can you see it, private = can others see it too"""
+        visible = await self.permission_allowed(
+            actor,
+            action,
+            resource=resource,
+            default=True,
+        )
+        if not visible:
+            return False, False
+        private = not await self.permission_allowed(
+            None,
+            action,
+            resource=resource,
+            default=True,
+        )
+        return visible, private
+
     async def execute(
         self,
         db_name,
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index c89b9d23..cd8e3d61 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1002,25 +1002,6 @@ def actor_matches_allow(actor, allow):
     return False
 
 
-async def check_visibility(datasette, actor, action, resource, default=True):
-    """Returns (visible, private) - visible = can you see it, private = can others see it too"""
-    visible = await datasette.permission_allowed(
-        actor,
-        action,
-        resource=resource,
-        default=default,
-    )
-    if not visible:
-        return False, False
-    private = not await datasette.permission_allowed(
-        None,
-        action,
-        resource=resource,
-        default=default,
-    )
-    return visible, private
-
-
 def resolve_env_secrets(config, environ):
     """Create copy that recursively replaces {"$env": "NAME"} with values from environ"""
     if isinstance(config, dict):
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 31a1839f..103bd575 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -10,7 +10,6 @@ import markupsafe
 from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
-    check_visibility,
     derive_named_parameters,
     tilde_decode,
     to_css_class,
@@ -62,8 +61,7 @@ class DatabaseView(DataView):
 
         views = []
         for view_name in await db.view_names():
-            visible, private = await check_visibility(
-                self.ds,
+            visible, private = await self.ds.check_visibility(
                 request.actor,
                 "view-table",
                 (database, view_name),
@@ -78,8 +76,7 @@ class DatabaseView(DataView):
 
         tables = []
         for table in table_counts:
-            visible, private = await check_visibility(
-                self.ds,
+            visible, private = await self.ds.check_visibility(
                 request.actor,
                 "view-table",
                 (database, table),
@@ -105,8 +102,7 @@ class DatabaseView(DataView):
         for query in (
             await self.ds.get_canned_queries(database, request.actor)
         ).values():
-            visible, private = await check_visibility(
-                self.ds,
+            visible, private = await self.ds.check_visibility(
                 request.actor,
                 "view-query",
                 (database, query["name"]),
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 1c391e26..aec78814 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -1,7 +1,7 @@
 import hashlib
 import json
 
-from datasette.utils import add_cors_headers, check_visibility, CustomJSONEncoder
+from datasette.utils import add_cors_headers, CustomJSONEncoder
 from datasette.utils.asgi import Response
 from datasette.version import __version__
 
@@ -23,8 +23,7 @@ class IndexView(BaseView):
         await self.ds.ensure_permissions(request.actor, ["view-instance"])
         databases = []
         for name, db in self.ds.databases.items():
-            visible, database_private = await check_visibility(
-                self.ds,
+            visible, database_private = await self.ds.check_visibility(
                 request.actor,
                 "view-database",
                 name,
@@ -36,8 +35,7 @@ class IndexView(BaseView):
 
             views = []
             for view_name in await db.view_names():
-                visible, private = await check_visibility(
-                    self.ds,
+                visible, private = await self.ds.check_visibility(
                     request.actor,
                     "view-table",
                     (name, view_name),
@@ -55,8 +53,7 @@ class IndexView(BaseView):
 
             tables = {}
             for table in table_names:
-                visible, private = await check_visibility(
-                    self.ds,
+                visible, private = await self.ds.check_visibility(
                     request.actor,
                     "view-table",
                     (name, table),
diff --git a/docs/internals.rst b/docs/internals.rst
index 12adde00..f9a24fea 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -295,7 +295,7 @@ If neither ``metadata.json`` nor any of the plugins provide an answer to the per
 
 See :ref:`permissions` for a full list of permission actions included in Datasette core.
 
-.. _datasette_permission_allowed:
+.. _datasette_ensure_permissions:
 
 await .ensure_permissions(actor, permissions)
 ---------------------------------------------
@@ -321,6 +321,32 @@ This is useful when you need to check multiple permissions at once. For example,
         ]
     )
 
+.. _datasette_check_visibilty:
+
+await .check_visibility(actor, action, resource=None)
+-----------------------------------------------------
+
+``actor`` - dictionary
+    The authenticated actor. This is usually ``request.actor``.
+
+``action`` - string
+    The name of the action that is being permission checked.
+
+``resource`` - string or tuple, optional
+    The resource, e.g. the name of the database, or a tuple of two strings containing the name of the database and the name of the table. Only some permissions apply to a resource.
+
+This convenience method can be used to answer the question "should this item be considered private, in that it is visible to me but it is not visible to anonymous users?"
+
+It returns a tuple of two booleans, ``(visible, private)``. ``visible`` indicates if the actor can see this resource. ``private`` will be ``True`` if an anonymous user would not be able to view the resource.
+
+This example checks if the user can access a specific table, and sets ``private`` so that a padlock icon can later be displayed:
+
+.. code-block:: python
+
+    visible, private = await self.ds.check_visibility(
+        request.actor, "view-table", (database, table)
+    )
+
 .. _datasette_get_database:
 
 .get_database(name)

From 72bfd75fb7241893c931348e6aca712edc67ab04 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 21 Mar 2022 14:55:50 -0700
Subject: [PATCH 1381/2303] Drop n=1 threshold down to <= 20ms, closes #1679

---
 datasette/utils/__init__.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index cd8e3d61..9109f823 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -182,15 +182,16 @@ class CustomJSONEncoder(json.JSONEncoder):
 def sqlite_timelimit(conn, ms):
     deadline = time.perf_counter() + (ms / 1000)
     # n is the number of SQLite virtual machine instructions that will be
-    # executed between each check. It's hard to know what to pick here.
-    # After some experimentation, I've decided to go with 1000 by default and
-    # 1 for time limits that are less than 50ms
+    # executed between each check. It takes about 0.08ms to execute 1000.
+    # https://github.com/simonw/datasette/issues/1679
     n = 1000
-    if ms < 50:
+    if ms <= 20:
+        # This mainly happens while executing our test suite
         n = 1
 
     def handler():
         if time.perf_counter() >= deadline:
+            # Returning 1 terminates the query with an error
             return 1
 
     conn.set_progress_handler(handler, n)

From 12f3ca79956ed9003c874f67748432adcacc6fd2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 21 Mar 2022 18:42:03 -0700
Subject: [PATCH 1382/2303] google-github-actions/setup-gcloud@v0

---
 .github/workflows/deploy-latest.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index 92aa1c6b..a61f6629 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -14,7 +14,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: "3.10"
     - uses: actions/cache@v2
       name: Configure pip caching
       with:
@@ -54,7 +54,7 @@ jobs:
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
     - name: Set up Cloud Run
-      uses: google-github-actions/setup-gcloud@master
+      uses: google-github-actions/setup-gcloud@v0
       with:
         version: '275.0.0'
         service_account_email: ${{ secrets.GCP_SA_EMAIL }}

From c4c9dbd0386e46d2bf199f0ed34e4895c98cb78c Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 22 Mar 2022 09:49:26 -0700
Subject: [PATCH 1383/2303] google-github-actions/setup-gcloud@v0

---
 .github/workflows/publish.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 3cfc67da..3e4f8146 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -85,7 +85,7 @@ jobs:
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up Cloud Run
-      uses: google-github-actions/setup-gcloud@master
+      uses: google-github-actions/setup-gcloud@v0
       with:
         version: '275.0.0'
         service_account_email: ${{ secrets.GCP_SA_EMAIL }}

From d7c793d7998388d915f8d270079c68a77a785051 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 23 Mar 2022 11:12:26 -0700
Subject: [PATCH 1384/2303] Release 0.61

Refs #957, #1228, #1533, #1545, #1576, #1577, #1587, #1601, #1603, #1607, #1612, #1621, #1649, #1654, #1657, #1661, #1668, #1675, #1678
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 22 ++++++++++++++--------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index ccc1e04b..f9b10696 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.61a0"
+__version__ = "0.61"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 05ad85f2..d2de8da1 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,30 +4,36 @@
 Changelog
 =========
 
-.. _v0_61_a0:
+.. _v0_61:
 
-0.61a0 (2022-03-19)
--------------------
+0.61 (2022-03-23)
+-----------------
 
+In preparation for Datasette 1.0, this release includes two potentially backwards-incompatible changes. Hashed URL mode has been moved to a separate plugin, and the way Datasette generates URLs to databases and tables with special characters in their name such as ``/`` and ``.`` has changed.
+
+Datasette also now requires Python 3.7 or higher.
+
+- URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`)
 - Removed hashed URL mode from Datasette. The new ``datasette-hashed-urls`` plugin can be used to achieve the same result, see :ref:`performance_hashed_urls` for details. (:issue:`1661`)
 - Databases can now have a custom path within the Datasette instance that is independent of the database name, using the ``db.route`` property. (:issue:`1668`)
-- URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`)
+- Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`)
+- Python 3.6 is no longer supported. (:issue:`1577`)
+- Tests now run against Python 3.11-dev. (:issue:`1621`)
+- New :ref:`datasette.ensure_permissions(actor, permissions) <datasette_ensure_permissions>` internal method for checking multiple permissions at once. (:issue:`1675`)
+- New :ref:`datasette.check_visibility(actor, action, resource=None) <datasette_check_visibilty>` internal method for checking if a user can see a resource that would otherwise be invisible to unauthenticated users. (:issue:`1678`)
 - Table and row HTML pages now include a ``<link rel="alternate" type="application/json+datasette" href="...">`` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`)
 - ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header.
 - Canned queries are now shown at the top of the database page, directly below the SQL editor. Previously they were shown at the bottom, below the list of tables. (:issue:`1612`)
 - Datasette now has a default favicon. (:issue:`1603`)
 - ``sqlite_stat`` tables are now hidden by default. (:issue:`1587`)
 - SpatiaLite tables ``data_licenses``, ``KNN`` and ``KNN2`` are now hidden by default. (:issue:`1601`)
-- Python 3.6 is no longer supported. (:issue:`1577`)
-- Tests now run against Python 3.11-dev. (:issue:`1621`)
-- Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`)
 - SQL query tracing mechanism now works for queries executed in ``asyncio`` sub-tasks, such as those created by ``asyncio.gather()``. (:issue:`1576`)
 - :ref:`internals_tracer` mechanism is now documented.
 - Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`)
 - ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`)
 - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__.
 - Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__)
-- Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`)
+- Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`)
 - Fixed error caused when a table had a column named ``n``. (:issue:`1228`)
 
 .. _v0_60_2:

From 0159662ab8ccb363c59647861360e0cb7a6f930d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 23 Mar 2022 11:48:10 -0700
Subject: [PATCH 1385/2303] Fix for bug running ?sql= against databases with a
 different route, closes #1682

---
 datasette/views/database.py | 7 ++++++-
 tests/test_routes.py        | 1 +
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index 103bd575..bdd433cc 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -203,7 +203,12 @@ class QueryView(DataView):
         named_parameters=None,
         write=False,
     ):
-        database = tilde_decode(request.url_vars["database"])
+        database_route = tilde_decode(request.url_vars["database"])
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
         params = {key: request.args.get(key) for key in request.args}
         if "sql" in params:
             params.pop("sql")
diff --git a/tests/test_routes.py b/tests/test_routes.py
index 211b77b5..5ae55d21 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -94,6 +94,7 @@ async def test_db_with_route_databases(ds_with_route):
         ("/original-name/t", 404),
         ("/original-name/t/1", 404),
         ("/custom-route-name", 200),
+        ("/custom-route-name?sql=select+id+from+t", 200),
         ("/custom-route-name/t", 200),
         ("/custom-route-name/t/1", 200),
     ),

From d431a9055e977aefe48689a2e5866ea8d3558a6c Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 23 Mar 2022 11:54:10 -0700
Subject: [PATCH 1386/2303] Release 0.61.1

Refs #1682

Refs https://github.com/simonw/datasette-hashed-urls/issues/13
---
 datasette/version.py | 2 +-
 docs/changelog.rst   | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index f9b10696..02451a1e 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.61"
+__version__ = "0.61.1"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index d2de8da1..03cf62b6 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,13 @@
 Changelog
 =========
 
+.. _v0_61_1:
+
+0.61.1 (2022-03-23)
+-------------------
+
+- Fixed a bug where databases with a different route from their name (as used by the `datasette-hashed-urls plugin <https://datasette.io/plugins/datasette-hashed-urls>`__) returned errors when executing custom SQL queries. (:issue:`1682`)
+
 .. _v0_61:
 
 0.61 (2022-03-23)

From c496f2b663ff0cef908ffaaa68b8cb63111fb5f2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 24 Mar 2022 12:16:19 -0700
Subject: [PATCH 1387/2303] Don't show facet in cog menu if not allow_facet,
 closes #1683

---
 datasette/static/table.js      | 10 ++++++++--
 datasette/templates/table.html |  1 +
 datasette/views/table.py       |  3 +++
 tests/test_table_html.py       | 14 ++++++++++++++
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/datasette/static/table.js b/datasette/static/table.js
index 3c88cc40..096a27ac 100644
--- a/datasette/static/table.js
+++ b/datasette/static/table.js
@@ -128,7 +128,8 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig
     } else {
       hideColumn.parentNode.style.display = "none";
     }
-    /* Only show facet if it's not the first column, not selected, not a single PK */
+    /* Only show "Facet by this" if it's not the first column, not selected,
+       not a single PK and the Datasette allow_facet setting is True */
     var displayedFacets = Array.from(
       document.querySelectorAll(".facet-info")
     ).map((el) => el.dataset.column);
@@ -137,7 +138,12 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig
     var isSinglePk =
       th.getAttribute("data-is-pk") == "1" &&
       document.querySelectorAll('th[data-is-pk="1"]').length == 1;
-    if (isFirstColumn || displayedFacets.includes(column) || isSinglePk) {
+    if (
+      !DATASETTE_ALLOW_FACET ||
+      isFirstColumn ||
+      displayedFacets.includes(column) ||
+      isSinglePk
+    ) {
       facetItem.parentNode.style.display = "none";
     } else {
       facetItem.parentNode.style.display = "block";
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index 81bd044a..a9e88330 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -5,6 +5,7 @@
 {% block extra_head %}
 {{- super() -}}
 <script src="{{ urls.static('table.js') }}" defer></script>
+<script>DATASETTE_ALLOW_FACET = {{ datasette_allow_facet }};</script>
 <style>
 @media only screen and (max-width: 576px) {
 {% for column in display_columns -%}
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 84169820..cd7afea6 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -888,6 +888,9 @@ class TableView(RowTableShared):
                 "metadata": metadata,
                 "view_definition": await db.get_view_definition(table),
                 "table_definition": await db.get_table_definition(table),
+                "datasette_allow_facet": "true"
+                if self.ds.setting("allow_facet")
+                else "false",
             }
             d.update(extra_context_from_filters)
             return d
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index d40f017a..6dc26434 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1075,3 +1075,17 @@ def test_table_page_title(app_client, path, expected):
     response = app_client.get(path)
     title = Soup(response.text, "html.parser").find("title").text
     assert title == expected
+
+
+@pytest.mark.parametrize("allow_facet", (True, False))
+def test_allow_facet_off(allow_facet):
+    with make_app_client(settings={"allow_facet": allow_facet}) as client:
+        response = client.get("/fixtures/facetable")
+        expected = "DATASETTE_ALLOW_FACET = {};".format(
+            "true" if allow_facet else "false"
+        )
+        assert expected in response.text
+        if allow_facet:
+            assert "Suggested facets" in response.text
+        else:
+            assert "Suggested facets" not in response.text

From 6b99e4a66ba0ed8fca8ee41ceb7206928b60d5d1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 25 Mar 2022 16:44:35 -0700
Subject: [PATCH 1388/2303] Added missing hookimpl import

Useful for copying and pasting to create a quick plugin
---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 92cf662f..9c1f4402 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -542,7 +542,7 @@ Return a list of ``(regex, view_function)`` pairs, something like this:
 
 .. code-block:: python
 
-    from datasette import Response
+    from datasette import hookimpl, Response
     import html
 
 

From bd8a58ae61b2c986ef04ea721897906e0852e33e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 26 Mar 2022 13:51:20 -0700
Subject: [PATCH 1389/2303] Fix message_type in documentation, closes #1689

---
 docs/internals.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index f9a24fea..0ba3fa69 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -458,8 +458,8 @@ Returns the original, decoded object that was passed to :ref:`datasette_sign`. I
 
 .. _datasette_add_message:
 
-.add_message(request, message, message_type=datasette.INFO)
------------------------------------------------------------
+.add_message(request, message, type=datasette.INFO)
+---------------------------------------------------
 
 ``request`` - Request
     The current Request object
@@ -467,7 +467,7 @@ Returns the original, decoded object that was passed to :ref:`datasette_sign`. I
 ``message`` - string
     The message string
 
-``message_type`` - constant, optional
+``type`` - constant, optional
     The message type - ``datasette.INFO``, ``datasette.WARNING`` or ``datasette.ERROR``
 
 Datasette's flash messaging mechanism allows you to add a message that will be displayed to the user on the next page that they visit. Messages are persisted in a ``ds_messages`` cookie. This method adds a message to that cookie.

From e73fa72917ca28c152208d62d07a490c81cadf52 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 26 Mar 2022 15:46:08 -0700
Subject: [PATCH 1390/2303] Fixed bug in httpx_mock example, closes #1691

---
 docs/testing_plugins.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index 1291a875..8e4e3f91 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -198,7 +198,7 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
     async def test_outbound_http_call(httpx_mock):
         httpx_mock.add_response(
             url='https://www.example.com/',
-            data='Hello world',
+            text='Hello world',
         )
         datasette = Datasette([], memory=True)
         response = await datasette.client.post("/-/fetch-url", data={

From 5c5e9b365790d7c75cf2611e650d1013f587d316 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 31 Mar 2022 19:01:58 -0700
Subject: [PATCH 1391/2303] Request.fake(... url_vars), plus .fake() is now
 documented

Also made 'from datasette import Request' shortcut work.

Closes #1697
---
 datasette/__init__.py           |  2 +-
 datasette/utils/asgi.py         |  4 +++-
 docs/internals.rst              | 27 +++++++++++++++++++++++++++
 tests/test_internals_request.py |  7 +++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/datasette/__init__.py b/datasette/__init__.py
index faa36051..ea10c13d 100644
--- a/datasette/__init__.py
+++ b/datasette/__init__.py
@@ -1,5 +1,5 @@
 from datasette.version import __version_info__, __version__  # noqa
-from datasette.utils.asgi import Forbidden, NotFound, Response  # noqa
+from datasette.utils.asgi import Forbidden, NotFound, Request, Response  # noqa
 from datasette.utils import actor_matches_allow  # noqa
 from .hookspecs import hookimpl  # noqa
 from .hookspecs import hookspec  # noqa
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index cd3ec654..8a2fa060 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -118,7 +118,7 @@ class Request:
         return dict(parse_qsl(body.decode("utf-8"), keep_blank_values=True))
 
     @classmethod
-    def fake(cls, path_with_query_string, method="GET", scheme="http"):
+    def fake(cls, path_with_query_string, method="GET", scheme="http", url_vars=None):
         """Useful for constructing Request objects for tests"""
         path, _, query_string = path_with_query_string.partition("?")
         scope = {
@@ -130,6 +130,8 @@ class Request:
             "scheme": scheme,
             "type": "http",
         }
+        if url_vars:
+            scope["url_route"] = {"kwargs": url_vars}
         return cls(scope, None)
 
 
diff --git a/docs/internals.rst b/docs/internals.rst
index 0ba3fa69..854b96f8 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -60,6 +60,33 @@ The object also has two awaitable methods:
 ``await request.post_body()`` - bytes
     Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data.
 
+And a class method that can be used to create fake request objects for use in tests:
+
+``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)``
+    Returns a ``Request`` instance for the specified path and method. For example:
+
+    .. code-block:: python
+
+        from datasette import Request
+        from pprint import pprint
+
+        request = Request.fake("/fixtures/facetable/", url_vars={
+            "database": "fixtures",
+            "table": "facetable"
+        })
+        pprint(request.scope)
+
+    This outputs::
+
+        {'http_version': '1.1',
+         'method': 'GET',
+         'path': '/fixtures/facetable/',
+         'query_string': b'',
+         'raw_path': b'/fixtures/facetable/',
+         'scheme': 'http',
+         'type': 'http',
+         'url_route': {'kwargs': {'database': 'fixtures', 'table': 'facetable'}}}
+
 .. _internals_multiparams:
 
 The MultiParams class
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index 44aaa153..d1ca1f46 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -75,6 +75,13 @@ def test_request_args():
         request.args["missing"]
 
 
+def test_request_fake_url_vars():
+    request = Request.fake("/")
+    assert request.url_vars == {}
+    request = Request.fake("/", url_vars={"database": "fixtures"})
+    assert request.url_vars == {"database": "fixtures"}
+
+
 def test_request_repr():
     request = Request.fake("/foo?multi=1&multi=2&single=3")
     assert (

From df88d03298fa34d141ace7d6d8c35ca5e70576da Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 2 Apr 2022 23:05:10 -0700
Subject: [PATCH 1392/2303] Warn about Cloud Run and bots

Refs #1698
---
 docs/publish.rst | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/publish.rst b/docs/publish.rst
index 1d9664e7..166f2883 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -20,7 +20,14 @@ You will need a hosting account with `Heroku <https://www.heroku.com/>`__ or `Go
 Publishing to Google Cloud Run
 ------------------------------
 
-`Google Cloud Run <https://cloud.google.com/run/>`__ launched as a GA in in November 2019. It allows you to publish data in a scale-to-zero environment, so your application will start running when the first request is received and will shut down again when traffic ceases. This means you only pay for time spent serving traffic.
+`Google Cloud Run <https://cloud.google.com/run/>`__ allows you to publish data in a scale-to-zero environment, so your application will start running when the first request is received and will shut down again when traffic ceases. This means you only pay for time spent serving traffic.
+
+.. warning::
+    Cloud Run is a great option for inexpensively hosting small, low traffic projects - but costs can add up for projects that serve a lot of requests.
+
+    Be particularly careful if your project has tables with large numbers of rows. Search engine crawlers that index a page for every row could result in a high bill.
+
+    The `datasette-block-robots <https://datasette.io/plugins/datasette-block-robots>`__ plugin can be used to request search engine crawlers omit crawling your site, which can help avoid this issue.
 
 You will first need to install and configure the Google Cloud CLI tools by following `these instructions <https://cloud.google.com/sdk/>`__.
 
@@ -171,4 +178,4 @@ You can customize the port that is exposed by the container using the ``--port``
 
 A full list of options can be seen by running ``datasette package --help``:
 
-See :ref:`cli_help_package___help` for the full list of options for this command.
\ No newline at end of file
+See :ref:`cli_help_package___help` for the full list of options for this command.

From 90d1be9952db9aaddc21a536e4d00a8de44765d7 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 6 Apr 2022 08:55:01 -0700
Subject: [PATCH 1393/2303] Tilde encoding now encodes space as plus, closes
 #1701

Refs #1657
---
 datasette/utils/__init__.py | 12 ++++++++++--
 docs/internals.rst          |  6 ++++--
 tests/test_html.py          |  6 +++---
 tests/test_utils.py         |  1 +
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 9109f823..4745254e 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1113,12 +1113,20 @@ _TILDE_ENCODING_SAFE = frozenset(
     # '.' and '~'
 )
 
+_space = ord(" ")
+
 
 class TildeEncoder(dict):
     # Keeps a cache internally, via __missing__
     def __missing__(self, b):
+        print("b is ", b)
         # Handle a cache miss, store encoded string in cache and return.
-        res = chr(b) if b in _TILDE_ENCODING_SAFE else "~{:02X}".format(b)
+        if b in _TILDE_ENCODING_SAFE:
+            res = chr(b)
+        elif b == _space:
+            res = "+"
+        else:
+            res = "~{:02X}".format(b)
         self[b] = res
         return res
 
@@ -1138,7 +1146,7 @@ def tilde_decode(s: str) -> str:
     # Avoid accidentally decoding a %2f style sequence
     temp = secrets.token_hex(16)
     s = s.replace("%", temp)
-    decoded = urllib.parse.unquote(s.replace("~", "%"))
+    decoded = urllib.parse.unquote_plus(s.replace("~", "%"))
     return decoded.replace(temp, "%")
 
 
diff --git a/docs/internals.rst b/docs/internals.rst
index 854b96f8..76e27e5f 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -980,15 +980,17 @@ Datasette uses a custom encoding scheme in some places, called **tilde encoding*
 
 Tilde encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``~`` tilde character used in place of ``%``.
 
-Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_-`` will be replaced by the numeric equivalent preceded by a tilde. For example:
+Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz0123456789_-`` will be replaced by the numeric equivalent preceded by a tilde. For example:
 
 - ``/`` becomes ``~2F``
 - ``.`` becomes ``~2E``
 - ``%`` becomes ``~25``
 - ``~`` becomes ``~7E``
-- Space character becomes ``~20``
+- Space becomes ``+``
 - ``polls/2022.primary`` becomes ``polls~2F2022~2Eprimary``
 
+Note that the space character is a special case: it will be replaced with a ``+`` symbol.
+
 .. _internals_utils_tilde_encode:
 
 .. autofunction:: datasette.utils.tilde_encode
diff --git a/tests/test_html.py b/tests/test_html.py
index 6e4c22b1..42f1a3ee 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -28,7 +28,7 @@ def test_homepage(app_client_two_attached_databases):
     )
     # Should be two attached databases
     assert [
-        {"href": "/extra~20database", "text": "extra database"},
+        {"href": "/extra+database", "text": "extra database"},
         {"href": "/fixtures", "text": "fixtures"},
     ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")]
     # Database should show count text and attached tables
@@ -43,8 +43,8 @@ def test_homepage(app_client_two_attached_databases):
         {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a")
     ]
     assert [
-        {"href": r"/extra~20database/searchable", "text": "searchable"},
-        {"href": r"/extra~20database/searchable_view", "text": "searchable_view"},
+        {"href": r"/extra+database/searchable", "text": "searchable"},
+        {"href": r"/extra+database/searchable_view", "text": "searchable_view"},
     ] == table_links
 
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 7b41a87f..df788767 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -618,6 +618,7 @@ async def test_derive_named_parameters(sql, expected):
         ("-/db-/table.csv", "-~2Fdb-~2Ftable~2Ecsv"),
         (r"%~-/", "~25~7E-~2F"),
         ("~25~7E~2D~2F", "~7E25~7E7E~7E2D~7E2F"),
+        ("with space", "with+space"),
     ),
 )
 def test_tilde_encoding(original, expected):

From 247e460e08bf823142f7b84058fe44e43626787f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 8 Apr 2022 15:51:04 -0700
Subject: [PATCH 1394/2303] Update beautifulsoup4 requirement (#1703)

Updates the requirements on [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) to permit the latest version.

---
updated-dependencies:
- dependency-name: beautifulsoup4
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4b58b8c4..77fca8cd 100644
--- a/setup.py
+++ b/setup.py
@@ -70,7 +70,7 @@ setup(
             "pytest>=5.2.2,<7.2.0",
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.17,<0.19",
-            "beautifulsoup4>=4.8.1,<4.11.0",
+            "beautifulsoup4>=4.8.1,<4.12.0",
             "black==22.1.0",
             "pytest-timeout>=1.4.2,<2.2",
             "trustme>=0.7,<0.10",

From 138e4d9a53e3982137294ba383303c3a848cfca4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 8 Apr 2022 16:05:09 -0700
Subject: [PATCH 1395/2303] Update click requirement from <8.1.0,>=7.1.1 to
 >=7.1.1,<8.2.0 (#1694)

Updates the requirements on [click](https://github.com/pallets/click) to permit the latest version.
- [Release notes](https://github.com/pallets/click/releases)
- [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/click/compare/7.1.1...8.1.0)

---
updated-dependencies:
- dependency-name: click
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 77fca8cd..e5dd55fd 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@ setup(
     python_requires=">=3.7",
     install_requires=[
         "asgiref>=3.2.10,<3.6.0",
-        "click>=7.1.1,<8.1.0",
+        "click>=7.1.1,<8.2.0",
         "click-default-group~=1.2.2",
         "Jinja2>=2.10.3,<3.1.0",
         "hupper~=1.9",

From 143c105f875f4c8d4512233fa856477a938b38ca Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 12 Apr 2022 11:43:32 -0700
Subject: [PATCH 1396/2303] Removed rogue print

---
 datasette/utils/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 4745254e..77768112 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1119,7 +1119,6 @@ _space = ord(" ")
 class TildeEncoder(dict):
     # Keeps a cache internally, via __missing__
     def __missing__(self, b):
-        print("b is ", b)
         # Handle a cache miss, store encoded string in cache and return.
         if b in _TILDE_ENCODING_SAFE:
             res = chr(b)

From 0bc5186b7bb4fc82392df08f99a9132f84dcb331 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 12 Apr 2022 11:44:12 -0700
Subject: [PATCH 1397/2303] Tooltip and commas for byte length display, closes
 #1712

---
 datasette/views/database.py | 12 +++++++++---
 datasette/views/table.py    |  8 +++++++-
 tests/test_table_html.py    | 26 ++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index bdd433cc..9a8aca32 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -11,6 +11,7 @@ from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
     derive_named_parameters,
+    format_bytes,
     tilde_decode,
     to_css_class,
     validate_sql_select,
@@ -399,13 +400,18 @@ class QueryView(DataView):
                                     ).hexdigest(),
                                 },
                             )
-                            display_value = Markup(
-                                '<a class="blob-download" href="{}"><Binary: {} byte{}></a>'.format(
+                            formatted = format_bytes(len(value))
+                            display_value = markupsafe.Markup(
+                                '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format(
                                     blob_url,
-                                    len(display_value),
+                                    ' title="{}"'.format(formatted)
+                                    if "bytes" not in formatted
+                                    else "",
+                                    len(value),
                                     "" if len(value) == 1 else "s",
                                 )
                             )
+
                     display_row.append(display_value)
                 display_rows.append(display_row)
 
diff --git a/datasette/views/table.py b/datasette/views/table.py
index cd7afea6..dc85165e 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -12,10 +12,12 @@ from datasette.utils import (
     MultiParams,
     append_querystring,
     compound_keys_after_sql,
+    format_bytes,
     tilde_decode,
     tilde_encode,
     escape_sqlite,
     filters_should_redirect,
+    format_bytes,
     is_url,
     path_from_row_pks,
     path_with_added_args,
@@ -175,14 +177,18 @@ class RowTableShared(DataView):
                 if plugin_display_value:
                     display_value = plugin_display_value
                 elif isinstance(value, bytes):
+                    formatted = format_bytes(len(value))
                     display_value = markupsafe.Markup(
-                        '<a class="blob-download" href="{}"><Binary: {} byte{}></a>'.format(
+                        '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format(
                             self.ds.urls.row_blob(
                                 database,
                                 table,
                                 path_from_row_pks(row, pks, not pks),
                                 column,
                             ),
+                            ' title="{}"'.format(formatted)
+                            if "bytes" not in formatted
+                            else "",
                             len(value),
                             "" if len(value) == 1 else "s",
                         )
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 6dc26434..d3cb3e17 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1,3 +1,4 @@
+from datasette.app import Datasette, Database
 from bs4 import BeautifulSoup as Soup
 from .fixtures import (  # noqa
     app_client,
@@ -1089,3 +1090,28 @@ def test_allow_facet_off(allow_facet):
             assert "Suggested facets" in response.text
         else:
             assert "Suggested facets" not in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "size,title,length_bytes",
+    (
+        (2000, ' title="2.0 KB"', "2,000"),
+        (20000, ' title="19.5 KB"', "20,000"),
+        (20, "", "20"),
+    ),
+)
+async def test_format_of_binary_links(size, title, length_bytes):
+    ds = Datasette()
+    db_name = "binary-links-{}".format(size)
+    db = ds.add_memory_database(db_name)
+    sql = "select zeroblob({}) as blob".format(size)
+    await db.execute_write("create table blobs as {}".format(sql))
+    response = await ds.client.get("/{}/blobs".format(db_name))
+    assert response.status_code == 200
+    expected = "{}><Binary: {} bytes></a>".format(title, length_bytes)
+    assert expected in response.text
+    # And test with arbitrary SQL query too
+    sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql})
+    assert sql_response.status_code == 200
+    assert expected in sql_response.text

From 8338c66a57502ef27c3d7afb2527fbc0663b2570 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 21 Apr 2022 11:05:43 -0700
Subject: [PATCH 1398/2303] datasette-geojson is an example of
 register_output_renderer

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 9c1f4402..67842fc4 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -526,7 +526,7 @@ And here is an example ``can_render`` function which returns ``True`` only if th
     def can_render_demo(columns):
         return {"atom_id", "atom_title", "atom_updated"}.issubset(columns)
 
-Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_
+Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__
 
 .. _plugin_register_routes:
 

From d57c347f35bcd8cff15f913da851b4b8eb030867 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 22 Apr 2022 14:58:46 -0700
Subject: [PATCH 1399/2303] Ignore Black commits in git blame, refs #1716

---
 .git-blame-ignore-revs | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 .git-blame-ignore-revs

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
new file mode 100644
index 00000000..84e574fd
--- /dev/null
+++ b/.git-blame-ignore-revs
@@ -0,0 +1,4 @@
+# Applying Black
+35d6ee2790e41e96f243c1ff58be0c9c0519a8ce
+368638555160fb9ac78f462d0f79b1394163fa30
+2b344f6a34d2adaa305996a1a580ece06397f6e4

From 3001e1e394b6cb605c2cd81eed671a7da419c1b3 Mon Sep 17 00:00:00 2001
From: Tim Sherratt <tim@discontents.com.au>
Date: Mon, 25 Apr 2022 00:03:08 +1000
Subject: [PATCH 1400/2303] Add timeout option to Cloudrun build (#1717)

* Add timeout option for build phase
* Make the --timeout setting optional
* Add test for --timeout setting

Thanks, @wragge
---
 datasette/publish/cloudrun.py  | 12 +++++++++++-
 tests/test_publish_cloudrun.py | 31 +++++++++++++++++++------------
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py
index a1e2f580..11a39fb2 100644
--- a/datasette/publish/cloudrun.py
+++ b/datasette/publish/cloudrun.py
@@ -41,6 +41,10 @@ def publish_subcommand(publish):
         type=click.Choice(["1", "2", "4"]),
         help="Number of vCPUs to allocate in Cloud Run",
     )
+    @click.option(
+        "--timeout",
+        help="Build timeout in seconds",
+    )
     @click.option(
         "--apt-get-install",
         "apt_get_extras",
@@ -72,6 +76,7 @@ def publish_subcommand(publish):
         show_files,
         memory,
         cpu,
+        timeout,
         apt_get_extras,
     ):
         "Publish databases to Datasette running on Cloud Run"
@@ -156,7 +161,12 @@ def publish_subcommand(publish):
                 print("\n====================\n")
 
             image_id = f"gcr.io/{project}/{name}"
-            check_call(f"gcloud builds submit --tag {image_id}", shell=True)
+            check_call(
+                "gcloud builds submit --tag {}{}".format(
+                    image_id, " --timeout {}".format(timeout) if timeout else ""
+                ),
+                shell=True,
+            )
         check_call(
             "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}{}".format(
                 image_id,
diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py
index 9c8c38cf..3427f4f7 100644
--- a/tests/test_publish_cloudrun.py
+++ b/tests/test_publish_cloudrun.py
@@ -105,18 +105,19 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory):
 @mock.patch("datasette.publish.cloudrun.check_output")
 @mock.patch("datasette.publish.cloudrun.check_call")
 @pytest.mark.parametrize(
-    "memory,cpu,expected_gcloud_args",
+    "memory,cpu,timeout,expected_gcloud_args",
     [
-        ["1Gi", None, "--memory 1Gi"],
-        ["2G", None, "--memory 2G"],
-        ["256Mi", None, "--memory 256Mi"],
-        ["4", None, None],
-        ["GB", None, None],
-        [None, 1, "--cpu 1"],
-        [None, 2, "--cpu 2"],
-        [None, 3, None],
-        [None, 4, "--cpu 4"],
-        ["2G", 4, "--memory 2G --cpu 4"],
+        ["1Gi", None, None, "--memory 1Gi"],
+        ["2G", None, None, "--memory 2G"],
+        ["256Mi", None, None, "--memory 256Mi"],
+        ["4", None, None, None],
+        ["GB", None, None, None],
+        [None, 1, None, "--cpu 1"],
+        [None, 2, None, "--cpu 2"],
+        [None, 3, None, None],
+        [None, 4, None, "--cpu 4"],
+        ["2G", 4, None, "--memory 2G --cpu 4"],
+        [None, None, 1800, "--timeout 1800"],
     ],
 )
 def test_publish_cloudrun_memory_cpu(
@@ -125,6 +126,7 @@ def test_publish_cloudrun_memory_cpu(
     mock_which,
     memory,
     cpu,
+    timeout,
     expected_gcloud_args,
     tmp_path_factory,
 ):
@@ -139,6 +141,8 @@ def test_publish_cloudrun_memory_cpu(
         args.extend(["--memory", memory])
     if cpu:
         args.extend(["--cpu", str(cpu)])
+    if timeout:
+        args.extend(["--timeout", str(timeout)])
     result = runner.invoke(cli.cli, args)
     if expected_gcloud_args is None:
         assert 2 == result.exit_code
@@ -149,13 +153,16 @@ def test_publish_cloudrun_memory_cpu(
         "gcloud run deploy --allow-unauthenticated --platform=managed"
         " --image {} test".format(tag)
     )
+    expected_build_call = f"gcloud builds submit --tag {tag}"
     if memory:
         expected_call += " --memory {}".format(memory)
     if cpu:
         expected_call += " --cpu {}".format(cpu)
+    if timeout:
+        expected_build_call += f" --timeout {timeout}"
     mock_call.assert_has_calls(
         [
-            mock.call(f"gcloud builds submit --tag {tag}", shell=True),
+            mock.call(expected_build_call, shell=True),
             mock.call(
                 expected_call,
                 shell=True,

From 4bd3a30e1ea460e17011c11c16408300b87d1106 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 07:04:11 -0700
Subject: [PATCH 1401/2303] Update cog docs for publish cloudrun, refs #1717

---
 docs/cli-reference.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 69670d8a..3ca48aa2 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -248,6 +248,7 @@ datasette publish cloudrun --help
                                       metadata.json
       --memory TEXT                   Memory to allocate in Cloud Run, e.g. 1Gi
       --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
+      --timeout TEXT                  Build timeout in seconds
       --apt-get-install TEXT          Additional packages to apt-get install
       --help                          Show this message and exit.
 

From e64d14e413a955a10df88e106a8b5f1572ec8613 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 07:09:08 -0700
Subject: [PATCH 1402/2303] Use type integer for --timeout, refs #1717

---
 datasette/publish/cloudrun.py | 1 +
 docs/cli-reference.rst        | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py
index 11a39fb2..50b2b2fd 100644
--- a/datasette/publish/cloudrun.py
+++ b/datasette/publish/cloudrun.py
@@ -43,6 +43,7 @@ def publish_subcommand(publish):
     )
     @click.option(
         "--timeout",
+        type=int,
         help="Build timeout in seconds",
     )
     @click.option(
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 3ca48aa2..2a6fbfc8 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -248,7 +248,7 @@ datasette publish cloudrun --help
                                       metadata.json
       --memory TEXT                   Memory to allocate in Cloud Run, e.g. 1Gi
       --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
-      --timeout TEXT                  Build timeout in seconds
+      --timeout INTEGER               Build timeout in seconds
       --apt-get-install TEXT          Additional packages to apt-get install
       --help                          Show this message and exit.
 

From 40ef8ebac2d83c34f467fd2d7bf80f0549b6f6c3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 07:10:13 -0700
Subject: [PATCH 1403/2303] Run tests on pull requests

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 478e1f34..c11bfa2e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,6 +1,6 @@
 name: Test
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   test:

From 36573638b0948174ae237d62e6369b7d55220d7f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 08:50:43 -0700
Subject: [PATCH 1404/2303] Apply Black to code examples in documentation, refs
 #1718

Uses blacken-docs. This has a deliberate error which I hope will fail CI.
---
 .github/workflows/test.yml |  5 ++++
 docs/contributing.rst      |  9 +++++++
 docs/spatialite.rst        | 50 ++++++++++++++++++++++------------
 docs/writing_plugins.rst   | 55 ++++++++++++++++++++++----------------
 setup.py                   |  3 ++-
 5 files changed, 81 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c11bfa2e..38b62995 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,3 +32,8 @@ jobs:
     - name: Check if cog needs to be run
       run: |
         cog --check docs/*.rst
+    - name: Check if blacken-docs needs to be run
+      run: |
+        blacken-docs -l 60 docs/*.rst
+        # This fails if a diff was generated:
+        git diff-index --quiet HEAD --
diff --git a/docs/contributing.rst b/docs/contributing.rst
index b74f2f36..c193ba49 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -144,6 +144,15 @@ If any of your code does not conform to Black you can run this to automatically
     All done! ✨ 🍰 ✨
     1 file reformatted, 94 files left unchanged.
 
+.. _contributing_formatting_blacken_docs:
+
+blacken-docs
+~~~~~~~~~~~~
+
+The `blacken-docs <https://pypi.org/project/blacken-docs/>`__ command applies Black formatting rules to code examples in the documentation. Run it like this::
+
+    blacken-docs -l 60 docs/*.rst
+
 .. _contributing_formatting_prettier:
 
 Prettier
diff --git a/docs/spatialite.rst b/docs/spatialite.rst
index d1b300b2..52b6747e 100644
--- a/docs/spatialite.rst
+++ b/docs/spatialite.rst
@@ -58,21 +58,28 @@ Here's a recipe for taking a table with existing latitude and longitude columns,
 .. code-block:: python
 
     import sqlite3
-    conn = sqlite3.connect('museums.db')
+
+    conn = sqlite3.connect("museums.db")
     # Lead the spatialite extension:
     conn.enable_load_extension(True)
-    conn.load_extension('/usr/local/lib/mod_spatialite.dylib')
+    conn.load_extension("/usr/local/lib/mod_spatialite.dylib")
     # Initialize spatial metadata for this database:
-    conn.execute('select InitSpatialMetadata(1)')
+    conn.execute("select InitSpatialMetadata(1)")
     # Add a geometry column called point_geom to our museums table:
-    conn.execute("SELECT AddGeometryColumn('museums', 'point_geom', 4326, 'POINT', 2);")
+    conn.execute(
+        "SELECT AddGeometryColumn('museums', 'point_geom', 4326, 'POINT', 2);"
+    )
     # Now update that geometry column with the lat/lon points
-    conn.execute('''
+    conn.execute(
+        """
         UPDATE museums SET
         point_geom = GeomFromText('POINT('||"longitude"||' '||"latitude"||')',4326);
-    ''')
+    """
+    )
     # Now add a spatial index to that column
-    conn.execute('select CreateSpatialIndex("museums", "point_geom");')
+    conn.execute(
+        'select CreateSpatialIndex("museums", "point_geom");'
+    )
     # If you don't commit your changes will not be persisted:
     conn.commit()
     conn.close()
@@ -186,28 +193,37 @@ Here's Python code to create a SQLite database, enable SpatiaLite, create a plac
 .. code-block:: python
 
     import sqlite3
-    conn = sqlite3.connect('places.db')
+
+    conn = sqlite3.connect("places.db")
     # Enable SpatialLite extension
     conn.enable_load_extension(True)
-    conn.load_extension('/usr/local/lib/mod_spatialite.dylib')
+    conn.load_extension("/usr/local/lib/mod_spatialite.dylib")
     # Create the masic countries table
-    conn.execute('select InitSpatialMetadata(1)')
-    conn.execute('create table places (id integer primary key, name text);')
+    conn.execute("select InitSpatialMetadata(1)")
+    conn.execute(
+        "create table places (id integer primary key, name text);"
+    )
     # Add a MULTIPOLYGON Geometry column
-    conn.execute("SELECT AddGeometryColumn('places', 'geom', 4326, 'MULTIPOLYGON', 2);")
+    conn.execute(
+        "SELECT AddGeometryColumn('places', 'geom', 4326, 'MULTIPOLYGON', 2);"
+    )
     # Add a spatial index against the new column
     conn.execute("SELECT CreateSpatialIndex('places', 'geom');")
     # Now populate the table
     from shapely.geometry.multipolygon import MultiPolygon
     from shapely.geometry import shape
     import requests
-    geojson = requests.get('https://data.whosonfirst.org/404/227/475/404227475.geojson').json()
+
+    geojson = requests.get(
+        "https://data.whosonfirst.org/404/227/475/404227475.geojson"
+    ).json()
     # Convert to "Well Known Text" format
-    wkt = shape(geojson['geometry']).wkt
+    wkt = shape(geojson["geometry"]).wkt
     # Insert and commit the record
-    conn.execute("INSERT INTO places (id, name, geom) VALUES(null, ?, GeomFromText(?, 4326))", (
-       "Wales", wkt
-    ))
+    conn.execute(
+        "INSERT INTO places (id, name, geom) VALUES(null, ?, GeomFromText(?, 4326))",
+        ("Wales", wkt),
+    )
     conn.commit()
 
 Querying polygons using within()
diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst
index bd60a4b6..89f7f5eb 100644
--- a/docs/writing_plugins.rst
+++ b/docs/writing_plugins.rst
@@ -18,9 +18,12 @@ The quickest way to start writing a plugin is to create a ``my_plugin.py`` file
 
     from datasette import hookimpl
 
+
     @hookimpl
     def prepare_connection(conn):
-        conn.create_function('hello_world', 0, lambda: 'Hello world!')
+        conn.create_function(
+            "hello_world", 0, lambda: "Hello world!"
+        )
 
 If you save this in ``plugins/my_plugin.py`` you can then start Datasette like this::
 
@@ -60,22 +63,22 @@ The example consists of two files: a ``setup.py`` file that defines the plugin:
 
     from setuptools import setup
 
-    VERSION = '0.1'
+    VERSION = "0.1"
 
     setup(
-        name='datasette-plugin-demos',
-        description='Examples of plugins for Datasette',
-        author='Simon Willison',
-        url='https://github.com/simonw/datasette-plugin-demos',
-        license='Apache License, Version 2.0',
+        name="datasette-plugin-demos",
+        description="Examples of plugins for Datasette",
+        author="Simon Willison",
+        url="https://github.com/simonw/datasette-plugin-demos",
+        license="Apache License, Version 2.0",
         version=VERSION,
-        py_modules=['datasette_plugin_demos'],
+        py_modules=["datasette_plugin_demos"],
         entry_points={
-            'datasette': [
-                'plugin_demos = datasette_plugin_demos'
+            "datasette": [
+                "plugin_demos = datasette_plugin_demos"
             ]
         },
-        install_requires=['datasette']
+        install_requires=["datasette"],
     )
 
 And a Python module file, ``datasette_plugin_demos.py``, that implements the plugin:
@@ -88,12 +91,14 @@ And a Python module file, ``datasette_plugin_demos.py``, that implements the plu
 
     @hookimpl
     def prepare_jinja2_environment(env):
-        env.filters['uppercase'] = lambda u: u.upper()
+        env.filters["uppercase"] = lambda u: u.upper()
 
 
     @hookimpl
     def prepare_connection(conn):
-        conn.create_function('random_integer', 2, random.randint)
+        conn.create_function(
+            "random_integer", 2, random.randint
+        )
 
 
 Having built a plugin in this way you can turn it into an installable package using the following command::
@@ -123,11 +128,13 @@ To bundle the static assets for a plugin in the package that you publish to PyPI
 
 .. code-block:: python
 
-        package_data={
-            'datasette_plugin_name': [
-                'static/plugin.js',
-            ],
-        },
+        package_data = (
+            {
+                "datasette_plugin_name": [
+                    "static/plugin.js",
+                ],
+            },
+        )
 
 Where ``datasette_plugin_name`` is the name of the plugin package (note that it uses underscores, not hyphens) and ``static/plugin.js`` is the path within that package to the static file.
 
@@ -152,11 +159,13 @@ Templates should be bundled for distribution using the same ``package_data`` mec
 
 .. code-block:: python
 
-        package_data={
-            'datasette_plugin_name': [
-                'templates/my_template.html',
-            ],
-        },
+        package_data = (
+            {
+                "datasette_plugin_name": [
+                    "templates/my_template.html",
+                ],
+            },
+        )
 
 You can also use wildcards here such as ``templates/*.html``. See `datasette-edit-schema <https://github.com/simonw/datasette-edit-schema>`__ for an example of this pattern.
 
diff --git a/setup.py b/setup.py
index e5dd55fd..7f0562fd 100644
--- a/setup.py
+++ b/setup.py
@@ -65,13 +65,14 @@ setup(
     """,
     setup_requires=["pytest-runner"],
     extras_require={
-        "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
+        "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell", "blacken-docs"],
         "test": [
             "pytest>=5.2.2,<7.2.0",
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.17,<0.19",
             "beautifulsoup4>=4.8.1,<4.12.0",
             "black==22.1.0",
+            "blacken-docs==1.12.1",
             "pytest-timeout>=1.4.2,<2.2",
             "trustme>=0.7,<0.10",
             "cogapp>=3.3.0",

From 92b26673d86a663050c9a40a8ffd5b56c25be85f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 08:51:09 -0700
Subject: [PATCH 1405/2303] Fix blacken-docs errors and warnings, refs #1718

---
 docs/authentication.rst |  25 ++--
 docs/internals.rst      |  98 ++++++++-----
 docs/json_api.rst       |   2 +-
 docs/plugin_hooks.rst   | 305 +++++++++++++++++++++++++++-------------
 4 files changed, 289 insertions(+), 141 deletions(-)

diff --git a/docs/authentication.rst b/docs/authentication.rst
index 0d98cf82..24960733 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -381,11 +381,10 @@ Authentication plugins can set signed ``ds_actor`` cookies themselves like so:
 .. code-block:: python
 
     response = Response.redirect("/")
-    response.set_cookie("ds_actor", datasette.sign({
-        "a": {
-            "id": "cleopaws"
-        }
-    }, "actor"))
+    response.set_cookie(
+        "ds_actor",
+        datasette.sign({"a": {"id": "cleopaws"}}, "actor"),
+    )
 
 Note that you need to pass ``"actor"`` as the namespace to :ref:`datasette_sign`.
 
@@ -412,12 +411,16 @@ To include an expiry, add a ``"e"`` key to the cookie value containing a `base62
     expires_at = int(time.time()) + (24 * 60 * 60)
 
     response = Response.redirect("/")
-    response.set_cookie("ds_actor", datasette.sign({
-        "a": {
-            "id": "cleopaws"
-        },
-        "e": baseconv.base62.encode(expires_at),
-    }, "actor"))
+    response.set_cookie(
+        "ds_actor",
+        datasette.sign(
+            {
+                "a": {"id": "cleopaws"},
+                "e": baseconv.base62.encode(expires_at),
+            },
+            "actor",
+        ),
+    )
 
 The resulting cookie will encode data that looks something like this:
 
diff --git a/docs/internals.rst b/docs/internals.rst
index 76e27e5f..aad608dc 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -70,10 +70,10 @@ And a class method that can be used to create fake request objects for use in te
         from datasette import Request
         from pprint import pprint
 
-        request = Request.fake("/fixtures/facetable/", url_vars={
-            "database": "fixtures",
-            "table": "facetable"
-        })
+        request = Request.fake(
+            "/fixtures/facetable/",
+            url_vars={"database": "fixtures", "table": "facetable"},
+        )
         pprint(request.scope)
 
     This outputs::
@@ -146,7 +146,7 @@ For example:
 
     response = Response(
         "<xml>This is XML</xml>",
-        content_type="application/xml; charset=utf-8"
+        content_type="application/xml; charset=utf-8",
     )
 
 The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods:
@@ -157,9 +157,13 @@ The quickest way to create responses is using the ``Response.text(...)``, ``Resp
 
     html_response = Response.html("This is HTML")
     json_response = Response.json({"this_is": "json"})
-    text_response = Response.text("This will become utf-8 encoded text")
+    text_response = Response.text(
+        "This will become utf-8 encoded text"
+    )
     # Redirects are served as 302, unless you pass status=301:
-    redirect_response = Response.redirect("https://latest.datasette.io/")
+    redirect_response = Response.redirect(
+        "https://latest.datasette.io/"
+    )
 
 Each of these responses will use the correct corresponding content-type - ``text/html; charset=utf-8``, ``application/json; charset=utf-8`` or ``text/plain; charset=utf-8`` respectively.
 
@@ -207,13 +211,17 @@ To set cookies on the response, use the ``response.set_cookie(...)`` method. The
         httponly=False,
         samesite="lax",
     ):
+        ...
 
 You can use this with :ref:`datasette.sign() <datasette_sign>` to set signed cookies. Here's how you would set the :ref:`ds_actor cookie <authentication_ds_actor>` for use with Datasette :ref:`authentication <authentication>`:
 
 .. code-block:: python
 
     response = Response.redirect("/")
-    response.set_cookie("ds_actor", datasette.sign({"a": {"id": "cleopaws"}}, "actor"))
+    response.set_cookie(
+        "ds_actor",
+        datasette.sign({"a": {"id": "cleopaws"}}, "actor"),
+    )
     return response
 
 .. _internals_datasette:
@@ -236,13 +244,16 @@ You can create your own instance of this - for example to help write tests for a
     datasette = Datasette(files=["/path/to/my-database.db"])
 
     # Pass metadata as a JSON dictionary like this
-    datasette = Datasette(files=["/path/to/my-database.db"], metadata={
-        "databases": {
-            "my-database": {
-                "description": "This is my database"
+    datasette = Datasette(
+        files=["/path/to/my-database.db"],
+        metadata={
+            "databases": {
+                "my-database": {
+                    "description": "This is my database"
+                }
             }
-        }
-    })
+        },
+    )
 
 Constructor parameters include:
 
@@ -345,7 +356,7 @@ This is useful when you need to check multiple permissions at once. For example,
             ("view-table", (database, table)),
             ("view-database", database),
             "view-instance",
-        ]
+        ],
     )
 
 .. _datasette_check_visibilty:
@@ -406,11 +417,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database`
 
     from datasette.database import Database
 
-    datasette.add_database(Database(
-        datasette,
-        path="path/to/my-new-database.db",
-        is_mutable=True
-    ))
+    datasette.add_database(
+        Database(
+            datasette,
+            path="path/to/my-new-database.db",
+            is_mutable=True,
+        )
+    )
 
 This will add a mutable database and serve it at ``/my-new-database``.
 
@@ -418,8 +431,12 @@ This will add a mutable database and serve it at ``/my-new-database``.
 
 .. code-block:: python
 
-    db = datasette.add_database(Database(datasette, memory_name="statistics"))
-    await db.execute_write("CREATE TABLE foo(id integer primary key)")
+    db = datasette.add_database(
+        Database(datasette, memory_name="statistics")
+    )
+    await db.execute_write(
+        "CREATE TABLE foo(id integer primary key)"
+    )
 
 .. _datasette_add_memory_database:
 
@@ -438,10 +455,9 @@ This is a shortcut for the following:
 
     from datasette.database import Database
 
-    datasette.add_database(Database(
-        datasette,
-        memory_name="statistics"
-    ))
+    datasette.add_database(
+        Database(datasette, memory_name="statistics")
+    )
 
 Using either of these pattern will result in the in-memory database being served at ``/statistics``.
 
@@ -516,7 +532,9 @@ Returns the absolute URL for the given path, including the protocol and host. Fo
 
 .. code-block:: python
 
-    absolute_url = datasette.absolute_url(request, "/dbname/table.json")
+    absolute_url = datasette.absolute_url(
+        request, "/dbname/table.json"
+    )
     # Would return "http://localhost:8001/dbname/table.json"
 
 The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`setting_force_https_urls` configuration setting is taken into account.
@@ -578,7 +596,9 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
 
     table_json = (
         await datasette.client.get(
-            datasette.urls.table("fixtures", "facetable", format="json")
+            datasette.urls.table(
+                "fixtures", "facetable", format="json"
+            )
         )
     ).json()
 
@@ -754,6 +774,7 @@ Example usage:
             "select sqlite_version()"
         ).fetchall()[0][0]
 
+
     version = await db.execute_fn(get_version)
 
 .. _database_execute_write:
@@ -789,7 +810,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d
 
     await db.execute_write_many(
         "insert into characters (id, name) values (?, ?)",
-        [(1, "Melanie"), (2, "Selma"), (2, "Viktor")]
+        [(1, "Melanie"), (2, "Selma"), (2, "Viktor")],
     )
 
 .. _database_execute_write_fn:
@@ -811,10 +832,15 @@ For example:
 
     def delete_and_return_count(conn):
         conn.execute("delete from some_table where id > 5")
-        return conn.execute("select count(*) from some_table").fetchone()[0]
+        return conn.execute(
+            "select count(*) from some_table"
+        ).fetchone()[0]
+
 
     try:
-        num_rows_left = await database.execute_write_fn(delete_and_return_count)
+        num_rows_left = await database.execute_write_fn(
+            delete_and_return_count
+        )
     except Exception as e:
         print("An error occurred:", e)
 
@@ -1021,6 +1047,7 @@ This example uses trace to record the start, end and duration of any HTTP GET re
     from datasette.tracer import trace
     import httpx
 
+
     async def fetch_url(url):
         with trace("fetch-url", url=url):
             async with httpx.AsyncClient() as client:
@@ -1051,9 +1078,9 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h
     from datasette import hookimpl
     from datasette import tracer
 
+
     @hookimpl
     def register_routes():
-
         async def parallel_queries(datasette):
             db = datasette.get_database()
             with tracer.trace_child_tasks():
@@ -1061,7 +1088,12 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h
                     db.execute("select 1"),
                     db.execute("select 2"),
                 )
-            return Response.json({"one": one.single_value(), "two": two.single_value()})
+            return Response.json(
+                {
+                    "one": one.single_value(),
+                    "two": two.single_value(),
+                }
+            )
 
         return [
             (r"/parallel-queries$", parallel_queries),
diff --git a/docs/json_api.rst b/docs/json_api.rst
index aa6fcdaa..d3fdb1e4 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -446,7 +446,7 @@ Most of the HTML pages served by Datasette provide a mechanism for discovering t
 
 You can find this near the top of the source code of those pages, looking like this:
 
-.. code-block:: python
+.. code-block:: html
 
     <link rel="alternate"
       type="application/json+datasette"
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 67842fc4..ace206b7 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -44,9 +44,12 @@ aggregates and collations. For example:
     from datasette import hookimpl
     import random
 
+
     @hookimpl
     def prepare_connection(conn):
-        conn.create_function('random_integer', 2, random.randint)
+        conn.create_function(
+            "random_integer", 2, random.randint
+        )
 
 This registers a SQL function called ``random_integer`` which takes two
 arguments and can be called like this::
@@ -72,9 +75,10 @@ example:
 
     from datasette import hookimpl
 
+
     @hookimpl
     def prepare_jinja2_environment(env):
-        env.filters['uppercase'] = lambda u: u.upper()
+        env.filters["uppercase"] = lambda u: u.upper()
 
 You can now use this filter in your custom templates like so::
 
@@ -127,9 +131,7 @@ Here's an example plugin that adds a ``"user_agent"`` variable to the template c
 
     @hookimpl
     def extra_template_vars(request):
-        return {
-            "user_agent": request.headers.get("user-agent")
-        }
+        return {"user_agent": request.headers.get("user-agent")}
 
 This example returns an awaitable function which adds a list of ``hidden_table_names`` to the context:
 
@@ -140,9 +142,12 @@ This example returns an awaitable function which adds a list of ``hidden_table_n
         async def hidden_table_names():
             if database:
                 db = datasette.databases[database]
-                return {"hidden_table_names": await db.hidden_table_names()}
+                return {
+                    "hidden_table_names": await db.hidden_table_names()
+                }
             else:
                 return {}
+
         return hidden_table_names
 
 And here's an example which adds a ``sql_first(sql_query)`` function which executes a SQL statement and returns the first column of the first row of results:
@@ -152,8 +157,15 @@ And here's an example which adds a ``sql_first(sql_query)`` function which execu
     @hookimpl
     def extra_template_vars(datasette, database):
         async def sql_first(sql, dbname=None):
-            dbname = dbname or database or next(iter(datasette.databases.keys()))
-            return (await datasette.execute(dbname, sql)).rows[0][0]
+            dbname = (
+                dbname
+                or database
+                or next(iter(datasette.databases.keys()))
+            )
+            return (await datasette.execute(dbname, sql)).rows[
+                0
+            ][0]
+
         return {"sql_first": sql_first}
 
 You can then use the new function in a template like so::
@@ -178,6 +190,7 @@ This can be a list of URLs:
 
     from datasette import hookimpl
 
+
     @hookimpl
     def extra_css_urls():
         return [
@@ -191,10 +204,12 @@ Or a list of dictionaries defining both a URL and an
 
     @hookimpl
     def extra_css_urls():
-        return [{
-            "url": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css",
-            "sri": "sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4",
-        }]
+        return [
+            {
+                "url": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css",
+                "sri": "sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4",
+            }
+        ]
 
 This function can also return an awaitable function, useful if it needs to run any async code:
 
@@ -204,7 +219,9 @@ This function can also return an awaitable function, useful if it needs to run a
     def extra_css_urls(datasette):
         async def inner():
             db = datasette.get_database()
-            results = await db.execute("select url from css_files")
+            results = await db.execute(
+                "select url from css_files"
+            )
             return [r[0] for r in results]
 
         return inner
@@ -225,12 +242,15 @@ return a list of URLs, a list of dictionaries or an awaitable function that retu
 
     from datasette import hookimpl
 
+
     @hookimpl
     def extra_js_urls():
-        return [{
-            "url": "https://code.jquery.com/jquery-3.3.1.slim.min.js",
-            "sri": "sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo",
-        }]
+        return [
+            {
+                "url": "https://code.jquery.com/jquery-3.3.1.slim.min.js",
+                "sri": "sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo",
+            }
+        ]
 
 You can also return URLs to files from your plugin's ``static/`` directory, if
 you have one:
@@ -239,9 +259,7 @@ you have one:
 
     @hookimpl
     def extra_js_urls():
-        return [
-            "/-/static-plugins/your-plugin/app.js"
-        ]
+        return ["/-/static-plugins/your-plugin/app.js"]
 
 Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page.
 
@@ -251,9 +269,11 @@ If your code uses `JavaScript modules <https://developer.mozilla.org/en-US/docs/
 
     @hookimpl
     def extra_js_urls():
-        return [{
-            "url": "/-/static-plugins/your-plugin/app.js",
-            "module": True
+        return [
+            {
+                "url": "/-/static-plugins/your-plugin/app.js",
+                "module": True,
+            }
         ]
 
 Examples: `datasette-cluster-map <https://datasette.io/plugins/datasette-cluster-map>`_, `datasette-vega <https://datasette.io/plugins/datasette-vega>`_
@@ -281,7 +301,7 @@ Use a dictionary if you want to specify that the code should be placed in a ``<s
     def extra_body_script():
         return {
             "module": True,
-            "script": "console.log('Your JavaScript goes here...')"
+            "script": "console.log('Your JavaScript goes here...')",
         }
 
 This will add the following to the end of your page:
@@ -311,7 +331,9 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_
 .. code-block:: python
 
     from datasette import hookimpl
-    from datasette.publish.common import add_common_publish_arguments_and_options
+    from datasette.publish.common import (
+        add_common_publish_arguments_and_options,
+    )
     import click
 
 
@@ -345,7 +367,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_
             about_url,
             api_key,
         ):
-            # Your implementation goes here
+            ...
 
 Examples: `datasette-publish-fly <https://datasette.io/plugins/datasette-publish-fly>`_, `datasette-publish-vercel <https://datasette.io/plugins/datasette-publish-vercel>`_
 
@@ -400,7 +422,9 @@ If the value matches that pattern, the plugin returns an HTML link element:
         if not isinstance(value, str):
             return None
         stripped = value.strip()
-        if not stripped.startswith("{") and stripped.endswith("}"):
+        if not stripped.startswith("{") and stripped.endswith(
+            "}"
+        ):
             return None
         try:
             data = json.loads(value)
@@ -412,14 +436,18 @@ If the value matches that pattern, the plugin returns an HTML link element:
             return None
         href = data["href"]
         if not (
-            href.startswith("/") or href.startswith("http://")
+            href.startswith("/")
+            or href.startswith("http://")
             or href.startswith("https://")
         ):
             return None
-        return markupsafe.Markup('<a href="{href}">{label}</a>'.format(
-            href=markupsafe.escape(data["href"]),
-            label=markupsafe.escape(data["label"] or "") or " "
-        ))
+        return markupsafe.Markup(
+            '<a href="{href}">{label}</a>'.format(
+                href=markupsafe.escape(data["href"]),
+                label=markupsafe.escape(data["label"] or "")
+                or " ",
+            )
+        )
 
 Examples: `datasette-render-binary <https://datasette.io/plugins/datasette-render-binary>`_, `datasette-render-markdown <https://datasette.io/plugins/datasette-render-markdown>`__, `datasette-json-html <https://datasette.io/plugins/datasette-json-html>`__
 
@@ -516,7 +544,7 @@ Here is a more complex example:
         return Response(
             "\n".join(lines),
             content_type="text/plain; charset=utf-8",
-            headers={"x-sqlite-version": result.first()[0]}
+            headers={"x-sqlite-version": result.first()[0]},
         )
 
 And here is an example ``can_render`` function which returns ``True`` only if the query results contain the columns ``atom_id``, ``atom_title`` and ``atom_updated``:
@@ -524,7 +552,11 @@ And here is an example ``can_render`` function which returns ``True`` only if th
 .. code-block:: python
 
     def can_render_demo(columns):
-        return {"atom_id", "atom_title", "atom_updated"}.issubset(columns)
+        return {
+            "atom_id",
+            "atom_title",
+            "atom_updated",
+        }.issubset(columns)
 
 Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__
 
@@ -548,16 +580,14 @@ Return a list of ``(regex, view_function)`` pairs, something like this:
 
     async def hello_from(request):
         name = request.url_vars["name"]
-        return Response.html("Hello from {}".format(
-            html.escape(name)
-        ))
+        return Response.html(
+            "Hello from {}".format(html.escape(name))
+        )
 
 
     @hookimpl
     def register_routes():
-        return [
-            (r"^/hello-from/(?P<name>.*)$", hello_from)
-        ]
+        return [(r"^/hello-from/(?P<name>.*)$", hello_from)]
 
 The view functions can take a number of different optional arguments. The corresponding argument will be passed to your function depending on its named parameters - a form of dependency injection.
 
@@ -606,10 +636,13 @@ This example registers a new ``datasette verify file1.db file2.db`` command that
     import click
     import sqlite3
 
+
     @hookimpl
     def register_commands(cli):
         @cli.command()
-        @click.argument("files", type=click.Path(exists=True), nargs=-1)
+        @click.argument(
+            "files", type=click.Path(exists=True), nargs=-1
+        )
         def verify(files):
             "Verify that files can be opened by Datasette"
             for file in files:
@@ -617,7 +650,9 @@ This example registers a new ``datasette verify file1.db file2.db`` command that
                 try:
                     conn.execute("select * from sqlite_master")
                 except sqlite3.DatabaseError:
-                    raise click.ClickException("Invalid database: {}".format(file))
+                    raise click.ClickException(
+                        "Invalid database: {}".format(file)
+                    )
 
 The new command can then be executed like so::
 
@@ -656,15 +691,18 @@ Each Facet subclass implements a new type of facet operation. The class should l
         async def suggest(self):
             # Use self.sql and self.params to suggest some facets
             suggested_facets = []
-            suggested_facets.append({
-                "name": column, # Or other unique name
-                # Construct the URL that will enable this facet:
-                "toggle_url": self.ds.absolute_url(
-                    self.request, path_with_added_args(
-                        self.request, {"_facet": column}
-                    )
-                ),
-            })
+            suggested_facets.append(
+                {
+                    "name": column,  # Or other unique name
+                    # Construct the URL that will enable this facet:
+                    "toggle_url": self.ds.absolute_url(
+                        self.request,
+                        path_with_added_args(
+                            self.request, {"_facet": column}
+                        ),
+                    ),
+                }
+            )
             return suggested_facets
 
         async def facet_results(self):
@@ -678,18 +716,25 @@ Each Facet subclass implements a new type of facet operation. The class should l
                 try:
                     facet_results_values = []
                     # More calculations...
-                    facet_results_values.append({
-                        "value": value,
-                        "label": label,
-                        "count": count,
-                        "toggle_url": self.ds.absolute_url(self.request, toggle_path),
-                        "selected": selected,
-                    })
-                    facet_results.append({
-                        "name": column,
-                        "results": facet_results_values,
-                        "truncated": len(facet_rows_results) > facet_size,
-                    })
+                    facet_results_values.append(
+                        {
+                            "value": value,
+                            "label": label,
+                            "count": count,
+                            "toggle_url": self.ds.absolute_url(
+                                self.request, toggle_path
+                            ),
+                            "selected": selected,
+                        }
+                    )
+                    facet_results.append(
+                        {
+                            "name": column,
+                            "results": facet_results_values,
+                            "truncated": len(facet_rows_results)
+                            > facet_size,
+                        }
+                    )
                 except QueryInterrupted:
                     facets_timed_out.append(column)
 
@@ -728,21 +773,33 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
     def asgi_wrapper(datasette):
         def wrap_with_databases_header(app):
             @wraps(app)
-            async def add_x_databases_header(scope, receive, send):
+            async def add_x_databases_header(
+                scope, receive, send
+            ):
                 async def wrapped_send(event):
                     if event["type"] == "http.response.start":
-                        original_headers = event.get("headers") or []
+                        original_headers = (
+                            event.get("headers") or []
+                        )
                         event = {
                             "type": event["type"],
                             "status": event["status"],
-                            "headers": original_headers + [
-                                [b"x-databases",
-                                ", ".join(datasette.databases.keys()).encode("utf-8")]
+                            "headers": original_headers
+                            + [
+                                [
+                                    b"x-databases",
+                                    ", ".join(
+                                        datasette.databases.keys()
+                                    ).encode("utf-8"),
+                                ]
                             ],
                         }
                     await send(event)
+
                 await app(scope, receive, wrapped_send)
+
             return add_x_databases_header
+
         return wrap_with_databases_header
 
 Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `datasette-pyinstrument <https://datasette.io/plugins/datasette-pyinstrument>`__
@@ -759,7 +816,9 @@ This hook fires when the Datasette application server first starts up. You can i
     @hookimpl
     def startup(datasette):
         config = datasette.plugin_config("my-plugin") or {}
-        assert "required-setting" in config, "my-plugin requires setting required-setting"
+        assert (
+            "required-setting" in config
+        ), "my-plugin requires setting required-setting"
 
 Or you can return an async function which will be awaited on startup. Use this option if you need to make any database queries:
 
@@ -770,9 +829,12 @@ Or you can return an async function which will be awaited on startup. Use this o
         async def inner():
             db = datasette.get_database()
             if "my_table" not in await db.table_names():
-                await db.execute_write("""
+                await db.execute_write(
+                    """
                     create table my_table (mycol text)
-                """)
+                """
+                )
+
         return inner
 
 Potential use-cases:
@@ -815,6 +877,7 @@ Ues this hook to return a dictionary of additional :ref:`canned query <canned_qu
 
     from datasette import hookimpl
 
+
     @hookimpl
     def canned_queries(datasette, database):
         if database == "mydb":
@@ -830,15 +893,20 @@ The hook can alternatively return an awaitable function that returns a list. Her
 
     from datasette import hookimpl
 
+
     @hookimpl
     def canned_queries(datasette, database):
         async def inner():
             db = datasette.get_database(database)
             if await db.table_exists("saved_queries"):
-                results = await db.execute("select name, sql from saved_queries")
-                return {result["name"]: {
-                    "sql": result["sql"]
-                } for result in results}
+                results = await db.execute(
+                    "select name, sql from saved_queries"
+                )
+                return {
+                    result["name"]: {"sql": result["sql"]}
+                    for result in results
+                }
+
         return inner
 
 The actor parameter can be used to include the currently authenticated actor in your decision. Here's an example that returns saved queries that were saved by that actor:
@@ -847,19 +915,23 @@ The actor parameter can be used to include the currently authenticated actor in
 
     from datasette import hookimpl
 
+
     @hookimpl
     def canned_queries(datasette, database, actor):
         async def inner():
             db = datasette.get_database(database)
-            if actor is not None and await db.table_exists("saved_queries"):
+            if actor is not None and await db.table_exists(
+                "saved_queries"
+            ):
                 results = await db.execute(
-                    "select name, sql from saved_queries where actor_id = :id", {
-                        "id": actor["id"]
-                    }
+                    "select name, sql from saved_queries where actor_id = :id",
+                    {"id": actor["id"]},
                 )
-                return {result["name"]: {
-                    "sql": result["sql"]
-                } for result in results}
+                return {
+                    result["name"]: {"sql": result["sql"]}
+                    for result in results
+                }
+
         return inner
 
 Example: `datasette-saved-queries <https://datasette.io/plugins/datasette-saved-queries>`__
@@ -888,9 +960,12 @@ Here's an example that authenticates the actor based on an incoming API key:
 
     SECRET_KEY = "this-is-a-secret"
 
+
     @hookimpl
     def actor_from_request(datasette, request):
-        authorization = request.headers.get("authorization") or ""
+        authorization = (
+            request.headers.get("authorization") or ""
+        )
         expected = "Bearer {}".format(SECRET_KEY)
 
         if secrets.compare_digest(authorization, expected):
@@ -906,6 +981,7 @@ Instead of returning a dictionary, this function can return an awaitable functio
 
     from datasette import hookimpl
 
+
     @hookimpl
     def actor_from_request(datasette, request):
         async def inner():
@@ -914,7 +990,8 @@ Instead of returning a dictionary, this function can return an awaitable functio
                 return None
             # Look up ?_token=xxx in sessions table
             result = await datasette.get_database().execute(
-                "select count(*) from sessions where token = ?", [token]
+                "select count(*) from sessions where token = ?",
+                [token],
             )
             if result.first()[0]:
                 return {"token": token}
@@ -952,7 +1029,7 @@ The hook should return an instance of ``datasette.filters.FilterArguments`` whic
         where_clauses=["id > :max_id"],
         params={"max_id": 5},
         human_descriptions=["max_id is greater than 5"],
-        extra_context={}
+        extra_context={},
     )
 
 The arguments to the ``FilterArguments`` class constructor are as follows:
@@ -973,10 +1050,13 @@ This example plugin causes 0 results to be returned if ``?_nothing=1`` is added
     from datasette import hookimpl
     from datasette.filters import FilterArguments
 
+
     @hookimpl
     def filters_from_request(self, request):
         if request.args.get("_nothing"):
-            return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"])
+            return FilterArguments(
+                ["1 = 0"], human_descriptions=["NOTHING"]
+            )
 
 Example: `datasette-leaflet-freedraw <https://datasette.io/plugins/datasette-leaflet-freedraw>`_
 
@@ -1006,6 +1086,7 @@ Here's an example plugin which randomly selects if a permission should be allowe
     from datasette import hookimpl
     import random
 
+
     @hookimpl
     def permission_allowed(action):
         if action != "view-instance":
@@ -1024,11 +1105,16 @@ Here's an example that allows users to view the ``admin_log`` table only if thei
         async def inner():
             if action == "execute-sql" and resource == "staff":
                 return False
-            if action == "view-table" and resource == ("staff", "admin_log"):
+            if action == "view-table" and resource == (
+                "staff",
+                "admin_log",
+            ):
                 if not actor:
                     return False
                 user_id = actor["id"]
-                return await datasette.get_database("staff").execute(
+                return await datasette.get_database(
+                    "staff"
+                ).execute(
                     "select count(*) from admin_users where user_id = :user_id",
                     {"user_id": user_id},
                 )
@@ -1059,18 +1145,21 @@ This example registers two new magic parameters: ``:_request_http_version`` retu
 
     from uuid import uuid4
 
+
     def uuid(key, request):
         if key == "new":
             return str(uuid4())
         else:
             raise KeyError
 
+
     def request(key, request):
         if key == "http_version":
             return request.scope["http_version"]
         else:
             raise KeyError
 
+
     @hookimpl
     def register_magic_parameters(datasette):
         return [
@@ -1103,9 +1192,12 @@ This example returns a redirect to a ``/-/login`` page:
     from datasette import hookimpl
     from urllib.parse import urlencode
 
+
     @hookimpl
     def forbidden(request, message):
-        return Response.redirect("/-/login?=" + urlencode({"message": message}))
+        return Response.redirect(
+            "/-/login?=" + urlencode({"message": message})
+        )
 
 The function can alternatively return an awaitable function if it needs to make any asynchronous method calls. This example renders a template:
 
@@ -1114,10 +1206,15 @@ The function can alternatively return an awaitable function if it needs to make
     from datasette import hookimpl
     from datasette.utils.asgi import Response
 
+
     @hookimpl
     def forbidden(datasette):
         async def inner():
-            return Response.html(await datasette.render_template("forbidden.html"))
+            return Response.html(
+                await datasette.render_template(
+                    "forbidden.html"
+                )
+            )
 
         return inner
 
@@ -1147,11 +1244,17 @@ This example adds a new menu item but only if the signed in user is ``"root"``:
 
     from datasette import hookimpl
 
+
     @hookimpl
     def menu_links(datasette, actor):
         if actor and actor.get("id") == "root":
             return [
-                {"href": datasette.urls.path("/-/edit-schema"), "label": "Edit schema"},
+                {
+                    "href": datasette.urls.path(
+                        "/-/edit-schema"
+                    ),
+                    "label": "Edit schema",
+                },
             ]
 
 Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account.
@@ -1188,13 +1291,20 @@ This example adds a new table action if the signed in user is ``"root"``:
 
     from datasette import hookimpl
 
+
     @hookimpl
     def table_actions(datasette, actor):
         if actor and actor.get("id") == "root":
-            return [{
-                "href": datasette.urls.path("/-/edit-schema/{}/{}".format(database, table)),
-                "label": "Edit schema for this table",
-            }]
+            return [
+                {
+                    "href": datasette.urls.path(
+                        "/-/edit-schema/{}/{}".format(
+                            database, table
+                        )
+                    ),
+                    "label": "Edit schema for this table",
+                }
+            ]
 
 Example: `datasette-graphql <https://datasette.io/plugins/datasette-graphql>`_
 
@@ -1238,6 +1348,7 @@ This example will disable CSRF protection for that specific URL path:
 
     from datasette import hookimpl
 
+
     @hookimpl
     def skip_csrf(scope):
         return scope["path"] == "/submit-comment"
@@ -1278,7 +1389,9 @@ This hook is responsible for returning a dictionary corresponding to Datasette :
             "description": get_instance_description(datasette),
             "databases": [],
         }
-        for db_name, db_data_dict in get_my_database_meta(datasette, database, table, key):
+        for db_name, db_data_dict in get_my_database_meta(
+            datasette, database, table, key
+        ):
             metadata["databases"][db_name] = db_data_dict
         # whatever we return here will be merged with any other plugins using this hook and
         # will be overwritten by a local metadata.yaml if one exists!

From 498e1536f5f3e69c50934c0c031055e0af770bf6 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 09:08:56 -0700
Subject: [PATCH 1406/2303] One more blacken-docs test, refs #1718

---
 docs/testing_plugins.rst | 45 ++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index 8e4e3f91..6361d744 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -19,7 +19,10 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
         response = await datasette.client.get("/-/plugins.json")
         assert response.status_code == 200
         installed_plugins = {p["name"] for p in response.json()}
-        assert "datasette-plugin-template-demo" in installed_plugins
+        assert (
+            "datasette-plugin-template-demo"
+            in installed_plugins
+        )
 
 
 This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
@@ -37,9 +40,7 @@ If you are building an installable package you can add them as test dependencies
     setup(
         name="datasette-my-plugin",
         # ...
-        extras_require={
-            "test": ["pytest", "pytest-asyncio"]
-        },
+        extras_require={"test": ["pytest", "pytest-asyncio"]},
         tests_require=["datasette-my-plugin[test]"],
     )
 
@@ -87,31 +88,34 @@ Here's an example that uses the `sqlite-utils library <https://sqlite-utils.data
     import pytest
     import sqlite_utils
 
+
     @pytest.fixture(scope="session")
     def datasette(tmp_path_factory):
         db_directory = tmp_path_factory.mktemp("dbs")
         db_path = db_directory / "test.db"
         db = sqlite_utils.Database(db_path)
-        db["dogs"].insert_all([
-            {"id": 1, "name": "Cleo", "age": 5},
-            {"id": 2, "name": "Pancakes", "age": 4}
-        ], pk="id")
+        db["dogs"].insert_all(
+            [
+                {"id": 1, "name": "Cleo", "age": 5},
+                {"id": 2, "name": "Pancakes", "age": 4},
+            ],
+            pk="id",
+        )
         datasette = Datasette(
             [db_path],
             metadata={
                 "databases": {
                     "test": {
                         "tables": {
-                            "dogs": {
-                                "title": "Some dogs"
-                            }
+                            "dogs": {"title": "Some dogs"}
                         }
                     }
                 }
-            }
+            },
         )
         return datasette
 
+
     @pytest.mark.asyncio
     async def test_example_table_json(datasette):
         response = await datasette.client.get("/test/dogs.json?_shape=array")
@@ -121,6 +125,7 @@ Here's an example that uses the `sqlite-utils library <https://sqlite-utils.data
             {"id": 2, "name": "Pancakes", "age": 4},
         ]
 
+
     @pytest.mark.asyncio
     async def test_example_table_html(datasette):
         response = await datasette.client.get("/test/dogs")
@@ -137,6 +142,7 @@ If you want to create that test database repeatedly for every individual test fu
     @pytest.fixture
     def datasette(tmp_path_factory):
         # This fixture will be executed repeatedly for every test
+        ...
 
 .. _testing_plugins_pytest_httpx:
 
@@ -197,14 +203,17 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
 
     async def test_outbound_http_call(httpx_mock):
         httpx_mock.add_response(
-            url='https://www.example.com/',
-            text='Hello world',
+            url="https://www.example.com/",
+            text="Hello world",
         )
         datasette = Datasette([], memory=True)
-        response = await datasette.client.post("/-/fetch-url", data={
-            "url": "https://www.example.com/"
-        })
+        response = await datasette.client.post(
+            "/-/fetch-url",
+            data={"url": "https://www.example.com/"},
+        )
         assert response.text == "Hello world"
 
         outbound_request = httpx_mock.get_request()
-        assert outbound_request.url == "https://www.example.com/"
+        assert (
+            outbound_request.url == "https://www.example.com/"
+        )

From 289e4cf80a14f05f791b218f092556148b49a0fa Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 09:17:59 -0700
Subject: [PATCH 1407/2303] Finished applying blacken-docs, closes #1718

---
 .github/workflows/test.yml | 3 +--
 docs/testing_plugins.rst   | 4 +++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 38b62995..8d916e49 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,6 +34,5 @@ jobs:
         cog --check docs/*.rst
     - name: Check if blacken-docs needs to be run
       run: |
+        # This fails on syntax errors, or a diff was applied
         blacken-docs -l 60 docs/*.rst
-        # This fails if a diff was generated:
-        git diff-index --quiet HEAD --
diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index 6361d744..1bbaaac1 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -118,7 +118,9 @@ Here's an example that uses the `sqlite-utils library <https://sqlite-utils.data
 
     @pytest.mark.asyncio
     async def test_example_table_json(datasette):
-        response = await datasette.client.get("/test/dogs.json?_shape=array")
+        response = await datasette.client.get(
+            "/test/dogs.json?_shape=array"
+        )
         assert response.status_code == 200
         assert response.json() == [
             {"id": 1, "name": "Cleo", "age": 5},

From 7463b051cf8d7f856df5eba9f7aa944183ebabe5 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 24 Apr 2022 09:59:20 -0700
Subject: [PATCH 1408/2303] Cosmetic tweaks after blacken-docs, refs #1718

---
 docs/plugin_hooks.rst | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index ace206b7..4560ec9a 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -162,9 +162,8 @@ And here's an example which adds a ``sql_first(sql_query)`` function which execu
                 or database
                 or next(iter(datasette.databases.keys()))
             )
-            return (await datasette.execute(dbname, sql)).rows[
-                0
-            ][0]
+            result = await datasette.execute(dbname, sql)
+            return result.rows[0][0]
 
         return {"sql_first": sql_first}
 
@@ -422,8 +421,8 @@ If the value matches that pattern, the plugin returns an HTML link element:
         if not isinstance(value, str):
             return None
         stripped = value.strip()
-        if not stripped.startswith("{") and stripped.endswith(
-            "}"
+        if not (
+            stripped.startswith("{") and stripped.endswith("}")
         ):
             return None
         try:

From 579f59dcec43a91dd7d404e00b87a00afd8515f2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 25 Apr 2022 11:33:35 -0700
Subject: [PATCH 1409/2303] Refactor to remove RowTableShared class, closes
 #1719

Refs #1715
---
 datasette/app.py         |   3 +-
 datasette/views/row.py   | 142 +++++++++++
 datasette/views/table.py | 497 +++++++++++++++------------------------
 3 files changed, 328 insertions(+), 314 deletions(-)
 create mode 100644 datasette/views/row.py

diff --git a/datasette/app.py b/datasette/app.py
index c9eede26..d269372c 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -40,7 +40,8 @@ from .views.special import (
     PermissionsDebugView,
     MessagesDebugView,
 )
-from .views.table import RowView, TableView
+from .views.table import TableView
+from .views.row import RowView
 from .renderer import json_renderer
 from .url_builder import Urls
 from .database import Database, QueryInterrupted
diff --git a/datasette/views/row.py b/datasette/views/row.py
new file mode 100644
index 00000000..b1c7362d
--- /dev/null
+++ b/datasette/views/row.py
@@ -0,0 +1,142 @@
+from datasette.utils.asgi import NotFound
+from datasette.database import QueryInterrupted
+from .base import DataView
+from datasette.utils import (
+    tilde_decode,
+    urlsafe_components,
+    to_css_class,
+    escape_sqlite,
+)
+from .table import _sql_params_pks, display_columns_and_rows
+
+
+class RowView(DataView):
+    name = "row"
+
+    async def data(self, request, default_labels=False):
+        database_route = tilde_decode(request.url_vars["database"])
+        table = tilde_decode(request.url_vars["table"])
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
+        await self.ds.ensure_permissions(
+            request.actor,
+            [
+                ("view-table", (database, table)),
+                ("view-database", database),
+                "view-instance",
+            ],
+        )
+        pk_values = urlsafe_components(request.url_vars["pks"])
+        try:
+            db = self.ds.get_database(route=database_route)
+        except KeyError:
+            raise NotFound("Database not found: {}".format(database_route))
+        database = db.name
+        sql, params, pks = await _sql_params_pks(db, table, pk_values)
+        results = await db.execute(sql, params, truncate=True)
+        columns = [r[0] for r in results.description]
+        rows = list(results.rows)
+        if not rows:
+            raise NotFound(f"Record not found: {pk_values}")
+
+        async def template_data():
+            display_columns, display_rows = await display_columns_and_rows(
+                self.ds,
+                database,
+                table,
+                results.description,
+                rows,
+                link_column=False,
+                truncate_cells=0,
+            )
+            for column in display_columns:
+                column["sortable"] = False
+            return {
+                "foreign_key_tables": await self.foreign_key_tables(
+                    database, table, pk_values
+                ),
+                "display_columns": display_columns,
+                "display_rows": display_rows,
+                "custom_table_templates": [
+                    f"_table-{to_css_class(database)}-{to_css_class(table)}.html",
+                    f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html",
+                    "_table.html",
+                ],
+                "metadata": (self.ds.metadata("databases") or {})
+                .get(database, {})
+                .get("tables", {})
+                .get(table, {}),
+            }
+
+        data = {
+            "database": database,
+            "table": table,
+            "rows": rows,
+            "columns": columns,
+            "primary_keys": pks,
+            "primary_key_values": pk_values,
+            "units": self.ds.table_metadata(database, table).get("units", {}),
+        }
+
+        if "foreign_key_tables" in (request.args.get("_extras") or "").split(","):
+            data["foreign_key_tables"] = await self.foreign_key_tables(
+                database, table, pk_values
+            )
+
+        return (
+            data,
+            template_data,
+            (
+                f"row-{to_css_class(database)}-{to_css_class(table)}.html",
+                "row.html",
+            ),
+        )
+
+    async def foreign_key_tables(self, database, table, pk_values):
+        if len(pk_values) != 1:
+            return []
+        db = self.ds.databases[database]
+        all_foreign_keys = await db.get_all_foreign_keys()
+        foreign_keys = all_foreign_keys[table]["incoming"]
+        if len(foreign_keys) == 0:
+            return []
+
+        sql = "select " + ", ".join(
+            [
+                "(select count(*) from {table} where {column}=:id)".format(
+                    table=escape_sqlite(fk["other_table"]),
+                    column=escape_sqlite(fk["other_column"]),
+                )
+                for fk in foreign_keys
+            ]
+        )
+        try:
+            rows = list(await db.execute(sql, {"id": pk_values[0]}))
+        except QueryInterrupted:
+            # Almost certainly hit the timeout
+            return []
+
+        foreign_table_counts = dict(
+            zip(
+                [(fk["other_table"], fk["other_column"]) for fk in foreign_keys],
+                list(rows[0]),
+            )
+        )
+        foreign_key_tables = []
+        for fk in foreign_keys:
+            count = (
+                foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0
+            )
+            key = fk["other_column"]
+            if key.startswith("_"):
+                key += "__exact"
+            link = "{}?{}={}".format(
+                self.ds.urls.table(database, fk["other_table"]),
+                key,
+                ",".join(pk_values),
+            )
+            foreign_key_tables.append({**fk, **{"count": count, "link": link}})
+        return foreign_key_tables
diff --git a/datasette/views/table.py b/datasette/views/table.py
index dc85165e..37fb2ebb 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -1,4 +1,3 @@
-import urllib
 import itertools
 import json
 
@@ -9,7 +8,6 @@ from datasette.database import QueryInterrupted
 from datasette.utils import (
     await_me_maybe,
     CustomRow,
-    MultiParams,
     append_querystring,
     compound_keys_after_sql,
     format_bytes,
@@ -21,7 +19,6 @@ from datasette.utils import (
     is_url,
     path_from_row_pks,
     path_with_added_args,
-    path_with_format,
     path_with_removed_args,
     path_with_replaced_args,
     to_css_class,
@@ -68,7 +65,9 @@ class Row:
         return json.dumps(d, default=repr, indent=2)
 
 
-class RowTableShared(DataView):
+class TableView(DataView):
+    name = "table"
+
     async def sortable_columns_for_table(self, database, table, use_rowid):
         db = self.ds.databases[database]
         table_metadata = self.ds.table_metadata(database, table)
@@ -89,193 +88,6 @@ class RowTableShared(DataView):
             expandables.append((fk, label_column))
         return expandables
 
-    async def display_columns_and_rows(
-        self, database, table, description, rows, link_column=False, truncate_cells=0
-    ):
-        """Returns columns, rows for specified table - including fancy foreign key treatment"""
-        db = self.ds.databases[database]
-        table_metadata = self.ds.table_metadata(database, table)
-        column_descriptions = table_metadata.get("columns") or {}
-        column_details = {col.name: col for col in await db.table_column_details(table)}
-        sortable_columns = await self.sortable_columns_for_table(database, table, True)
-        pks = await db.primary_keys(table)
-        pks_for_display = pks
-        if not pks_for_display:
-            pks_for_display = ["rowid"]
-
-        columns = []
-        for r in description:
-            if r[0] == "rowid" and "rowid" not in column_details:
-                type_ = "integer"
-                notnull = 0
-            else:
-                type_ = column_details[r[0]].type
-                notnull = column_details[r[0]].notnull
-            columns.append(
-                {
-                    "name": r[0],
-                    "sortable": r[0] in sortable_columns,
-                    "is_pk": r[0] in pks_for_display,
-                    "type": type_,
-                    "notnull": notnull,
-                    "description": column_descriptions.get(r[0]),
-                }
-            )
-
-        column_to_foreign_key_table = {
-            fk["column"]: fk["other_table"]
-            for fk in await db.foreign_keys_for_table(table)
-        }
-
-        cell_rows = []
-        base_url = self.ds.setting("base_url")
-        for row in rows:
-            cells = []
-            # Unless we are a view, the first column is a link - either to the rowid
-            # or to the simple or compound primary key
-            if link_column:
-                is_special_link_column = len(pks) != 1
-                pk_path = path_from_row_pks(row, pks, not pks, False)
-                cells.append(
-                    {
-                        "column": pks[0] if len(pks) == 1 else "Link",
-                        "value_type": "pk",
-                        "is_special_link_column": is_special_link_column,
-                        "raw": pk_path,
-                        "value": markupsafe.Markup(
-                            '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format(
-                                base_url=base_url,
-                                table_path=self.ds.urls.table(database, table),
-                                flat_pks=str(markupsafe.escape(pk_path)),
-                                flat_pks_quoted=path_from_row_pks(row, pks, not pks),
-                            )
-                        ),
-                    }
-                )
-
-            for value, column_dict in zip(row, columns):
-                column = column_dict["name"]
-                if link_column and len(pks) == 1 and column == pks[0]:
-                    # If there's a simple primary key, don't repeat the value as it's
-                    # already shown in the link column.
-                    continue
-
-                # First let the plugins have a go
-                # pylint: disable=no-member
-                plugin_display_value = None
-                for candidate in pm.hook.render_cell(
-                    value=value,
-                    column=column,
-                    table=table,
-                    database=database,
-                    datasette=self.ds,
-                ):
-                    candidate = await await_me_maybe(candidate)
-                    if candidate is not None:
-                        plugin_display_value = candidate
-                        break
-                if plugin_display_value:
-                    display_value = plugin_display_value
-                elif isinstance(value, bytes):
-                    formatted = format_bytes(len(value))
-                    display_value = markupsafe.Markup(
-                        '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format(
-                            self.ds.urls.row_blob(
-                                database,
-                                table,
-                                path_from_row_pks(row, pks, not pks),
-                                column,
-                            ),
-                            ' title="{}"'.format(formatted)
-                            if "bytes" not in formatted
-                            else "",
-                            len(value),
-                            "" if len(value) == 1 else "s",
-                        )
-                    )
-                elif isinstance(value, dict):
-                    # It's an expanded foreign key - display link to other row
-                    label = value["label"]
-                    value = value["value"]
-                    # The table we link to depends on the column
-                    other_table = column_to_foreign_key_table[column]
-                    link_template = (
-                        LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE
-                    )
-                    display_value = markupsafe.Markup(
-                        link_template.format(
-                            database=database,
-                            base_url=base_url,
-                            table=tilde_encode(other_table),
-                            link_id=tilde_encode(str(value)),
-                            id=str(markupsafe.escape(value)),
-                            label=str(markupsafe.escape(label)) or "-",
-                        )
-                    )
-                elif value in ("", None):
-                    display_value = markupsafe.Markup(" ")
-                elif is_url(str(value).strip()):
-                    display_value = markupsafe.Markup(
-                        '<a href="{url}">{url}</a>'.format(
-                            url=markupsafe.escape(value.strip())
-                        )
-                    )
-                elif column in table_metadata.get("units", {}) and value != "":
-                    # Interpret units using pint
-                    value = value * ureg(table_metadata["units"][column])
-                    # Pint uses floating point which sometimes introduces errors in the compact
-                    # representation, which we have to round off to avoid ugliness. In the vast
-                    # majority of cases this rounding will be inconsequential. I hope.
-                    value = round(value.to_compact(), 6)
-                    display_value = markupsafe.Markup(
-                        f"{value:~P}".replace(" ", " ")
-                    )
-                else:
-                    display_value = str(value)
-                    if truncate_cells and len(display_value) > truncate_cells:
-                        display_value = display_value[:truncate_cells] + "\u2026"
-
-                cells.append(
-                    {
-                        "column": column,
-                        "value": display_value,
-                        "raw": value,
-                        "value_type": "none"
-                        if value is None
-                        else str(type(value).__name__),
-                    }
-                )
-            cell_rows.append(Row(cells))
-
-        if link_column:
-            # Add the link column header.
-            # If it's a simple primary key, we have to remove and re-add that column name at
-            # the beginning of the header row.
-            first_column = None
-            if len(pks) == 1:
-                columns = [col for col in columns if col["name"] != pks[0]]
-                first_column = {
-                    "name": pks[0],
-                    "sortable": len(pks) == 1,
-                    "is_pk": True,
-                    "type": column_details[pks[0]].type,
-                    "notnull": column_details[pks[0]].notnull,
-                }
-            else:
-                first_column = {
-                    "name": "Link",
-                    "sortable": False,
-                    "is_pk": False,
-                    "type": "",
-                    "notnull": 0,
-                }
-            columns = [first_column] + columns
-        return columns, cell_rows
-
-
-class TableView(RowTableShared):
-    name = "table"
-
     async def post(self, request):
         database_route = tilde_decode(request.url_vars["database"])
         try:
@@ -807,13 +619,17 @@ class TableView(RowTableShared):
         async def extra_template():
             nonlocal sort
 
-            display_columns, display_rows = await self.display_columns_and_rows(
+            display_columns, display_rows = await display_columns_and_rows(
+                self.ds,
                 database,
                 table,
                 results.description,
                 rows,
                 link_column=not is_view,
                 truncate_cells=self.ds.setting("truncate_cells_html"),
+                sortable_columns=await self.sortable_columns_for_table(
+                    database, table, use_rowid=True
+                ),
             )
             metadata = (
                 (self.ds.metadata("databases") or {})
@@ -948,132 +764,187 @@ async def _sql_params_pks(db, table, pk_values):
     return sql, params, pks
 
 
-class RowView(RowTableShared):
-    name = "row"
+async def display_columns_and_rows(
+    datasette,
+    database,
+    table,
+    description,
+    rows,
+    link_column=False,
+    truncate_cells=0,
+    sortable_columns=None,
+):
+    """Returns columns, rows for specified table - including fancy foreign key treatment"""
+    sortable_columns = sortable_columns or set()
+    db = datasette.databases[database]
+    table_metadata = datasette.table_metadata(database, table)
+    column_descriptions = table_metadata.get("columns") or {}
+    column_details = {col.name: col for col in await db.table_column_details(table)}
+    pks = await db.primary_keys(table)
+    pks_for_display = pks
+    if not pks_for_display:
+        pks_for_display = ["rowid"]
 
-    async def data(self, request, default_labels=False):
-        database_route = tilde_decode(request.url_vars["database"])
-        table = tilde_decode(request.url_vars["table"])
-        try:
-            db = self.ds.get_database(route=database_route)
-        except KeyError:
-            raise NotFound("Database not found: {}".format(database_route))
-        database = db.name
-        await self.ds.ensure_permissions(
-            request.actor,
-            [
-                ("view-table", (database, table)),
-                ("view-database", database),
-                "view-instance",
-            ],
-        )
-        pk_values = urlsafe_components(request.url_vars["pks"])
-        try:
-            db = self.ds.get_database(route=database_route)
-        except KeyError:
-            raise NotFound("Database not found: {}".format(database_route))
-        database = db.name
-        sql, params, pks = await _sql_params_pks(db, table, pk_values)
-        results = await db.execute(sql, params, truncate=True)
-        columns = [r[0] for r in results.description]
-        rows = list(results.rows)
-        if not rows:
-            raise NotFound(f"Record not found: {pk_values}")
-
-        async def template_data():
-            display_columns, display_rows = await self.display_columns_and_rows(
-                database,
-                table,
-                results.description,
-                rows,
-                link_column=False,
-                truncate_cells=0,
-            )
-            for column in display_columns:
-                column["sortable"] = False
-            return {
-                "foreign_key_tables": await self.foreign_key_tables(
-                    database, table, pk_values
-                ),
-                "display_columns": display_columns,
-                "display_rows": display_rows,
-                "custom_table_templates": [
-                    f"_table-{to_css_class(database)}-{to_css_class(table)}.html",
-                    f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html",
-                    "_table.html",
-                ],
-                "metadata": (self.ds.metadata("databases") or {})
-                .get(database, {})
-                .get("tables", {})
-                .get(table, {}),
+    columns = []
+    for r in description:
+        if r[0] == "rowid" and "rowid" not in column_details:
+            type_ = "integer"
+            notnull = 0
+        else:
+            type_ = column_details[r[0]].type
+            notnull = column_details[r[0]].notnull
+        columns.append(
+            {
+                "name": r[0],
+                "sortable": r[0] in sortable_columns,
+                "is_pk": r[0] in pks_for_display,
+                "type": type_,
+                "notnull": notnull,
+                "description": column_descriptions.get(r[0]),
             }
-
-        data = {
-            "database": database,
-            "table": table,
-            "rows": rows,
-            "columns": columns,
-            "primary_keys": pks,
-            "primary_key_values": pk_values,
-            "units": self.ds.table_metadata(database, table).get("units", {}),
-        }
-
-        if "foreign_key_tables" in (request.args.get("_extras") or "").split(","):
-            data["foreign_key_tables"] = await self.foreign_key_tables(
-                database, table, pk_values
-            )
-
-        return (
-            data,
-            template_data,
-            (
-                f"row-{to_css_class(database)}-{to_css_class(table)}.html",
-                "row.html",
-            ),
         )
 
-    async def foreign_key_tables(self, database, table, pk_values):
-        if len(pk_values) != 1:
-            return []
-        db = self.ds.databases[database]
-        all_foreign_keys = await db.get_all_foreign_keys()
-        foreign_keys = all_foreign_keys[table]["incoming"]
-        if len(foreign_keys) == 0:
-            return []
+    column_to_foreign_key_table = {
+        fk["column"]: fk["other_table"] for fk in await db.foreign_keys_for_table(table)
+    }
 
-        sql = "select " + ", ".join(
-            [
-                "(select count(*) from {table} where {column}=:id)".format(
-                    table=escape_sqlite(fk["other_table"]),
-                    column=escape_sqlite(fk["other_column"]),
+    cell_rows = []
+    base_url = datasette.setting("base_url")
+    for row in rows:
+        cells = []
+        # Unless we are a view, the first column is a link - either to the rowid
+        # or to the simple or compound primary key
+        if link_column:
+            is_special_link_column = len(pks) != 1
+            pk_path = path_from_row_pks(row, pks, not pks, False)
+            cells.append(
+                {
+                    "column": pks[0] if len(pks) == 1 else "Link",
+                    "value_type": "pk",
+                    "is_special_link_column": is_special_link_column,
+                    "raw": pk_path,
+                    "value": markupsafe.Markup(
+                        '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format(
+                            base_url=base_url,
+                            table_path=datasette.urls.table(database, table),
+                            flat_pks=str(markupsafe.escape(pk_path)),
+                            flat_pks_quoted=path_from_row_pks(row, pks, not pks),
+                        )
+                    ),
+                }
+            )
+
+        for value, column_dict in zip(row, columns):
+            column = column_dict["name"]
+            if link_column and len(pks) == 1 and column == pks[0]:
+                # If there's a simple primary key, don't repeat the value as it's
+                # already shown in the link column.
+                continue
+
+            # First let the plugins have a go
+            # pylint: disable=no-member
+            plugin_display_value = None
+            for candidate in pm.hook.render_cell(
+                value=value,
+                column=column,
+                table=table,
+                database=database,
+                datasette=datasette,
+            ):
+                candidate = await await_me_maybe(candidate)
+                if candidate is not None:
+                    plugin_display_value = candidate
+                    break
+            if plugin_display_value:
+                display_value = plugin_display_value
+            elif isinstance(value, bytes):
+                formatted = format_bytes(len(value))
+                display_value = markupsafe.Markup(
+                    '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format(
+                        datasette.urls.row_blob(
+                            database,
+                            table,
+                            path_from_row_pks(row, pks, not pks),
+                            column,
+                        ),
+                        ' title="{}"'.format(formatted)
+                        if "bytes" not in formatted
+                        else "",
+                        len(value),
+                        "" if len(value) == 1 else "s",
+                    )
                 )
-                for fk in foreign_keys
-            ]
-        )
-        try:
-            rows = list(await db.execute(sql, {"id": pk_values[0]}))
-        except QueryInterrupted:
-            # Almost certainly hit the timeout
-            return []
+            elif isinstance(value, dict):
+                # It's an expanded foreign key - display link to other row
+                label = value["label"]
+                value = value["value"]
+                # The table we link to depends on the column
+                other_table = column_to_foreign_key_table[column]
+                link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE
+                display_value = markupsafe.Markup(
+                    link_template.format(
+                        database=database,
+                        base_url=base_url,
+                        table=tilde_encode(other_table),
+                        link_id=tilde_encode(str(value)),
+                        id=str(markupsafe.escape(value)),
+                        label=str(markupsafe.escape(label)) or "-",
+                    )
+                )
+            elif value in ("", None):
+                display_value = markupsafe.Markup(" ")
+            elif is_url(str(value).strip()):
+                display_value = markupsafe.Markup(
+                    '<a href="{url}">{url}</a>'.format(
+                        url=markupsafe.escape(value.strip())
+                    )
+                )
+            elif column in table_metadata.get("units", {}) and value != "":
+                # Interpret units using pint
+                value = value * ureg(table_metadata["units"][column])
+                # Pint uses floating point which sometimes introduces errors in the compact
+                # representation, which we have to round off to avoid ugliness. In the vast
+                # majority of cases this rounding will be inconsequential. I hope.
+                value = round(value.to_compact(), 6)
+                display_value = markupsafe.Markup(f"{value:~P}".replace(" ", " "))
+            else:
+                display_value = str(value)
+                if truncate_cells and len(display_value) > truncate_cells:
+                    display_value = display_value[:truncate_cells] + "\u2026"
 
-        foreign_table_counts = dict(
-            zip(
-                [(fk["other_table"], fk["other_column"]) for fk in foreign_keys],
-                list(rows[0]),
+            cells.append(
+                {
+                    "column": column,
+                    "value": display_value,
+                    "raw": value,
+                    "value_type": "none"
+                    if value is None
+                    else str(type(value).__name__),
+                }
             )
-        )
-        foreign_key_tables = []
-        for fk in foreign_keys:
-            count = (
-                foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0
-            )
-            key = fk["other_column"]
-            if key.startswith("_"):
-                key += "__exact"
-            link = "{}?{}={}".format(
-                self.ds.urls.table(database, fk["other_table"]),
-                key,
-                ",".join(pk_values),
-            )
-            foreign_key_tables.append({**fk, **{"count": count, "link": link}})
-        return foreign_key_tables
+        cell_rows.append(Row(cells))
+
+    if link_column:
+        # Add the link column header.
+        # If it's a simple primary key, we have to remove and re-add that column name at
+        # the beginning of the header row.
+        first_column = None
+        if len(pks) == 1:
+            columns = [col for col in columns if col["name"] != pks[0]]
+            first_column = {
+                "name": pks[0],
+                "sortable": len(pks) == 1,
+                "is_pk": True,
+                "type": column_details[pks[0]].type,
+                "notnull": column_details[pks[0]].notnull,
+            }
+        else:
+            first_column = {
+                "name": "Link",
+                "sortable": False,
+                "is_pk": False,
+                "type": "",
+                "notnull": 0,
+            }
+        columns = [first_column] + columns
+    return columns, cell_rows

From c101f0efeec4f6e49298a542c5e2b59236cfa0ff Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 26 Apr 2022 15:34:29 -0700
Subject: [PATCH 1410/2303] datasette-total-page-time example of asgi_wrapper

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 4560ec9a..3c9ae2e2 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -801,7 +801,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
 
         return wrap_with_databases_header
 
-Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `datasette-pyinstrument <https://datasette.io/plugins/datasette-pyinstrument>`__
+Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `datasette-pyinstrument <https://datasette.io/plugins/datasette-pyinstrument>`__, `datasette-total-page-time <https://datasette.io/plugins/datasette-total-page-time>`__
 
 .. _plugin_hook_startup:
 

From 8a0c38f0b89543e652a968a90d480859cb102510 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 26 Apr 2022 13:56:27 -0700
Subject: [PATCH 1411/2303] Rename database->database_name and table->
 table_name, refs #1715

---
 datasette/views/table.py | 143 +++++++++++++++++++++------------------
 1 file changed, 76 insertions(+), 67 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 37fb2ebb..d66adb82 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -68,22 +68,22 @@ class Row:
 class TableView(DataView):
     name = "table"
 
-    async def sortable_columns_for_table(self, database, table, use_rowid):
-        db = self.ds.databases[database]
-        table_metadata = self.ds.table_metadata(database, table)
+    async def sortable_columns_for_table(self, database_name, table_name, use_rowid):
+        db = self.ds.databases[database_name]
+        table_metadata = self.ds.table_metadata(database_name, table_name)
         if "sortable_columns" in table_metadata:
             sortable_columns = set(table_metadata["sortable_columns"])
         else:
-            sortable_columns = set(await db.table_columns(table))
+            sortable_columns = set(await db.table_columns(table_name))
         if use_rowid:
             sortable_columns.add("rowid")
         return sortable_columns
 
-    async def expandable_columns(self, database, table):
+    async def expandable_columns(self, database_name, table_name):
         # Returns list of (fk_dict, label_column-or-None) pairs for that table
         expandables = []
-        db = self.ds.databases[database]
-        for fk in await db.foreign_keys_for_table(table):
+        db = self.ds.databases[database_name]
+        for fk in await db.foreign_keys_for_table(table_name):
             label_column = await db.label_column_for_table(fk["other_table"])
             expandables.append((fk, label_column))
         return expandables
@@ -94,17 +94,19 @@ class TableView(DataView):
             db = self.ds.get_database(route=database_route)
         except KeyError:
             raise NotFound("Database not found: {}".format(database_route))
-        database = db.name
-        table = tilde_decode(request.url_vars["table"])
+        database_name = db.name
+        table_name = tilde_decode(request.url_vars["table"])
         # Handle POST to a canned query
-        canned_query = await self.ds.get_canned_query(database, table, request.actor)
+        canned_query = await self.ds.get_canned_query(
+            database_name, table_name, request.actor
+        )
         assert canned_query, "You may only POST to a canned query"
         return await QueryView(self.ds).data(
             request,
             canned_query["sql"],
             metadata=canned_query,
             editable=False,
-            canned_query=table,
+            canned_query=table_name,
             named_parameters=canned_query.get("params"),
             write=bool(canned_query.get("write")),
         )
@@ -150,45 +152,47 @@ class TableView(DataView):
         _size=None,
     ):
         database_route = tilde_decode(request.url_vars["database"])
-        table = tilde_decode(request.url_vars["table"])
+        table_name = tilde_decode(request.url_vars["table"])
         try:
             db = self.ds.get_database(route=database_route)
         except KeyError:
             raise NotFound("Database not found: {}".format(database_route))
-        database = db.name
+        database_name = db.name
 
         # If this is a canned query, not a table, then dispatch to QueryView instead
-        canned_query = await self.ds.get_canned_query(database, table, request.actor)
+        canned_query = await self.ds.get_canned_query(
+            database_name, table_name, request.actor
+        )
         if canned_query:
             return await QueryView(self.ds).data(
                 request,
                 canned_query["sql"],
                 metadata=canned_query,
                 editable=False,
-                canned_query=table,
+                canned_query=table_name,
                 named_parameters=canned_query.get("params"),
                 write=bool(canned_query.get("write")),
             )
 
-        is_view = bool(await db.get_view_definition(table))
-        table_exists = bool(await db.table_exists(table))
+        is_view = bool(await db.get_view_definition(table_name))
+        table_exists = bool(await db.table_exists(table_name))
 
         # If table or view not found, return 404
         if not is_view and not table_exists:
-            raise NotFound(f"Table not found: {table}")
+            raise NotFound(f"Table not found: {table_name}")
 
         # Ensure user has permission to view this table
         await self.ds.ensure_permissions(
             request.actor,
             [
-                ("view-table", (database, table)),
-                ("view-database", database),
+                ("view-table", (database_name, table_name)),
+                ("view-database", database_name),
                 "view-instance",
             ],
         )
 
         private = not await self.ds.permission_allowed(
-            None, "view-table", (database, table), default=True
+            None, "view-table", (database_name, table_name), default=True
         )
 
         # Handle ?_filter_column and redirect, if present
@@ -216,8 +220,8 @@ class TableView(DataView):
             )
 
         # Introspect columns and primary keys for table
-        pks = await db.primary_keys(table)
-        table_columns = await db.table_columns(table)
+        pks = await db.primary_keys(table_name)
+        table_columns = await db.table_columns(table_name)
 
         # Take ?_col= and ?_nocol= into account
         specified_columns = await self.columns_to_select(table_columns, pks, request)
@@ -248,7 +252,7 @@ class TableView(DataView):
             nocount = True
             nofacet = True
 
-        table_metadata = self.ds.table_metadata(database, table)
+        table_metadata = self.ds.table_metadata(database_name, table_name)
         units = table_metadata.get("units", {})
 
         # Arguments that start with _ and don't contain a __ are
@@ -262,7 +266,7 @@ class TableView(DataView):
 
         # Build where clauses from query string arguments
         filters = Filters(sorted(filter_args), units, ureg)
-        where_clauses, params = filters.build_where_clauses(table)
+        where_clauses, params = filters.build_where_clauses(table_name)
 
         # Execute filters_from_request plugin hooks - including the default
         # ones that live in datasette/filters.py
@@ -271,8 +275,8 @@ class TableView(DataView):
 
         for hook in pm.hook.filters_from_request(
             request=request,
-            table=table,
-            database=database,
+            table=table_name,
+            database=database_name,
             datasette=self.ds,
         ):
             filter_arguments = await await_me_maybe(hook)
@@ -284,7 +288,7 @@ class TableView(DataView):
 
         # Deal with custom sort orders
         sortable_columns = await self.sortable_columns_for_table(
-            database, table, use_rowid
+            database_name, table_name, use_rowid
         )
         sort = request.args.get("_sort")
         sort_desc = request.args.get("_sort_desc")
@@ -309,7 +313,7 @@ class TableView(DataView):
             order_by = f"{escape_sqlite(sort_desc)} desc"
 
         from_sql = "from {table_name} {where}".format(
-            table_name=escape_sqlite(table),
+            table_name=escape_sqlite(table_name),
             where=("where {} ".format(" and ".join(where_clauses)))
             if where_clauses
             else "",
@@ -422,7 +426,7 @@ class TableView(DataView):
         sql_no_order_no_limit = (
             "select {select_all_columns} from {table_name} {where}".format(
                 select_all_columns=select_all_columns,
-                table_name=escape_sqlite(table),
+                table_name=escape_sqlite(table_name),
                 where=where_clause,
             )
         )
@@ -430,7 +434,7 @@ class TableView(DataView):
         # This is the SQL that populates the main table on the page
         sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format(
             select_specified_columns=select_specified_columns,
-            table_name=escape_sqlite(table),
+            table_name=escape_sqlite(table_name),
             where=where_clause,
             order_by=order_by,
             page_size=page_size + 1,
@@ -448,13 +452,13 @@ class TableView(DataView):
         if (
             not db.is_mutable
             and self.ds.inspect_data
-            and count_sql == f"select count(*) from {table} "
+            and count_sql == f"select count(*) from {table_name} "
         ):
             # We can use a previously cached table row count
             try:
-                filtered_table_rows_count = self.ds.inspect_data[database]["tables"][
-                    table
-                ]["count"]
+                filtered_table_rows_count = self.ds.inspect_data[database_name][
+                    "tables"
+                ][table_name]["count"]
             except KeyError:
                 pass
 
@@ -484,10 +488,10 @@ class TableView(DataView):
                 klass(
                     self.ds,
                     request,
-                    database,
+                    database_name,
                     sql=sql_no_order_no_limit,
                     params=params,
-                    table=table,
+                    table=table_name,
                     metadata=table_metadata,
                     row_count=filtered_table_rows_count,
                 )
@@ -527,7 +531,7 @@ class TableView(DataView):
 
         # Expand labeled columns if requested
         expanded_columns = []
-        expandable_columns = await self.expandable_columns(database, table)
+        expandable_columns = await self.expandable_columns(database_name, table_name)
         columns_to_expand = None
         try:
             all_labels = value_as_boolean(request.args.get("_labels", ""))
@@ -554,7 +558,9 @@ class TableView(DataView):
                 values = [row[column_index] for row in rows]
                 # Expand them
                 expanded_labels.update(
-                    await self.ds.expand_foreign_keys(database, table, column, values)
+                    await self.ds.expand_foreign_keys(
+                        database_name, table_name, column, values
+                    )
                 )
             if expanded_labels:
                 # Rewrite the rows
@@ -621,21 +627,21 @@ class TableView(DataView):
 
             display_columns, display_rows = await display_columns_and_rows(
                 self.ds,
-                database,
-                table,
+                database_name,
+                table_name,
                 results.description,
                 rows,
                 link_column=not is_view,
                 truncate_cells=self.ds.setting("truncate_cells_html"),
                 sortable_columns=await self.sortable_columns_for_table(
-                    database, table, use_rowid=True
+                    database_name, table_name, use_rowid=True
                 ),
             )
             metadata = (
                 (self.ds.metadata("databases") or {})
-                .get(database, {})
+                .get(database_name, {})
                 .get("tables", {})
-                .get(table, {})
+                .get(table_name, {})
             )
             self.ds.update_with_inherited_metadata(metadata)
 
@@ -661,8 +667,8 @@ class TableView(DataView):
                 links = []
                 for hook in pm.hook.table_actions(
                     datasette=self.ds,
-                    table=table,
-                    database=database,
+                    table=table_name,
+                    database=database_name,
                     actor=request.actor,
                     request=request,
                 ):
@@ -703,13 +709,13 @@ class TableView(DataView):
                 "sort_desc": sort_desc,
                 "disable_sort": is_view,
                 "custom_table_templates": [
-                    f"_table-{to_css_class(database)}-{to_css_class(table)}.html",
-                    f"_table-table-{to_css_class(database)}-{to_css_class(table)}.html",
+                    f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html",
+                    f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html",
                     "_table.html",
                 ],
                 "metadata": metadata,
-                "view_definition": await db.get_view_definition(table),
-                "table_definition": await db.get_table_definition(table),
+                "view_definition": await db.get_view_definition(table_name),
+                "table_definition": await db.get_table_definition(table_name),
                 "datasette_allow_facet": "true"
                 if self.ds.setting("allow_facet")
                 else "false",
@@ -719,8 +725,8 @@ class TableView(DataView):
 
         return (
             {
-                "database": database,
-                "table": table,
+                "database": database_name,
+                "table": table_name,
                 "is_view": is_view,
                 "human_description_en": human_description_en,
                 "rows": rows[:page_size],
@@ -738,12 +744,12 @@ class TableView(DataView):
                 "next_url": next_url,
                 "private": private,
                 "allow_execute_sql": await self.ds.permission_allowed(
-                    request.actor, "execute-sql", database, default=True
+                    request.actor, "execute-sql", database_name, default=True
                 ),
             },
             extra_template,
             (
-                f"table-{to_css_class(database)}-{to_css_class(table)}.html",
+                f"table-{to_css_class(database_name)}-{to_css_class(table_name)}.html",
                 "table.html",
             ),
         )
@@ -766,8 +772,8 @@ async def _sql_params_pks(db, table, pk_values):
 
 async def display_columns_and_rows(
     datasette,
-    database,
-    table,
+    database_name,
+    table_name,
     description,
     rows,
     link_column=False,
@@ -776,11 +782,13 @@ async def display_columns_and_rows(
 ):
     """Returns columns, rows for specified table - including fancy foreign key treatment"""
     sortable_columns = sortable_columns or set()
-    db = datasette.databases[database]
-    table_metadata = datasette.table_metadata(database, table)
+    db = datasette.databases[database_name]
+    table_metadata = datasette.table_metadata(database_name, table_name)
     column_descriptions = table_metadata.get("columns") or {}
-    column_details = {col.name: col for col in await db.table_column_details(table)}
-    pks = await db.primary_keys(table)
+    column_details = {
+        col.name: col for col in await db.table_column_details(table_name)
+    }
+    pks = await db.primary_keys(table_name)
     pks_for_display = pks
     if not pks_for_display:
         pks_for_display = ["rowid"]
@@ -805,7 +813,8 @@ async def display_columns_and_rows(
         )
 
     column_to_foreign_key_table = {
-        fk["column"]: fk["other_table"] for fk in await db.foreign_keys_for_table(table)
+        fk["column"]: fk["other_table"]
+        for fk in await db.foreign_keys_for_table(table_name)
     }
 
     cell_rows = []
@@ -826,7 +835,7 @@ async def display_columns_and_rows(
                     "value": markupsafe.Markup(
                         '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format(
                             base_url=base_url,
-                            table_path=datasette.urls.table(database, table),
+                            table_path=datasette.urls.table(database_name, table_name),
                             flat_pks=str(markupsafe.escape(pk_path)),
                             flat_pks_quoted=path_from_row_pks(row, pks, not pks),
                         )
@@ -847,8 +856,8 @@ async def display_columns_and_rows(
             for candidate in pm.hook.render_cell(
                 value=value,
                 column=column,
-                table=table,
-                database=database,
+                table=table_name,
+                database=database_name,
                 datasette=datasette,
             ):
                 candidate = await await_me_maybe(candidate)
@@ -862,8 +871,8 @@ async def display_columns_and_rows(
                 display_value = markupsafe.Markup(
                     '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format(
                         datasette.urls.row_blob(
-                            database,
-                            table,
+                            database_name,
+                            table_name,
                             path_from_row_pks(row, pks, not pks),
                             column,
                         ),
@@ -883,7 +892,7 @@ async def display_columns_and_rows(
                 link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE
                 display_value = markupsafe.Markup(
                     link_template.format(
-                        database=database,
+                        database=database_name,
                         base_url=base_url,
                         table=tilde_encode(other_table),
                         link_id=tilde_encode(str(value)),

From 942411ef946e9a34a2094944d3423cddad27efd3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 26 Apr 2022 15:48:56 -0700
Subject: [PATCH 1412/2303] Execute some TableView queries in parallel

Use ?_noparallel=1 to opt out (undocumented, useful for benchmark comparisons)

Refs #1723, #1715
---
 datasette/views/table.py | 91 +++++++++++++++++++++++++++++-----------
 1 file changed, 66 insertions(+), 25 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index d66adb82..23289b29 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -1,3 +1,4 @@
+import asyncio
 import itertools
 import json
 
@@ -5,6 +6,7 @@ import markupsafe
 
 from datasette.plugins import pm
 from datasette.database import QueryInterrupted
+from datasette import tracer
 from datasette.utils import (
     await_me_maybe,
     CustomRow,
@@ -150,6 +152,16 @@ class TableView(DataView):
         default_labels=False,
         _next=None,
         _size=None,
+    ):
+        with tracer.trace_child_tasks():
+            return await self._data_traced(request, default_labels, _next, _size)
+
+    async def _data_traced(
+        self,
+        request,
+        default_labels=False,
+        _next=None,
+        _size=None,
     ):
         database_route = tilde_decode(request.url_vars["database"])
         table_name = tilde_decode(request.url_vars["table"])
@@ -159,6 +171,20 @@ class TableView(DataView):
             raise NotFound("Database not found: {}".format(database_route))
         database_name = db.name
 
+        # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather
+        async def _gather_parallel(*args):
+            return await asyncio.gather(*args)
+
+        async def _gather_sequential(*args):
+            results = []
+            for fn in args:
+                results.append(await fn)
+            return results
+
+        gather = (
+            _gather_sequential if request.args.get("_noparallel") else _gather_parallel
+        )
+
         # If this is a canned query, not a table, then dispatch to QueryView instead
         canned_query = await self.ds.get_canned_query(
             database_name, table_name, request.actor
@@ -174,8 +200,12 @@ class TableView(DataView):
                 write=bool(canned_query.get("write")),
             )
 
-        is_view = bool(await db.get_view_definition(table_name))
-        table_exists = bool(await db.table_exists(table_name))
+        is_view, table_exists = map(
+            bool,
+            await gather(
+                db.get_view_definition(table_name), db.table_exists(table_name)
+            ),
+        )
 
         # If table or view not found, return 404
         if not is_view and not table_exists:
@@ -497,33 +527,44 @@ class TableView(DataView):
                 )
             )
 
-        if not nofacet:
-            for facet in facet_instances:
-                (
+        async def execute_facets():
+            if not nofacet:
+                # Run them in parallel
+                facet_awaitables = [facet.facet_results() for facet in facet_instances]
+                facet_awaitable_results = await gather(*facet_awaitables)
+                for (
                     instance_facet_results,
                     instance_facets_timed_out,
-                ) = await facet.facet_results()
-                for facet_info in instance_facet_results:
-                    base_key = facet_info["name"]
-                    key = base_key
-                    i = 1
-                    while key in facet_results:
-                        i += 1
-                        key = f"{base_key}_{i}"
-                    facet_results[key] = facet_info
-                facets_timed_out.extend(instance_facets_timed_out)
+                ) in facet_awaitable_results:
+                    for facet_info in instance_facet_results:
+                        base_key = facet_info["name"]
+                        key = base_key
+                        i = 1
+                        while key in facet_results:
+                            i += 1
+                            key = f"{base_key}_{i}"
+                        facet_results[key] = facet_info
+                    facets_timed_out.extend(instance_facets_timed_out)
 
-        # Calculate suggested facets
         suggested_facets = []
-        if (
-            self.ds.setting("suggest_facets")
-            and self.ds.setting("allow_facet")
-            and not _next
-            and not nofacet
-            and not nosuggest
-        ):
-            for facet in facet_instances:
-                suggested_facets.extend(await facet.suggest())
+
+        async def execute_suggested_facets():
+            # Calculate suggested facets
+            if (
+                self.ds.setting("suggest_facets")
+                and self.ds.setting("allow_facet")
+                and not _next
+                and not nofacet
+                and not nosuggest
+            ):
+                # Run them in parallel
+                facet_suggest_awaitables = [
+                    facet.suggest() for facet in facet_instances
+                ]
+                for suggest_result in await gather(*facet_suggest_awaitables):
+                    suggested_facets.extend(suggest_result)
+
+        await gather(execute_facets(), execute_suggested_facets())
 
         # Figure out columns and rows for the query
         columns = [r[0] for r in results.description]

From 94a3171b01fde5c52697aeeff052e3ad4bab5391 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 28 Apr 2022 13:29:11 -0700
Subject: [PATCH 1413/2303] .plugin_config() can return None

---
 docs/internals.rst       | 4 ++++
 docs/writing_plugins.rst | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index aad608dc..18822d47 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -288,6 +288,10 @@ All databases are listed, irrespective of user permissions. This means that the
 
 This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used.
 
+The return value will be the value from the configuration file - usually a dictionary.
+
+If the plugin is not configured the return value will be ``None``.
+
 .. _datasette_render_template:
 
 await .render_template(template, context=None, request=None)
diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst
index 89f7f5eb..9aee70f6 100644
--- a/docs/writing_plugins.rst
+++ b/docs/writing_plugins.rst
@@ -182,6 +182,8 @@ When you are writing plugins, you can access plugin configuration like this usin
 
 This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` in the above example.
 
+If there is no configuration for that plugin, the method will return ``None``.
+
 If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so::
 
     {

From 4afc1afc721ac0d14f58b0f8339c1bf431d5313c Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 12:13:11 -0700
Subject: [PATCH 1414/2303] Depend on click-default-group-wheel>=1.2.2

Refs #1733
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 7f0562fd..fcb43aa1 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ setup(
     install_requires=[
         "asgiref>=3.2.10,<3.6.0",
         "click>=7.1.1,<8.2.0",
-        "click-default-group~=1.2.2",
+        "click-default-group-wheel>=1.2.2",
         "Jinja2>=2.10.3,<3.1.0",
         "hupper~=1.9",
         "httpx>=0.20",

From 7e03394734307a5761e4c98d902b6a8cab188562 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 12:20:14 -0700
Subject: [PATCH 1415/2303] Optional uvicorn import for Pyodide, refs #1733

---
 datasette/app.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index d269372c..a5330458 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -26,7 +26,6 @@ from itsdangerous import URLSafeSerializer
 from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
 from jinja2.environment import Template
 from jinja2.exceptions import TemplateNotFound
-import uvicorn
 
 from .views.base import DatasetteError, ureg
 from .views.database import DatabaseDownload, DatabaseView
@@ -806,6 +805,15 @@ class Datasette:
         datasette_version = {"version": __version__}
         if self.version_note:
             datasette_version["note"] = self.version_note
+
+        try:
+            # Optional import to avoid breaking Pyodide
+            # https://github.com/simonw/datasette/issues/1733#issuecomment-1115268245
+            import uvicorn
+
+            uvicorn_version = uvicorn.__version__
+        except ImportError:
+            uvicorn_version = None
         info = {
             "python": {
                 "version": ".".join(map(str, sys.version_info[:3])),
@@ -813,7 +821,7 @@ class Datasette:
             },
             "datasette": datasette_version,
             "asgi": "3.0",
-            "uvicorn": uvicorn.__version__,
+            "uvicorn": uvicorn_version,
             "sqlite": {
                 "version": sqlite_version,
                 "fts_versions": fts_versions,

From 687907aa2b1bde4de6ae7155b0e2a949ca015ca9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 12:39:06 -0700
Subject: [PATCH 1416/2303] Remove python-baseconv dependency, refs #1733,
 closes #1734

---
 datasette/actor_auth_cookie.py |  2 +-
 datasette/utils/baseconv.py    | 59 ++++++++++++++++++++++++++++++++++
 docs/authentication.rst        |  4 +--
 setup.py                       |  1 -
 tests/test_auth.py             |  2 +-
 5 files changed, 63 insertions(+), 5 deletions(-)
 create mode 100644 datasette/utils/baseconv.py

diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py
index 15ecd331..368213af 100644
--- a/datasette/actor_auth_cookie.py
+++ b/datasette/actor_auth_cookie.py
@@ -1,6 +1,6 @@
 from datasette import hookimpl
 from itsdangerous import BadSignature
-import baseconv
+from datasette.utils import baseconv
 import time
 
 
diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py
new file mode 100644
index 00000000..27e4fb00
--- /dev/null
+++ b/datasette/utils/baseconv.py
@@ -0,0 +1,59 @@
+"""
+Convert numbers from base 10 integers to base X strings and back again.
+
+Sample usage:
+
+>>> base20 = BaseConverter('0123456789abcdefghij')
+>>> base20.from_decimal(1234)
+'31e'
+>>> base20.to_decimal('31e')
+1234
+
+Originally shared here: https://www.djangosnippets.org/snippets/1431/
+"""
+
+
+class BaseConverter(object):
+    decimal_digits = "0123456789"
+
+    def __init__(self, digits):
+        self.digits = digits
+
+    def from_decimal(self, i):
+        return self.convert(i, self.decimal_digits, self.digits)
+
+    def to_decimal(self, s):
+        return int(self.convert(s, self.digits, self.decimal_digits))
+
+    def convert(number, fromdigits, todigits):
+        # Based on http://code.activestate.com/recipes/111286/
+        if str(number)[0] == "-":
+            number = str(number)[1:]
+            neg = 1
+        else:
+            neg = 0
+
+        # make an integer out of the number
+        x = 0
+        for digit in str(number):
+            x = x * len(fromdigits) + fromdigits.index(digit)
+
+        # create the result in base 'len(todigits)'
+        if x == 0:
+            res = todigits[0]
+        else:
+            res = ""
+            while x > 0:
+                digit = x % len(todigits)
+                res = todigits[digit] + res
+                x = int(x / len(todigits))
+            if neg:
+                res = "-" + res
+        return res
+
+    convert = staticmethod(convert)
+
+
+bin = BaseConverter("01")
+hexconv = BaseConverter("0123456789ABCDEF")
+base62 = BaseConverter("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz")
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 24960733..685dab15 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -401,12 +401,12 @@ Including an expiry time
 
 ``ds_actor`` cookies can optionally include a signed expiry timestamp, after which the cookies will no longer be valid. Authentication plugins may chose to use this mechanism to limit the lifetime of the cookie. For example, if a plugin implements single-sign-on against another source it may decide to set short-lived cookies so that if the user is removed from the SSO system their existing Datasette cookies will stop working shortly afterwards.
 
-To include an expiry, add a ``"e"`` key to the cookie value containing a `base62-encoded integer <https://pypi.org/project/python-baseconv/>`__ representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours:
+To include an expiry, add a ``"e"`` key to the cookie value containing a base62-encoded integer representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours:
 
 .. code-block:: python
 
     import time
-    import baseconv
+    from datasette.utils import baseconv
 
     expires_at = int(time.time()) + (24 * 60 * 60)
 
diff --git a/setup.py b/setup.py
index fcb43aa1..ca449f02 100644
--- a/setup.py
+++ b/setup.py
@@ -57,7 +57,6 @@ setup(
         "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",
         "itsdangerous>=1.1,<3.0",
-        "python-baseconv==1.2.2",
     ],
     entry_points="""
         [console_scripts]
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 974f89ea..4ef35a76 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -1,5 +1,5 @@
 from .fixtures import app_client
-import baseconv
+from datasette.utils import baseconv
 import pytest
 import time
 

From a29c1277896b6a7905ef5441c42a37bc15f67599 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 12:44:09 -0700
Subject: [PATCH 1417/2303] Rename to_decimal/from_decimal to decode/encode,
 refs #1734

---
 datasette/utils/baseconv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py
index 27e4fb00..c4b64908 100644
--- a/datasette/utils/baseconv.py
+++ b/datasette/utils/baseconv.py
@@ -19,10 +19,10 @@ class BaseConverter(object):
     def __init__(self, digits):
         self.digits = digits
 
-    def from_decimal(self, i):
+    def encode(self, i):
         return self.convert(i, self.decimal_digits, self.digits)
 
-    def to_decimal(self, s):
+    def decode(self, s):
         return int(self.convert(s, self.digits, self.decimal_digits))
 
     def convert(number, fromdigits, todigits):

From 3f00a29141bdea5be747f6d1c93871ccdb792167 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 13:15:27 -0700
Subject: [PATCH 1418/2303] Clean up compatibility with Pyodide (#1736)

* Optional uvicorn import for Pyodide, refs #1733
* --setting num_sql_threads 0 to disable threading, refs #1735
---
 datasette/app.py                  | 11 ++++++++---
 datasette/database.py             | 19 +++++++++++++++++++
 docs/settings.rst                 |  2 ++
 tests/test_internals_datasette.py | 14 +++++++++++++-
 4 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index a5330458..b7b84371 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -288,9 +288,12 @@ class Datasette:
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
-        self.executor = futures.ThreadPoolExecutor(
-            max_workers=self.setting("num_sql_threads")
-        )
+        if self.setting("num_sql_threads") == 0:
+            self.executor = None
+        else:
+            self.executor = futures.ThreadPoolExecutor(
+                max_workers=self.setting("num_sql_threads")
+            )
         self.max_returned_rows = self.setting("max_returned_rows")
         self.sql_time_limit_ms = self.setting("sql_time_limit_ms")
         self.page_size = self.setting("default_page_size")
@@ -862,6 +865,8 @@ class Datasette:
         ]
 
     def _threads(self):
+        if self.setting("num_sql_threads") == 0:
+            return {"num_threads": 0, "threads": []}
         threads = list(threading.enumerate())
         d = {
             "num_threads": len(threads),
diff --git a/datasette/database.py b/datasette/database.py
index ba594a8c..44d32667 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -45,6 +45,9 @@ class Database:
         self._cached_table_counts = None
         self._write_thread = None
         self._write_queue = None
+        # These are used when in non-threaded mode:
+        self._read_connection = None
+        self._write_connection = None
         if not self.is_mutable and not self.is_memory:
             p = Path(path)
             self.hash = inspect_hash(p)
@@ -134,6 +137,14 @@ class Database:
         return results
 
     async def execute_write_fn(self, fn, block=True):
+        if self.ds.executor is None:
+            # non-threaded mode
+            if self._write_connection is None:
+                self._write_connection = self.connect(write=True)
+                self.ds._prepare_connection(self._write_connection, self.name)
+            return fn(self._write_connection)
+
+        # threaded mode
         task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
         if self._write_queue is None:
             self._write_queue = queue.Queue()
@@ -177,6 +188,14 @@ class Database:
             task.reply_queue.sync_q.put(result)
 
     async def execute_fn(self, fn):
+        if self.ds.executor is None:
+            # non-threaded mode
+            if self._read_connection is None:
+                self._read_connection = self.connect()
+                self.ds._prepare_connection(self._read_connection, self.name)
+            return fn(self._read_connection)
+
+        # threaded mode
         def in_thread():
             conn = getattr(connections, self.name, None)
             if not conn:
diff --git a/docs/settings.rst b/docs/settings.rst
index 60c4b36d..8437fb04 100644
--- a/docs/settings.rst
+++ b/docs/settings.rst
@@ -107,6 +107,8 @@ Maximum number of threads in the thread pool Datasette uses to execute SQLite qu
 
     datasette mydatabase.db --setting num_sql_threads 10
 
+Setting this to 0 turns off threaded SQL queries entirely - useful for environments that do not support threading such as `Pyodide <https://pyodide.org/>`__.
+
 .. _setting_allow_facet:
 
 allow_facet
diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py
index cc200a2d..1dc14cab 100644
--- a/tests/test_internals_datasette.py
+++ b/tests/test_internals_datasette.py
@@ -1,7 +1,7 @@
 """
 Tests for the datasette.app.Datasette class
 """
-from datasette.app import Datasette
+from datasette.app import Datasette, Database
 from itsdangerous import BadSignature
 from .fixtures import app_client
 import pytest
@@ -63,3 +63,15 @@ async def test_datasette_constructor():
             "hash": None,
         }
     ]
+
+
+@pytest.mark.asyncio
+async def test_num_sql_threads_zero():
+    ds = Datasette([], memory=True, settings={"num_sql_threads": 0})
+    db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero"))
+    await db.execute_write("create table t(id integer primary key)")
+    await db.execute_write("insert into t (id) values (1)")
+    response = await ds.client.get("/-/threads.json")
+    assert response.json() == {"num_threads": 0, "threads": []}
+    response2 = await ds.client.get("/test_num_sql_threads_zero/t.json?_shape=array")
+    assert response2.json() == [{"id": 1}]

From 943aa2e1f7341cb51e60332cde46bde650c64217 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 14:38:34 -0700
Subject: [PATCH 1419/2303] Release 0.62a0

Refs #1683, #1701, #1712, #1717, #1718, #1733
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 02451a1e..cf18c441 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.61.1"
+__version__ = "0.62a0"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 03cf62b6..74814fcb 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,20 @@
 Changelog
 =========
 
+.. _v0_62a0:
+
+0.62a0 (2022-05-02)
+-------------------
+
+- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details.
+- Datasette should now be compatible with Pyodide. (:issue:`1733`)
+- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__)
+- Spaces in database names are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
+- ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
+- Don't show the facet option in the cog menu if faceting is not allowed. (:issue:`1683`)
+- Code examples in the documentation are now all formatted using Black. (:issue:`1718`)
+- ``Request.fake()`` method is now documented, see :ref:`internals_request`.
+
 .. _v0_61_1:
 
 0.61.1 (2022-03-23)

From 847d6b1aac38c3e776e8c600eed07ba4c9ac9942 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 16:32:24 -0700
Subject: [PATCH 1420/2303] Test wheel against Pyodide, refs #1737, #1733

---
 .github/workflows/test-pyodide.yml   | 28 ++++++++++++++++++
 test-in-pyodide-with-shot-scraper.sh | 43 ++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 .github/workflows/test-pyodide.yml
 create mode 100755 test-in-pyodide-with-shot-scraper.sh

diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml
new file mode 100644
index 00000000..3715d055
--- /dev/null
+++ b/.github/workflows/test-pyodide.yml
@@ -0,0 +1,28 @@
+name: Test in Pyodide with shot-scraper
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+        cache-dependency-path: '**/setup.py'
+    - name: Cache Playwright browsers
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/ms-playwright/
+        key: ${{ runner.os }}-browsers
+    - name: Install Playwright dependencies
+      run: |
+        pip install shot-scraper
+        shot-scraper install
+    - name: Run test
+      run: |
+        ./test-in-pyodide-with-shot-scraper.sh
diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh
new file mode 100755
index 00000000..0f29c0e0
--- /dev/null
+++ b/test-in-pyodide-with-shot-scraper.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Build the wheel
+python3 -m build
+
+# Find name of wheel
+wheel=$(basename $(ls dist/*.whl))
+# strip off the dist/
+
+
+# Create a blank index page
+echo '
+<script src="https://cdn.jsdelivr.net/pyodide/v0.20.0/full/pyodide.js"></script>
+' > dist/index.html
+
+# Run a server for that dist/ folder
+cd dist
+python3 -m http.server 8529 &
+cd ..
+
+shot-scraper javascript http://localhost:8529/ "
+async () => {
+  let pyodide = await loadPyodide();
+  await pyodide.loadPackage(['micropip', 'ssl', 'setuptools']);
+  let output = await pyodide.runPythonAsync(\`
+    import micropip
+    await micropip.install('h11==0.12.0')
+    await micropip.install('http://localhost:8529/$wheel')
+    import ssl
+    import setuptools
+    from datasette.app import Datasette
+    ds = Datasette(memory=True, settings={'num_sql_threads': 0})
+    (await ds.client.get('/_memory.json?sql=select+55+as+itworks&_shape=array')).text
+  \`);
+  if (JSON.parse(output)[0].itworks != 55) {
+    throw 'Got ' + output + ', expected itworks: 55';
+  }
+  return 'Test passed!';
+}
+"
+
+# Shut down the server
+pkill -f 'http.server 8529'

From c0cbcf2aba0d8393ba464acc515803ebf2eeda12 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 16:36:58 -0700
Subject: [PATCH 1421/2303] Tweaks to test scripts, refs #1737

---
 .github/workflows/test-pyodide.yml   | 2 +-
 test-in-pyodide-with-shot-scraper.sh | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml
index 3715d055..beb6a5fb 100644
--- a/.github/workflows/test-pyodide.yml
+++ b/.github/workflows/test-pyodide.yml
@@ -21,7 +21,7 @@ jobs:
         key: ${{ runner.os }}-browsers
     - name: Install Playwright dependencies
       run: |
-        pip install shot-scraper
+        pip install shot-scraper build
         shot-scraper install
     - name: Run test
       run: |
diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh
index 0f29c0e0..e5df7398 100755
--- a/test-in-pyodide-with-shot-scraper.sh
+++ b/test-in-pyodide-with-shot-scraper.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
+set -e
+# So the script fails if there are any errors
 
 # Build the wheel
 python3 -m build
 
-# Find name of wheel
+# Find name of wheel, strip off the dist/
 wheel=$(basename $(ls dist/*.whl))
-# strip off the dist/
-
 
 # Create a blank index page
 echo '

From d60f163528f466b1127b2935c3b6869c34fd6545 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 2 May 2022 16:40:49 -0700
Subject: [PATCH 1422/2303] Run on push and PR, closes #1737

---
 .github/workflows/test-pyodide.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml
index beb6a5fb..1b75aade 100644
--- a/.github/workflows/test-pyodide.yml
+++ b/.github/workflows/test-pyodide.yml
@@ -1,6 +1,8 @@
 name: Test in Pyodide with shot-scraper
 
 on:
+  push:
+  pull_request:
   workflow_dispatch:
 
 jobs:

From 280ff372ab30df244f6c54f6f3002da57334b3d7 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 3 May 2022 07:59:18 -0700
Subject: [PATCH 1423/2303] ETag support for .db downloads, closes #1739

---
 datasette/utils/testing.py  | 20 ++++++++++++++++++--
 datasette/views/database.py |  7 +++++++
 tests/test_html.py          | 10 +++++++++-
 3 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py
index 94750b1f..640c94e6 100644
--- a/datasette/utils/testing.py
+++ b/datasette/utils/testing.py
@@ -55,10 +55,21 @@ class TestClient:
 
     @async_to_sync
     async def get(
-        self, path, follow_redirects=False, redirect_count=0, method="GET", cookies=None
+        self,
+        path,
+        follow_redirects=False,
+        redirect_count=0,
+        method="GET",
+        cookies=None,
+        if_none_match=None,
     ):
         return await self._request(
-            path, follow_redirects, redirect_count, method, cookies
+            path=path,
+            follow_redirects=follow_redirects,
+            redirect_count=redirect_count,
+            method=method,
+            cookies=cookies,
+            if_none_match=if_none_match,
         )
 
     @async_to_sync
@@ -110,6 +121,7 @@ class TestClient:
         headers=None,
         post_body=None,
         content_type=None,
+        if_none_match=None,
     ):
         return await self._request(
             path,
@@ -120,6 +132,7 @@ class TestClient:
             headers=headers,
             post_body=post_body,
             content_type=content_type,
+            if_none_match=if_none_match,
         )
 
     async def _request(
@@ -132,10 +145,13 @@ class TestClient:
         headers=None,
         post_body=None,
         content_type=None,
+        if_none_match=None,
     ):
         headers = headers or {}
         if content_type:
             headers["content-type"] = content_type
+        if if_none_match:
+            headers["if-none-match"] = if_none_match
         httpx_response = await self.ds.client.request(
             method,
             path,
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 9a8aca32..bc08ba05 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -183,6 +183,13 @@ class DatabaseDownload(DataView):
         headers = {}
         if self.ds.cors:
             add_cors_headers(headers)
+        if db.hash:
+            etag = '"{}"'.format(db.hash)
+            headers["Etag"] = etag
+            # Has user seen this already?
+            if_none_match = request.headers.get("if-none-match")
+            if if_none_match and if_none_match == etag:
+                return Response("", status=304)
         headers["Transfer-Encoding"] = "chunked"
         return AsgiFileDownload(
             filepath,
diff --git a/tests/test_html.py b/tests/test_html.py
index 42f1a3ee..409fec68 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -401,7 +401,7 @@ def test_database_download_for_immutable():
         assert len(soup.findAll("a", {"href": re.compile(r"\.db$")}))
         # Check we can actually download it
         download_response = client.get("/fixtures.db")
-        assert 200 == download_response.status
+        assert download_response.status == 200
         # Check the content-length header exists
         assert "content-length" in download_response.headers
         content_length = download_response.headers["content-length"]
@@ -413,6 +413,14 @@ def test_database_download_for_immutable():
             == 'attachment; filename="fixtures.db"'
         )
         assert download_response.headers["transfer-encoding"] == "chunked"
+        # ETag header should be present and match db.hash
+        assert "etag" in download_response.headers
+        etag = download_response.headers["etag"]
+        assert etag == '"{}"'.format(client.ds.databases["fixtures"].hash)
+        # Try a second download with If-None-Match: current-etag
+        download_response2 = client.get("/fixtures.db", if_none_match=etag)
+        assert download_response2.body == b""
+        assert download_response2.status == 304
 
 
 def test_database_download_disallowed_for_mutable(app_client):

From a5acfff4bd364d30ce8878e19f9839890371ef14 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 16 May 2022 17:06:40 -0700
Subject: [PATCH 1424/2303] Empty Datasette([]) list is no longer required

---
 docs/testing_plugins.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index 1bbaaac1..41046bfb 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -15,7 +15,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
 
     @pytest.mark.asyncio
     async def test_plugin_is_installed():
-        datasette = Datasette([], memory=True)
+        datasette = Datasette(memory=True)
         response = await datasette.client.get("/-/plugins.json")
         assert response.status_code == 200
         installed_plugins = {p["name"] for p in response.json()}

From 3508bf7875f8d62b2725222f3b07747974d54b97 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 17 May 2022 12:40:05 -0700
Subject: [PATCH 1425/2303] --nolock mode to ignore locked files, closes #1744

---
 datasette/app.py         | 2 ++
 datasette/cli.py         | 7 +++++++
 datasette/database.py    | 2 ++
 docs/cli-reference.rst   | 1 +
 docs/getting_started.rst | 4 +++-
 5 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index b7b84371..f43700d4 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -213,6 +213,7 @@ class Datasette:
         config_dir=None,
         pdb=False,
         crossdb=False,
+        nolock=False,
     ):
         assert config_dir is None or isinstance(
             config_dir, Path
@@ -238,6 +239,7 @@ class Datasette:
         self.databases = collections.OrderedDict()
         self._refresh_schemas_lock = asyncio.Lock()
         self.crossdb = crossdb
+        self.nolock = nolock
         if memory or crossdb or not self.files:
             self.add_database(Database(self, is_memory=True), name="_memory")
         # memory_name is a random string so that each Datasette instance gets its own
diff --git a/datasette/cli.py b/datasette/cli.py
index 3c6e1b2c..8781747c 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -452,6 +452,11 @@ def uninstall(packages, yes):
     is_flag=True,
     help="Enable cross-database joins using the /_memory database",
 )
+@click.option(
+    "--nolock",
+    is_flag=True,
+    help="Ignore locking, open locked files in read-only mode",
+)
 @click.option(
     "--ssl-keyfile",
     help="SSL key file",
@@ -486,6 +491,7 @@ def serve(
     open_browser,
     create,
     crossdb,
+    nolock,
     ssl_keyfile,
     ssl_certfile,
     return_instance=False,
@@ -545,6 +551,7 @@ def serve(
         version_note=version_note,
         pdb=pdb,
         crossdb=crossdb,
+        nolock=nolock,
     )
 
     # if files is a single directory, use that as config_dir=
diff --git a/datasette/database.py b/datasette/database.py
index 44d32667..fa558045 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -89,6 +89,8 @@ class Database:
         # mode=ro or immutable=1?
         if self.is_mutable:
             qs = "?mode=ro"
+            if self.ds.nolock:
+                qs += "&nolock=1"
         else:
             qs = "?immutable=1"
         assert not (write and not self.is_mutable)
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 2a6fbfc8..1c1aff15 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -115,6 +115,7 @@ datasette serve --help
       --create                  Create database files if they do not exist
       --crossdb                 Enable cross-database joins using the /_memory
                                 database
+      --nolock                  Ignore locking, open locked files in read-only mode
       --ssl-keyfile TEXT        SSL key file
       --ssl-certfile TEXT       SSL certificate file
       --help                    Show this message and exit.
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 3e357afb..502a9e5a 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -56,7 +56,9 @@ like so:
 
 ::
 
-     datasette ~/Library/Application\ Support/Google/Chrome/Default/History
+     datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock
+
+The `--nolock` option ignores any file locks. This is safe as Datasette will open the file in read-only mode.
 
 Now visiting http://localhost:8001/History/downloads will show you a web
 interface to browse your downloads data:

From 5555bc8aef043f75d2200f66de90c54aeeaa08c3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 17 May 2022 12:43:44 -0700
Subject: [PATCH 1426/2303] How to run cog, closes #1745

---
 docs/contributing.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index c193ba49..bddceafe 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -211,6 +211,17 @@ For added productivity, you can use use `sphinx-autobuild <https://pypi.org/proj
 
 Now browse to ``http://localhost:8000/`` to view the documentation. Any edits you make should be instantly reflected in your browser.
 
+.. _contributing_documentation_cog:
+
+Running Cog
+~~~~~~~~~~~
+
+Some pages of documentation (in particular the :ref:`cli_reference`) are automatically updated using `Cog <https://github.com/nedbat/cog>`__.
+
+To update these pages, run the following command::
+
+    cog -r docs/*.rst
+
 .. _contributing_continuous_deployment:
 
 Continuously deployed demo instances

From b393e164dc9e962702546d6f1ad9c857b5788dc0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 17 May 2022 12:45:28 -0700
Subject: [PATCH 1427/2303] ReST fix

---
 docs/getting_started.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 502a9e5a..af3a1385 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -58,7 +58,7 @@ like so:
 
      datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock
 
-The `--nolock` option ignores any file locks. This is safe as Datasette will open the file in read-only mode.
+The ``--nolock`` option ignores any file locks. This is safe as Datasette will open the file in read-only mode.
 
 Now visiting http://localhost:8001/History/downloads will show you a web
 interface to browse your downloads data:

From 7d1e004ff679b3fb4dca36d1d751a1ad16688fe6 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 17 May 2022 12:59:28 -0700
Subject: [PATCH 1428/2303] Fix test I broke in #1744

---
 tests/test_cli.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_cli.py b/tests/test_cli.py
index dca65f26..d0f6e26c 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -150,6 +150,7 @@ def test_metadata_yaml():
         help_settings=False,
         pdb=False,
         crossdb=False,
+        nolock=False,
         open_browser=False,
         create=False,
         ssl_keyfile=None,

From 0e2f6f1f82f4445a63f1251470a7778a34f5c8b9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 18 May 2022 17:37:46 -0700
Subject: [PATCH 1429/2303] datasette-copyable is an example of
 register_output_renderer

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 3c9ae2e2..c0d88964 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -557,7 +557,7 @@ And here is an example ``can_render`` function which returns ``True`` only if th
             "atom_updated",
         }.issubset(columns)
 
-Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__
+Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__, `datasette-copyable <https://datasette.io/plugins/datasette-copyable>`__
 
 .. _plugin_register_routes:
 

From 18a6e05887abf1ac946a6e0d36ce662dfd8aeff1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 20 May 2022 12:05:33 -0700
Subject: [PATCH 1430/2303] Added "follow a tutorial" to getting started docs

Closes #1747
---
 docs/getting_started.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index af3a1385..00b753a9 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -1,6 +1,8 @@
 Getting started
 ===============
 
+.. _getting_started_demo:
+
 Play with a live demo
 ---------------------
 
@@ -9,6 +11,16 @@ The best way to experience Datasette for the first time is with a demo:
 * `global-power-plants.datasettes.com <https://global-power-plants.datasettes.com/global-power-plants/global-power-plants>`__ provides a searchable database of power plants around the world, using data from the `World Resources Institude <https://www.wri.org/publication/global-power-plant-database>`__ rendered using the `datasette-cluster-map <https://github.com/simonw/datasette-cluster-map>`__ plugin.
 * `fivethirtyeight.datasettes.com <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__ shows Datasette running against over 400 datasets imported from the `FiveThirtyEight GitHub repository <https://github.com/fivethirtyeight/data>`__.
 
+.. _getting_started_tutorial:
+
+Follow a tutorial
+-----------------
+
+Datasette has several `tutorials <https://datasette.io/tutorials>`__ to help you get started with the tool. Try one of the following:
+
+- `Exploring a database with Datasette <https://datasette.io/tutorials/explore>`__ shows how to use the Datasette web interface to explore a new database.
+- `Learn SQL with Datasette <https://datasette.io/tutorials/learn-sql>`__ introduces SQL, and shows how to use that query language to ask questions of your data.
+
 .. _getting_started_glitch:
 
 Try Datasette without installing anything using Glitch

From 1465fea4798599eccfe7e8f012bd8d9adfac3039 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 20 May 2022 12:11:08 -0700
Subject: [PATCH 1431/2303] sphinx-copybutton for docs, closes #1748

---
 docs/conf.py | 2 +-
 setup.py     | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index d114bc52..351cb1b1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -31,7 +31,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"]
+extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"]
 
 extlinks = {
     "issue": ("https://github.com/simonw/datasette/issues/%s", "#"),
diff --git a/setup.py b/setup.py
index ca449f02..aad05840 100644
--- a/setup.py
+++ b/setup.py
@@ -64,7 +64,13 @@ setup(
     """,
     setup_requires=["pytest-runner"],
     extras_require={
-        "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell", "blacken-docs"],
+        "docs": [
+            "sphinx_rtd_theme",
+            "sphinx-autobuild",
+            "codespell",
+            "blacken-docs",
+            "sphinx-copybutton",
+        ],
         "test": [
             "pytest>=5.2.2,<7.2.0",
             "pytest-xdist>=2.2.1,<2.6",

From 1d33fd03b3c211e0f48a8f3bde83880af89e4e69 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 20 May 2022 13:34:51 -0700
Subject: [PATCH 1432/2303] Switch docs theme to Furo, refs #1746

---
 docs/_static/css/custom.css                   |  7 ++--
 .../layout.html => _static/js/custom.js}      | 34 -------------------
 docs/_templates/base.html                     |  6 ++++
 docs/_templates/sidebar/brand.html            | 16 +++++++++
 docs/_templates/sidebar/navigation.html       | 11 ++++++
 docs/conf.py                                  | 24 +++----------
 docs/installation.rst                         |  1 +
 docs/plugin_hooks.rst                         |  1 +
 setup.py                                      |  2 +-
 9 files changed, 45 insertions(+), 57 deletions(-)
 rename docs/{_templates/layout.html => _static/js/custom.js} (55%)
 create mode 100644 docs/_templates/base.html
 create mode 100644 docs/_templates/sidebar/brand.html
 create mode 100644 docs/_templates/sidebar/navigation.html

diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css
index 4dabb725..0a6f8799 100644
--- a/docs/_static/css/custom.css
+++ b/docs/_static/css/custom.css
@@ -1,7 +1,8 @@
 a.external {
     overflow-wrap: anywhere;
 }
-
-div .wy-side-nav-search > div.version {
-    color: rgba(0,0,0,0.75);
+body[data-theme="dark"] .sidebar-logo-container {
+    background-color: white;
+    padding: 5px;
+    opacity: 0.6;
 }
diff --git a/docs/_templates/layout.html b/docs/_static/js/custom.js
similarity index 55%
rename from docs/_templates/layout.html
rename to docs/_static/js/custom.js
index 785cdc7c..efca33ed 100644
--- a/docs/_templates/layout.html
+++ b/docs/_static/js/custom.js
@@ -1,35 +1,3 @@
-{%- extends "!layout.html" %}
-
-{% block htmltitle %}
-{{ super() }}
-<script defer data-domain="docs.datasette.io" src="https://plausible.io/js/plausible.js"></script>
-{% endblock %}
-
-{% block sidebartitle %}
-
-<a href="https://datasette.io/">
-  <img src="{{ pathto('_static/' + logo, 1) }}" class="logo" alt="{{ _('Logo') }}"/>
-</a>
-
-{% if theme_display_version %}
-  {%- set nav_version = version %}
-  {% if READTHEDOCS and current_version %}
-    {%- set nav_version = current_version %}
-  {% endif %}
-  {% if nav_version %}
-    <div class="version">
-      {{ nav_version }}
-    </div>
-  {% endif %}
-{% endif %}
-
-{% include "searchbox.html" %}
-
-{% endblock %}
-
-{% block footer %}
-{{ super() }}
-<script>
 jQuery(function ($) {
   // Show banner linking to /stable/ if this is a /latest/ page
   if (!/\/latest\//.test(location.pathname)) {
@@ -57,5 +25,3 @@ jQuery(function ($) {
     }
   });
 });
-</script>
-{% endblock %}
diff --git a/docs/_templates/base.html b/docs/_templates/base.html
new file mode 100644
index 00000000..969de5ab
--- /dev/null
+++ b/docs/_templates/base.html
@@ -0,0 +1,6 @@
+{%- extends "!base.html" %}
+
+{% block site_meta %}
+{{ super() }}
+<script defer data-domain="docs.datasette.io" src="https://plausible.io/js/plausible.js"></script>
+{% endblock %}
diff --git a/docs/_templates/sidebar/brand.html b/docs/_templates/sidebar/brand.html
new file mode 100644
index 00000000..8be9e8ee
--- /dev/null
+++ b/docs/_templates/sidebar/brand.html
@@ -0,0 +1,16 @@
+<div class="sidebar-brand centered">
+  {% block brand_content %}
+  <div class="sidebar-logo-container">
+    <a href="https://datasette.io/"><img class="sidebar-logo" src="{{ logo_url }}" alt="Datasette"></a>
+  </div>
+  {%- set nav_version = version %}
+  {% if READTHEDOCS and current_version %}
+    {%- set nav_version = current_version %}
+  {% endif %}
+  {% if nav_version %}
+    <div class="version">
+      {{ nav_version }}
+    </div>
+  {% endif %}
+  {% endblock brand_content %}
+</div>
diff --git a/docs/_templates/sidebar/navigation.html b/docs/_templates/sidebar/navigation.html
new file mode 100644
index 00000000..c460a17e
--- /dev/null
+++ b/docs/_templates/sidebar/navigation.html
@@ -0,0 +1,11 @@
+<div class="sidebar-tree">
+  <ul>
+    <li class="toctree-l1"><a class="reference internal" href="{{ pathto(master_doc) }}">Contents</a></li>
+  </ul>
+  {{ toctree(
+    collapse=True,
+    titles_only=False,
+    maxdepth=3,
+    includehidden=True,
+) }}
+</div>
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 351cb1b1..25d2acfe 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -90,18 +90,15 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = "sphinx_rtd_theme"
+html_theme = "furo"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 html_theme_options = {
-    "logo_only": True,
-    "style_nav_header_background": "white",
-    "prev_next_buttons_location": "both",
+    "sidebar_hide_name": True,
 }
 
-
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
@@ -112,20 +109,9 @@ html_logo = "datasette-logo.svg"
 html_css_files = [
     "css/custom.css",
 ]
-
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# This is required for the alabaster theme
-# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
-html_sidebars = {
-    "**": [
-        "relations.html",  # needs 'show_related': True theme option to display
-        "searchbox.html",
-    ]
-}
-
+html_js_files = [
+    "js/custom.js"
+]
 
 # -- Options for HTMLHelp output ------------------------------------------
 
diff --git a/docs/installation.rst b/docs/installation.rst
index e8bef9cd..a4757736 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -13,6 +13,7 @@ If you want to start making contributions to the Datasette project by installing
 
 .. contents::
    :local:
+   :class: this-will-duplicate-information-and-it-is-still-useful-here
 
 .. _installation_basic:
 
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index c0d88964..7d10fe37 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -20,6 +20,7 @@ For example, you can implement the ``render_cell`` plugin hook like this even th
 
 .. contents:: List of plugin hooks
    :local:
+   :class: this-will-duplicate-information-and-it-is-still-useful-here
 
 .. _plugin_hook_prepare_connection:
 
diff --git a/setup.py b/setup.py
index aad05840..d3fcdbd1 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ setup(
     setup_requires=["pytest-runner"],
     extras_require={
         "docs": [
-            "sphinx_rtd_theme",
+            "furo==2022.4.7",
             "sphinx-autobuild",
             "codespell",
             "blacken-docs",

From 4446075334ea7231beb56b630bc7ec363afc2d08 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 20 May 2022 13:44:23 -0700
Subject: [PATCH 1433/2303] Append warning to the write element, refs #1746

---
 docs/_static/js/custom.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/docs/_static/js/custom.js b/docs/_static/js/custom.js
index efca33ed..91c3e306 100644
--- a/docs/_static/js/custom.js
+++ b/docs/_static/js/custom.js
@@ -17,11 +17,7 @@ jQuery(function ($) {
         </div>`
       );
       warning.find("a").attr("href", stableUrl);
-      var body = $("div.body");
-      if (!body.length) {
-        body = $("div.document");
-      }
-      body.prepend(warning);
+      $("article[role=main]").prepend(warning);
     }
   });
 });

From b010af7bb85856aeb44f69e7e980f617c1fc0db1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 20 May 2022 15:23:09 -0700
Subject: [PATCH 1434/2303] Updated copyright years in documentation footer

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 25d2acfe..7ffeedd0 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -51,7 +51,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "Datasette"
-copyright = "2017-2021, Simon Willison"
+copyright = "2017-2022, Simon Willison"
 author = "Simon Willison"
 
 # Disable -- turning into –

From adedd85b68ec66e03b97fb62ff4da8987734436e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 28 May 2022 18:42:31 -0700
Subject: [PATCH 1435/2303] Clarify that request.headers names are converted to
 lowercase

---
 docs/internals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index 18822d47..da135282 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -26,7 +26,7 @@ The request object is passed to various plugin hooks. It represents an incoming
     The request scheme - usually ``https`` or ``http``.
 
 ``.headers`` - dictionary (str -> str)
-    A dictionary of incoming HTTP request headers.
+    A dictionary of incoming HTTP request headers. Header names have been converted to lowercase.
 
 ``.cookies`` - dictionary (str -> str)
     A dictionary of incoming cookies

From 8dd816bc76937f1e37f86acce10dc2cb4fa31e52 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 30 May 2022 15:42:38 -0700
Subject: [PATCH 1436/2303] Applied Black

---
 docs/conf.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 7ffeedd0..4ef6b768 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -109,9 +109,7 @@ html_logo = "datasette-logo.svg"
 html_css_files = [
     "css/custom.css",
 ]
-html_js_files = [
-    "js/custom.js"
-]
+html_js_files = ["js/custom.js"]
 
 # -- Options for HTMLHelp output ------------------------------------------
 

From 2e9751672d4fe329b3c359d5b7b1992283185820 Mon Sep 17 00:00:00 2001
From: Naveen <172697+naveensrinivasan@users.noreply.github.com>
Date: Tue, 31 May 2022 14:28:40 -0500
Subject: [PATCH 1437/2303] chore: Set permissions for GitHub actions (#1740)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Restrict the GitHub token permissions only to the required ones; this way, even if the attackers will succeed in compromising your workflow, they won’t be able to do much.

- Included permissions for the action. https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions

https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions

https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs

[Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)

Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com>
---
 .github/workflows/deploy-latest.yml   | 3 +++
 .github/workflows/prettier.yml        | 3 +++
 .github/workflows/publish.yml         | 3 +++
 .github/workflows/push_docker_tag.yml | 3 +++
 .github/workflows/spellcheck.yml      | 3 +++
 .github/workflows/test-coverage.yml   | 3 +++
 .github/workflows/test-pyodide.yml    | 3 +++
 .github/workflows/test.yml            | 3 +++
 .github/workflows/tmate-mac.yml       | 3 +++
 .github/workflows/tmate.yml           | 3 +++
 10 files changed, 30 insertions(+)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index a61f6629..2b94a7f1 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -5,6 +5,9 @@ on:
     branches:
       - main
 
+permissions:
+  contents: read
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml
index 9dfe7ee0..ded41040 100644
--- a/.github/workflows/prettier.yml
+++ b/.github/workflows/prettier.yml
@@ -2,6 +2,9 @@ name: Check JavaScript for conformance with Prettier
 
 on: [push]
 
+permissions:
+  contents: read
+
 jobs:
   prettier:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 3e4f8146..9ef09d2e 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -4,6 +4,9 @@ on:
   release:
     types: [created]
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml
index 9a3969f0..afe8d6b2 100644
--- a/.github/workflows/push_docker_tag.yml
+++ b/.github/workflows/push_docker_tag.yml
@@ -6,6 +6,9 @@ on:
       version_tag:
         description: Tag to build and push
 
+permissions:
+  contents: read
+
 jobs:
   deploy_docker:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml
index 2e24d3eb..a2621ecc 100644
--- a/.github/workflows/spellcheck.yml
+++ b/.github/workflows/spellcheck.yml
@@ -2,6 +2,9 @@ name: Check spelling in documentation
 
 on: [push, pull_request]
 
+permissions:
+  contents: read
+
 jobs:
   spellcheck:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml
index 1d1cf332..bd720664 100644
--- a/.github/workflows/test-coverage.yml
+++ b/.github/workflows/test-coverage.yml
@@ -7,6 +7,9 @@ on:
   pull_request:
     branches:
       - main
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml
index 1b75aade..bc9593a8 100644
--- a/.github/workflows/test-pyodide.yml
+++ b/.github/workflows/test-pyodide.yml
@@ -5,6 +5,9 @@ on:
   pull_request:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8d916e49..90b6555e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,6 +2,9 @@ name: Test
 
 on: [push, pull_request]
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml
index 46be117e..fcee0f21 100644
--- a/.github/workflows/tmate-mac.yml
+++ b/.github/workflows/tmate-mac.yml
@@ -3,6 +3,9 @@ name: tmate session mac
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: macos-latest
diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml
index 02e7bd33..9792245d 100644
--- a/.github/workflows/tmate.yml
+++ b/.github/workflows/tmate.yml
@@ -3,6 +3,9 @@ name: tmate session
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest

From e780b2f5d662ef3579d801d33567440055d4e84d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 20 Jun 2022 10:54:23 -0700
Subject: [PATCH 1438/2303] Trying out one-sentence-per-line

As suggested here: https://sive.rs/1s

Markdown and reStructuredText will display this as if it is a single paragraph, even though the sentences themselves are separated by newlines.

This could result in more useful diffs. Trying it out on this page first.
---
 docs/facets.rst | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/docs/facets.rst b/docs/facets.rst
index 0228aa84..2a2eb039 100644
--- a/docs/facets.rst
+++ b/docs/facets.rst
@@ -3,7 +3,9 @@
 Facets
 ======
 
-Datasette facets can be used to add a faceted browse interface to any database table. With facets, tables are displayed along with a summary showing the most common values in specified columns. These values can be selected to further filter the table.
+Datasette facets can be used to add a faceted browse interface to any database table.
+With facets, tables are displayed along with a summary showing the most common values in specified columns.
+These values can be selected to further filter the table.
 
 .. image:: facets.png
 
@@ -12,11 +14,13 @@ Facets can be specified in two ways: using query string parameters, or in ``meta
 Facets in query strings
 -----------------------
 
-To turn on faceting for specific columns on a Datasette table view, add one or more ``_facet=COLUMN`` parameters to the URL. For example, if you want to turn on facets for the ``city_id`` and ``state`` columns, construct a URL that looks like this::
+To turn on faceting for specific columns on a Datasette table view, add one or more ``_facet=COLUMN`` parameters to the URL.
+For example, if you want to turn on facets for the ``city_id`` and ``state`` columns, construct a URL that looks like this::
 
     /dbname/tablename?_facet=state&_facet=city_id
 
-This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this:
+This works for both the HTML interface and the ``.json`` view.
+When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this:
 
 .. code-block:: json
 
@@ -86,7 +90,8 @@ This works for both the HTML interface and the ``.json`` view. When enabled, fac
 
 If Datasette detects that a column is a foreign key, the ``"label"`` property will be automatically derived from the detected label column on the referenced table.
 
-The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000).
+The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting.
+You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000).
 
 .. _facets_metadata:
 
@@ -137,12 +142,14 @@ For the currently filtered data are there any columns which, if applied as a fac
 * Will return less unique options than the total number of filtered rows
 * And the query used to evaluate this criteria can be completed in under 50ms
 
-That last point is particularly important: Datasette runs a query for every column that is displayed on a page, which could get expensive - so to avoid slow load times it sets a time limit of just 50ms for each of those queries. This means suggested facets are unlikely to appear for tables with millions of records in them.
+That last point is particularly important: Datasette runs a query for every column that is displayed on a page, which could get expensive - so to avoid slow load times it sets a time limit of just 50ms for each of those queries.
+This means suggested facets are unlikely to appear for tables with millions of records in them.
 
 Speeding up facets with indexes
 -------------------------------
 
-The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``::
+The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by.
+Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``::
 
     $ sqlite3 mydatabase.db
     SQLite version 3.19.3 2017-06-27 16:48:08
@@ -169,6 +176,7 @@ Example here: `latest.datasette.io/fixtures/facetable?_facet_array=tags <https:/
 Facet by date
 -------------
 
-If Datasette finds any columns that contain dates in the first 100 values, it will offer a faceting interface against the dates of those values. This works especially well against timestamp values such as ``2019-03-01 12:44:00``.
+If Datasette finds any columns that contain dates in the first 100 values, it will offer a faceting interface against the dates of those values.
+This works especially well against timestamp values such as ``2019-03-01 12:44:00``.
 
 Example here: `latest.datasette.io/fixtures/facetable?_facet_date=created <https://latest.datasette.io/fixtures/facetable?_facet_date=created>`__

From 00e59ec461dc0150772b999c7cc15fcb9b507d58 Mon Sep 17 00:00:00 2001
From: "M. Nasimul Haque" <nasim.haque@gmail.com>
Date: Mon, 20 Jun 2022 19:05:44 +0100
Subject: [PATCH 1439/2303] Extract facet pieces of table.html into included
 templates

Thanks, @nsmgr8
---
 datasette/templates/_facet_results.html    | 28 ++++++++++++++++++
 datasette/templates/_suggested_facets.html |  3 ++
 datasette/templates/table.html             | 33 ++--------------------
 3 files changed, 33 insertions(+), 31 deletions(-)
 create mode 100644 datasette/templates/_facet_results.html
 create mode 100644 datasette/templates/_suggested_facets.html

diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html
new file mode 100644
index 00000000..d0cbcf77
--- /dev/null
+++ b/datasette/templates/_facet_results.html
@@ -0,0 +1,28 @@
+<div class="facet-results">
+    {% for facet_info in sorted_facet_results %}
+        <div class="facet-info facet-{{ database|to_css_class }}-{{ table|to_css_class }}-{{ facet_info.name|to_css_class }}" id="facet-{{ facet_info.name|to_css_class }}" data-column="{{ facet_info.name }}">
+            <p class="facet-info-name">
+                <strong>{{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
+                    <span class="facet-info-total">{% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}</span>
+                </strong>
+                {% if facet_info.hideable %}
+                    <a href="{{ facet_info.toggle_url }}" class="cross">✖</a>
+                {% endif %}
+            </p>
+            <ul class="tight-bullets">
+                {% for facet_value in facet_info.results %}
+                    {% if not facet_value.selected %}
+                        <li><a href="{{ facet_value.toggle_url }}">{{ (facet_value.label | string()) or "-" }}</a> {{ "{:,}".format(facet_value.count) }}</li>
+                    {% else %}
+                        <li>{{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} <a href="{{ facet_value.toggle_url }}" class="cross">✖</a></li>
+                    {% endif %}
+                {% endfor %}
+                {% if facet_info.truncated %}
+                    <li class="facet-truncated">{% if request.args._facet_size != "max" -%}
+                        <a href="{{ path_with_replaced_args(request, {"_facet_size": "max"}) }}">…</a>{% else -%}…{% endif %}
+                    </li>
+                {% endif %}
+            </ul>
+        </div>
+    {% endfor %}
+</div>
diff --git a/datasette/templates/_suggested_facets.html b/datasette/templates/_suggested_facets.html
new file mode 100644
index 00000000..ec98fb36
--- /dev/null
+++ b/datasette/templates/_suggested_facets.html
@@ -0,0 +1,3 @@
+<p class="suggested-facets">
+    Suggested facets: {% for facet in suggested_facets %}<a href="{{ facet.toggle_url }}#facet-{{ facet.name|to_css_class }}">{{ facet.name }}</a>{% if facet.type %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
+</p>
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index a9e88330..a86398ea 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -142,9 +142,7 @@
 <p class="export-links">This data as {% for name, url in renderers.items() %}<a href="{{ url }}">{{ name }}</a>{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, <a href="{{ url_csv }}">CSV</a> (<a href="#export">advanced</a>){% endif %}</p>
 
 {% if suggested_facets %}
-    <p class="suggested-facets">
-        Suggested facets: {% for facet in suggested_facets %}<a href="{{ facet.toggle_url }}#facet-{{ facet.name|to_css_class }}">{{ facet.name }}</a>{% if facet.type %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
-    </p>
+    {% include "_suggested_facets.html" %}
 {% endif %}
 
 {% if facets_timed_out %}
@@ -152,34 +150,7 @@
 {% endif %}
 
 {% if facet_results %}
-    <div class="facet-results">
-        {% for facet_info in sorted_facet_results %}
-            <div class="facet-info facet-{{ database|to_css_class }}-{{ table|to_css_class }}-{{ facet_info.name|to_css_class }}" id="facet-{{ facet_info.name|to_css_class }}" data-column="{{ facet_info.name }}">
-                <p class="facet-info-name">
-                    <strong>{{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
-                        <span class="facet-info-total">{% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}</span>
-                    </strong>
-                    {% if facet_info.hideable %}
-                        <a href="{{ facet_info.toggle_url }}" class="cross">✖</a>
-                    {% endif %}
-                </p>
-                <ul class="tight-bullets">
-                    {% for facet_value in facet_info.results %}
-                        {% if not facet_value.selected %}
-                            <li><a href="{{ facet_value.toggle_url }}">{{ (facet_value.label | string()) or "-" }}</a> {{ "{:,}".format(facet_value.count) }}</li>
-                        {% else %}
-                            <li>{{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} <a href="{{ facet_value.toggle_url }}" class="cross">✖</a></li>
-                        {% endif %}
-                    {% endfor %}
-                    {% if facet_info.truncated %}
-                        <li class="facet-truncated">{% if request.args._facet_size != "max" -%}
-                            <a href="{{ path_with_replaced_args(request, {"_facet_size": "max"}) }}">…</a>{% else -%}…{% endif %}
-                        </li>
-                    {% endif %}
-                </ul>
-            </div>
-        {% endfor %}
-    </div>
+    {% include "_facet_results.html" %}
 {% endif %}
 
 {% include custom_table_templates %}

From 9f1eb0d4eac483b953392157bd9fd6cc4df37de7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 28 Jun 2022 10:40:24 -0700
Subject: [PATCH 1440/2303] Bump black from 22.1.0 to 22.6.0 (#1763)

Bumps [black](https://github.com/psf/black) from 22.1.0 to 22.6.0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/22.1.0...22.6.0)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index d3fcdbd1..29cb77bf 100644
--- a/setup.py
+++ b/setup.py
@@ -76,7 +76,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.6",
             "pytest-asyncio>=0.17,<0.19",
             "beautifulsoup4>=4.8.1,<4.12.0",
-            "black==22.1.0",
+            "black==22.6.0",
             "blacken-docs==1.12.1",
             "pytest-timeout>=1.4.2,<2.2",
             "trustme>=0.7,<0.10",

From 6373bb341457e5becfd5b67792ac2c8b9ed7c384 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 7 Jul 2022 09:30:49 -0700
Subject: [PATCH 1441/2303] Expose current SQLite row to render_cell hook,
 closes #1300

---
 datasette/hookspecs.py      | 2 +-
 datasette/views/database.py | 1 +
 datasette/views/table.py    | 1 +
 docs/plugin_hooks.rst       | 9 ++++++---
 tests/plugins/my_plugin.py  | 3 ++-
 tests/test_plugins.py       | 5 +++--
 6 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 8f4fecab..c84db0a3 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -60,7 +60,7 @@ def publish_subcommand(publish):
 
 
 @hookspec
-def render_cell(value, column, table, database, datasette):
+def render_cell(row, value, column, table, database, datasette):
     """Customize rendering of HTML table cell values"""
 
 
diff --git a/datasette/views/database.py b/datasette/views/database.py
index bc08ba05..42058752 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -375,6 +375,7 @@ class QueryView(DataView):
                     # pylint: disable=no-member
                     plugin_display_value = None
                     for candidate in pm.hook.render_cell(
+                        row=row,
                         value=value,
                         column=column,
                         table=None,
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 23289b29..cd4be823 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -895,6 +895,7 @@ async def display_columns_and_rows(
             # pylint: disable=no-member
             plugin_display_value = None
             for candidate in pm.hook.render_cell(
+                row=row,
                 value=value,
                 column=column,
                 table=table_name,
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 7d10fe37..f5c3ee83 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -373,12 +373,15 @@ Examples: `datasette-publish-fly <https://datasette.io/plugins/datasette-publish
 
 .. _plugin_hook_render_cell:
 
-render_cell(value, column, table, database, datasette)
-------------------------------------------------------
+render_cell(row, value, column, table, database, datasette)
+-----------------------------------------------------------
 
 Lets you customize the display of values within table cells in the HTML table view.
 
-``value`` - string, integer or None
+``row`` - ``sqlite.Row``
+    The SQLite row object that the value being rendered is part of
+
+``value`` - string, integer, float, bytes or None
     The value that was loaded from the database
 
 ``column`` - string
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 1c9b0575..53613b7d 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -98,12 +98,13 @@ def extra_body_script(
 
 
 @hookimpl
-def render_cell(value, column, table, database, datasette):
+def render_cell(row, value, column, table, database, datasette):
     async def inner():
         # Render some debug output in cell with value RENDER_CELL_DEMO
         if value == "RENDER_CELL_DEMO":
             return json.dumps(
                 {
+                    "row": dict(row),
                     "column": column,
                     "table": table,
                     "database": database,
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 15bde962..4a7ad7c6 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -181,12 +181,13 @@ def test_hook_render_cell_demo(app_client):
     response = app_client.get("/fixtures/simple_primary_key?id=4")
     soup = Soup(response.body, "html.parser")
     td = soup.find("td", {"class": "col-content"})
-    assert {
+    assert json.loads(td.string) == {
+        "row": {"id": "4", "content": "RENDER_CELL_DEMO"},
         "column": "content",
         "table": "simple_primary_key",
         "database": "fixtures",
         "config": {"depth": "table", "special": "this-is-simple_primary_key"},
-    } == json.loads(td.string)
+    }
 
 
 @pytest.mark.parametrize(

From 035dc5e7b95142d4a700819a8cc4ff64aefe4efe Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 9 Jul 2022 10:25:37 -0700
Subject: [PATCH 1442/2303] More than 90 plugins now

---
 docs/writing_plugins.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst
index 9aee70f6..01ee8c90 100644
--- a/docs/writing_plugins.rst
+++ b/docs/writing_plugins.rst
@@ -5,7 +5,7 @@ Writing plugins
 
 You can write one-off plugins that apply to just one Datasette instance, or you can write plugins which can be installed using ``pip`` and can be shipped to the Python Package Index (`PyPI <https://pypi.org/>`__) for other people to install.
 
-Want to start by looking at an example? The `Datasette plugins directory <https://datasette.io/plugins>`__ lists more than 50 open source plugins with code you can explore. The :ref:`plugin hooks <plugin_hooks>` page includes links to example plugins for each of the documented hooks.
+Want to start by looking at an example? The `Datasette plugins directory <https://datasette.io/plugins>`__ lists more than 90 open source plugins with code you can explore. The :ref:`plugin hooks <plugin_hooks>` page includes links to example plugins for each of the documented hooks.
 
 .. _writing_plugins_one_off:
 

From 5d76c1f81b2d978f48b85c70d041a2142cf8ee26 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 14 Jul 2022 15:03:33 -0700
Subject: [PATCH 1443/2303] Discord badge

Refs https://github.com/simonw/datasette.io/issues/112
---
 README.md      | 1 +
 docs/index.rst | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 557d9290..c57ee604 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@
 [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE)
 [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette)
+[![discord](https://img.shields.io/discord/823971286308356157?label=Discord)](https://discord.gg/ktd74dm5mw)
 
 *An open source multi-tool for exploring and publishing data*
 
diff --git a/docs/index.rst b/docs/index.rst
index a2888822..62ed70f8 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -16,6 +16,8 @@ datasette|
    :target: https://github.com/simonw/datasette/blob/main/LICENSE
 .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue
    :target: https://hub.docker.com/r/datasetteproject/datasette
+.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=Discord
+   :target: https://discord.gg/ktd74dm5mw
 
 *An open source multi-tool for exploring and publishing data*
 

From c133545fe9c7ac2d509e55bf4bf6164bfbe892ad Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 14 Jul 2022 15:04:38 -0700
Subject: [PATCH 1444/2303] Make discord badge lowercase

Refs https://github.com/simonw/datasette.io/issues/112
---
 README.md      | 2 +-
 docs/index.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c57ee604..032180aa 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
 [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE)
 [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette)
-[![discord](https://img.shields.io/discord/823971286308356157?label=Discord)](https://discord.gg/ktd74dm5mw)
+[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/ktd74dm5mw)
 
 *An open source multi-tool for exploring and publishing data*
 
diff --git a/docs/index.rst b/docs/index.rst
index 62ed70f8..051898b1 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -16,7 +16,7 @@ datasette|
    :target: https://github.com/simonw/datasette/blob/main/LICENSE
 .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue
    :target: https://hub.docker.com/r/datasetteproject/datasette
-.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=Discord
+.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord
    :target: https://discord.gg/ktd74dm5mw
 
 *An open source multi-tool for exploring and publishing data*

From 950cc7677f65aa2543067b3bbfc2b6acb98b62c8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 14 Jul 2022 15:18:28 -0700
Subject: [PATCH 1445/2303] Fix missing Discord image

Refs https://github.com/simonw/datasette.io/issues/112
---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index 051898b1..efe196b3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -2,7 +2,7 @@ Datasette
 =========
 
 |PyPI| |Changelog| |Python 3.x| |Tests| |License| |docker:
-datasette|
+datasette| |discord|
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/datasette.svg
    :target: https://pypi.org/project/datasette/

From 8188f55efc0fcca1be692b0d0c875f2d1ee99f17 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 17 Jul 2022 15:24:16 -0700
Subject: [PATCH 1446/2303] Rename handle_500 to handle_exception, refs #1770

---
 datasette/app.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index f43700d4..43e60dbc 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1275,7 +1275,7 @@ class DatasetteRouter:
         except NotFound as exception:
             return await self.handle_404(request, send, exception)
         except Exception as exception:
-            return await self.handle_500(request, send, exception)
+            return await self.handle_exception(request, send, exception)
 
     async def handle_404(self, request, send, exception=None):
         # If path contains % encoding, redirect to tilde encoding
@@ -1354,7 +1354,7 @@ class DatasetteRouter:
                         view_name="page",
                     )
                 except NotFoundExplicit as e:
-                    await self.handle_500(request, send, e)
+                    await self.handle_exception(request, send, e)
                     return
                 # Pull content-type out into separate parameter
                 content_type = "text/html; charset=utf-8"
@@ -1369,9 +1369,9 @@ class DatasetteRouter:
                     content_type=content_type,
                 )
             else:
-                await self.handle_500(request, send, exception or NotFound("404"))
+                await self.handle_exception(request, send, exception or NotFound("404"))
 
-    async def handle_500(self, request, send, exception):
+    async def handle_exception(self, request, send, exception):
         if self.ds.pdb:
             import pdb
 

From c09c53f3455a7b9574cf7695478f2b87d20897db Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 17 Jul 2022 16:24:39 -0700
Subject: [PATCH 1447/2303] New handle_exception plugin hook, refs #1770

Also refs:
- https://github.com/simonw/datasette-sentry/issues/1
- https://github.com/simonw/datasette-show-errors/issues/2
---
 datasette/app.py              | 97 +++++++++--------------------------
 datasette/forbidden.py        | 20 ++++++++
 datasette/handle_exception.py | 74 ++++++++++++++++++++++++++
 datasette/hookspecs.py        |  5 ++
 datasette/plugins.py          |  2 +
 docs/plugin_hooks.rst         | 78 ++++++++++++++++++++--------
 tests/fixtures.py             |  1 +
 tests/plugins/my_plugin_2.py  | 18 +++++++
 tests/test_permissions.py     |  1 +
 tests/test_plugins.py         | 14 +++++
 10 files changed, 215 insertions(+), 95 deletions(-)
 create mode 100644 datasette/forbidden.py
 create mode 100644 datasette/handle_exception.py

diff --git a/datasette/app.py b/datasette/app.py
index 43e60dbc..edd05bb3 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -16,7 +16,6 @@ import re
 import secrets
 import sys
 import threading
-import traceback
 import urllib.parse
 from concurrent import futures
 from pathlib import Path
@@ -27,7 +26,7 @@ from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
 from jinja2.environment import Template
 from jinja2.exceptions import TemplateNotFound
 
-from .views.base import DatasetteError, ureg
+from .views.base import ureg
 from .views.database import DatabaseDownload, DatabaseView
 from .views.index import IndexView
 from .views.special import (
@@ -49,7 +48,6 @@ from .utils import (
     PrefixedUrlString,
     SPATIALITE_FUNCTIONS,
     StartupError,
-    add_cors_headers,
     async_call_with_supported_arguments,
     await_me_maybe,
     call_with_supported_arguments,
@@ -87,11 +85,6 @@ from .tracer import AsgiTracer
 from .plugins import pm, DEFAULT_PLUGINS, get_plugins
 from .version import __version__
 
-try:
-    import rich
-except ImportError:
-    rich = None
-
 app_root = Path(__file__).parent.parent
 
 # https://github.com/simonw/datasette/issues/283#issuecomment-781591015
@@ -1274,6 +1267,16 @@ class DatasetteRouter:
             return
         except NotFound as exception:
             return await self.handle_404(request, send, exception)
+        except Forbidden as exception:
+            # Try the forbidden() plugin hook
+            for custom_response in pm.hook.forbidden(
+                datasette=self.ds, request=request, message=exception.args[0]
+            ):
+                custom_response = await await_me_maybe(custom_response)
+                assert (
+                    custom_response
+                ), "Default forbidden() hook should have been called"
+                return await custom_response.asgi_send(send)
         except Exception as exception:
             return await self.handle_exception(request, send, exception)
 
@@ -1372,72 +1375,20 @@ class DatasetteRouter:
                 await self.handle_exception(request, send, exception or NotFound("404"))
 
     async def handle_exception(self, request, send, exception):
-        if self.ds.pdb:
-            import pdb
+        responses = []
+        for hook in pm.hook.handle_exception(
+            datasette=self.ds,
+            request=request,
+            exception=exception,
+        ):
+            response = await await_me_maybe(hook)
+            if response is not None:
+                responses.append(response)
 
-            pdb.post_mortem(exception.__traceback__)
-
-        if rich is not None:
-            rich.get_console().print_exception(show_locals=True)
-
-        title = None
-        if isinstance(exception, Forbidden):
-            status = 403
-            info = {}
-            message = exception.args[0]
-            # Try the forbidden() plugin hook
-            for custom_response in pm.hook.forbidden(
-                datasette=self.ds, request=request, message=message
-            ):
-                custom_response = await await_me_maybe(custom_response)
-                if custom_response is not None:
-                    await custom_response.asgi_send(send)
-                    return
-        elif isinstance(exception, Base400):
-            status = exception.status
-            info = {}
-            message = exception.args[0]
-        elif isinstance(exception, DatasetteError):
-            status = exception.status
-            info = exception.error_dict
-            message = exception.message
-            if exception.message_is_html:
-                message = Markup(message)
-            title = exception.title
-        else:
-            status = 500
-            info = {}
-            message = str(exception)
-            traceback.print_exc()
-        templates = [f"{status}.html", "error.html"]
-        info.update(
-            {
-                "ok": False,
-                "error": message,
-                "status": status,
-                "title": title,
-            }
-        )
-        headers = {}
-        if self.ds.cors:
-            add_cors_headers(headers)
-        if request.path.split("?")[0].endswith(".json"):
-            await asgi_send_json(send, info, status=status, headers=headers)
-        else:
-            template = self.ds.jinja_env.select_template(templates)
-            await asgi_send_html(
-                send,
-                await template.render_async(
-                    dict(
-                        info,
-                        urls=self.ds.urls,
-                        app_css_hash=self.ds.app_css_hash(),
-                        menu_links=lambda: [],
-                    )
-                ),
-                status=status,
-                headers=headers,
-            )
+        assert responses, "Default exception handler should have returned something"
+        # Even if there are multiple responses use just the first one
+        response = responses[0]
+        await response.asgi_send(send)
 
 
 _cleaner_task_str_re = re.compile(r"\S*site-packages/")
diff --git a/datasette/forbidden.py b/datasette/forbidden.py
new file mode 100644
index 00000000..156a44d4
--- /dev/null
+++ b/datasette/forbidden.py
@@ -0,0 +1,20 @@
+from os import stat
+from datasette import hookimpl, Response
+
+
+@hookimpl(trylast=True)
+def forbidden(datasette, request, message):
+    async def inner():
+        return Response.html(
+            await datasette.render_template(
+                "error.html",
+                {
+                    "title": "Forbidden",
+                    "error": message,
+                },
+                request=request,
+            ),
+            status=403,
+        )
+
+    return inner
diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py
new file mode 100644
index 00000000..8b7e83e3
--- /dev/null
+++ b/datasette/handle_exception.py
@@ -0,0 +1,74 @@
+from datasette import hookimpl, Response
+from .utils import await_me_maybe, add_cors_headers
+from .utils.asgi import (
+    Base400,
+    Forbidden,
+)
+from .views.base import DatasetteError
+from markupsafe import Markup
+import pdb
+import traceback
+from .plugins import pm
+
+try:
+    import rich
+except ImportError:
+    rich = None
+
+
+@hookimpl(trylast=True)
+def handle_exception(datasette, request, exception):
+    async def inner():
+        if datasette.pdb:
+            pdb.post_mortem(exception.__traceback__)
+
+        if rich is not None:
+            rich.get_console().print_exception(show_locals=True)
+
+        title = None
+        if isinstance(exception, Base400):
+            status = exception.status
+            info = {}
+            message = exception.args[0]
+        elif isinstance(exception, DatasetteError):
+            status = exception.status
+            info = exception.error_dict
+            message = exception.message
+            if exception.message_is_html:
+                message = Markup(message)
+            title = exception.title
+        else:
+            status = 500
+            info = {}
+            message = str(exception)
+            traceback.print_exc()
+        templates = [f"{status}.html", "error.html"]
+        info.update(
+            {
+                "ok": False,
+                "error": message,
+                "status": status,
+                "title": title,
+            }
+        )
+        headers = {}
+        if datasette.cors:
+            add_cors_headers(headers)
+        if request.path.split("?")[0].endswith(".json"):
+            return Response.json(info, status=status, headers=headers)
+        else:
+            template = datasette.jinja_env.select_template(templates)
+            return Response.html(
+                await template.render_async(
+                    dict(
+                        info,
+                        urls=datasette.urls,
+                        app_css_hash=datasette.app_css_hash(),
+                        menu_links=lambda: [],
+                    )
+                ),
+                status=status,
+                headers=headers,
+            )
+
+    return inner
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index c84db0a3..a5fb536f 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -138,3 +138,8 @@ def database_actions(datasette, actor, database, request):
 @hookspec
 def skip_csrf(datasette, scope):
     """Mechanism for skipping CSRF checks for certain requests"""
+
+
+@hookspec
+def handle_exception(datasette, request, exception):
+    """Handle an uncaught exception. Can return a Response or None."""
diff --git a/datasette/plugins.py b/datasette/plugins.py
index 76b46a47..fef0c8e9 100644
--- a/datasette/plugins.py
+++ b/datasette/plugins.py
@@ -15,6 +15,8 @@ DEFAULT_PLUGINS = (
     "datasette.default_magic_parameters",
     "datasette.blob_renderer",
     "datasette.default_menu_links",
+    "datasette.handle_exception",
+    "datasette.forbidden",
 )
 
 pm = pluggy.PluginManager("datasette")
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index f5c3ee83..6020a941 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -107,8 +107,8 @@ Extra template variables that should be made available in the rendered template
 ``view_name`` - string
     The name of the view being displayed. (``index``, ``database``, ``table``, and ``row`` are the most important ones.)
 
-``request`` - object or None
-    The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available.
+``request`` - :ref:`internals_request` or None
+    The current HTTP request. This can be ``None`` if the request object is not available.
 
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
@@ -504,7 +504,7 @@ When a request is received, the ``"render"`` callback function is called with ze
     The table or view, if one is being rendered.
 
 ``request`` - :ref:`internals_request`
-    The incoming HTTP request.
+    The current HTTP request.
 
 ``view_name`` - string
     The name of the current view being called. ``index``, ``database``, ``table``, and ``row`` are the most important ones.
@@ -599,8 +599,8 @@ The optional view function arguments are as follows:
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
 
-``request`` - Request object
-    The current HTTP :ref:`internals_request`.
+``request`` - :ref:`internals_request`
+    The current HTTP request.
 
 ``scope`` - dictionary
     The incoming ASGI scope dictionary.
@@ -947,8 +947,8 @@ actor_from_request(datasette, request)
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
 
-``request`` - object
-    The current HTTP :ref:`internals_request`.
+``request`` - :ref:`internals_request`
+    The current HTTP request.
 
 This is part of Datasette's :ref:`authentication and permissions system <authentication>`. The function should attempt to authenticate an actor (either a user or an API actor of some sort) based on information in the request.
 
@@ -1010,8 +1010,8 @@ Example: `datasette-auth-tokens <https://datasette.io/plugins/datasette-auth-tok
 filters_from_request(request, database, table, datasette)
 ---------------------------------------------------------
 
-``request`` - object
-    The current HTTP :ref:`internals_request`.
+``request`` - :ref:`internals_request`
+    The current HTTP request.
 
 ``database`` - string
     The name of the database.
@@ -1178,8 +1178,8 @@ forbidden(datasette, request, message)
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
 
-``request`` - object
-    The current HTTP :ref:`internals_request`.
+``request`` - :ref:`internals_request`
+    The current HTTP request.
 
 ``message`` - string
     A message hinting at why the request was forbidden.
@@ -1206,21 +1206,55 @@ The function can alternatively return an awaitable function if it needs to make
 
 .. code-block:: python
 
-    from datasette import hookimpl
-    from datasette.utils.asgi import Response
+    from datasette import hookimpl, Response
 
 
     @hookimpl
     def forbidden(datasette):
         async def inner():
             return Response.html(
-                await datasette.render_template(
-                    "forbidden.html"
-                )
+                await datasette.render_template("render_message.html", request=request)
             )
 
         return inner
 
+.. _plugin_hook_handle_exception:
+
+handle_exception(datasette, request, exception)
+-----------------------------------------------
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
+
+``request`` - :ref:`internals_request`
+    The current HTTP request.
+
+``exception`` - ``Exception``
+    The exception that was raised.
+
+This hook is called any time an unexpected exception is raised. You can use it to record the exception.
+
+If your handler returns a ``Response`` object it will be returned to the client in place of the default Datasette error page.
+
+The handler can return a response directly, or it can return return an awaitable function that returns a response.
+
+This example logs an error to `Sentry <https://sentry.io/>`__ and then renders a custom error page:
+
+.. code-block:: python
+
+    from datasette import hookimpl, Response
+    import sentry_sdk
+
+
+    @hookimpl
+    def handle_exception(datasette, exception):
+        sentry_sdk.capture_exception(exception)
+        async def inner():
+            return Response.html(
+                await datasette.render_template("custom_error.html", request=request)
+            )
+        return inner
+
 .. _plugin_hook_menu_links:
 
 menu_links(datasette, actor, request)
@@ -1232,8 +1266,8 @@ menu_links(datasette, actor, request)
 ``actor`` - dictionary or None
     The currently authenticated :ref:`actor <authentication_actor>`.
 
-``request`` - object or None
-    The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available.
+``request`` - :ref:`internals_request`
+    The current HTTP request. This can be ``None`` if the request object is not available.
 
 This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon.
 
@@ -1281,8 +1315,8 @@ table_actions(datasette, actor, database, table, request)
 ``table`` - string
     The name of the table.
 
-``request`` - object
-    The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available.
+``request`` - :ref:`internals_request`
+    The current HTTP request. This can be ``None`` if the request object is not available.
 
 This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items.
 
@@ -1325,8 +1359,8 @@ database_actions(datasette, actor, database, request)
 ``database`` - string
     The name of the database.
 
-``request`` - object
-    The current HTTP :ref:`internals_request`.
+``request`` - :ref:`internals_request`
+    The current HTTP request.
 
 This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page.
 
diff --git a/tests/fixtures.py b/tests/fixtures.py
index e0e4ec7b..c145ac78 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -68,6 +68,7 @@ EXPECTED_PLUGINS = [
             "canned_queries",
             "extra_js_urls",
             "extra_template_vars",
+            "handle_exception",
             "menu_links",
             "permission_allowed",
             "register_routes",
diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index f5ce36b3..4df02343 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -185,3 +185,21 @@ def register_routes(datasette):
         # Also serves to demonstrate over-ride of default paths:
         (r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)", new_table),
     ]
+
+
+@hookimpl
+def handle_exception(datasette, request, exception):
+    datasette._exception_hook_fired = (request, exception)
+    if request.args.get("_custom_error"):
+        return Response.text("_custom_error")
+    elif request.args.get("_custom_error_async"):
+
+        async def inner():
+            return Response.text("_custom_error_async")
+
+        return inner
+
+
+@hookimpl(specname="register_routes")
+def register_triger_error():
+    return ((r"/trigger-error", lambda: 1 / 0),)
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index f4169dbe..2a519e76 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -332,6 +332,7 @@ def test_permissions_debug(app_client):
     assert checks == [
         {"action": "permissions-debug", "result": True, "used_default": False},
         {"action": "view-instance", "result": None, "used_default": True},
+        {"action": "debug-menu", "result": False, "used_default": True},
         {"action": "permissions-debug", "result": False, "used_default": True},
         {"action": "view-instance", "result": None, "used_default": True},
     ]
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 4a7ad7c6..948a40b8 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -824,6 +824,20 @@ def test_hook_forbidden(restore_working_directory):
         assert "view-database" == client.ds._last_forbidden_message
 
 
+def test_hook_handle_exception(app_client):
+    app_client.get("/trigger-error?x=123")
+    assert hasattr(app_client.ds, "_exception_hook_fired")
+    request, exception = app_client.ds._exception_hook_fired
+    assert request.url == "http://localhost/trigger-error?x=123"
+    assert isinstance(exception, ZeroDivisionError)
+
+
+@pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async"))
+def test_hook_handle_exception_custom_response(app_client, param):
+    response = app_client.get("/trigger-error?{}=1".format(param))
+    assert response.text == param
+
+
 def test_hook_menu_links(app_client):
     def get_menu_links(html):
         soup = Soup(html, "html.parser")

From 58fd1e33ec7ac5ed85431d5c86d60600cd5280fb Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 17 Jul 2022 16:30:58 -0700
Subject: [PATCH 1448/2303] Hint that you can render templates for these hooks,
 refs #1770

---
 docs/plugin_hooks.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 6020a941..b4869606 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -1176,7 +1176,7 @@ forbidden(datasette, request, message)
 --------------------------------------
 
 ``datasette`` - :ref:`internals_datasette`
-    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to render templates or execute SQL queries.
 
 ``request`` - :ref:`internals_request`
     The current HTTP request.
@@ -1224,7 +1224,7 @@ handle_exception(datasette, request, exception)
 -----------------------------------------------
 
 ``datasette`` - :ref:`internals_datasette`
-    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries.
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to render templates or execute SQL queries.
 
 ``request`` - :ref:`internals_request`
     The current HTTP request.

From e543a095cc4c1ca895b082cfd1263ca25203a7c0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 17 Jul 2022 17:57:41 -0700
Subject: [PATCH 1449/2303] Updated default plugins in docs, refs #1770

---
 docs/plugins.rst | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/plugins.rst b/docs/plugins.rst
index f2ed02f7..29078054 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -172,6 +172,24 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "filters_from_request"
             ]
         },
+        {
+            "name": "datasette.forbidden",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "forbidden"
+            ]
+        },
+        {
+            "name": "datasette.handle_exception",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "handle_exception"
+            ]
+        },
         {
             "name": "datasette.publish.cloudrun",
             "static": false,

From 6d5e1955470424cf4faf5d35788d328ebdd6d463 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 17 Jul 2022 17:59:20 -0700
Subject: [PATCH 1450/2303] Release 0.62a1

Refs #1300, #1739, #1744, #1746, #1748, #1759, #1770
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index cf18c441..86f4cf7e 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.62a0"
+__version__ = "0.62a1"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 74814fcb..3f105811 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,20 @@
 Changelog
 =========
 
+.. _v0_62a1:
+
+0.62a1 (2022-07-17)
+-------------------
+
+- New plugin hook: :ref:`handle_exception() <plugin_hook_handle_exception>`, for custom handling of exceptions caught by Datasette. (:issue:`1770`)
+- The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`)
+- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
+- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`1746`)
+- Datasette now has a `Discord community <https://discord.gg/ktd74dm5mw>`__.
+- Database file downloads now implement conditional GET using ETags. (:issue:`1739`)
+- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`)
+- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__)
+
 .. _v0_62a0:
 
 0.62a0 (2022-05-02)

From ed1ebc0f1d4153e3e0934f2af19f82e5fdf137d3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 17 Jul 2022 18:03:33 -0700
Subject: [PATCH 1451/2303] Run blacken-docs, refs #1770

---
 docs/plugin_hooks.rst | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index b4869606..aec1df56 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -1213,7 +1213,9 @@ The function can alternatively return an awaitable function if it needs to make
     def forbidden(datasette):
         async def inner():
             return Response.html(
-                await datasette.render_template("render_message.html", request=request)
+                await datasette.render_template(
+                    "render_message.html", request=request
+                )
             )
 
         return inner
@@ -1249,10 +1251,14 @@ This example logs an error to `Sentry <https://sentry.io/>`__ and then renders a
     @hookimpl
     def handle_exception(datasette, exception):
         sentry_sdk.capture_exception(exception)
+
         async def inner():
             return Response.html(
-                await datasette.render_template("custom_error.html", request=request)
+                await datasette.render_template(
+                    "custom_error.html", request=request
+                )
             )
+
         return inner
 
 .. _plugin_hook_menu_links:

From ea6161f8475d9fa41c4879049511c58f692cce04 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 17 Jul 2022 18:06:26 -0700
Subject: [PATCH 1452/2303] Bump furo from 2022.4.7 to 2022.6.21 (#1760)

Bumps [furo](https://github.com/pradyunsg/furo) from 2022.4.7 to 2022.6.21.
- [Release notes](https://github.com/pradyunsg/furo/releases)
- [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md)
- [Commits](https://github.com/pradyunsg/furo/compare/2022.04.07...2022.06.21)

---
updated-dependencies:
- dependency-name: furo
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 29cb77bf..558b5c87 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ setup(
     setup_requires=["pytest-runner"],
     extras_require={
         "docs": [
-            "furo==2022.4.7",
+            "furo==2022.6.21",
             "sphinx-autobuild",
             "codespell",
             "blacken-docs",

From 22354c48ce4d514d7a1b321e5651c7f1340e3f5e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 17 Jul 2022 18:06:37 -0700
Subject: [PATCH 1453/2303] Update pytest-asyncio requirement from <0.19,>=0.17
 to >=0.17,<0.20 (#1769)

Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Changelog](https://github.com/pytest-dev/pytest-asyncio/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.17.0...v0.19.0)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 558b5c87..a1c51d0b 100644
--- a/setup.py
+++ b/setup.py
@@ -74,7 +74,7 @@ setup(
         "test": [
             "pytest>=5.2.2,<7.2.0",
             "pytest-xdist>=2.2.1,<2.6",
-            "pytest-asyncio>=0.17,<0.19",
+            "pytest-asyncio>=0.17,<0.20",
             "beautifulsoup4>=4.8.1,<4.12.0",
             "black==22.6.0",
             "blacken-docs==1.12.1",

From 01369176b0a8943ab45292ffc6f9c929b80a00e8 Mon Sep 17 00:00:00 2001
From: Chris Amico <eyeseast@gmail.com>
Date: Sun, 17 Jul 2022 21:12:45 -0400
Subject: [PATCH 1454/2303] Keep track of datasette.config_dir (#1766)

Thanks, @eyeseast - closes #1764
---
 datasette/app.py         | 1 +
 tests/test_config_dir.py | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/datasette/app.py b/datasette/app.py
index edd05bb3..1a9afc10 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -211,6 +211,7 @@ class Datasette:
         assert config_dir is None or isinstance(
             config_dir, Path
         ), "config_dir= should be a pathlib.Path"
+        self.config_dir = config_dir
         self.pdb = pdb
         self._secret = secret or secrets.token_hex(32)
         self.files = tuple(files or []) + tuple(immutables or [])
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index 015c6ace..fe927c42 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -1,4 +1,5 @@
 import json
+import pathlib
 import pytest
 
 from datasette.app import Datasette
@@ -150,3 +151,11 @@ def test_metadata_yaml(tmp_path_factory, filename):
     response = client.get("/-/metadata.json")
     assert 200 == response.status
     assert {"title": "Title from metadata"} == response.json
+
+
+def test_store_config_dir(config_dir_client):
+    ds = config_dir_client.ds
+
+    assert hasattr(ds, "config_dir")
+    assert ds.config_dir is not None
+    assert isinstance(ds.config_dir, pathlib.Path)

From 7af67b54b7d9bca43e948510fc62f6db2b748fa8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 18 Jul 2022 14:31:09 -0700
Subject: [PATCH 1455/2303] How to register temporary plugins in tests, closes
 #903

---
 docs/testing_plugins.rst | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index 41046bfb..d02003a9 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -219,3 +219,39 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
         assert (
             outbound_request.url == "https://www.example.com/"
         )
+
+.. _testing_plugins_register_in_test:
+
+Registering a plugin for the duration of a test
+-----------------------------------------------
+
+When writing tests for plugins you may find it useful to register a test plugin just for the duration of a single test. You can do this using ``pm.register()`` and ``pm.unregister()`` like this:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette.app import Datasette
+    from datasette.plugins import pm
+    import pytest
+
+
+    @pytest.mark.asyncio
+    async def test_using_test_plugin():
+        class TestPlugin:
+            __name__ = "TestPlugin"
+
+            # Use hookimpl and method names to register hooks
+            @hookimpl
+            def register_routes(self):
+                return [
+                    (r"^/error$", lambda: 1/0),
+                ]
+
+        pm.register(TestPlugin(), name="undo")
+        try:
+            # The test implementation goes here
+            datasette = Datasette()
+            response = await datasette.client.get("/error")
+            assert response.status_code == 500
+        finally:
+            pm.unregister(name="undo")

From bca2d95d0228f80a108e13408f8e72b2c06c2c7b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 2 Aug 2022 16:38:02 -0700
Subject: [PATCH 1456/2303] Configure readthedocs/readthedocs-preview

---
 .github/workflows/documentation-links.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 .github/workflows/documentation-links.yml

diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml
new file mode 100644
index 00000000..e7062a46
--- /dev/null
+++ b/.github/workflows/documentation-links.yml
@@ -0,0 +1,16 @@
+name: Read the Docs Pull Request Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/readthedocs-preview@main
+        with:
+          project-slug: "datasette"

From 8cfc72336878dd846d149658e99cc598e835b661 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 9 Aug 2022 11:21:53 -0700
Subject: [PATCH 1457/2303] Ran blacken-docs

---
 docs/testing_plugins.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index d02003a9..992b4b0e 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -244,7 +244,7 @@ When writing tests for plugins you may find it useful to register a test plugin
             @hookimpl
             def register_routes(self):
                 return [
-                    (r"^/error$", lambda: 1/0),
+                    (r"^/error$", lambda: 1 / 0),
                 ]
 
         pm.register(TestPlugin(), name="undo")

From 05d9c682689a0f1d23cbb502e027364ab3363910 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 08:16:53 -0700
Subject: [PATCH 1458/2303] Promote Discord more in the README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 032180aa..7ebbca57 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 * Comprehensive documentation: https://docs.datasette.io/
 * Examples: https://datasette.io/examples
 * Live demo of current main: https://latest.datasette.io/
-* Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions)
+* Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw)
 
 Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem.
 

From db00c00f6397287749331e8042fe998ee7f3b919 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 08:19:30 -0700
Subject: [PATCH 1459/2303] Promote Datasette Lite in the README, refs #1781

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7ebbca57..1af20129 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover
 * Latest [Datasette News](https://datasette.io/news)
 * Comprehensive documentation: https://docs.datasette.io/
 * Examples: https://datasette.io/examples
-* Live demo of current main: https://latest.datasette.io/
+* Live demo of current `main` branch: https://latest.datasette.io/
 * Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw)
 
 Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem.
@@ -85,3 +85,7 @@ Or:
 This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API.
 
 See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details.
+
+## Datasette Lite
+
+[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md).

From 8eb699de7becdefc6d72555d9fb17c9f06235dc4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 08:24:39 -0700
Subject: [PATCH 1460/2303] Datasette Lite in Getting Started docs, closes
 #1781

---
 docs/getting_started.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 00b753a9..571540cf 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -21,6 +21,17 @@ Datasette has several `tutorials <https://datasette.io/tutorials>`__ to help you
 - `Exploring a database with Datasette <https://datasette.io/tutorials/explore>`__ shows how to use the Datasette web interface to explore a new database.
 - `Learn SQL with Datasette <https://datasette.io/tutorials/learn-sql>`__ introduces SQL, and shows how to use that query language to ask questions of your data.
 
+.. _getting_started_datasette_lite:
+
+Datasette in your browser with Datasette Lite
+---------------------------------------------
+
+`Datasette Lite <https://lite.datasette.io/>`__ is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required.
+
+You can pass a URL to a CSV, SQLite or raw SQL file directly to Datasette Lite to explore that data in your browser.
+
+This `example link <https://lite.datasette.io/?url=https%3A%2F%2Fraw.githubusercontent.com%2FNUKnightLab%2Fsql-mysteries%2Fmaster%2Fsql-murder-mystery.db#/sql-murder-mystery>`__ opens Datasette Lite and loads the SQL Murder Mystery example database from `Northwestern University Knight Lab <https://github.com/NUKnightLab/sql-mysteries>`__. 
+
 .. _getting_started_glitch:
 
 Try Datasette without installing anything using Glitch

From df4fd2d7ddca8956d8a51c72ce007b8c75227f32 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 08:44:02 -0700
Subject: [PATCH 1461/2303] _sort= works even if sort column not selected,
 closes #1773

---
 datasette/views/table.py | 22 +++++++++++++++++++++-
 tests/test_table_api.py  |  2 ++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index cd4be823..94d2673b 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -630,7 +630,27 @@ class TableView(DataView):
                 next_value = path_from_row_pks(rows[-2], pks, use_rowid)
             # If there's a sort or sort_desc, add that value as a prefix
             if (sort or sort_desc) and not is_view:
-                prefix = rows[-2][sort or sort_desc]
+                try:
+                    prefix = rows[-2][sort or sort_desc]
+                except IndexError:
+                    # sort/sort_desc column missing from SELECT - look up value by PK instead
+                    prefix_where_clause = " and ".join(
+                        "[{}] = :pk{}".format(pk, i) for i, pk in enumerate(pks)
+                    )
+                    prefix_lookup_sql = "select [{}] from [{}] where {}".format(
+                        sort or sort_desc, table_name, prefix_where_clause
+                    )
+                    prefix = (
+                        await db.execute(
+                            prefix_lookup_sql,
+                            {
+                                **{
+                                    "pk{}".format(i): rows[-2][pk]
+                                    for i, pk in enumerate(pks)
+                                }
+                            },
+                        )
+                    ).single_value()
                 if isinstance(prefix, dict) and "value" in prefix:
                     prefix = prefix["value"]
                 if prefix is None:
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 9db383c3..e56a72b5 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -288,6 +288,8 @@ def test_paginate_compound_keys_with_extra_filters(app_client):
         ),
         # text column contains '$null' - ensure it doesn't confuse pagination:
         ("_sort=text", lambda row: row["text"], "sorted by text"),
+        # Still works if sort column removed using _col=
+        ("_sort=text&_col=content", lambda row: row["text"], "sorted by text"),
     ],
 )
 def test_sortable(app_client, query_string, sort_key, human_description_en):

From 668415df9f6334bd255c22ab02018bed5bc14edd Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 08:47:17 -0700
Subject: [PATCH 1462/2303] Upgrade Docker baes to 3.10.6-slim-bullseye - refs
 #1768

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 42f5529b..ee7ed957 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9.7-slim-bullseye as build
+FROM python:3.10.6-slim-bullseye as build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

From 080d4b3e065d78faf977c6ded6ead31aae24e2ae Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 08:49:14 -0700
Subject: [PATCH 1463/2303] Switch to python:3.10.6-slim-bullseye for datasette
 publish - refs #1768

---
 datasette/utils/__init__.py    | 2 +-
 demos/apache-proxy/Dockerfile  | 2 +-
 docs/publish.rst               | 2 +-
 tests/test_package.py          | 2 +-
 tests/test_publish_cloudrun.py | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 77768112..d148cc2c 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -390,7 +390,7 @@ def make_dockerfile(
             "SQLITE_EXTENSIONS"
         ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so"
     return """
-FROM python:3.8
+FROM python:3.10.6-slim-bullseye
 COPY . /app
 WORKDIR /app
 {apt_get_extras}
diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 6c921963..70b33bec 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9.7-slim-bullseye
+FROM python:3.10.6-slim-bullseye
 
 RUN apt-get update && \
     apt-get install -y apache2 supervisor && \
diff --git a/docs/publish.rst b/docs/publish.rst
index 166f2883..9c7c99cc 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -144,7 +144,7 @@ Here's example output for the package command::
 
     $ datasette package parlgov.db --extra-options="--setting sql_time_limit_ms 2500"
     Sending build context to Docker daemon  4.459MB
-    Step 1/7 : FROM python:3
+    Step 1/7 : FROM python:3.10.6-slim-bullseye
      ---> 79e1dc9af1c1
     Step 2/7 : COPY . /app
      ---> Using cache
diff --git a/tests/test_package.py b/tests/test_package.py
index 02ed1775..ac15e61e 100644
--- a/tests/test_package.py
+++ b/tests/test_package.py
@@ -12,7 +12,7 @@ class CaptureDockerfile:
 
 
 EXPECTED_DOCKERFILE = """
-FROM python:3.8
+FROM python:3.10.6-slim-bullseye
 COPY . /app
 WORKDIR /app
 
diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py
index 3427f4f7..60079ab3 100644
--- a/tests/test_publish_cloudrun.py
+++ b/tests/test_publish_cloudrun.py
@@ -223,7 +223,7 @@ def test_publish_cloudrun_plugin_secrets(
     )
     expected = textwrap.dedent(
         r"""
-    FROM python:3.8
+    FROM python:3.10.6-slim-bullseye
     COPY . /app
     WORKDIR /app
 
@@ -290,7 +290,7 @@ def test_publish_cloudrun_apt_get_install(
     )
     expected = textwrap.dedent(
         r"""
-    FROM python:3.8
+    FROM python:3.10.6-slim-bullseye
     COPY . /app
     WORKDIR /app
 

From 1563c22a8c65e6cff5194aa07df54d0ab8d4eecb Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 09:13:12 -0700
Subject: [PATCH 1464/2303] Don't duplicate _sort_desc, refs #1738

---
 datasette/views/table.py | 2 +-
 tests/test_table_html.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 94d2673b..49c30c9c 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -710,7 +710,7 @@ class TableView(DataView):
             for key in request.args:
                 if (
                     key.startswith("_")
-                    and key not in ("_sort", "_search", "_next")
+                    and key not in ("_sort", "_sort_desc", "_search", "_next")
                     and "__" not in key
                 ):
                     for value in request.args.getlist(key):
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index d3cb3e17..f3808ea3 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -828,6 +828,7 @@ def test_other_hidden_form_fields(app_client, path, expected_hidden):
     [
         ("/fixtures/searchable?_search=terry", []),
         ("/fixtures/searchable?_sort=text2", []),
+        ("/fixtures/searchable?_sort_desc=text2", []),
         ("/fixtures/searchable?_sort=text2&_where=1", [("_where", "1")]),
     ],
 )

From c1396bf86033a7bd99fa0c0431f585475391a11a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 09:34:31 -0700
Subject: [PATCH 1465/2303] Don't allow canned write queries on immutable DBs,
 closes #1728

---
 datasette/templates/query.html |  6 ++++-
 datasette/views/database.py    |  4 ++++
 tests/test_canned_queries.py   | 40 ++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/query.html b/datasette/templates/query.html
index 8c920527..cee779fc 100644
--- a/datasette/templates/query.html
+++ b/datasette/templates/query.html
@@ -28,6 +28,10 @@
 
 {% block content %}
 
+{% if canned_write and db_is_immutable %}
+    <p class="message-error">This query cannot be executed because the database is immutable.</p>
+{% endif %}
+
 <h1 style="padding-left: 10px; border-left: 10px solid #{{ database_color(database) }}">{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}</h1>
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
@@ -61,7 +65,7 @@
     <p>
         {% if not hide_sql %}<button id="sql-format" type="button" hidden>Format SQL</button>{% endif %}
         {% if canned_write %}<input type="hidden" name="csrftoken" value="{{ csrftoken() }}">{% endif %}
-        <input type="submit" value="Run SQL">
+        <input type="submit" value="Run SQL"{% if canned_write and db_is_immutable %} disabled{% endif %}>
         {{ show_hide_hidden }}
         {% if canned_query and edit_sql_url %}<a href="{{ edit_sql_url }}" class="canned-query-edit-sql">Edit SQL</a>{% endif %}
     </p>
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 42058752..77632b9d 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -273,6 +273,9 @@ class QueryView(DataView):
         # Execute query - as write or as read
         if write:
             if request.method == "POST":
+                # If database is immutable, return an error
+                if not db.is_mutable:
+                    raise Forbidden("Database is immutable")
                 body = await request.post_body()
                 body = body.decode("utf-8").strip()
                 if body.startswith("{") and body.endswith("}"):
@@ -326,6 +329,7 @@ class QueryView(DataView):
                 async def extra_template():
                     return {
                         "request": request,
+                        "db_is_immutable": not db.is_mutable,
                         "path_with_added_args": path_with_added_args,
                         "path_with_removed_args": path_with_removed_args,
                         "named_parameter_values": named_parameter_values,
diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index 5abffdcc..976aa0db 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -53,6 +53,26 @@ def canned_write_client(tmpdir):
         yield client
 
 
+@pytest.fixture
+def canned_write_immutable_client():
+    with make_app_client(
+        is_immutable=True,
+        metadata={
+            "databases": {
+                "fixtures": {
+                    "queries": {
+                        "add": {
+                            "sql": "insert into sortable (text) values (:text)",
+                            "write": True,
+                        },
+                    }
+                }
+            }
+        },
+    ) as client:
+        yield client
+
+
 def test_canned_query_with_named_parameter(app_client):
     response = app_client.get("/fixtures/neighborhood_search.json?text=town")
     assert [
@@ -373,3 +393,23 @@ def test_canned_write_custom_template(canned_write_client):
         response.headers["link"]
         == 'http://localhost/data/update_name.json; rel="alternate"; type="application/json+datasette"'
     )
+
+
+def test_canned_write_query_disabled_for_immutable_database(
+    canned_write_immutable_client,
+):
+    response = canned_write_immutable_client.get("/fixtures/add")
+    assert response.status == 200
+    assert (
+        "This query cannot be executed because the database is immutable."
+        in response.text
+    )
+    assert '<input type="submit" value="Run SQL" disabled>' in response.text
+    # Submitting form should get a forbidden error
+    response = canned_write_immutable_client.post(
+        "/fixtures/add",
+        {"text": "text"},
+        csrftoken_from=True,
+    )
+    assert response.status == 403
+    assert "Database is immutable" in response.text

From 82167105ee699c850cc106ea927de1ad09276cfe Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 10:07:30 -0700
Subject: [PATCH 1466/2303] --min-instances and --max-instances Cloud Run
 publish options, closes #1779

---
 datasette/publish/cloudrun.py  | 26 +++++++++++++++++---
 docs/cli-reference.rst         |  2 ++
 tests/test_publish_cloudrun.py | 43 ++++++++++++++++++++++++----------
 3 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py
index 50b2b2fd..77274eb0 100644
--- a/datasette/publish/cloudrun.py
+++ b/datasette/publish/cloudrun.py
@@ -52,6 +52,16 @@ def publish_subcommand(publish):
         multiple=True,
         help="Additional packages to apt-get install",
     )
+    @click.option(
+        "--max-instances",
+        type=int,
+        help="Maximum Cloud Run instances",
+    )
+    @click.option(
+        "--min-instances",
+        type=int,
+        help="Minimum Cloud Run instances",
+    )
     def cloudrun(
         files,
         metadata,
@@ -79,6 +89,8 @@ def publish_subcommand(publish):
         cpu,
         timeout,
         apt_get_extras,
+        max_instances,
+        min_instances,
     ):
         "Publish databases to Datasette running on Cloud Run"
         fail_if_publish_binary_not_installed(
@@ -168,12 +180,20 @@ def publish_subcommand(publish):
                 ),
                 shell=True,
             )
+        extra_deploy_options = []
+        for option, value in (
+            ("--memory", memory),
+            ("--cpu", cpu),
+            ("--max-instances", max_instances),
+            ("--min-instances", min_instances),
+        ):
+            if value:
+                extra_deploy_options.append("{} {}".format(option, value))
         check_call(
-            "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}{}".format(
+            "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format(
                 image_id,
                 service,
-                " --memory {}".format(memory) if memory else "",
-                " --cpu {}".format(cpu) if cpu else "",
+                " " + " ".join(extra_deploy_options) if extra_deploy_options else "",
             ),
             shell=True,
         )
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 1c1aff15..415af13c 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -251,6 +251,8 @@ datasette publish cloudrun --help
       --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
       --timeout INTEGER               Build timeout in seconds
       --apt-get-install TEXT          Additional packages to apt-get install
+      --max-instances INTEGER         Maximum Cloud Run instances
+      --min-instances INTEGER         Minimum Cloud Run instances
       --help                          Show this message and exit.
 
 
diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py
index 60079ab3..e64534d2 100644
--- a/tests/test_publish_cloudrun.py
+++ b/tests/test_publish_cloudrun.py
@@ -105,19 +105,36 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory):
 @mock.patch("datasette.publish.cloudrun.check_output")
 @mock.patch("datasette.publish.cloudrun.check_call")
 @pytest.mark.parametrize(
-    "memory,cpu,timeout,expected_gcloud_args",
+    "memory,cpu,timeout,min_instances,max_instances,expected_gcloud_args",
     [
-        ["1Gi", None, None, "--memory 1Gi"],
-        ["2G", None, None, "--memory 2G"],
-        ["256Mi", None, None, "--memory 256Mi"],
-        ["4", None, None, None],
-        ["GB", None, None, None],
-        [None, 1, None, "--cpu 1"],
-        [None, 2, None, "--cpu 2"],
-        [None, 3, None, None],
-        [None, 4, None, "--cpu 4"],
-        ["2G", 4, None, "--memory 2G --cpu 4"],
-        [None, None, 1800, "--timeout 1800"],
+        ["1Gi", None, None, None, None, "--memory 1Gi"],
+        ["2G", None, None, None, None, "--memory 2G"],
+        ["256Mi", None, None, None, None, "--memory 256Mi"],
+        [
+            "4",
+            None,
+            None,
+            None,
+            None,
+            None,
+        ],
+        [
+            "GB",
+            None,
+            None,
+            None,
+            None,
+            None,
+        ],
+        [None, 1, None, None, None, "--cpu 1"],
+        [None, 2, None, None, None, "--cpu 2"],
+        [None, 3, None, None, None, None],
+        [None, 4, None, None, None, "--cpu 4"],
+        ["2G", 4, None, None, None, "--memory 2G --cpu 4"],
+        [None, None, 1800, None, None, "--timeout 1800"],
+        [None, None, None, 2, None, "--min-instances 2"],
+        [None, None, None, 2, 4, "--min-instances 2 --max-instances 4"],
+        [None, 2, None, None, 4, "--cpu 2 --max-instances 4"],
     ],
 )
 def test_publish_cloudrun_memory_cpu(
@@ -127,6 +144,8 @@ def test_publish_cloudrun_memory_cpu(
     memory,
     cpu,
     timeout,
+    min_instances,
+    max_instances,
     expected_gcloud_args,
     tmp_path_factory,
 ):

From 5e6c5c9e3191a80f17a91c5205d9d69efdebb73f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 10:18:47 -0700
Subject: [PATCH 1467/2303] Document datasette.config_dir, refs #1766

---
 docs/internals.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index da135282..20797e98 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -260,6 +260,7 @@ Constructor parameters include:
 - ``files=[...]`` - a list of database files to open
 - ``immutables=[...]`` - a list of database files to open in immutable mode
 - ``metadata={...}`` - a dictionary of :ref:`metadata`
+- ``config_dir=...`` - the :ref:`configuration directory <config_dir>` to use, stored in ``datasette.config_dir``
 
 .. _datasette_databases:
 

From 815162cf029fab9f1c9308c1d6ecdba7ee369ebe Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 10:32:42 -0700
Subject: [PATCH 1468/2303] Release 0.62

Refs #903, #1300, #1683, #1701, #1712, #1717, #1718, #1728, #1733, #1738, #1739, #1744, #1746, #1748, #1759, #1766, #1768, #1770, #1773, #1779

Closes #1782
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 53 ++++++++++++++++++++++++++++++--------------
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index 86f4cf7e..0453346c 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.62a1"
+__version__ = "0.62"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 3f105811..1225c63f 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,33 +4,52 @@
 Changelog
 =========
 
-.. _v0_62a1:
+.. _v0_62:
 
-0.62a1 (2022-07-17)
+0.62 (2022-08-14)
 -------------------
 
+Datasette can now run entirely in your browser using WebAssembly. Try out `Datasette Lite <https://lite.datasette.io/>`__, take a look `at the code <https://github.com/simonw/datasette-lite>`__ or read more about it in `Datasette Lite: a server-side Python web application running in a browser <https://simonwillison.net/2022/May/4/datasette-lite/>`__.
+
+Datasette now has a `Discord community <https://discord.gg/ktd74dm5mw>`__ for questions and discussions about Datasette and its ecosystem of projects.
+
+Features
+~~~~~~~~
+
+- Datasette is now compatible with `Pyodide <https://pyodide.org/>`__.  This is the enabling technology behind `Datasette Lite <https://lite.datasette.io/>`__. (:issue:`1733`)
+- Database file downloads now implement conditional GET using ETags. (:issue:`1739`)
+- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__)
+- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details.
+- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
+- Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
+- ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
+- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
+- Canned writable queries against immutable databases now show a warning message. (:issue:`1728`)
+- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__)
+- ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`)
+
+Plugin hooks
+~~~~~~~~~~~~
+
 - New plugin hook: :ref:`handle_exception() <plugin_hook_handle_exception>`, for custom handling of exceptions caught by Datasette. (:issue:`1770`)
 - The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`)
-- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
-- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`1746`)
-- Datasette now has a `Discord community <https://discord.gg/ktd74dm5mw>`__.
-- Database file downloads now implement conditional GET using ETags. (:issue:`1739`)
-- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`)
-- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__)
+- The :ref:`configuration directory <config_dir>` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (`#1766 <https://github.com/simonw/datasette/pull/1766>`__)
 
-.. _v0_62a0:
+Bug fixes
+~~~~~~~~~
 
-0.62a0 (2022-05-02)
--------------------
-
-- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details.
-- Datasette should now be compatible with Pyodide. (:issue:`1733`)
-- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__)
-- Spaces in database names are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
-- ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
 - Don't show the facet option in the cog menu if faceting is not allowed. (:issue:`1683`)
+- ``?_sort`` and ``?_sort_desc`` now work if the column that is being sorted has been excluded from the query using ``?_col=`` or ``?_nocol=``. (:issue:`1773`)
+- Fixed bug where ``?_sort_desc`` was duplicated in the URL every time the Apply button was clicked. (:issue:`1738`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`)
+- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`1746`)
 - Code examples in the documentation are now all formatted using Black. (:issue:`1718`)
 - ``Request.fake()`` method is now documented, see :ref:`internals_request`.
+- New documentation for plugin authors: :ref:`testing_plugins_register_in_test`. (:issue:`903`)
 
 .. _v0_61_1:
 

From a107e3a028923c1ab3911c0f880011283f93f368 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Aug 2022 16:07:46 -0700
Subject: [PATCH 1469/2303] datasette-sentry is an example of handle_exception

---
 docs/plugin_hooks.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index aec1df56..c6f35d06 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -1261,6 +1261,8 @@ This example logs an error to `Sentry <https://sentry.io/>`__ and then renders a
 
         return inner
 
+Example: `datasette-sentry <https://datasette.io/plugins/datasette-sentry>`_
+
 .. _plugin_hook_menu_links:
 
 menu_links(datasette, actor, request)

From 481eb96d85291cdfa5767a83884a1525dfc382d8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 15 Aug 2022 13:17:28 -0700
Subject: [PATCH 1470/2303] https://datasette.io/tutorials/clean-data tutorial

Refs #1783
---
 docs/getting_started.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 571540cf..a9eaa404 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -20,6 +20,7 @@ Datasette has several `tutorials <https://datasette.io/tutorials>`__ to help you
 
 - `Exploring a database with Datasette <https://datasette.io/tutorials/explore>`__ shows how to use the Datasette web interface to explore a new database.
 - `Learn SQL with Datasette <https://datasette.io/tutorials/learn-sql>`__ introduces SQL, and shows how to use that query language to ask questions of your data.
+- `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__ guides you through using `sqlite-utils <https://sqlite-utils.datasette.io/>`__ to turn a CSV file into a database that you can explore using Datasette.
 
 .. _getting_started_datasette_lite:
 

From a3e6f1b16757fb2d39e7ddba4e09eda2362508bf Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 18 Aug 2022 09:06:02 -0700
Subject: [PATCH 1471/2303] Increase height of non-JS textarea to fit query

Closes #1786
---
 datasette/templates/query.html | 3 ++-
 tests/test_html.py             | 6 ++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/datasette/templates/query.html b/datasette/templates/query.html
index cee779fc..a35e3afe 100644
--- a/datasette/templates/query.html
+++ b/datasette/templates/query.html
@@ -45,7 +45,8 @@
     {% endif %}
     {% if not hide_sql %}
         {% if editable and allow_execute_sql %}
-            <p><textarea id="sql-editor" name="sql">{% if query and query.sql %}{{ query.sql }}{% else %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}</textarea></p>
+            <p><textarea id="sql-editor" name="sql"{% if query and query.sql %} style="height: {{ query.sql.split("\n")|length + 2 }}em"{% endif %}
+            >{% if query and query.sql %}{{ query.sql }}{% else %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}</textarea></p>
         {% else %}
             <pre id="sql-query">{% if query %}{{ query.sql }}{% endif %}</pre>
         {% endif %}
diff --git a/tests/test_html.py b/tests/test_html.py
index 409fec68..be21bd84 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -695,10 +695,8 @@ def test_query_error(app_client):
     response = app_client.get("/fixtures?sql=select+*+from+notatable")
     html = response.text
     assert '<p class="message-error">no such table: notatable</p>' in html
-    assert (
-        '<textarea id="sql-editor" name="sql">select * from notatable</textarea>'
-        in html
-    )
+    assert '<textarea id="sql-editor" name="sql" style="height: 3em' in html
+    assert ">select * from notatable</textarea>" in html
     assert "0 results" not in html
 
 

From 09a41662e70b788469157bb58ed9ca4acdf2f904 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 18 Aug 2022 09:10:48 -0700
Subject: [PATCH 1472/2303] Fix typo

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index c6f35d06..30bd75b7 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -874,7 +874,7 @@ canned_queries(datasette, database, actor)
 ``actor`` - dictionary or None
     The currently authenticated :ref:`actor <authentication_actor>`.
 
-Ues this hook to return a dictionary of additional :ref:`canned query <canned_queries>` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query <canned_queries>` documentation.
+Use this hook to return a dictionary of additional :ref:`canned query <canned_queries>` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query <canned_queries>` documentation.
 
 .. code-block:: python
 

From 6c0ba7c00c2ae3ecbb5309efa59079cea1c850b3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 18 Aug 2022 14:52:04 -0700
Subject: [PATCH 1473/2303] Improved CLI reference documentation, refs #1787

---
 datasette/cli.py         |   2 +-
 docs/changelog.rst       |   2 +-
 docs/cli-reference.rst   | 325 ++++++++++++++++++++++++++++++---------
 docs/getting_started.rst |  50 ------
 docs/index.rst           |   2 +-
 docs/publish.rst         |   2 +
 6 files changed, 259 insertions(+), 124 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 8781747c..f2a03d53 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -282,7 +282,7 @@ def package(
     port,
     **extra_metadata,
 ):
-    """Package specified SQLite files into a new datasette Docker container"""
+    """Package SQLite files into a Datasette Docker container"""
     if not shutil.which("docker"):
         click.secho(
             ' The package command requires "docker" to be installed and configured ',
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1225c63f..f9dcc980 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -621,7 +621,7 @@ See also `Datasette 0.49: The annotated release notes <https://simonwillison.net
 - Datasette now has `a GitHub discussions forum <https://github.com/simonw/datasette/discussions>`__ for conversations about the project that go beyond just bug reports and issues.
 - Datasette can now be installed on macOS using Homebrew! Run ``brew install simonw/datasette/datasette``. See :ref:`installation_homebrew`. (:issue:`335`)
 - Two new commands: ``datasette install name-of-plugin`` and ``datasette uninstall name-of-plugin``. These are equivalent to ``pip install`` and ``pip uninstall`` but automatically run in the same virtual environment as Datasette, so users don't have to figure out where that virtual environment is - useful for installations created using Homebrew or ``pipx``. See :ref:`plugins_installing`. (:issue:`925`)
-- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`getting_started_datasette_get` for an example. (:issue:`926`)
+- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`cli_datasette_get` for an example. (:issue:`926`)
 
 .. _v0_46:
 
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 415af13c..a1e56774 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -4,44 +4,34 @@
  CLI reference
 ===============
 
-This page lists the ``--help`` for every ``datasette`` CLI command.
+The ``datasette`` CLI tool provides a number of commands.
+
+Running ``datasette`` without specifying a command runs the default command, ``datasette serve``.  See :ref:`cli_help_serve___help` for the full list of options for that command.
 
 .. [[[cog
     from datasette import cli
     from click.testing import CliRunner
     import textwrap
-    commands = [
-        ["--help"],
-        ["serve", "--help"],
-        ["serve", "--help-settings"],
-        ["plugins", "--help"],
-        ["publish", "--help"],
-        ["publish", "cloudrun", "--help"],
-        ["publish", "heroku", "--help"],
-        ["package", "--help"],
-        ["inspect", "--help"],
-        ["install", "--help"],
-        ["uninstall", "--help"],
-    ]
-    cog.out("\n")
-    for command in commands:
-        title = "datasette " + " ".join(command)
-        ref = "_cli_help_" + ("_".join(command).replace("-", "_"))
-        cog.out(".. {}:\n\n".format(ref))
-        cog.out(title + "\n")
-        cog.out(("=" * len(title)) + "\n\n")
+    def help(args):
+        title = "datasette " + " ".join(args)
         cog.out("::\n\n")
-        result = CliRunner().invoke(cli.cli, command)
+        result = CliRunner().invoke(cli.cli, args)
         output = result.output.replace("Usage: cli ", "Usage: datasette ")
         cog.out(textwrap.indent(output, '    '))
         cog.out("\n\n")
 .. ]]]
+.. [[[end]]]
 
 .. _cli_help___help:
 
 datasette --help
 ================
 
+Running ``datasette --help`` shows a list of all of the available commands.
+
+.. [[[cog
+    help(["--help"])
+.. ]]]
 ::
 
     Usage: datasette [OPTIONS] COMMAND [ARGS]...
@@ -59,17 +49,34 @@ datasette --help
       serve*     Serve up specified SQLite database files with a web UI
       inspect    Generate JSON summary of provided database files
       install    Install plugins and packages from PyPI into the same...
-      package    Package specified SQLite files into a new datasette Docker...
+      package    Package SQLite files into a Datasette Docker container
       plugins    List currently installed plugins
       publish    Publish specified SQLite database files to the internet along...
       uninstall  Uninstall plugins and Python packages from the Datasette...
 
 
+.. [[[end]]]
+
+Additional commands added by plugins that use the :ref:`plugin_hook_register_commands` hook will be listed here as well.
+
 .. _cli_help_serve___help:
 
-datasette serve --help
-======================
+datasette serve
+===============
 
+This command starts the Datasette web application running on your machine::
+
+    datasette serve mydatabase.db
+
+Or since this is the default command you can run this instead::
+
+    datasette mydatabase.db
+
+Once started you can access it at ``http://localhost:8001``
+
+.. [[[cog
+    help(["serve", "--help"])
+.. ]]]
 ::
 
     Usage: datasette serve [OPTIONS] [FILES]...
@@ -121,11 +128,75 @@ datasette serve --help
       --help                    Show this message and exit.
 
 
+.. [[[end]]]
+
+
+.. _cli_datasette_get:
+
+datasette --get
+---------------
+
+The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies the path to a page within Datasette and causes Datasette to output the content from that path without starting the web server.
+
+This means that all of Datasette's functionality can be accessed directly from the command-line.
+
+For example::
+
+    $ datasette --get '/-/versions.json' | jq .
+    {
+      "python": {
+        "version": "3.8.5",
+        "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]"
+      },
+      "datasette": {
+        "version": "0.46+15.g222a84a.dirty"
+      },
+      "asgi": "3.0",
+      "uvicorn": "0.11.8",
+      "sqlite": {
+        "version": "3.32.3",
+        "fts_versions": [
+          "FTS5",
+          "FTS4",
+          "FTS3"
+        ],
+        "extensions": {
+          "json1": null
+        },
+        "compile_options": [
+          "COMPILER=clang-11.0.3",
+          "ENABLE_COLUMN_METADATA",
+          "ENABLE_FTS3",
+          "ENABLE_FTS3_PARENTHESIS",
+          "ENABLE_FTS4",
+          "ENABLE_FTS5",
+          "ENABLE_GEOPOLY",
+          "ENABLE_JSON1",
+          "ENABLE_PREUPDATE_HOOK",
+          "ENABLE_RTREE",
+          "ENABLE_SESSION",
+          "MAX_VARIABLE_NUMBER=250000",
+          "THREADSAFE=1"
+        ]
+      }
+    }
+
+The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error.
+
+This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions.
+
 .. _cli_help_serve___help_settings:
 
 datasette serve --help-settings
-===============================
+-------------------------------
 
+This command outputs all of the available Datasette :ref:`settings <settings>`.
+
+These can be passed to ``datasette serve`` using ``datasette serve --setting name value``.
+
+.. [[[cog
+    help(["--help-settings"])
+.. ]]]
 ::
 
     Settings:
@@ -170,11 +241,18 @@ datasette serve --help-settings
 
 
 
+.. [[[end]]]
+
 .. _cli_help_plugins___help:
 
-datasette plugins --help
-========================
+datasette plugins
+=================
 
+Output JSON showing all currently installed plugins, their versions, whether they include static files or templates and which :ref:`plugin_hooks` they use.
+
+.. [[[cog
+    help(["plugins", "--help"])
+.. ]]]
 ::
 
     Usage: datasette plugins [OPTIONS]
@@ -187,11 +265,110 @@ datasette plugins --help
       --help                   Show this message and exit.
 
 
+.. [[[end]]]
+
+Example output:
+
+.. code-block:: json
+
+    [
+        {
+            "name": "datasette-geojson",
+            "static": false,
+            "templates": false,
+            "version": "0.3.1",
+            "hooks": [
+                "register_output_renderer"
+            ]
+        },
+        {
+            "name": "datasette-geojson-map",
+            "static": true,
+            "templates": false,
+            "version": "0.4.0",
+            "hooks": [
+                "extra_body_script",
+                "extra_css_urls",
+                "extra_js_urls"
+            ]
+        },
+        {
+            "name": "datasette-leaflet",
+            "static": true,
+            "templates": false,
+            "version": "0.2.2",
+            "hooks": [
+                "extra_body_script",
+                "extra_template_vars"
+            ]
+        }
+    ]
+
+
+.. _cli_help_install___help:
+
+datasette install
+=================
+
+Install new Datasette plugins. This command works like ``pip install`` but ensures that your plugins will be installed into the same environment as Datasette.
+
+This command::
+
+    datasette install datasette-cluster-map
+
+Would install the `datasette-cluster-map <https://datasette.io/plugins/datasette-cluster-map>`__ plugin.
+
+.. [[[cog
+    help(["install", "--help"])
+.. ]]]
+::
+
+    Usage: datasette install [OPTIONS] PACKAGES...
+
+      Install plugins and packages from PyPI into the same environment as Datasette
+
+    Options:
+      -U, --upgrade  Upgrade packages to latest version
+      --help         Show this message and exit.
+
+
+.. [[[end]]]
+
+.. _cli_help_uninstall___help:
+
+datasette uninstall
+===================
+
+Uninstall one or more plugins.
+
+.. [[[cog
+    help(["uninstall", "--help"])
+.. ]]]
+::
+
+    Usage: datasette uninstall [OPTIONS] PACKAGES...
+
+      Uninstall plugins and Python packages from the Datasette environment
+
+    Options:
+      -y, --yes  Don't ask for confirmation
+      --help     Show this message and exit.
+
+
+.. [[[end]]]
+
 .. _cli_help_publish___help:
 
-datasette publish --help
-========================
+datasette publish
+=================
 
+Shows a list of available deployment targets for :ref:`publishing data <publishing>` with Datasette.
+
+Additional deployment targets can be added by plugins that use the :ref:`plugin_hook_publish_subcommand` hook.
+
+.. [[[cog
+    help(["publish", "--help"])
+.. ]]]
 ::
 
     Usage: datasette publish [OPTIONS] COMMAND [ARGS]...
@@ -207,11 +384,19 @@ datasette publish --help
       heroku    Publish databases to Datasette running on Heroku
 
 
+.. [[[end]]]
+
+
 .. _cli_help_publish_cloudrun___help:
 
-datasette publish cloudrun --help
-=================================
+datasette publish cloudrun
+==========================
 
+See :ref:`publish_cloud_run`.
+
+.. [[[cog
+    help(["publish", "cloudrun", "--help"])
+.. ]]]
 ::
 
     Usage: datasette publish cloudrun [OPTIONS] [FILES]...
@@ -256,11 +441,19 @@ datasette publish cloudrun --help
       --help                          Show this message and exit.
 
 
+.. [[[end]]]
+
+
 .. _cli_help_publish_heroku___help:
 
-datasette publish heroku --help
-===============================
+datasette publish heroku
+========================
 
+See :ref:`publish_heroku`.
+
+.. [[[cog
+    help(["publish", "heroku", "--help"])
+.. ]]]
 ::
 
     Usage: datasette publish heroku [OPTIONS] [FILES]...
@@ -297,16 +490,23 @@ datasette publish heroku --help
       --help                          Show this message and exit.
 
 
+.. [[[end]]]
+
 .. _cli_help_package___help:
 
-datasette package --help
-========================
+datasette package
+=================
 
+Package SQLite files into a Datasette Docker container, see :ref:`cli_package`.
+
+.. [[[cog
+    help(["package", "--help"])
+.. ]]]
 ::
 
     Usage: datasette package [OPTIONS] FILES...
 
-      Package specified SQLite files into a new datasette Docker container
+      Package SQLite files into a Datasette Docker container
 
     Options:
       -t, --tag TEXT            Name for the resulting Docker container, can
@@ -335,11 +535,26 @@ datasette package --help
       --help                    Show this message and exit.
 
 
+.. [[[end]]]
+
+
 .. _cli_help_inspect___help:
 
-datasette inspect --help
-========================
+datasette inspect
+=================
 
+Outputs JSON representing introspected data about one or more SQLite database files.
+
+If you are opening an immutable database, you can pass this file to the ``--inspect-data`` option to improve Datasette's performance by allowing it to skip running row counts against the database when it first starts running::
+
+    datasette inspect mydatabase.db > inspect-data.json
+    datasette serve -i mydatabase.db --inspect-file inspect-data.json
+
+This performance optimization is used automatically by some of the ``datasette publish`` commands. You are unlikely to need to apply this optimization manually.
+
+.. [[[cog
+    help(["inspect", "--help"])
+.. ]]]
 ::
 
     Usage: datasette inspect [OPTIONS] [FILES]...
@@ -355,36 +570,4 @@ datasette inspect --help
       --help                 Show this message and exit.
 
 
-.. _cli_help_install___help:
-
-datasette install --help
-========================
-
-::
-
-    Usage: datasette install [OPTIONS] PACKAGES...
-
-      Install plugins and packages from PyPI into the same environment as Datasette
-
-    Options:
-      -U, --upgrade  Upgrade packages to latest version
-      --help         Show this message and exit.
-
-
-.. _cli_help_uninstall___help:
-
-datasette uninstall --help
-==========================
-
-::
-
-    Usage: datasette uninstall [OPTIONS] PACKAGES...
-
-      Uninstall plugins and Python packages from the Datasette environment
-
-    Options:
-      -y, --yes  Don't ask for confirmation
-      --help     Show this message and exit.
-
-
 .. [[[end]]]
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index a9eaa404..6515ef8d 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -138,53 +138,3 @@ JSON in a more convenient format:
             }
         ]
     }
-
-.. _getting_started_datasette_get:
-
-datasette --get
----------------
-
-The ``--get`` option can specify the path to a page within Datasette and cause Datasette to output the content from that path without starting the web server. This means that all of Datasette's functionality can be accessed directly from the command-line. For example::
-
-    $ datasette --get '/-/versions.json' | jq .
-    {
-      "python": {
-        "version": "3.8.5",
-        "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]"
-      },
-      "datasette": {
-        "version": "0.46+15.g222a84a.dirty"
-      },
-      "asgi": "3.0",
-      "uvicorn": "0.11.8",
-      "sqlite": {
-        "version": "3.32.3",
-        "fts_versions": [
-          "FTS5",
-          "FTS4",
-          "FTS3"
-        ],
-        "extensions": {
-          "json1": null
-        },
-        "compile_options": [
-          "COMPILER=clang-11.0.3",
-          "ENABLE_COLUMN_METADATA",
-          "ENABLE_FTS3",
-          "ENABLE_FTS3_PARENTHESIS",
-          "ENABLE_FTS4",
-          "ENABLE_FTS5",
-          "ENABLE_GEOPOLY",
-          "ENABLE_JSON1",
-          "ENABLE_PREUPDATE_HOOK",
-          "ENABLE_RTREE",
-          "ENABLE_SESSION",
-          "MAX_VARIABLE_NUMBER=250000",
-          "THREADSAFE=1"
-        ]
-      }
-    }
-
-The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This means you can use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions.
-
-Running ``datasette`` without specifying a command runs the default command, ``datasette serve``.  See :ref:`cli_help_serve___help` for the full list of options for that command.
diff --git a/docs/index.rst b/docs/index.rst
index efe196b3..5a9cc7ed 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -40,6 +40,7 @@ Contents
    getting_started
    installation
    ecosystem
+   cli-reference
    pages
    publish
    deploying
@@ -61,6 +62,5 @@ Contents
    plugin_hooks
    testing_plugins
    internals
-   cli-reference
    contributing
    changelog
diff --git a/docs/publish.rst b/docs/publish.rst
index 9c7c99cc..dd8566ed 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -56,6 +56,8 @@ Cloud Run provides a URL on the ``.run.app`` domain, but you can also point your
 
 See :ref:`cli_help_publish_cloudrun___help` for the full list of options for this command.
 
+.. _publish_heroku:
+
 Publishing to Heroku
 --------------------
 

From aff3df03d4fe0806ce432d1818f6643cdb2a854e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 18 Aug 2022 14:55:08 -0700
Subject: [PATCH 1474/2303] Ignore ro which stands for read only

Refs #1787 where it caused tests to break
---
 docs/codespell-ignore-words.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt
index a625cde5..d6744d05 100644
--- a/docs/codespell-ignore-words.txt
+++ b/docs/codespell-ignore-words.txt
@@ -1 +1 @@
-AddWordsToIgnoreHere
+ro

From 0d9d33955b503c88a2c712144d97f094baa5d46d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 18 Aug 2022 16:06:12 -0700
Subject: [PATCH 1475/2303] Clarify you can publish multiple files, closes
 #1788

---
 docs/publish.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/publish.rst b/docs/publish.rst
index dd8566ed..d817ed31 100644
--- a/docs/publish.rst
+++ b/docs/publish.rst
@@ -31,7 +31,7 @@ Publishing to Google Cloud Run
 
 You will first need to install and configure the Google Cloud CLI tools by following `these instructions <https://cloud.google.com/sdk/>`__.
 
-You can then publish a database to Google Cloud Run using the following command::
+You can then publish one or more SQLite database files to Google Cloud Run using the following command::
 
     datasette publish cloudrun mydatabase.db --service=my-database
 
@@ -63,7 +63,7 @@ Publishing to Heroku
 
 To publish your data using `Heroku <https://www.heroku.com/>`__, first create an account there and install and configure the `Heroku CLI tool <https://devcenter.heroku.com/articles/heroku-cli>`_.
 
-You can publish a database to Heroku using the following command::
+You can publish one or more databases to Heroku using the following command::
 
     datasette publish heroku mydatabase.db
 
@@ -138,7 +138,7 @@ If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plug
 datasette package
 =================
 
-If you have docker installed (e.g. using `Docker for Mac <https://www.docker.com/docker-mac>`_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with your selected SQLite databases::
+If you have docker installed (e.g. using `Docker for Mac <https://www.docker.com/docker-mac>`_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with one or more SQLite databases::
 
     datasette package mydatabase.db
 

From 663ac431fe7202c85967568d82b2034f92b9aa43 Mon Sep 17 00:00:00 2001
From: Manuel Kaufmann <humitos@gmail.com>
Date: Sat, 20 Aug 2022 02:04:16 +0200
Subject: [PATCH 1476/2303] Use Read the Docs action v1 (#1778)

Read the Docs repository was renamed from `readthedocs/readthedocs-preview` to `readthedocs/actions/`. Now, the `preview` action is under `readthedocs/actions/preview` and is tagged as `v1`
---
 .github/workflows/documentation-links.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml
index e7062a46..a54bd83a 100644
--- a/.github/workflows/documentation-links.yml
+++ b/.github/workflows/documentation-links.yml
@@ -11,6 +11,6 @@ jobs:
   documentation-links:
     runs-on: ubuntu-latest
     steps:
-      - uses: readthedocs/readthedocs-preview@main
+      - uses: readthedocs/actions/preview@v1
         with:
           project-slug: "datasette"

From 1d64c9a8dac45b9a3452acf8e76dfadea2b0bc49 Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Tue, 23 Aug 2022 11:34:30 -0700
Subject: [PATCH 1477/2303] Add new entrypoint option to --load-extensions.
 (#1789)

Thanks, @asg017
---
 .gitignore                    |  6 ++++
 datasette/app.py              |  8 ++++-
 datasette/cli.py              |  4 ++-
 datasette/utils/__init__.py   | 11 ++++++
 tests/ext.c                   | 48 ++++++++++++++++++++++++++
 tests/test_load_extensions.py | 65 +++++++++++++++++++++++++++++++++++
 6 files changed, 140 insertions(+), 2 deletions(-)
 create mode 100644 tests/ext.c
 create mode 100644 tests/test_load_extensions.py

diff --git a/.gitignore b/.gitignore
index 066009f0..277ff653 100644
--- a/.gitignore
+++ b/.gitignore
@@ -118,3 +118,9 @@ ENV/
 .DS_Store
 node_modules
 .*.swp
+
+# In case someone compiled tests/ext.c for test_load_extensions, don't
+# include it in source control.
+tests/*.dylib
+tests/*.so
+tests/*.dll
\ No newline at end of file
diff --git a/datasette/app.py b/datasette/app.py
index 1a9afc10..bb9232c9 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -559,7 +559,13 @@ class Datasette:
         if self.sqlite_extensions:
             conn.enable_load_extension(True)
             for extension in self.sqlite_extensions:
-                conn.execute("SELECT load_extension(?)", [extension])
+                # "extension" is either a string path to the extension 
+                # or a 2-item tuple that specifies which entrypoint to load. 
+                if isinstance(extension, tuple):
+                    path, entrypoint = extension
+                    conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
+                else:
+                    conn.execute("SELECT load_extension(?)", [extension])
         if self.setting("cache_size_kb"):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
diff --git a/datasette/cli.py b/datasette/cli.py
index f2a03d53..6eb42712 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -21,6 +21,7 @@ from .app import (
     pm,
 )
 from .utils import (
+    LoadExtension,
     StartupError,
     check_connection,
     find_spatialite,
@@ -128,9 +129,10 @@ def sqlite_extensions(fn):
     return click.option(
         "sqlite_extensions",
         "--load-extension",
+        type=LoadExtension(),
         envvar="SQLITE_EXTENSIONS",
         multiple=True,
-        help="Path to a SQLite extension to load",
+        help="Path to a SQLite extension to load, and optional entrypoint",
     )(fn)
 
 
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index d148cc2c..0fc87d51 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -833,6 +833,17 @@ class StaticMount(click.ParamType):
             self.fail(f"{value} is not a valid directory path", param, ctx)
         return path, dirpath
 
+# The --load-extension parameter can optionally include a specific entrypoint.
+# This is done by appending ":entrypoint_name" after supplying the path to the extension
+class LoadExtension(click.ParamType):
+    name = "path:entrypoint?"
+
+    def convert(self, value, param, ctx):
+        if ":" not in value:
+            return value
+        path, entrypoint = value.split(":", 1)
+        return path, entrypoint
+
 
 def format_bytes(bytes):
     current = float(bytes)
diff --git a/tests/ext.c b/tests/ext.c
new file mode 100644
index 00000000..5fe970d9
--- /dev/null
+++ b/tests/ext.c
@@ -0,0 +1,48 @@
+/*
+** This file implements a SQLite extension with multiple entrypoints.
+**
+** The default entrypoint, sqlite3_ext_init, has a single function "a".
+** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b".
+** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c".
+**
+** Compiling instructions: 
+**     https://www.sqlite.org/loadext.html#compiling_a_loadable_extension
+**
+*/
+
+#include "sqlite3ext.h"
+
+SQLITE_EXTENSION_INIT1
+
+// SQL function that returns back the value supplied during sqlite3_create_function()
+static void func(sqlite3_context *context, int argc, sqlite3_value **argv) {
+  sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC);
+}
+
+
+// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0);
+}
+
+// Alternate entrypoint #1
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0);
+}
+
+// Alternate entrypoint #2
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0);
+}
diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py
new file mode 100644
index 00000000..360bc8f3
--- /dev/null
+++ b/tests/test_load_extensions.py
@@ -0,0 +1,65 @@
+from datasette.app import Datasette
+import pytest
+from pathlib import Path
+
+# not necessarily a full path - the full compiled path looks like "ext.dylib"
+# or another suffix, but sqlite will, under the hood, decide which file
+# extension to use based on the operating system (apple=dylib, windows=dll etc)
+# this resolves to "./ext", which is enough for SQLite to calculate the rest
+COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext")
+
+# See if ext.c has been compiled, based off the different possible suffixes.
+def has_compiled_ext():
+    for ext in ["dylib", "so", "dll"]:
+        path = Path(__file__).parent / f"ext.{ext}"
+        if path.is_file():
+            return True
+    return False
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
+async def test_load_extension_default_entrypoint():
+
+    # The default entrypoint only loads a() and NOT b() or c(), so those
+    # should fail.
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+
+    response = await ds.client.get("/_memory.json?sql=select+a()")
+    assert response.status_code == 200
+    assert response.json()["rows"][0][0] == "a"
+
+    response = await ds.client.get("/_memory.json?sql=select+b()")
+    assert response.status_code == 400
+    assert response.json()["error"] == "no such function: b"
+
+    response = await ds.client.get("/_memory.json?sql=select+c()")
+    assert response.status_code == 400
+    assert response.json()["error"] == "no such function: c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
+async def test_load_extension_multiple_entrypoints():
+
+    # Load in the default entrypoint and the other 2 custom entrypoints, now
+    # all a(), b(), and c() should run successfully.
+    ds = Datasette(
+        sqlite_extensions=[
+            COMPILED_EXTENSION_PATH,
+            (COMPILED_EXTENSION_PATH, "sqlite3_ext_b_init"),
+            (COMPILED_EXTENSION_PATH, "sqlite3_ext_c_init"),
+        ]
+    )
+
+    response = await ds.client.get("/_memory.json?sql=select+a()")
+    assert response.status_code == 200
+    assert response.json()["rows"][0][0] == "a"
+
+    response = await ds.client.get("/_memory.json?sql=select+b()")
+    assert response.status_code == 200
+    assert response.json()["rows"][0][0] == "b"
+
+    response = await ds.client.get("/_memory.json?sql=select+c()")
+    assert response.status_code == 200
+    assert response.json()["rows"][0][0] == "c"

From fd1086c6867f3e3582b1eca456e4ea95f6cecf8b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 23 Aug 2022 11:35:41 -0700
Subject: [PATCH 1478/2303] Applied Black, refs #1789

---
 datasette/app.py            | 4 ++--
 datasette/utils/__init__.py | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index bb9232c9..f2a6763a 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -559,8 +559,8 @@ class Datasette:
         if self.sqlite_extensions:
             conn.enable_load_extension(True)
             for extension in self.sqlite_extensions:
-                # "extension" is either a string path to the extension 
-                # or a 2-item tuple that specifies which entrypoint to load. 
+                # "extension" is either a string path to the extension
+                # or a 2-item tuple that specifies which entrypoint to load.
                 if isinstance(extension, tuple):
                     path, entrypoint = extension
                     conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 0fc87d51..bbaa0510 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -833,6 +833,7 @@ class StaticMount(click.ParamType):
             self.fail(f"{value} is not a valid directory path", param, ctx)
         return path, dirpath
 
+
 # The --load-extension parameter can optionally include a specific entrypoint.
 # This is done by appending ":entrypoint_name" after supplying the path to the extension
 class LoadExtension(click.ParamType):

From 456dc155d491a009942ace71a4e1827cddc6b93d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 23 Aug 2022 11:40:36 -0700
Subject: [PATCH 1479/2303] Ran cog, refs #1789

---
 docs/cli-reference.rst | 95 +++++++++++++++++++++++-------------------
 1 file changed, 51 insertions(+), 44 deletions(-)

diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index a1e56774..f8419d58 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -84,48 +84,53 @@ Once started you can access it at ``http://localhost:8001``
       Serve up specified SQLite database files with a web UI
 
     Options:
-      -i, --immutable PATH      Database files to open in immutable mode
-      -h, --host TEXT           Host for server. Defaults to 127.0.0.1 which means
-                                only connections from the local machine will be
-                                allowed. Use 0.0.0.0 to listen to all IPs and allow
-                                access from other machines.
-      -p, --port INTEGER RANGE  Port for server, defaults to 8001. Use -p 0 to
-                                automatically assign an available port.
-                                [0<=x<=65535]
-      --uds TEXT                Bind to a Unix domain socket
-      --reload                  Automatically reload if code or metadata change
-                                detected - useful for development
-      --cors                    Enable CORS by serving Access-Control-Allow-Origin:
-                                *
-      --load-extension TEXT     Path to a SQLite extension to load
-      --inspect-file TEXT       Path to JSON file created using "datasette inspect"
-      -m, --metadata FILENAME   Path to JSON/YAML file containing license/source
-                                metadata
-      --template-dir DIRECTORY  Path to directory containing custom templates
-      --plugins-dir DIRECTORY   Path to directory containing custom plugins
-      --static MOUNT:DIRECTORY  Serve static files from this directory at /MOUNT/...
-      --memory                  Make /_memory database available
-      --config CONFIG           Deprecated: set config option using
-                                configname:value. Use --setting instead.
-      --setting SETTING...      Setting, see
-                                docs.datasette.io/en/stable/settings.html
-      --secret TEXT             Secret used for signing secure values, such as
-                                signed cookies
-      --root                    Output URL that sets a cookie authenticating the
-                                root user
-      --get TEXT                Run an HTTP GET request against this path, print
-                                results and exit
-      --version-note TEXT       Additional note to show on /-/versions
-      --help-settings           Show available settings
-      --pdb                     Launch debugger on any errors
-      -o, --open                Open Datasette in your web browser
-      --create                  Create database files if they do not exist
-      --crossdb                 Enable cross-database joins using the /_memory
-                                database
-      --nolock                  Ignore locking, open locked files in read-only mode
-      --ssl-keyfile TEXT        SSL key file
-      --ssl-certfile TEXT       SSL certificate file
-      --help                    Show this message and exit.
+      -i, --immutable PATH            Database files to open in immutable mode
+      -h, --host TEXT                 Host for server. Defaults to 127.0.0.1 which
+                                      means only connections from the local machine
+                                      will be allowed. Use 0.0.0.0 to listen to all
+                                      IPs and allow access from other machines.
+      -p, --port INTEGER RANGE        Port for server, defaults to 8001. Use -p 0 to
+                                      automatically assign an available port.
+                                      [0<=x<=65535]
+      --uds TEXT                      Bind to a Unix domain socket
+      --reload                        Automatically reload if code or metadata
+                                      change detected - useful for development
+      --cors                          Enable CORS by serving Access-Control-Allow-
+                                      Origin: *
+      --load-extension PATH:ENTRYPOINT?
+                                      Path to a SQLite extension to load, and
+                                      optional entrypoint
+      --inspect-file TEXT             Path to JSON file created using "datasette
+                                      inspect"
+      -m, --metadata FILENAME         Path to JSON/YAML file containing
+                                      license/source metadata
+      --template-dir DIRECTORY        Path to directory containing custom templates
+      --plugins-dir DIRECTORY         Path to directory containing custom plugins
+      --static MOUNT:DIRECTORY        Serve static files from this directory at
+                                      /MOUNT/...
+      --memory                        Make /_memory database available
+      --config CONFIG                 Deprecated: set config option using
+                                      configname:value. Use --setting instead.
+      --setting SETTING...            Setting, see
+                                      docs.datasette.io/en/stable/settings.html
+      --secret TEXT                   Secret used for signing secure values, such as
+                                      signed cookies
+      --root                          Output URL that sets a cookie authenticating
+                                      the root user
+      --get TEXT                      Run an HTTP GET request against this path,
+                                      print results and exit
+      --version-note TEXT             Additional note to show on /-/versions
+      --help-settings                 Show available settings
+      --pdb                           Launch debugger on any errors
+      -o, --open                      Open Datasette in your web browser
+      --create                        Create database files if they do not exist
+      --crossdb                       Enable cross-database joins using the /_memory
+                                      database
+      --nolock                        Ignore locking, open locked files in read-only
+                                      mode
+      --ssl-keyfile TEXT              SSL key file
+      --ssl-certfile TEXT             SSL certificate file
+      --help                          Show this message and exit.
 
 
 .. [[[end]]]
@@ -566,8 +571,10 @@ This performance optimization is used automatically by some of the ``datasette p
 
     Options:
       --inspect-file TEXT
-      --load-extension TEXT  Path to a SQLite extension to load
-      --help                 Show this message and exit.
+      --load-extension PATH:ENTRYPOINT?
+                                      Path to a SQLite extension to load, and
+                                      optional entrypoint
+      --help                          Show this message and exit.
 
 
 .. [[[end]]]

From ba35105eee2d3ba620e4f230028a02b2e2571df2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 23 Aug 2022 17:11:45 -0700
Subject: [PATCH 1480/2303] Test `--load-extension` in GitHub Actions (#1792)

* Run the --load-extension test, refs #1789
* Ran cog, refs #1789
---
 .github/workflows/test.yml | 3 +++
 tests/test_api.py          | 2 +-
 tests/test_html.py         | 4 ++--
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 90b6555e..e38d5ee9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,6 +24,9 @@ jobs:
         key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
         restore-keys: |
           ${{ runner.os }}-pip-
+    - name: Build extension for --load-extension test
+      run: |-
+        (cd tests && gcc ext.c -fPIC -shared -o ext.so)
     - name: Install dependencies
       run: |
         pip install -e '.[test]'
diff --git a/tests/test_api.py b/tests/test_api.py
index 253c1718..f6db2f9d 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -36,7 +36,7 @@ def test_homepage(app_client):
     # 4 hidden FTS tables + no_primary_key (hidden in metadata)
     assert d["hidden_tables_count"] == 6
     # 201 in no_primary_key, plus 6 in other hidden tables:
-    assert d["hidden_table_rows_sum"] == 207
+    assert d["hidden_table_rows_sum"] == 207, response.json
     assert d["views_count"] == 4
 
 
diff --git a/tests/test_html.py b/tests/test_html.py
index be21bd84..d6e969ad 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -115,7 +115,7 @@ def test_database_page(app_client):
         assert fragment in response.text
 
     # And views
-    views_ul = soup.find("h2", text="Views").find_next_sibling("ul")
+    views_ul = soup.find("h2", string="Views").find_next_sibling("ul")
     assert views_ul is not None
     assert [
         ("/fixtures/paginated_view", "paginated_view"),
@@ -128,7 +128,7 @@ def test_database_page(app_client):
     ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")])
 
     # And a list of canned queries
-    queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul")
+    queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul")
     assert queries_ul is not None
     assert [
         ("/fixtures/from_async_hook", "from_async_hook"),

From 51030df1869b3b574dd3584d1563415776b9cd4e Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Sep 2022 11:35:40 -0700
Subject: [PATCH 1481/2303] Don't use upper bound dependencies any more

See https://iscinumpy.dev/post/bound-version-constraints/ for the rationale behind this change.

Closes #1800
---
 setup.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/setup.py b/setup.py
index a1c51d0b..b2e50b38 100644
--- a/setup.py
+++ b/setup.py
@@ -42,21 +42,21 @@ setup(
     include_package_data=True,
     python_requires=">=3.7",
     install_requires=[
-        "asgiref>=3.2.10,<3.6.0",
-        "click>=7.1.1,<8.2.0",
+        "asgiref>=3.2.10",
+        "click>=7.1.1",
         "click-default-group-wheel>=1.2.2",
-        "Jinja2>=2.10.3,<3.1.0",
-        "hupper~=1.9",
+        "Jinja2>=2.10.3",
+        "hupper>=1.9",
         "httpx>=0.20",
-        "pint~=0.9",
-        "pluggy>=1.0,<1.1",
-        "uvicorn~=0.11",
-        "aiofiles>=0.4,<0.9",
-        "janus>=0.6.2,<1.1",
+        "pint>=0.9",
+        "pluggy>=1.0",
+        "uvicorn>=0.11",
+        "aiofiles>=0.4",
+        "janus>=0.6.2",
         "asgi-csrf>=0.9",
-        "PyYAML>=5.3,<7.0",
-        "mergedeep>=1.1.1,<1.4.0",
-        "itsdangerous>=1.1,<3.0",
+        "PyYAML>=5.3",
+        "mergedeep>=1.1.1",
+        "itsdangerous>=1.1",
     ],
     entry_points="""
         [console_scripts]
@@ -72,14 +72,14 @@ setup(
             "sphinx-copybutton",
         ],
         "test": [
-            "pytest>=5.2.2,<7.2.0",
-            "pytest-xdist>=2.2.1,<2.6",
-            "pytest-asyncio>=0.17,<0.20",
-            "beautifulsoup4>=4.8.1,<4.12.0",
+            "pytest>=5.2.2",
+            "pytest-xdist>=2.2.1",
+            "pytest-asyncio>=0.17",
+            "beautifulsoup4>=4.8.1",
             "black==22.6.0",
             "blacken-docs==1.12.1",
-            "pytest-timeout>=1.4.2,<2.2",
-            "trustme>=0.7,<0.10",
+            "pytest-timeout>=1.4.2",
+            "trustme>=0.7",
             "cogapp>=3.3.0",
         ],
         "rich": ["rich"],

From 294ecd45f7801971dbeef383d0c5456ee95ab839 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Sep 2022 11:51:51 -0700
Subject: [PATCH 1482/2303] Bump black from 22.6.0 to 22.8.0 (#1797)

Bumps [black](https://github.com/psf/black) from 22.6.0 to 22.8.0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/22.6.0...22.8.0)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b2e50b38..92fa60d0 100644
--- a/setup.py
+++ b/setup.py
@@ -76,7 +76,7 @@ setup(
             "pytest-xdist>=2.2.1",
             "pytest-asyncio>=0.17",
             "beautifulsoup4>=4.8.1",
-            "black==22.6.0",
+            "black==22.8.0",
             "blacken-docs==1.12.1",
             "pytest-timeout>=1.4.2",
             "trustme>=0.7",

From b91e17280c05bbb9cf97432081bdcea8665879f9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Sep 2022 16:50:53 -0700
Subject: [PATCH 1483/2303] Run tests in serial, refs #1802

---
 .github/workflows/test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e38d5ee9..9c8c48ef 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,8 +33,7 @@ jobs:
         pip freeze
     - name: Run tests
       run: |
-        pytest -n auto -m "not serial"
-        pytest -m "serial"
+        pytest
     - name: Check if cog needs to be run
       run: |
         cog --check docs/*.rst

From b2b901e8c4b939e50ee1117ffcd2881ed8a8e3bf Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Sep 2022 17:05:23 -0700
Subject: [PATCH 1484/2303] Skip SpatiaLite test if no
 conn.enable_load_extension()

Ran into this problem while working on #1802
---
 tests/test_spatialite.py | 2 ++
 tests/utils.py           | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/tests/test_spatialite.py b/tests/test_spatialite.py
index 8b98c5d6..c07a30e8 100644
--- a/tests/test_spatialite.py
+++ b/tests/test_spatialite.py
@@ -1,5 +1,6 @@
 from datasette.app import Datasette
 from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS
+from .utils import has_load_extension
 import pytest
 
 
@@ -13,6 +14,7 @@ def has_spatialite():
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_spatialite(), reason="Requires SpatiaLite")
+@pytest.mark.skipif(not has_load_extension(), reason="Requires enable_load_extension")
 async def test_spatialite_version_info():
     ds = Datasette(sqlite_extensions=["spatialite"])
     response = await ds.client.get("/-/versions.json")
diff --git a/tests/utils.py b/tests/utils.py
index 972300db..191ead9b 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -1,3 +1,6 @@
+from datasette.utils.sqlite import sqlite3
+
+
 def assert_footer_links(soup):
     footer_links = soup.find("footer").findAll("a")
     assert 4 == len(footer_links)
@@ -22,3 +25,8 @@ def inner_html(soup):
     # This includes the parent tag - so remove that
     inner_html = html.split(">", 1)[1].rsplit("<", 1)[0]
     return inner_html.strip()
+
+
+def has_load_extension():
+    conn = sqlite3.connect(":memory:")
+    return hasattr(conn, "enable_load_extension")

From 1c29b925d300d1ee17047504473f2517767aa05b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Sep 2022 17:10:52 -0700
Subject: [PATCH 1485/2303] Run tests in serial again

Because this didn't fix the issue I'm seeing in #1802

Revert "Run tests in serial, refs #1802"

This reverts commit b91e17280c05bbb9cf97432081bdcea8665879f9.
---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9c8c48ef..e38d5ee9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,7 +33,8 @@ jobs:
         pip freeze
     - name: Run tests
       run: |
-        pytest
+        pytest -n auto -m "not serial"
+        pytest -m "serial"
     - name: Check if cog needs to be run
       run: |
         cog --check docs/*.rst

From 64288d827f7ff97f825e10f714da3f781ecf9345 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Sep 2022 17:40:19 -0700
Subject: [PATCH 1486/2303] Workaround for test failure: RuntimeError: There is
 no current event loop (#1803)

* Remove ensure_eventloop hack
* Hack to recover from intermittent RuntimeError calling asyncio.Lock()
---
 datasette/app.py  | 10 +++++++++-
 tests/test_cli.py | 27 ++++++++++-----------------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index f2a6763a..c6bbdaf0 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -231,7 +231,15 @@ class Datasette:
         self.inspect_data = inspect_data
         self.immutables = set(immutables or [])
         self.databases = collections.OrderedDict()
-        self._refresh_schemas_lock = asyncio.Lock()
+        try:
+            self._refresh_schemas_lock = asyncio.Lock()
+        except RuntimeError as rex:
+            # Workaround for intermittent test failure, see:
+            # https://github.com/simonw/datasette/issues/1802
+            if "There is no current event loop in thread" in str(rex):
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                self._refresh_schemas_lock = asyncio.Lock()
         self.crossdb = crossdb
         self.nolock = nolock
         if memory or crossdb or not self.files:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d0f6e26c..f0d28037 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -22,13 +22,6 @@ from unittest import mock
 import urllib
 
 
-@pytest.fixture
-def ensure_eventloop():
-    # Workaround for "Event loop is closed" error
-    if asyncio.get_event_loop().is_closed():
-        asyncio.set_event_loop(asyncio.new_event_loop())
-
-
 def test_inspect_cli(app_client):
     runner = CliRunner()
     result = runner.invoke(cli, ["inspect", "fixtures.db"])
@@ -72,7 +65,7 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache():
     ),
 )
 def test_spatialite_error_if_attempt_to_open_spatialite(
-    ensure_eventloop, spatialite_paths, should_suggest_load_extension
+    spatialite_paths, should_suggest_load_extension
 ):
     with mock.patch("datasette.utils.SPATIALITE_PATHS", spatialite_paths):
         runner = CliRunner()
@@ -199,14 +192,14 @@ def test_version():
 
 
 @pytest.mark.parametrize("invalid_port", ["-1", "0.5", "dog", "65536"])
-def test_serve_invalid_ports(ensure_eventloop, invalid_port):
+def test_serve_invalid_ports(invalid_port):
     runner = CliRunner(mix_stderr=False)
     result = runner.invoke(cli, ["--port", invalid_port])
     assert result.exit_code == 2
     assert "Invalid value for '-p'" in result.stderr
 
 
-def test_setting(ensure_eventloop):
+def test_setting():
     runner = CliRunner()
     result = runner.invoke(
         cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"]
@@ -215,14 +208,14 @@ def test_setting(ensure_eventloop):
     assert json.loads(result.output)["default_page_size"] == 5
 
 
-def test_setting_type_validation(ensure_eventloop):
+def test_setting_type_validation():
     runner = CliRunner(mix_stderr=False)
     result = runner.invoke(cli, ["--setting", "default_page_size", "dog"])
     assert result.exit_code == 2
     assert '"default_page_size" should be an integer' in result.stderr
 
 
-def test_config_deprecated(ensure_eventloop):
+def test_config_deprecated():
     # The --config option should show a deprecation message
     runner = CliRunner(mix_stderr=False)
     result = runner.invoke(
@@ -233,14 +226,14 @@ def test_config_deprecated(ensure_eventloop):
     assert "will be deprecated in" in result.stderr
 
 
-def test_sql_errors_logged_to_stderr(ensure_eventloop):
+def test_sql_errors_logged_to_stderr():
     runner = CliRunner(mix_stderr=False)
     result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"])
     assert result.exit_code == 1
     assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr
 
 
-def test_serve_create(ensure_eventloop, tmpdir):
+def test_serve_create(tmpdir):
     runner = CliRunner()
     db_path = tmpdir / "does_not_exist_yet.db"
     assert not db_path.exists()
@@ -258,7 +251,7 @@ def test_serve_create(ensure_eventloop, tmpdir):
     assert db_path.exists()
 
 
-def test_serve_duplicate_database_names(ensure_eventloop, tmpdir):
+def test_serve_duplicate_database_names(tmpdir):
     "'datasette db.db nested/db.db' should attach two databases, /db and /db_2"
     runner = CliRunner()
     db_1_path = str(tmpdir / "db.db")
@@ -273,7 +266,7 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir):
     assert {db["name"] for db in databases} == {"db", "db_2"}
 
 
-def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir):
+def test_serve_deduplicate_same_database_path(tmpdir):
     "'datasette db.db db.db' should only attach one database, /db"
     runner = CliRunner()
     db_path = str(tmpdir / "db.db")
@@ -287,7 +280,7 @@ def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir):
 @pytest.mark.parametrize(
     "filename", ["test-database (1).sqlite", "database (1).sqlite"]
 )
-def test_weird_database_names(ensure_eventloop, tmpdir, filename):
+def test_weird_database_names(tmpdir, filename):
     # https://github.com/simonw/datasette/issues/1181
     runner = CliRunner()
     db_path = str(tmpdir / filename)

From c9d1943aede436fa3413fd49bc56335cbda4ad07 Mon Sep 17 00:00:00 2001
From: Daniel Rech <dr@netsyno.com>
Date: Tue, 6 Sep 2022 02:45:41 +0200
Subject: [PATCH 1487/2303] Fix word break in facets by adding ul.tight-bullets
 li word-break: break-all (#1794)

Thanks, @dmr
---
 datasette/static/app.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/datasette/static/app.css b/datasette/static/app.css
index af3e14d5..712b9925 100644
--- a/datasette/static/app.css
+++ b/datasette/static/app.css
@@ -260,6 +260,7 @@ ul.bullets li {
 ul.tight-bullets li {
     list-style-type: disc;
     margin-bottom: 0;
+    word-break: break-all;
 }
 a.not-underlined {
     text-decoration: none;

From d80775a48d20917633792fdc9525f075d3bc2c7a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Sep 2022 17:44:44 -0700
Subject: [PATCH 1488/2303] Raise error if it's not about loops, refs #1802

---
 datasette/app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/datasette/app.py b/datasette/app.py
index c6bbdaf0..aeb81687 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -240,6 +240,8 @@ class Datasette:
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
                 self._refresh_schemas_lock = asyncio.Lock()
+            else:
+                raise
         self.crossdb = crossdb
         self.nolock = nolock
         if memory or crossdb or not self.files:

From 8430c3bc7dd22b173c1a8c6cd7180e3b31240cd1 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 08:59:19 -0700
Subject: [PATCH 1489/2303] table facet_size in metadata, refs #1804

---
 datasette/facets.py  | 14 +++++++++++---
 tests/test_facets.py | 17 +++++++++++++++++
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index b15a758c..e70d42df 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -102,11 +102,19 @@ class Facet:
     def get_facet_size(self):
         facet_size = self.ds.setting("default_facet_size")
         max_returned_rows = self.ds.setting("max_returned_rows")
+        table_facet_size = None
+        if self.table:
+            tables_metadata = self.ds.metadata("tables", database=self.database) or {}
+            table_metadata = tables_metadata.get(self.table) or {}
+            if table_metadata:
+                table_facet_size = table_metadata.get("facet_size")
         custom_facet_size = self.request.args.get("_facet_size")
-        if custom_facet_size == "max":
-            facet_size = max_returned_rows
-        elif custom_facet_size and custom_facet_size.isdigit():
+        if custom_facet_size and custom_facet_size.isdigit():
             facet_size = int(custom_facet_size)
+        elif table_facet_size:
+            facet_size = table_facet_size
+        if facet_size == "max":
+            facet_size = max_returned_rows
         return min(facet_size, max_returned_rows)
 
     async def suggest(self):
diff --git a/tests/test_facets.py b/tests/test_facets.py
index c28dc43c..cbee23b0 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -581,6 +581,23 @@ async def test_facet_size():
     )
     data5 = response5.json()
     assert len(data5["facet_results"]["city"]["results"]) == 20
+    # Now try messing with facet_size in the table metadata
+    ds._metadata_local = {
+        "databases": {
+            "test_facet_size": {"tables": {"neighbourhoods": {"facet_size": 6}}}
+        }
+    }
+    response6 = await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city")
+    data6 = response6.json()
+    assert len(data6["facet_results"]["city"]["results"]) == 6
+    # Setting it to max bumps it up to 50 again
+    ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][
+        "facet_size"
+    ] = "max"
+    data7 = (
+        await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city")
+    ).json()
+    assert len(data7["facet_results"]["city"]["results"]) == 20
 
 
 def test_other_types_of_facet_in_metadata():

From 303c6c733d95a6133558ec1b468f5bea5827d0d2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 11:05:00 -0700
Subject: [PATCH 1490/2303] Fix for incorrectly handled _facet_size=max, refs
 #1804

---
 datasette/facets.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index e70d42df..7fb0c68b 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -109,12 +109,19 @@ class Facet:
             if table_metadata:
                 table_facet_size = table_metadata.get("facet_size")
         custom_facet_size = self.request.args.get("_facet_size")
-        if custom_facet_size and custom_facet_size.isdigit():
-            facet_size = int(custom_facet_size)
-        elif table_facet_size:
-            facet_size = table_facet_size
-        if facet_size == "max":
-            facet_size = max_returned_rows
+        if custom_facet_size:
+            if custom_facet_size == "max":
+                facet_size = max_returned_rows
+            elif custom_facet_size.isdigit():
+                facet_size = int(custom_facet_size)
+            else:
+                # Invalid value, ignore it
+                custom_facet_size = None
+        if table_facet_size and not custom_facet_size:
+            if table_facet_size == "max":
+                facet_size = max_returned_rows
+            else:
+                facet_size = table_facet_size
         return min(facet_size, max_returned_rows)
 
     async def suggest(self):

From 0a7815d2038255a0834c955066a2a16c01f707b2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 11:06:49 -0700
Subject: [PATCH 1491/2303] Documentation for facet_size in metadata, closes
 #1804

---
 docs/facets.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/facets.rst b/docs/facets.rst
index 2a2eb039..6c9d99bd 100644
--- a/docs/facets.rst
+++ b/docs/facets.rst
@@ -129,6 +129,22 @@ You can specify :ref:`array <facet_by_json_array>` or :ref:`date <facet_by_date>
     ]
   }
 
+You can change the default facet size (the number of results shown for each facet) for a table using ``facet_size``:
+
+.. code-block:: json
+
+    {
+      "databases": {
+        "sf-trees": {
+          "tables": {
+            "Street_Tree_List": {
+              "facets": ["qLegalStatus"],
+              "facet_size": 10
+            }
+          }
+        }
+      }
+    }
 
 Suggested facets
 ----------------

From d0476897e10249bb4867473722270d02491c2c1f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 11:24:30 -0700
Subject: [PATCH 1492/2303] Fixed Sphinx warning about language = None

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 4ef6b768..8965974a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -71,7 +71,7 @@ release = ""
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

From ff9c87197dde8b09f9787ee878804cb6842ea5dc Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 11:26:21 -0700
Subject: [PATCH 1493/2303] Fixed Sphinx warnings on cli-reference page

---
 docs/cli-reference.rst | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index f8419d58..4a8465cb 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -14,7 +14,7 @@ Running ``datasette`` without specifying a command runs the default command, ``d
     import textwrap
     def help(args):
         title = "datasette " + " ".join(args)
-        cog.out("::\n\n")
+        cog.out("\n::\n\n")
         result = CliRunner().invoke(cli.cli, args)
         output = result.output.replace("Usage: cli ", "Usage: datasette ")
         cog.out(textwrap.indent(output, '    '))
@@ -32,6 +32,7 @@ Running ``datasette --help`` shows a list of all of the available commands.
 .. [[[cog
     help(["--help"])
 .. ]]]
+
 ::
 
     Usage: datasette [OPTIONS] COMMAND [ARGS]...
@@ -77,6 +78,7 @@ Once started you can access it at ``http://localhost:8001``
 .. [[[cog
     help(["serve", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette serve [OPTIONS] [FILES]...
@@ -202,6 +204,7 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
 .. [[[cog
     help(["--help-settings"])
 .. ]]]
+
 ::
 
     Settings:
@@ -258,6 +261,7 @@ Output JSON showing all currently installed plugins, their versions, whether the
 .. [[[cog
     help(["plugins", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette plugins [OPTIONS]
@@ -326,6 +330,7 @@ Would install the `datasette-cluster-map <https://datasette.io/plugins/datasette
 .. [[[cog
     help(["install", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette install [OPTIONS] PACKAGES...
@@ -349,6 +354,7 @@ Uninstall one or more plugins.
 .. [[[cog
     help(["uninstall", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette uninstall [OPTIONS] PACKAGES...
@@ -374,6 +380,7 @@ Additional deployment targets can be added by plugins that use the :ref:`plugin_
 .. [[[cog
     help(["publish", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette publish [OPTIONS] COMMAND [ARGS]...
@@ -402,6 +409,7 @@ See :ref:`publish_cloud_run`.
 .. [[[cog
     help(["publish", "cloudrun", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette publish cloudrun [OPTIONS] [FILES]...
@@ -459,6 +467,7 @@ See :ref:`publish_heroku`.
 .. [[[cog
     help(["publish", "heroku", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette publish heroku [OPTIONS] [FILES]...
@@ -507,6 +516,7 @@ Package SQLite files into a Datasette Docker container, see :ref:`cli_package`.
 .. [[[cog
     help(["package", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette package [OPTIONS] FILES...
@@ -560,6 +570,7 @@ This performance optimization is used automatically by some of the ``datasette p
 .. [[[cog
     help(["inspect", "--help"])
 .. ]]]
+
 ::
 
     Usage: datasette inspect [OPTIONS] [FILES]...

From d0737e4de51ce178e556fc011ccb8cc46bbb6359 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 16:50:43 -0700
Subject: [PATCH 1494/2303] truncate_cells_html now affects URLs too, refs
 #1805

---
 datasette/utils/__init__.py | 10 ++++++++++
 datasette/views/database.py | 11 ++++++++---
 datasette/views/table.py    |  8 ++++++--
 tests/fixtures.py           |  9 +++++----
 tests/test_api.py           |  2 +-
 tests/test_table_api.py     | 11 +++++++----
 tests/test_table_html.py    | 11 +++++++++++
 tests/test_utils.py         | 20 ++++++++++++++++++++
 8 files changed, 68 insertions(+), 14 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index bbaa0510..2bdea673 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1167,3 +1167,13 @@ def resolve_routes(routes, path):
         if match is not None:
             return match, view
     return None, None
+
+
+def truncate_url(url, length):
+    if (not length) or (len(url) <= length):
+        return url
+    bits = url.rsplit(".", 1)
+    if len(bits) == 2 and 1 <= len(bits[1]) <= 4 and "/" not in bits[1]:
+        rest, ext = bits
+        return rest[: length - 1 - len(ext)] + "…." + ext
+    return url[: length - 1] + "…"
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 77632b9d..fc344245 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -20,6 +20,7 @@ from datasette.utils import (
     path_with_format,
     path_with_removed_args,
     sqlite3,
+    truncate_url,
     InvalidSql,
 )
 from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden
@@ -371,6 +372,7 @@ class QueryView(DataView):
 
         async def extra_template():
             display_rows = []
+            truncate_cells = self.ds.setting("truncate_cells_html")
             for row in results.rows if results else []:
                 display_row = []
                 for column, value in zip(results.columns, row):
@@ -396,9 +398,12 @@ class QueryView(DataView):
                         if value in ("", None):
                             display_value = Markup(" ")
                         elif is_url(str(display_value).strip()):
-                            display_value = Markup(
-                                '<a href="{url}">{url}</a>'.format(
-                                    url=escape(value.strip())
+                            display_value = markupsafe.Markup(
+                                '<a href="{url}">{truncated_url}</a>'.format(
+                                    url=markupsafe.escape(value.strip()),
+                                    truncated_url=markupsafe.escape(
+                                        truncate_url(value.strip(), truncate_cells)
+                                    ),
                                 )
                             )
                         elif isinstance(display_value, bytes):
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 49c30c9c..60c092f9 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -24,6 +24,7 @@ from datasette.utils import (
     path_with_removed_args,
     path_with_replaced_args,
     to_css_class,
+    truncate_url,
     urlsafe_components,
     value_as_boolean,
 )
@@ -966,8 +967,11 @@ async def display_columns_and_rows(
                 display_value = markupsafe.Markup(" ")
             elif is_url(str(value).strip()):
                 display_value = markupsafe.Markup(
-                    '<a href="{url}">{url}</a>'.format(
-                        url=markupsafe.escape(value.strip())
+                    '<a href="{url}">{truncated_url}</a>'.format(
+                        url=markupsafe.escape(value.strip()),
+                        truncated_url=markupsafe.escape(
+                            truncate_url(value.strip(), truncate_cells)
+                        ),
                     )
                 )
             elif column in table_metadata.get("units", {}) and value != "":
diff --git a/tests/fixtures.py b/tests/fixtures.py
index c145ac78..82d8452e 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -598,23 +598,24 @@ CREATE TABLE roadside_attractions (
     pk integer primary key,
     name text,
     address text,
+    url text,
     latitude real,
     longitude real
 );
 INSERT INTO roadside_attractions VALUES (
-    1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065",
+    1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/",
     37.0167, -122.0024
 );
 INSERT INTO roadside_attractions VALUES (
-    2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128",
+    2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/",
     37.3184, -121.9511
 );
 INSERT INTO roadside_attractions VALUES (
-    3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010",
+    3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null,
     37.5793, -122.3442
 );
 INSERT INTO roadside_attractions VALUES (
-    4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018",
+    4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/",
     37.0414, -122.0725
 );
 
diff --git a/tests/test_api.py b/tests/test_api.py
index f6db2f9d..7a2bf91f 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -339,7 +339,7 @@ def test_database_page(app_client):
         },
         {
             "name": "roadside_attractions",
-            "columns": ["pk", "name", "address", "latitude", "longitude"],
+            "columns": ["pk", "name", "address", "url", "latitude", "longitude"],
             "primary_keys": ["pk"],
             "count": 4,
             "hidden": False,
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index e56a72b5..0db04434 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -615,11 +615,12 @@ def test_table_through(app_client):
     response = app_client.get(
         '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
     )
-    assert [
+    assert response.json["rows"] == [
         [
             3,
             "Burlingame Museum of PEZ Memorabilia",
             "214 California Drive, Burlingame, CA 94010",
+            None,
             37.5793,
             -122.3442,
         ],
@@ -627,13 +628,15 @@ def test_table_through(app_client):
             4,
             "Bigfoot Discovery Museum",
             "5497 Highway 9, Felton, CA 95018",
+            "https://www.bigfootdiscoveryproject.com/",
             37.0414,
             -122.0725,
         ],
-    ] == response.json["rows"]
+    ]
+
     assert (
-        'where roadside_attraction_characteristics.characteristic_id = "1"'
-        == response.json["human_description_en"]
+        response.json["human_description_en"]
+        == 'where roadside_attraction_characteristics.characteristic_id = "1"'
     )
 
 
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index f3808ea3..8e37468f 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -69,6 +69,17 @@ def test_table_cell_truncation():
             td.string
             for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
         ]
+        # URLs should be truncated too
+        response2 = client.get("/fixtures/roadside_attractions")
+        assert response2.status == 200
+        table = Soup(response2.body, "html.parser").find("table")
+        tds = table.findAll("td", {"class": "col-url"})
+        assert [str(td) for td in tds] == [
+            '<td class="col-url type-str"><a href="https://www.mysteryspot.com/">http…</a></td>',
+            '<td class="col-url type-str"><a href="https://winchestermysteryhouse.com/">http…</a></td>',
+            '<td class="col-url type-none">\xa0</td>',
+            '<td class="col-url type-str"><a href="https://www.bigfootdiscoveryproject.com/">http…</a></td>',
+        ]
 
 
 def test_add_filter_redirects(app_client):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index df788767..d71a612d 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -626,3 +626,23 @@ def test_tilde_encoding(original, expected):
     assert actual == expected
     # And test round-trip
     assert original == utils.tilde_decode(actual)
+
+
+@pytest.mark.parametrize(
+    "url,length,expected",
+    (
+        ("https://example.com/", 5, "http…"),
+        ("https://example.com/foo/bar", 15, "https://exampl…"),
+        ("https://example.com/foo/bar/baz.jpg", 30, "https://example.com/foo/ba….jpg"),
+        # Extensions longer than 4 characters are not treated specially:
+        ("https://example.com/foo/bar/baz.jpeg2", 30, "https://example.com/foo/bar/b…"),
+        (
+            "https://example.com/foo/bar/baz.jpeg2",
+            None,
+            "https://example.com/foo/bar/baz.jpeg2",
+        ),
+    ),
+)
+def test_truncate_url(url, length, expected):
+    actual = utils.truncate_url(url, length)
+    assert actual == expected

From 5aa359b86907d11b3ee601510775a85a90224da8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 16:58:30 -0700
Subject: [PATCH 1495/2303] Apply cell truncation on query page too, refs #1805

---
 datasette/views/database.py |  7 ++++++-
 tests/test_html.py          | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index fc344245..affbc540 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -428,7 +428,12 @@ class QueryView(DataView):
                                     "" if len(value) == 1 else "s",
                                 )
                             )
-
+                        else:
+                            display_value = str(value)
+                            if truncate_cells and len(display_value) > truncate_cells:
+                                display_value = (
+                                    display_value[:truncate_cells] + "\u2026"
+                                )
                     display_row.append(display_value)
                 display_rows.append(display_row)
 
diff --git a/tests/test_html.py b/tests/test_html.py
index d6e969ad..bf915247 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -186,6 +186,25 @@ def test_row_page_does_not_truncate():
         ]
 
 
+def test_query_page_truncates():
+    with make_app_client(settings={"truncate_cells_html": 5}) as client:
+        response = client.get(
+            "/fixtures?"
+            + urllib.parse.urlencode(
+                {
+                    "sql": "select 'this is longer than 5' as a, 'https://example.com/' as b"
+                }
+            )
+        )
+        assert response.status == 200
+        table = Soup(response.body, "html.parser").find("table")
+        tds = table.findAll("td")
+        assert [str(td) for td in tds] == [
+            '<td class="col-a">this …</td>',
+            '<td class="col-b"><a href="https://example.com/">http…</a></td>',
+        ]
+
+
 @pytest.mark.parametrize(
     "path,expected_classes",
     [

From bf8d84af5422606597be893cedd375020cb2b369 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 6 Sep 2022 20:34:59 -0700
Subject: [PATCH 1496/2303] word-wrap: anywhere on links in cells, refs #1805

---
 datasette/static/app.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/datasette/static/app.css b/datasette/static/app.css
index 712b9925..08b724f6 100644
--- a/datasette/static/app.css
+++ b/datasette/static/app.css
@@ -446,6 +446,7 @@ th {
 }
 table a:link {
     text-decoration: none;
+    word-wrap: anywhere;
 }
 .rows-and-columns td:before {
     display: block;

From fb7e70d5e72a951efe4b29ad999d8915c032d021 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 9 Sep 2022 09:19:20 -0700
Subject: [PATCH 1497/2303] Database(is_mutable=) now defaults to True, closes
 #1808

Refs https://github.com/simonw/datasette-upload-dbs/issues/6
---
 datasette/database.py             | 3 +--
 docs/internals.rst                | 9 +++++----
 tests/test_internals_database.py  | 1 +
 tests/test_internals_datasette.py | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index fa558045..44467370 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -28,7 +28,7 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file"))
 
 class Database:
     def __init__(
-        self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None
+        self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None
     ):
         self.name = None
         self.route = None
@@ -39,7 +39,6 @@ class Database:
         self.memory_name = memory_name
         if memory_name is not None:
             self.is_memory = True
-            self.is_mutable = True
         self.hash = None
         self.cached_size = None
         self._cached_table_counts = None
diff --git a/docs/internals.rst b/docs/internals.rst
index 20797e98..adeec1d8 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -426,12 +426,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database`
         Database(
             datasette,
             path="path/to/my-new-database.db",
-            is_mutable=True,
         )
     )
 
 This will add a mutable database and serve it at ``/my-new-database``.
 
+Use ``is_mutable=False`` to add an immutable database.
+
 ``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example:
 
 .. code-block:: python
@@ -671,8 +672,8 @@ Instances of the ``Database`` class can be used to execute queries against attac
 
 .. _database_constructor:
 
-Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None)
-----------------------------------------------------------------------------
+Database(ds, path=None, is_mutable=True, is_memory=False, memory_name=None)
+---------------------------------------------------------------------------
 
 The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases.
 
@@ -685,7 +686,7 @@ The arguments are as follows:
     Path to a SQLite database file on disk.
 
 ``is_mutable`` - boolean
-    Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior.
+    Set this to ``False`` to cause Datasette to open the file in immutable mode.
 
 ``is_memory`` - boolean
     Use this to create non-shared memory connections.
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 551f67e1..9e81c1d6 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -499,6 +499,7 @@ def test_mtime_ns_is_none_for_memory(app_client):
 
 
 def test_is_mutable(app_client):
+    assert Database(app_client.ds, is_memory=True).is_mutable is True
     assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True
     assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False
 
diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py
index 1dc14cab..249920fe 100644
--- a/tests/test_internals_datasette.py
+++ b/tests/test_internals_datasette.py
@@ -58,7 +58,7 @@ async def test_datasette_constructor():
             "route": "_memory",
             "path": None,
             "size": 0,
-            "is_mutable": False,
+            "is_mutable": True,
             "is_memory": True,
             "hash": None,
         }

From 610425460b519e9c16d386cb81aa081c9d730ef0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 10 Sep 2022 14:24:26 -0700
Subject: [PATCH 1498/2303] Add --nolock to the README Chrome demo

Refs #1744
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1af20129..af95b85e 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce
 
 Use Chrome on OS X? You can run datasette against your browser history like so:
 
-     datasette ~/Library/Application\ Support/Google/Chrome/Default/History
+     datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock
 
 Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data:
 

From b40872f5e5ae5dad331c58f75451e2d206565196 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 14 Sep 2022 14:31:54 -0700
Subject: [PATCH 1499/2303] prepare_jinja2_environment(datasette) argument,
 refs #1809

---
 datasette/app.py           | 2 +-
 datasette/hookspecs.py     | 2 +-
 docs/plugin_hooks.rst      | 9 +++++++--
 tests/plugins/my_plugin.py | 3 ++-
 tests/test_plugins.py      | 5 +++--
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index aeb81687..db686670 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -345,7 +345,7 @@ class Datasette:
         self.jinja_env.filters["escape_sqlite"] = escape_sqlite
         self.jinja_env.filters["to_css_class"] = to_css_class
         # pylint: disable=no-member
-        pm.hook.prepare_jinja2_environment(env=self.jinja_env)
+        pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self)
 
         self._register_renderers()
         self._permission_checks = collections.deque(maxlen=200)
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index a5fb536f..34e19664 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -26,7 +26,7 @@ def prepare_connection(conn, database, datasette):
 
 
 @hookspec
-def prepare_jinja2_environment(env):
+def prepare_jinja2_environment(env, datasette):
     """Modify Jinja2 template environment e.g. register custom template tags"""
 
 
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 30bd75b7..62ec5c90 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -61,12 +61,15 @@ Examples: `datasette-jellyfish <https://datasette.io/plugins/datasette-jellyfish
 
 .. _plugin_hook_prepare_jinja2_environment:
 
-prepare_jinja2_environment(env)
--------------------------------
+prepare_jinja2_environment(env, datasette)
+------------------------------------------
 
 ``env`` - jinja2 Environment
     The template environment that is being prepared
 
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
+
 This hook is called with the Jinja2 environment that is used to evaluate
 Datasette HTML templates. You can use it to do things like `register custom
 template filters <http://jinja.pocoo.org/docs/2.10/api/#custom-filters>`_, for
@@ -85,6 +88,8 @@ You can now use this filter in your custom templates like so::
 
     Table name: {{ table|uppercase }}
 
+Examples: `datasette-edit-templates <https://datasette.io/plugins/datasette-edit-templates>`_
+
 .. _plugin_hook_extra_template_vars:
 
 extra_template_vars(template, database, table, columns, view_name, request, datasette)
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 53613b7d..d49a7a34 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -142,8 +142,9 @@ def extra_template_vars(
 
 
 @hookimpl
-def prepare_jinja2_environment(env):
+def prepare_jinja2_environment(env, datasette):
     env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}"
+    env.filters["to_hello"] = lambda s: datasette._HELLO
 
 
 @hookimpl
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 948a40b8..590d88f6 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -545,11 +545,12 @@ def test_hook_register_output_renderer_can_render(app_client):
 
 @pytest.mark.asyncio
 async def test_hook_prepare_jinja2_environment(app_client):
+    app_client.ds._HELLO = "HI"
     template = app_client.ds.jinja_env.from_string(
-        "Hello there, {{ a|format_numeric }}", {"a": 3412341}
+        "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341}
     )
     rendered = await app_client.ds.render_template(template)
-    assert "Hello there, 3,412,341" == rendered
+    assert "Hello there, 3,412,341, HI" == rendered
 
 
 def test_hook_publish_subcommand():

From 2ebcffe2226ece2a5a86722790d486a480338632 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 16 Sep 2022 12:50:52 -0700
Subject: [PATCH 1500/2303] Bump furo from 2022.6.21 to 2022.9.15 (#1812)

Bumps [furo](https://github.com/pradyunsg/furo) from 2022.6.21 to 2022.9.15.
- [Release notes](https://github.com/pradyunsg/furo/releases)
- [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md)
- [Commits](https://github.com/pradyunsg/furo/compare/2022.06.21...2022.09.15)

---
updated-dependencies:
- dependency-name: furo
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 92fa60d0..afcba1f0 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ setup(
     setup_requires=["pytest-runner"],
     extras_require={
         "docs": [
-            "furo==2022.6.21",
+            "furo==2022.9.15",
             "sphinx-autobuild",
             "codespell",
             "blacken-docs",

From ddc999ad1296e8c69cffede3e367dda059b8adad Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 16 Sep 2022 20:38:15 -0700
Subject: [PATCH 1501/2303] Async support for prepare_jinja2_environment,
 closes #1809

---
 datasette/app.py                         | 22 ++++++++++++++---
 datasette/utils/testing.py               |  1 +
 docs/plugin_hooks.rst                    |  2 ++
 docs/testing_plugins.rst                 | 30 ++++++++++++++++++++++++
 tests/fixtures.py                        |  1 +
 tests/plugins/my_plugin.py               | 10 ++++++--
 tests/plugins/my_plugin_2.py             |  6 +++++
 tests/test_internals_datasette_client.py |  6 +++--
 tests/test_plugins.py                    |  6 +++--
 tests/test_routes.py                     |  1 +
 10 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index db686670..ea3e7b43 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -208,6 +208,7 @@ class Datasette:
         crossdb=False,
         nolock=False,
     ):
+        self._startup_invoked = False
         assert config_dir is None or isinstance(
             config_dir, Path
         ), "config_dir= should be a pathlib.Path"
@@ -344,9 +345,6 @@ class Datasette:
         self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus
         self.jinja_env.filters["escape_sqlite"] = escape_sqlite
         self.jinja_env.filters["to_css_class"] = to_css_class
-        # pylint: disable=no-member
-        pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self)
-
         self._register_renderers()
         self._permission_checks = collections.deque(maxlen=200)
         self._root_token = secrets.token_hex(32)
@@ -389,8 +387,16 @@ class Datasette:
         return Urls(self)
 
     async def invoke_startup(self):
+        # This must be called for Datasette to be in a usable state
+        if self._startup_invoked:
+            return
+        for hook in pm.hook.prepare_jinja2_environment(
+            env=self.jinja_env, datasette=self
+        ):
+            await await_me_maybe(hook)
         for hook in pm.hook.startup(datasette=self):
             await await_me_maybe(hook)
+        self._startup_invoked = True
 
     def sign(self, value, namespace="default"):
         return URLSafeSerializer(self._secret, namespace).dumps(value)
@@ -933,6 +939,8 @@ class Datasette:
     async def render_template(
         self, templates, context=None, request=None, view_name=None
     ):
+        if not self._startup_invoked:
+            raise Exception("render_template() called before await ds.invoke_startup()")
         context = context or {}
         if isinstance(templates, Template):
             template = templates
@@ -1495,34 +1503,42 @@ class DatasetteClient:
         return path
 
     async def get(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.get(self._fix(path), **kwargs)
 
     async def options(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.options(self._fix(path), **kwargs)
 
     async def head(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.head(self._fix(path), **kwargs)
 
     async def post(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.post(self._fix(path), **kwargs)
 
     async def put(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.put(self._fix(path), **kwargs)
 
     async def patch(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.patch(self._fix(path), **kwargs)
 
     async def delete(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.delete(self._fix(path), **kwargs)
 
     async def request(self, method, path, **kwargs):
+        await self.ds.invoke_startup()
         avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None)
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.request(
diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py
index 640c94e6..b28fc575 100644
--- a/datasette/utils/testing.py
+++ b/datasette/utils/testing.py
@@ -147,6 +147,7 @@ class TestClient:
         content_type=None,
         if_none_match=None,
     ):
+        await self.ds.invoke_startup()
         headers = headers or {}
         if content_type:
             headers["content-type"] = content_type
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 62ec5c90..f208e727 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -88,6 +88,8 @@ You can now use this filter in your custom templates like so::
 
     Table name: {{ table|uppercase }}
 
+This function can return an awaitable function if it needs to run any async code.
+
 Examples: `datasette-edit-templates <https://datasette.io/plugins/datasette-edit-templates>`_
 
 .. _plugin_hook_extra_template_vars:
diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst
index 992b4b0e..41f50e56 100644
--- a/docs/testing_plugins.rst
+++ b/docs/testing_plugins.rst
@@ -52,6 +52,36 @@ Then run the tests using pytest like so::
 
     pytest
 
+.. _testing_plugins_datasette_test_instance:
+
+Setting up a Datasette test instance
+------------------------------------
+
+The above example shows the easiest way to start writing tests against a Datasette instance:
+
+.. code-block:: python
+
+    from datasette.app import Datasette
+    import pytest
+
+
+    @pytest.mark.asyncio
+    async def test_plugin_is_installed():
+        datasette = Datasette(memory=True)
+        response = await datasette.client.get("/-/plugins.json")
+        assert response.status_code == 200
+
+Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this:
+
+.. code-block:: python
+
+    datasette = Datasette(memory=True)
+    await datasette.invoke_startup()
+
+This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls.
+
+If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you.
+
 .. _testing_plugins_pdb:
 
 Using pdb for errors thrown inside Datasette
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 82d8452e..5a875cd2 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -71,6 +71,7 @@ EXPECTED_PLUGINS = [
             "handle_exception",
             "menu_links",
             "permission_allowed",
+            "prepare_jinja2_environment",
             "register_routes",
             "render_cell",
             "startup",
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index d49a7a34..1a41de38 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -143,8 +143,14 @@ def extra_template_vars(
 
 @hookimpl
 def prepare_jinja2_environment(env, datasette):
-    env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}"
-    env.filters["to_hello"] = lambda s: datasette._HELLO
+    async def select_times_three(s):
+        db = datasette.get_database()
+        return (await db.execute("select 3 * ?", [int(s)])).first()[0]
+
+    async def inner():
+        env.filters["select_times_three"] = select_times_three
+
+    return inner
 
 
 @hookimpl
diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index 4df02343..cee80703 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -126,6 +126,12 @@ def permission_allowed(datasette, actor, action):
     return inner
 
 
+@hookimpl
+def prepare_jinja2_environment(env, datasette):
+    env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}"
+    env.filters["to_hello"] = lambda s: datasette._HELLO
+
+
 @hookimpl
 def startup(datasette):
     async def inner():
diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py
index 8c5b5bd3..497bf475 100644
--- a/tests/test_internals_datasette_client.py
+++ b/tests/test_internals_datasette_client.py
@@ -1,10 +1,12 @@
 from .fixtures import app_client
 import httpx
 import pytest
+import pytest_asyncio
 
 
-@pytest.fixture
-def datasette(app_client):
+@pytest_asyncio.fixture
+async def datasette(app_client):
+    await app_client.ds.invoke_startup()
     return app_client.ds
 
 
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 590d88f6..0ae3abf3 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -546,11 +546,13 @@ def test_hook_register_output_renderer_can_render(app_client):
 @pytest.mark.asyncio
 async def test_hook_prepare_jinja2_environment(app_client):
     app_client.ds._HELLO = "HI"
+    await app_client.ds.invoke_startup()
     template = app_client.ds.jinja_env.from_string(
-        "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341}
+        "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}",
+        {"a": 3412341, "b": 5},
     )
     rendered = await app_client.ds.render_template(template)
-    assert "Hello there, 3,412,341, HI" == rendered
+    assert "Hello there, 3,412,341, HI, 15" == rendered
 
 
 def test_hook_publish_subcommand():
diff --git a/tests/test_routes.py b/tests/test_routes.py
index 5ae55d21..d467abe1 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -59,6 +59,7 @@ def test_routes(routes, path, expected_class, expected_matches):
 @pytest_asyncio.fixture
 async def ds_with_route():
     ds = Datasette()
+    await ds.invoke_startup()
     ds.remove_database("_memory")
     db = Database(ds, is_memory=True, memory_name="route-name-db")
     ds.add_database(db, name="original-name", route="custom-route-name")

From df851c117db031dec50dd4ef1ca34745920ac77a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 19 Sep 2022 16:46:39 -0700
Subject: [PATCH 1502/2303] Validate settings.json keys on startup, closes
 #1816

Refs #1814
---
 datasette/app.py         |  4 ++++
 tests/test_config_dir.py | 20 ++++++++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index ea3e7b43..8873ce28 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -292,6 +292,10 @@ class Datasette:
             raise StartupError("config.json should be renamed to settings.json")
         if config_dir and (config_dir / "settings.json").exists() and not settings:
             settings = json.loads((config_dir / "settings.json").read_text())
+            # Validate those settings
+            for key in settings:
+                if key not in DEFAULT_SETTINGS:
+                    raise StartupError("Invalid setting '{key}' in settings.json")
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index fe927c42..e365515b 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -5,6 +5,7 @@ import pytest
 from datasette.app import Datasette
 from datasette.cli import cli
 from datasette.utils.sqlite import sqlite3
+from datasette.utils import StartupError
 from .fixtures import TestClient as _TestClient
 from click.testing import CliRunner
 
@@ -27,9 +28,8 @@ body { margin-top: 3em}
 
 
 @pytest.fixture(scope="session")
-def config_dir_client(tmp_path_factory):
+def config_dir(tmp_path_factory):
     config_dir = tmp_path_factory.mktemp("config-dir")
-
     plugins_dir = config_dir / "plugins"
     plugins_dir.mkdir()
     (plugins_dir / "hooray.py").write_text(PLUGIN, "utf-8")
@@ -77,7 +77,23 @@ def config_dir_client(tmp_path_factory):
         ),
         "utf-8",
     )
+    return config_dir
 
+
+def test_invalid_settings(config_dir):
+    previous = (config_dir / "settings.json").read_text("utf-8")
+    (config_dir / "settings.json").write_text(
+        json.dumps({"invalid": "invalid-setting"}), "utf-8"
+    )
+    try:
+        with pytest.raises(StartupError):
+            ds = Datasette([], config_dir=config_dir)
+    finally:
+        (config_dir / "settings.json").write_text(previous, "utf-8")
+
+
+@pytest.fixture(scope="session")
+def config_dir_client(config_dir):
     ds = Datasette([], config_dir=config_dir)
     yield _TestClient(ds)
 

From cb1e093fd361b758120aefc1a444df02462389a3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 19 Sep 2022 18:15:40 -0700
Subject: [PATCH 1503/2303] Fixed error message, closes #1816

---
 datasette/app.py         | 4 +++-
 tests/test_config_dir.py | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 8873ce28..03d1dacc 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -295,7 +295,9 @@ class Datasette:
             # Validate those settings
             for key in settings:
                 if key not in DEFAULT_SETTINGS:
-                    raise StartupError("Invalid setting '{key}' in settings.json")
+                    raise StartupError(
+                        "Invalid setting '{}' in settings.json".format(key)
+                    )
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index e365515b..f5ecf0d6 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -86,8 +86,9 @@ def test_invalid_settings(config_dir):
         json.dumps({"invalid": "invalid-setting"}), "utf-8"
     )
     try:
-        with pytest.raises(StartupError):
+        with pytest.raises(StartupError) as ex:
             ds = Datasette([], config_dir=config_dir)
+        assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json"
     finally:
         (config_dir / "settings.json").write_text(previous, "utf-8")
 

From 212137a90b4291db9605e039f198564dae59c5d0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 26 Sep 2022 14:14:25 -0700
Subject: [PATCH 1504/2303] Release 0.63a0

Refs #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 0453346c..e5ad585f 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.62"
+__version__ = "0.63a0"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index f9dcc980..bd93f4cb 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,23 @@
 Changelog
 =========
 
+.. _v0_63a0:
+
+0.63a0 (2022-09-26)
+-------------------
+
+- The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`)
+- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 <https://github.com/simonw/datasette/pull/1789>`__)
+- New tutorial: `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__.
+- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`)
+- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`)
+- ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`)
+- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`)
+- More detailed command descriptions on the :ref:`CLI reference <cli_reference>` page. (:issue:`1787`)
+- Datasette no longer enforces upper bounds on its depenedencies. (:issue:`1800`)
+- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 <https://github.com/simonw/datasette/pull/1794>`__)
+- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`)
+
 .. _v0_62:
 
 0.62 (2022-08-14)

From 5f9f567acbc58c9fcd88af440e68034510fb5d2b Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 26 Sep 2022 16:06:01 -0700
Subject: [PATCH 1505/2303] Show SQL query when reporting time limit error,
 closes #1819

---
 datasette/database.py   |  5 ++++-
 datasette/views/base.py | 21 +++++++++++++--------
 tests/test_api.py       | 12 +++++++++++-
 tests/test_html.py      | 10 +++++++---
 4 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 44467370..46094bd7 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -476,7 +476,10 @@ class WriteTask:
 
 
 class QueryInterrupted(Exception):
-    pass
+    def __init__(self, e, sql, params):
+        self.e = e
+        self.sql = sql
+        self.params = params
 
 
 class MultipleValues(Exception):
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 221e1882..67aa3a42 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -1,10 +1,12 @@
 import asyncio
 import csv
 import hashlib
-import re
 import sys
+import textwrap
 import time
 import urllib
+from markupsafe import escape
+
 
 import pint
 
@@ -24,11 +26,9 @@ from datasette.utils import (
     path_with_removed_args,
     path_with_format,
     sqlite3,
-    HASH_LENGTH,
 )
 from datasette.utils.asgi import (
     AsgiStream,
-    Forbidden,
     NotFound,
     Response,
     BadRequest,
@@ -371,13 +371,18 @@ class DataView(BaseView):
                 ) = response_or_template_contexts
             else:
                 data, extra_template_data, templates = response_or_template_contexts
-        except QueryInterrupted:
+        except QueryInterrupted as ex:
             raise DatasetteError(
-                """
-                SQL query took too long. The time limit is controlled by the
+                textwrap.dedent(
+                    """
+                <p>SQL query took too long. The time limit is controlled by the
                 <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
-                configuration option.
-            """,
+                configuration option.</p>
+                <pre>{}</pre>
+            """.format(
+                        escape(ex.sql)
+                    )
+                ).strip(),
                 title="SQL Interrupted",
                 status=400,
                 message_is_html=True,
diff --git a/tests/test_api.py b/tests/test_api.py
index 7a2bf91f..ad74d16e 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -656,7 +656,17 @@ def test_custom_sql(app_client):
 def test_sql_time_limit(app_client_shorter_time_limit):
     response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)")
     assert 400 == response.status
-    assert "SQL Interrupted" == response.json["title"]
+    assert response.json == {
+        "ok": False,
+        "error": (
+            "<p>SQL query took too long. The time limit is controlled by the\n"
+            '<a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>\n'
+            "configuration option.</p>\n"
+            "<pre>select sleep(0.5)</pre>"
+        ),
+        "status": 400,
+        "title": "SQL Interrupted",
+    }
 
 
 def test_custom_sql_time_limit(app_client):
diff --git a/tests/test_html.py b/tests/test_html.py
index bf915247..a99b0b6c 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -168,10 +168,14 @@ def test_disallowed_custom_sql_pragma(app_client):
 def test_sql_time_limit(app_client_shorter_time_limit):
     response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)")
     assert 400 == response.status
-    expected_html_fragment = """
+    expected_html_fragments = [
+        """
         <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
-    """.strip()
-    assert expected_html_fragment in response.text
+    """.strip(),
+        "<pre>select sleep(0.5)</pre>",
+    ]
+    for expected_html_fragment in expected_html_fragments:
+        assert expected_html_fragment in response.text
 
 
 def test_row_page_does_not_truncate():

From 7fb4ea4e39a15e1f7d3202949794d98af1cfa272 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 27 Sep 2022 21:06:40 -0700
Subject: [PATCH 1506/2303] Update note about render_cell signature, refs #1826

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index f208e727..c9cab8ab 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -9,7 +9,7 @@ Each plugin can implement one or more hooks using the ``@hookimpl`` decorator ag
 
 When you implement a plugin hook you can accept any or all of the parameters that are documented as being passed to that hook.
 
-For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(value, column, table, database, datasette)``:
+For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(row, value, column, table, database, datasette)``:
 
 .. code-block:: python
 

From 984b1df12cf19a6731889fc0665bb5f622e07b7c Mon Sep 17 00:00:00 2001
From: Adam Simpson <adam@adamsimpson.net>
Date: Wed, 28 Sep 2022 00:21:36 -0400
Subject: [PATCH 1507/2303] Add documentation for serving via OpenRC (#1825)

* Add documentation for serving via OpenRC
---
 docs/deploying.rst | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/docs/deploying.rst b/docs/deploying.rst
index d4ad8836..c8552758 100644
--- a/docs/deploying.rst
+++ b/docs/deploying.rst
@@ -74,18 +74,30 @@ Once the service has started you can confirm that Datasette is running on port 8
     curl 127.0.0.1:8000/-/versions.json
     # Should output JSON showing the installed version
 
-Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``.
+Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx`` - see :ref:`deploying_proxy`.
 
-Ubuntu offer `a tutorial on installing nginx <https://ubuntu.com/tutorials/install-and-configure-nginx#1-overview>`__. Once it is installed you can add configuration to proxy traffic through to Datasette that looks like this::
+.. _deploying_openrc:
 
-    server {
-        server_name mysubdomain.myhost.net;
+Running Datasette using OpenRC
+===============================
+OpenRC is the service manager on non-systemd Linux distributions like `Alpine Linux <https://www.alpinelinux.org/>`__ and `Gentoo <https://www.gentoo.org/>`__.
 
-        location / {
-            proxy_pass http://127.0.0.1:8000/;
-            proxy_set_header Host $host;
-        }
-    }
+Create an init script at ``/etc/init.d/datasette`` with the following contents:
+
+.. code-block:: sh
+
+    #!/sbin/openrc-run
+
+    name="datasette"
+    command="datasette"
+    command_args="serve -h 0.0.0.0 /path/to/db.db"
+    command_background=true
+    pidfile="/run/${RC_SVCNAME}.pid"
+
+You then need to configure the service to run at boot and start it::
+
+    rc-update add datasette
+    rc-service datasette start
 
 .. _deploying_buildpacks:
 

From 34defdc10aa293294ca01cfab70780755447e1d7 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 28 Sep 2022 17:39:36 -0700
Subject: [PATCH 1508/2303] Browse the plugins directory

---
 docs/writing_plugins.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst
index 01ee8c90..a3fc88ec 100644
--- a/docs/writing_plugins.rst
+++ b/docs/writing_plugins.rst
@@ -234,7 +234,7 @@ To avoid accidentally conflicting with a database file that may be loaded into D
 
 - ``/-/upload-excel``
 
-Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `datasette-plugin topic <https://github.com/topics/datasette-plugin>`__ on GitHub.
+Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory <https://datasette.io/plugins>`.
 
 If your plugin includes functionality that relates to a specific database you could also register a URL route like this:
 

From c92c4318e9892101f75fa158410c0a12c1d80b6e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 30 Sep 2022 10:55:40 -0700
Subject: [PATCH 1509/2303] Bump furo from 2022.9.15 to 2022.9.29 (#1827)

Bumps [furo](https://github.com/pradyunsg/furo) from 2022.9.15 to 2022.9.29.
- [Release notes](https://github.com/pradyunsg/furo/releases)
- [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md)
- [Commits](https://github.com/pradyunsg/furo/compare/2022.09.15...2022.09.29)

---
updated-dependencies:
- dependency-name: furo
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index afcba1f0..fe258adb 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ setup(
     setup_requires=["pytest-runner"],
     extras_require={
         "docs": [
-            "furo==2022.9.15",
+            "furo==2022.9.29",
             "sphinx-autobuild",
             "codespell",
             "blacken-docs",

From 883e326dd6ef95f854f7750ef2d4b0e17082fa96 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 2 Oct 2022 14:26:16 -0700
Subject: [PATCH 1510/2303] Drop word-wrap: anywhere, refs #1828, #1805

---
 datasette/static/app.css | 1 -
 1 file changed, 1 deletion(-)

diff --git a/datasette/static/app.css b/datasette/static/app.css
index 08b724f6..712b9925 100644
--- a/datasette/static/app.css
+++ b/datasette/static/app.css
@@ -446,7 +446,6 @@ th {
 }
 table a:link {
     text-decoration: none;
-    word-wrap: anywhere;
 }
 .rows-and-columns td:before {
     display: block;

From 4218c9cd742b79b1e3cb80878e42b7e39d16ded2 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 4 Oct 2022 11:45:36 -0700
Subject: [PATCH 1511/2303] reST markup fix

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index c9cab8ab..832a76b0 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -268,7 +268,7 @@ you have one:
     def extra_js_urls():
         return ["/-/static-plugins/your-plugin/app.js"]
 
-Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page.
+Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page.
 
 If your code uses `JavaScript modules <https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules>`__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details.
 

From b6ba117b7978b58b40e3c3c2b723b92c3010ed53 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 4 Oct 2022 18:25:52 -0700
Subject: [PATCH 1512/2303] Clarify request or None for two hooks

---
 docs/plugin_hooks.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 832a76b0..b61f953a 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -1281,7 +1281,7 @@ menu_links(datasette, actor, request)
 ``actor`` - dictionary or None
     The currently authenticated :ref:`actor <authentication_actor>`.
 
-``request`` - :ref:`internals_request`
+``request`` - :ref:`internals_request` or None
     The current HTTP request. This can be ``None`` if the request object is not available.
 
 This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon.
@@ -1330,7 +1330,7 @@ table_actions(datasette, actor, database, table, request)
 ``table`` - string
     The name of the table.
 
-``request`` - :ref:`internals_request`
+``request`` - :ref:`internals_request` or None
     The current HTTP request. This can be ``None`` if the request object is not available.
 
 This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items.

From bbf33a763537a1d913180b22bd3b5fe4a5e5b252 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 4 Oct 2022 21:32:11 -0700
Subject: [PATCH 1513/2303] Test for bool(results), closes #1832

---
 tests/test_internals_database.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 9e81c1d6..4e33beed 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -30,6 +30,14 @@ async def test_results_first(db):
     assert isinstance(row, sqlite3.Row)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("expected", (True, False))
+async def test_results_bool(db, expected):
+    where = "" if expected else "where pk = 0"
+    results = await db.execute("select * from facetable {}".format(where))
+    assert bool(results) is expected
+
+
 @pytest.mark.parametrize(
     "query,expected",
     [

From eff112498ecc499323c26612d707908831446d25 Mon Sep 17 00:00:00 2001
From: Forest Gregg <fgregg@users.noreply.github.com>
Date: Thu, 6 Oct 2022 16:06:06 -0400
Subject: [PATCH 1514/2303] Useuse inspect data for hash and file size on
 startup

Thanks, @fgregg

Closes #1834
---
 datasette/database.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 46094bd7..d75bd70c 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -48,9 +48,13 @@ class Database:
         self._read_connection = None
         self._write_connection = None
         if not self.is_mutable and not self.is_memory:
-            p = Path(path)
-            self.hash = inspect_hash(p)
-            self.cached_size = p.stat().st_size
+            if self.ds.inspect_data and self.ds.inspect_data.get(self.name):
+                self.hash = self.ds.inspect_data[self.name]["hash"]
+                self.cached_size = self.ds.inspect_data[self.name]["size"]
+            else:
+                p = Path(path)
+                self.hash = inspect_hash(p)
+                self.cached_size = p.stat().st_size
 
     @property
     def cached_table_counts(self):

From b7fec7f9020b79c1fe60cc5a2def86b50eeb5af9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 7 Oct 2022 16:03:09 -0700
Subject: [PATCH 1515/2303] .sqlite/.sqlite3 extensions for config directory
 mode

Closes #1646
---
 datasette/app.py         |  5 ++++-
 docs/settings.rst        |  2 +-
 tests/test_config_dir.py | 11 +++++------
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 03d1dacc..32a911c2 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -217,7 +217,10 @@ class Datasette:
         self._secret = secret or secrets.token_hex(32)
         self.files = tuple(files or []) + tuple(immutables or [])
         if config_dir:
-            self.files += tuple([str(p) for p in config_dir.glob("*.db")])
+            db_files = []
+            for ext in ("db", "sqlite", "sqlite3"):
+                db_files.extend(config_dir.glob("*.{}".format(ext)))
+            self.files += tuple(str(f) for f in db_files)
         if (
             config_dir
             and (config_dir / "inspect-data.json").exists()
diff --git a/docs/settings.rst b/docs/settings.rst
index 8437fb04..a6d50543 100644
--- a/docs/settings.rst
+++ b/docs/settings.rst
@@ -46,7 +46,7 @@ Datasette will detect the files in that directory and automatically configure it
 
 The files that can be included in this directory are as follows. All are optional.
 
-* ``*.db`` - SQLite database files that will be served by Datasette
+* ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette
 * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well
 * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running
 * ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index f5ecf0d6..c2af3836 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -49,7 +49,7 @@ def config_dir(tmp_path_factory):
     (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8")
     (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8")
 
-    for dbname in ("demo.db", "immutable.db"):
+    for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"):
         db = sqlite3.connect(str(config_dir / dbname))
         db.executescript(
             """
@@ -151,12 +151,11 @@ def test_databases(config_dir_client):
     response = config_dir_client.get("/-/databases.json")
     assert 200 == response.status
     databases = response.json
-    assert 2 == len(databases)
+    assert 4 == len(databases)
     databases.sort(key=lambda d: d["name"])
-    assert "demo" == databases[0]["name"]
-    assert databases[0]["is_mutable"]
-    assert "immutable" == databases[1]["name"]
-    assert not databases[1]["is_mutable"]
+    for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")):
+        assert expected_name == db["name"]
+        assert db["is_mutable"] == (expected_name != "immutable")
 
 
 @pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml"))

From 1a5e5f2aa951e5bd731067a49819efba68fbe8ef Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Oct 2022 14:42:52 -0700
Subject: [PATCH 1516/2303] Refactor breadcrumbs to respect permissions, refs
 #1831

---
 datasette/app.py                           | 40 ++++++++++++++++++++++
 datasette/templates/_crumbs.html           | 15 ++++++++
 datasette/templates/base.html              |  4 +--
 datasette/templates/database.html          |  9 -----
 datasette/templates/error.html             |  7 ----
 datasette/templates/logout.html            |  7 ----
 datasette/templates/permissions_debug.html |  7 ----
 datasette/templates/query.html             |  8 ++---
 datasette/templates/row.html               |  9 ++---
 datasette/templates/show_json.html         |  7 ----
 datasette/templates/table.html             |  8 ++---
 tests/test_permissions.py                  |  1 +
 tests/test_plugins.py                      |  2 +-
 13 files changed, 65 insertions(+), 59 deletions(-)
 create mode 100644 datasette/templates/_crumbs.html

diff --git a/datasette/app.py b/datasette/app.py
index 32a911c2..5fa4955c 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -631,6 +631,44 @@ class Datasette:
         else:
             return []
 
+    async def _crumb_items(self, request, table=None, database=None):
+        crumbs = []
+        # Top-level link
+        if await self.permission_allowed(
+            actor=request.actor, action="view-instance", default=True
+        ):
+            crumbs.append({"href": self.urls.instance(), "label": "home"})
+        # Database link
+        if database:
+            if await self.permission_allowed(
+                actor=request.actor,
+                action="view-database",
+                resource=database,
+                default=True,
+            ):
+                crumbs.append(
+                    {
+                        "href": self.urls.database(database),
+                        "label": database,
+                    }
+                )
+        # Table link
+        if table:
+            assert database, "table= requires database="
+            if await self.permission_allowed(
+                actor=request.actor,
+                action="view-table",
+                resource=(database, table),
+                default=True,
+            ):
+                crumbs.append(
+                    {
+                        "href": self.urls.table(database, table),
+                        "label": table,
+                    }
+                )
+        return crumbs
+
     async def permission_allowed(self, actor, action, resource=None, default=False):
         """Check permissions using the permissions_allowed plugin hook"""
         result = None
@@ -1009,6 +1047,8 @@ class Datasette:
         template_context = {
             **context,
             **{
+                "request": request,
+                "crumb_items": self._crumb_items,
                 "urls": self.urls,
                 "actor": request.actor if request else None,
                 "menu_links": menu_links,
diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html
new file mode 100644
index 00000000..bd1ff0da
--- /dev/null
+++ b/datasette/templates/_crumbs.html
@@ -0,0 +1,15 @@
+{% macro nav(request, database=None, table=None) -%}
+{% if crumb_items is defined %}
+  {% set items=crumb_items(request=request, database=database, table=table) %}
+  {% if items %}
+    <p class="crumbs">
+      {% for item in items %}
+        <a href="{{ item.href }}">{{ item.label }}</a>
+        {% if not loop.last %}
+          /
+        {% endif %}
+      {% endfor %}
+    </p>
+  {% endif %}
+{% endif %}
+{%- endmacro %}
diff --git a/datasette/templates/base.html b/datasette/templates/base.html
index c3a71acb..87c939ac 100644
--- a/datasette/templates/base.html
+++ b/datasette/templates/base.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+{% import "_crumbs.html" as crumbs with context %}<!DOCTYPE html>
 <html>
 <head>
     <title>{% block title %}{% endblock %}
@@ -17,7 +17,7 @@
 
 
 
      
-
      "
+    anon_response = padlock_client.get(path)
+    assert expected_anon == anon_response.status
+    if allow and anon_response.status == 200:
+        # Should be no padlock
+        assert fragment not in anon_response.text
+    auth_response = padlock_client.get(
+        path,
+        cookies={"ds_actor": padlock_client.actor_cookie({"id": "root"})},
+    )
+    assert expected_auth == auth_response.status
+    # Check for the padlock
+    if allow and expected_anon == 403 and expected_auth == 200:
+        assert fragment in auth_response.text
+    del padlock_client.ds._metadata_local["allow"]
 
 
 @pytest.mark.parametrize(
@@ -467,6 +487,10 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta
             path,
             cookies={"ds_actor": cascade_app_client.actor_cookie(actor)},
         )
-        assert expected_status == response.status
+        assert (
+            response.status == expected_status
+        ), "path: {}, permissions: {}, expected_status: {}, status: {}".format(
+            path, permissions, expected_status, response.status
+        )
     finally:
-        cascade_app_client.ds._metadata_local = previous_metadata
+        cascade_app_client.ds._local_metadata = previous_metadata
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 02cac132..e0a7bc76 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -823,8 +823,14 @@ def test_hook_forbidden(restore_working_directory):
         assert 403 == response.status
         response2 = client.get("/data2")
         assert 302 == response2.status
-        assert "/login?message=view-database" == response2.headers["Location"]
-        assert "view-database" == client.ds._last_forbidden_message
+        assert (
+            response2.headers["Location"]
+            == "/login?message=You do not have permission to view this database"
+        )
+        assert (
+            client.ds._last_forbidden_message
+            == "You do not have permission to view this database"
+        )
 
 
 def test_hook_handle_exception(app_client):

From 5be86d48b2e31565faca208fc4aeb0ddfaca71f9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 23 Oct 2022 19:42:30 -0700
Subject: [PATCH 1521/2303] Fix display of padlocks on database page, closes
 #1848

---
 datasette/views/database.py | 21 +++++++++++++++------
 tests/test_permissions.py   | 35 ++++++++++++++++++++++++++++++++++-
 2 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/datasette/views/database.py b/datasette/views/database.py
index bd9e4a7c..8e08c3b1 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -68,8 +68,11 @@ class DatabaseView(DataView):
         for view_name in await db.view_names():
             view_visible, view_private = await self.ds.check_visibility(
                 request.actor,
-                "view-table",
-                (database, view_name),
+                permissions=[
+                    ("view-table", (database, view_name)),
+                    ("view-database", database),
+                    "view-instance",
+                ],
             )
             if view_visible:
                 views.append(
@@ -83,8 +86,11 @@ class DatabaseView(DataView):
         for table in table_counts:
             table_visible, table_private = await self.ds.check_visibility(
                 request.actor,
-                "view-table",
-                (database, table),
+                permissions=[
+                    ("view-table", (database, table)),
+                    ("view-database", database),
+                    "view-instance",
+                ],
             )
             if not table_visible:
                 continue
@@ -109,8 +115,11 @@ class DatabaseView(DataView):
         ).values():
             query_visible, query_private = await self.ds.check_visibility(
                 request.actor,
-                "view-query",
-                (database, query["name"]),
+                permissions=[
+                    ("view-query", (database, query["name"])),
+                    ("view-database", database),
+                    "view-instance",
+                ],
             )
             if query_visible:
                 canned_queries.append(dict(query, private=query_private))
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 2d48431a..8812d0f7 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -493,4 +493,37 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta
             path, permissions, expected_status, response.status
         )
     finally:
-        cascade_app_client.ds._local_metadata = previous_metadata
+        cascade_app_client.ds._metadata_local = previous_metadata
+
+
+def test_padlocks_on_database_page(cascade_app_client):
+    metadata = {
+        "databases": {
+            "fixtures": {
+                "allow": {"id": "test"},
+                "tables": {
+                    "123_starts_with_digits": {"allow": True},
+                    "simple_view": {"allow": True},
+                },
+                "queries": {"query_two": {"allow": True, "sql": "select 2"}},
+            }
+        }
+    }
+    previous_metadata = cascade_app_client.ds._metadata_local
+    try:
+        cascade_app_client.ds._metadata_local = metadata
+        response = cascade_app_client.get(
+            "/fixtures",
+            cookies={"ds_actor": cascade_app_client.actor_cookie({"id": "test"})},
+        )
+        # Tables
+        assert ">123_starts_with_digits

    " in response.text + assert ">Table With Space In Name 🔒

    " in response.text + # Queries + assert ">from_async_hook 🔒" in response.text + assert ">query_two" in response.text + # Views + assert ">paginated_view 🔒" in response.text + assert ">simple_view" in response.text + finally: + cascade_app_client.ds._metadata_local = previous_metadata From 602c0888ce633000cfae42be00de474ef681bda7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 23 Oct 2022 20:07:09 -0700 Subject: [PATCH 1522/2303] Release 0.63a1 Refs #1646, #1819, #1825, #1829, #1831, #1832, #1834, #1844, #1848 --- datasette/version.py | 2 +- docs/changelog.rst | 16 +++++++++++++++- docs/internals.rst | 2 +- docs/performance.rst | 2 ++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index e5ad585f..eb36da45 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.63a0" +__version__ = "0.63a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index f5cf03e8..dd4c20b7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_63a1: + +0.63a1 (2022-10-23) +------------------- + +- SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`) +- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) +- The :ref:`inspect data ` mechanism is now used to speed up server startup - thanks, Forest Gregg. (:issue:`1834`) +- In :ref:`config_dir` databases with filenames ending in ``.sqlite`` or ``.sqlite3`` are now automatically added to the Datasette instance. (:issue:`1646`) +- Breadcrumb navigation display now respects the current user's permissions. (:issue:`1831`) +- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) +- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) + + .. _v0_63a0: 0.63a0 (2022-09-26) @@ -91,7 +105,7 @@ Datasette also now requires Python 3.7 or higher. - Python 3.6 is no longer supported. (:issue:`1577`) - Tests now run against Python 3.11-dev. (:issue:`1621`) - New :ref:`datasette.ensure_permissions(actor, permissions) ` internal method for checking multiple permissions at once. (:issue:`1675`) -- New :ref:`datasette.check_visibility(actor, action, resource=None) ` internal method for checking if a user can see a resource that would otherwise be invisible to unauthenticated users. (:issue:`1678`) +- New :ref:`datasette.check_visibility(actor, action, resource=None) ` internal method for checking if a user can see a resource that would otherwise be invisible to unauthenticated users. (:issue:`1678`) - Table and row HTML pages now include a ```` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`) - ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header. - Canned queries are now shown at the top of the database page, directly below the SQL editor. Previously they were shown at the bottom, below the list of tables. (:issue:`1612`) diff --git a/docs/internals.rst b/docs/internals.rst index 92f4efee..c3892a7c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -364,7 +364,7 @@ This is useful when you need to check multiple permissions at once. For example, ], ) -.. _datasette_check_visibilty: +.. _datasette_check_visibility: await .check_visibility(actor, action=None, resource=None, permissions=None) ---------------------------------------------------------------------------- diff --git a/docs/performance.rst b/docs/performance.rst index 89bbf5ae..4427757c 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -24,6 +24,8 @@ To open a file in immutable mode pass it to the datasette command using the ``-i When you open a file in immutable mode like this Datasette will also calculate and cache the row counts for each table in that database when it first starts up, further improving performance. +.. _performance_inspect: + Using "datasette inspect" ------------------------- From a0dd5fa02fb1e6d5477b962a2062f1a4be3354a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 23 Oct 2022 20:14:49 -0700 Subject: [PATCH 1523/2303] Fixed typo in release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dd4c20b7..2255dcce 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,7 +31,7 @@ Changelog - ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) - Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) - More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) -- Datasette no longer enforces upper bounds on its depenedencies. (:issue:`1800`) +- Datasette no longer enforces upper bounds on its dependencies. (:issue:`1800`) - Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) - The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) From 83adf55b2da83fd9a227f7e4c8506d72def72294 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 23 Oct 2022 20:28:15 -0700 Subject: [PATCH 1524/2303] Deploy one-dot-zero branch preview --- .github/workflows/deploy-latest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 2b94a7f1..43a843ed 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -3,7 +3,8 @@ name: Deploy latest.datasette.io on: push: branches: - - main + - main + - 1.0-dev permissions: contents: read @@ -68,6 +69,8 @@ jobs: gcloud config set project datasette-222320 export SUFFIX="-${GITHUB_REF#refs/heads/}" export SUFFIX=${SUFFIX#-main} + # Replace 1.0 with one-dot-zero in SUFFIX + export SUFFIX=${SUFFIX//1.0/one-dot-zero} datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ -m fixtures.json \ --plugins-dir=plugins \ From e135da8efe8fccecf9a137a941cc1f1db0db583a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 07:13:43 -0700 Subject: [PATCH 1525/2303] Python 3.11 in CI --- .github/workflows/publish.yml | 16 ++++++++-------- .github/workflows/test.yml | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9ef09d2e..fa608055 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,14 +12,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip @@ -37,12 +37,12 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.10' - - uses: actions/cache@v2 + python-version: '3.11' + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e38d5ee9..886f649a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,14 +10,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip From 02ae1a002918eb91f794e912c32742559da34cf5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 11:59:03 -0700 Subject: [PATCH 1526/2303] Upgrade Docker images to Python 3.11, closes #1853 --- Dockerfile | 2 +- datasette/utils/__init__.py | 2 +- demos/apache-proxy/Dockerfile | 2 +- docs/publish.rst | 2 +- tests/test_package.py | 2 +- tests/test_publish_cloudrun.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee7ed957..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.6-slim-bullseye as build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2bdea673..803ba96d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -390,7 +390,7 @@ def make_dockerfile( "SQLITE_EXTENSIONS" ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" return """ -FROM python:3.10.6-slim-bullseye +FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app {apt_get_extras} diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 70b33bec..9a8448da 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.6-slim-bullseye +FROM python:3.11.0-slim-bullseye RUN apt-get update && \ apt-get install -y apache2 supervisor && \ diff --git a/docs/publish.rst b/docs/publish.rst index d817ed31..4ba94792 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -146,7 +146,7 @@ Here's example output for the package command:: $ datasette package parlgov.db --extra-options="--setting sql_time_limit_ms 2500" Sending build context to Docker daemon 4.459MB - Step 1/7 : FROM python:3.10.6-slim-bullseye + Step 1/7 : FROM python:3.11.0-slim-bullseye ---> 79e1dc9af1c1 Step 2/7 : COPY . /app ---> Using cache diff --git a/tests/test_package.py b/tests/test_package.py index ac15e61e..f05f3ece 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -12,7 +12,7 @@ class CaptureDockerfile: EXPECTED_DOCKERFILE = """ -FROM python:3.10.6-slim-bullseye +FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index e64534d2..158a090e 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -242,7 +242,7 @@ def test_publish_cloudrun_plugin_secrets( ) expected = textwrap.dedent( r""" - FROM python:3.10.6-slim-bullseye + FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app @@ -309,7 +309,7 @@ def test_publish_cloudrun_apt_get_install( ) expected = textwrap.dedent( r""" - FROM python:3.10.6-slim-bullseye + FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app From 9676b2deb07cff20247ba91dad3e84a4ab0b00d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 11:59:03 -0700 Subject: [PATCH 1527/2303] Upgrade Docker images to Python 3.11, closes #1853 --- Dockerfile | 2 +- datasette/utils/__init__.py | 2 +- demos/apache-proxy/Dockerfile | 2 +- docs/publish.rst | 2 +- tests/test_package.py | 2 +- tests/test_publish_cloudrun.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee7ed957..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.6-slim-bullseye as build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2bdea673..803ba96d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -390,7 +390,7 @@ def make_dockerfile( "SQLITE_EXTENSIONS" ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" return """ -FROM python:3.10.6-slim-bullseye +FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app {apt_get_extras} diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 70b33bec..9a8448da 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.6-slim-bullseye +FROM python:3.11.0-slim-bullseye RUN apt-get update && \ apt-get install -y apache2 supervisor && \ diff --git a/docs/publish.rst b/docs/publish.rst index d817ed31..4ba94792 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -146,7 +146,7 @@ Here's example output for the package command:: $ datasette package parlgov.db --extra-options="--setting sql_time_limit_ms 2500" Sending build context to Docker daemon 4.459MB - Step 1/7 : FROM python:3.10.6-slim-bullseye + Step 1/7 : FROM python:3.11.0-slim-bullseye ---> 79e1dc9af1c1 Step 2/7 : COPY . /app ---> Using cache diff --git a/tests/test_package.py b/tests/test_package.py index ac15e61e..f05f3ece 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -12,7 +12,7 @@ class CaptureDockerfile: EXPECTED_DOCKERFILE = """ -FROM python:3.10.6-slim-bullseye +FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index e64534d2..158a090e 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -242,7 +242,7 @@ def test_publish_cloudrun_plugin_secrets( ) expected = textwrap.dedent( r""" - FROM python:3.10.6-slim-bullseye + FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app @@ -309,7 +309,7 @@ def test_publish_cloudrun_apt_get_install( ) expected = textwrap.dedent( r""" - FROM python:3.10.6-slim-bullseye + FROM python:3.11.0-slim-bullseye COPY . /app WORKDIR /app From 613ad05c095f92653221db267ef53d54d00cdfbb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 12:16:48 -0700 Subject: [PATCH 1528/2303] Don't need pysqlite3-binary any more, refs #1853 --- .github/workflows/deploy-latest.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 2b94a7f1..e423b8fa 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: "3.10" - - uses: actions/cache@v2 + python-version: "3.11" + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip @@ -74,7 +74,6 @@ jobs: --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ - --install=pysqlite3-binary \ --service "datasette-latest$SUFFIX" - name: Deploy to docs as well (only for main) if: ${{ github.ref == 'refs/heads/main' }} From c7dd76c26257ded5bcdfd0570e12412531b8b88f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 12:42:21 -0700 Subject: [PATCH 1529/2303] Poll until servers start, refs #1854 --- tests/conftest.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 215853b3..f4638a14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import httpx import os import pathlib import pytest @@ -110,8 +111,13 @@ def ds_localhost_http_server(): # Avoid FileNotFoundError: [Errno 2] No such file or directory: cwd=tempfile.gettempdir(), ) - # Give the server time to start - time.sleep(1.5) + # Loop until port 8041 serves traffic + while True: + try: + httpx.get("http://localhost:8041/") + break + except httpx.ConnectError: + time.sleep(0.1) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc @@ -146,8 +152,12 @@ def ds_localhost_https_server(tmp_path_factory): stderr=subprocess.STDOUT, cwd=tempfile.gettempdir(), ) - # Give the server time to start - time.sleep(1.5) + while True: + try: + httpx.get("https://localhost:8042/", verify=client_cert) + break + except httpx.ConnectError: + time.sleep(0.1) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc, client_cert @@ -168,8 +178,15 @@ def ds_unix_domain_socket_server(tmp_path_factory): stderr=subprocess.STDOUT, cwd=tempfile.gettempdir(), ) - # Give the server time to start - time.sleep(1.5) + # Poll until available + transport = httpx.HTTPTransport(uds=uds) + client = httpx.Client(transport=transport) + while True: + try: + client.get("http://localhost/_memory.json") + break + except httpx.ConnectError: + time.sleep(0.1) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc, uds From 6d085af28c63c28ecda388fc0552c91f756be0c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 07:13:43 -0700 Subject: [PATCH 1530/2303] Python 3.11 in CI --- .github/workflows/publish.yml | 16 ++++++++-------- .github/workflows/test.yml | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9ef09d2e..fa608055 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,14 +12,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip @@ -37,12 +37,12 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.10' - - uses: actions/cache@v2 + python-version: '3.11' + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e38d5ee9..886f649a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,14 +10,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip From 05b479224fa57af3ab2d03769edd5081dad62a19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 12:16:48 -0700 Subject: [PATCH 1531/2303] Don't need pysqlite3-binary any more, refs #1853 --- .github/workflows/deploy-latest.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 43a843ed..5598dc12 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,12 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: "3.10" - - uses: actions/cache@v2 + python-version: "3.11" + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip @@ -77,7 +77,6 @@ jobs: --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ - --install=pysqlite3-binary \ --service "datasette-latest$SUFFIX" - name: Deploy to docs as well (only for main) if: ${{ github.ref == 'refs/heads/main' }} From f9ae92b37796f7f559d57b1ee9718aa4d43547e8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 12:42:21 -0700 Subject: [PATCH 1532/2303] Poll until servers start, refs #1854 --- tests/conftest.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 215853b3..f4638a14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import httpx import os import pathlib import pytest @@ -110,8 +111,13 @@ def ds_localhost_http_server(): # Avoid FileNotFoundError: [Errno 2] No such file or directory: cwd=tempfile.gettempdir(), ) - # Give the server time to start - time.sleep(1.5) + # Loop until port 8041 serves traffic + while True: + try: + httpx.get("http://localhost:8041/") + break + except httpx.ConnectError: + time.sleep(0.1) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc @@ -146,8 +152,12 @@ def ds_localhost_https_server(tmp_path_factory): stderr=subprocess.STDOUT, cwd=tempfile.gettempdir(), ) - # Give the server time to start - time.sleep(1.5) + while True: + try: + httpx.get("https://localhost:8042/", verify=client_cert) + break + except httpx.ConnectError: + time.sleep(0.1) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc, client_cert @@ -168,8 +178,15 @@ def ds_unix_domain_socket_server(tmp_path_factory): stderr=subprocess.STDOUT, cwd=tempfile.gettempdir(), ) - # Give the server time to start - time.sleep(1.5) + # Poll until available + transport = httpx.HTTPTransport(uds=uds) + client = httpx.Client(transport=transport) + while True: + try: + client.get("http://localhost/_memory.json") + break + except httpx.ConnectError: + time.sleep(0.1) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc, uds From 42f8b402e6aa56af4bbe921e346af8df42acd50f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 17:07:58 -0700 Subject: [PATCH 1533/2303] Initial prototype of create API token page, refs #1852 --- datasette/app.py | 5 ++ datasette/templates/create_token.html | 83 +++++++++++++++++++++++++++ datasette/views/special.py | 54 +++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 datasette/templates/create_token.html diff --git a/datasette/app.py b/datasette/app.py index 9df16558..cab9d142 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,6 +33,7 @@ from .views.special import ( JsonDataView, PatternPortfolioView, AuthTokenView, + CreateTokenView, LogoutView, AllowDebugView, PermissionsDebugView, @@ -1212,6 +1213,10 @@ class Datasette: AuthTokenView.as_view(self), r"/-/auth-token$", ) + add_route( + CreateTokenView.as_view(self), + r"/-/create-token$", + ) add_route( LogoutView.as_view(self), r"/-/logout$", diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html new file mode 100644 index 00000000..a94881ed --- /dev/null +++ b/datasette/templates/create_token.html @@ -0,0 +1,83 @@ +{% extends "base.html" %} + +{% block title %}Create an API token{% endblock %} + +{% block content %} + +

    Create an API token

    + +

    This token will allow API access with the same abilities as your current user.

    + +{% if errors %} + {% for error in errors %} +

    {{ error }}

    + {% endfor %} +{% endif %} + + +
    +
    + +
    + + + +
    + + +{% if token %} +
    +

    Your API token

    +
    + + +
    + +
    + Token details + 
    {{ token_bits|tojson }}
    +
    +
    + {% endif %} + + + +{% endblock %} diff --git a/datasette/views/special.py b/datasette/views/special.py index dd834528..f2e69412 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -3,6 +3,7 @@ from datasette.utils.asgi import Response, Forbidden from datasette.utils import actor_matches_allow, add_cors_headers from .base import BaseView import secrets +import time class JsonDataView(BaseView): @@ -163,3 +164,56 @@ class MessagesDebugView(BaseView): else: datasette.add_message(request, message, getattr(datasette, message_type)) return Response.redirect(self.ds.urls.instance()) + + +class CreateTokenView(BaseView): + name = "create_token" + has_json_alternate = False + + async def get(self, request): + if not request.actor: + raise Forbidden("You must be logged in to create a token") + return await self.render( + ["create_token.html"], + request, + {"actor": request.actor}, + ) + + async def post(self, request): + if not request.actor: + raise Forbidden("You must be logged in to create a token") + post = await request.post_vars() + expires = None + errors = [] + if post.get("expire_type"): + duration = post.get("expire_duration") + if not duration or not duration.isdigit() or not int(duration) > 0: + errors.append("Invalid expire duration") + else: + unit = post["expire_type"] + if unit == "minutes": + expires = int(duration) * 60 + elif unit == "hours": + expires = int(duration) * 60 * 60 + elif unit == "days": + expires = int(duration) * 60 * 60 * 24 + else: + errors.append("Invalid expire duration unit") + token_bits = None + token = None + if not errors: + token_bits = { + "a": request.actor, + "e": (int(time.time()) + expires) if expires else None, + } + token = self.ds.sign(token_bits, "token") + return await self.render( + ["create_token.html"], + request, + { + "actor": request.actor, + "errors": errors, + "token": token, + "token_bits": token_bits, + }, + ) From 68ccb7578b5d3bf68b86fb2f5cf8753098dfe075 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 18:40:07 -0700 Subject: [PATCH 1534/2303] dstoke_ prefix for tokens Refs https://github.com/simonw/datasette/issues/1852#issuecomment-1291290451 --- datasette/views/special.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index f2e69412..d3f202f4 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -206,7 +206,7 @@ class CreateTokenView(BaseView): "a": request.actor, "e": (int(time.time()) + expires) if expires else None, } - token = self.ds.sign(token_bits, "token") + token = "dstok_{}".format(self.ds.sign(token_bits, "token")) return await self.render( ["create_token.html"], request, From 7ab091e8ef8d3af1e23b5a81ffad2bd8c96cc47c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 19:04:05 -0700 Subject: [PATCH 1535/2303] Tests and docs for /-/create-token, refs #1852 --- datasette/views/special.py | 14 +++++--- docs/authentication.rst | 15 +++++++++ tests/test_auth.py | 68 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index d3f202f4..7f70eb1f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -170,9 +170,16 @@ class CreateTokenView(BaseView): name = "create_token" has_json_alternate = False - async def get(self, request): + def check_permission(self, request): if not request.actor: raise Forbidden("You must be logged in to create a token") + if not request.actor.get("id"): + raise Forbidden( + "You must be logged in as an actor with an ID to create a token" + ) + + async def get(self, request): + self.check_permission(request) return await self.render( ["create_token.html"], request, @@ -180,8 +187,7 @@ class CreateTokenView(BaseView): ) async def post(self, request): - if not request.actor: - raise Forbidden("You must be logged in to create a token") + self.check_permission(request) post = await request.post_vars() expires = None errors = [] @@ -203,7 +209,7 @@ class CreateTokenView(BaseView): token = None if not errors: token_bits = { - "a": request.actor, + "a": request.actor["id"], "e": (int(time.time()) + expires) if expires else None, } token = "dstok_{}".format(self.ds.sign(token_bits, "token")) diff --git a/docs/authentication.rst b/docs/authentication.rst index 685dab15..fc903fbb 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -333,6 +333,21 @@ To limit this ability for just one specific database, use this: } } +.. _CreateTokenView: + +API Tokens +========== + +Datasette includes a default mechanism for generating API tokens that can be used to authenticate requests. + +Authenticated users can create new API tokens using a form on the ``/-/create-token`` page. + +Created tokens can then be passed in the ``Authorization: Bearer token_here`` header of HTTP requests to Datasette. + +A token created by a user will include that user's ``"id"`` in the token payload, so any permissions granted to that user based on their ID will be made available to the token as well. + +Coming soon: a mechanism for creating tokens that can only perform a subset of the actions available to the user who created them. + .. _permissions_plugins: Checking permissions in plugins diff --git a/tests/test_auth.py b/tests/test_auth.py index 4ef35a76..3aaab50d 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -110,3 +110,71 @@ def test_no_logout_button_in_navigation_if_no_ds_actor_cookie(app_client, path): response = app_client.get(path + "?_bot=1") assert "bot" in response.text assert '
    ' not in response.text + + +@pytest.mark.parametrize( + "post_data,errors,expected_duration", + ( + ({"expire_type": ""}, [], None), + ({"expire_type": "x"}, ["Invalid expire duration"], None), + ({"expire_type": "minutes"}, ["Invalid expire duration"], None), + ( + {"expire_type": "minutes", "expire_duration": "x"}, + ["Invalid expire duration"], + None, + ), + ( + {"expire_type": "minutes", "expire_duration": "-1"}, + ["Invalid expire duration"], + None, + ), + ( + {"expire_type": "minutes", "expire_duration": "0"}, + ["Invalid expire duration"], + None, + ), + ( + {"expire_type": "minutes", "expire_duration": "10"}, + [], + 600, + ), + ( + {"expire_type": "hours", "expire_duration": "10"}, + [], + 10 * 60 * 60, + ), + ( + {"expire_type": "days", "expire_duration": "3"}, + [], + 60 * 60 * 24 * 3, + ), + ), +) +def test_auth_create_token(app_client, post_data, errors, expected_duration): + assert app_client.get("/-/create-token").status == 403 + ds_actor = app_client.actor_cookie({"id": "test"}) + response = app_client.get("/-/create-token", cookies={"ds_actor": ds_actor}) + assert response.status == 200 + assert ">Create an API token<" in response.text + # Now try actually creating one + response2 = app_client.post( + "/-/create-token", + post_data, + csrftoken_from=True, + cookies={"ds_actor": ds_actor}, + ) + assert response2.status == 200 + if errors: + for error in errors: + assert '

    {}

    '.format(error) in response2.text + else: + # Extract token from page + token = response2.text.split('value="dstok_')[1].split('"')[0] + details = app_client.ds.unsign(token, "token") + assert details.keys() == {"a", "e"} + assert details["a"] == "test" + if expected_duration is None: + assert details["e"] is None + else: + about_right = int(time.time()) + expected_duration + assert about_right - 2 < details["e"] < about_right + 2 From b29e487bc3fde6418bf45bda7cfed2e081ff03fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 19:18:41 -0700 Subject: [PATCH 1536/2303] actor_from_request for dstok_ tokens, refs #1852 --- datasette/default_permissions.py | 25 +++++++++++++++++++++++++ datasette/utils/testing.py | 2 ++ tests/test_auth.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index b58d8d1b..4d836ddc 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -1,5 +1,7 @@ from datasette import hookimpl from datasette.utils import actor_matches_allow +import itsdangerous +import time @hookimpl(tryfirst=True) @@ -45,3 +47,26 @@ def permission_allowed(datasette, actor, action, resource): return actor_matches_allow(actor, database_allow_sql) return inner + + +@hookimpl +def actor_from_request(datasette, request): + prefix = "dstok_" + authorization = request.headers.get("authorization") + if not authorization: + return None + if not authorization.startswith("Bearer "): + return None + token = authorization[len("Bearer ") :] + if not token.startswith(prefix): + return None + token = token[len(prefix) :] + try: + decoded = datasette.unsign(token, namespace="token") + except itsdangerous.BadSignature: + return None + expires_at = decoded.get("e") + if expires_at is not None: + if expires_at < time.time(): + return None + return {"id": decoded["a"], "dstok": True} diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index b28fc575..4f76a799 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -62,6 +62,7 @@ class TestClient: method="GET", cookies=None, if_none_match=None, + headers=None, ): return await self._request( path=path, @@ -70,6 +71,7 @@ class TestClient: method=method, cookies=cookies, if_none_match=if_none_match, + headers=headers, ) @async_to_sync diff --git a/tests/test_auth.py b/tests/test_auth.py index 3aaab50d..be21d6a5 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -178,3 +178,35 @@ def test_auth_create_token(app_client, post_data, errors, expected_duration): else: about_right = int(time.time()) + expected_duration assert about_right - 2 < details["e"] < about_right + 2 + + +@pytest.mark.parametrize( + "scenario,should_work", + ( + ("no_token", False), + ("invalid_token", False), + ("expired_token", False), + ("valid_unlimited_token", True), + ("valid_expiring_token", True), + ), +) +def test_auth_with_dstok_token(app_client, scenario, should_work): + token = None + if scenario == "valid_unlimited_token": + token = app_client.ds.sign({"a": "test"}, "token") + elif scenario == "valid_expiring_token": + token = app_client.ds.sign({"a": "test", "e": int(time.time()) + 1000}, "token") + elif scenario == "expired_token": + token = app_client.ds.sign({"a": "test", "e": int(time.time()) - 1000}, "token") + elif scenario == "invalid_token": + token = "invalid" + if token: + token = "dstok_{}".format(token) + headers = {} + if token: + headers["Authorization"] = "Bearer {}".format(token) + response = app_client.get("/-/actor.json", headers=headers) + if should_work: + assert response.json == {"actor": {"id": "test", "dstok": True}} + else: + assert response.json == {"actor": None} From 0f013ff497df62e1dd2075777b9817555646010e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 19:43:55 -0700 Subject: [PATCH 1537/2303] Mechanism to prevent tokens creating tokens, closes #1857 --- datasette/default_permissions.py | 2 +- datasette/views/special.py | 4 ++++ docs/authentication.rst | 2 ++ tests/test_auth.py | 11 ++++++++++- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 4d836ddc..d908af7a 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -69,4 +69,4 @@ def actor_from_request(datasette, request): if expires_at is not None: if expires_at < time.time(): return None - return {"id": decoded["a"], "dstok": True} + return {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/views/special.py b/datasette/views/special.py index 7f70eb1f..91130353 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -177,6 +177,10 @@ class CreateTokenView(BaseView): raise Forbidden( "You must be logged in as an actor with an ID to create a token" ) + if request.actor.get("token"): + raise Forbidden( + "Token authentication cannot be used to create additional tokens" + ) async def get(self, request): self.check_permission(request) diff --git a/docs/authentication.rst b/docs/authentication.rst index fc903fbb..cbecd296 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -348,6 +348,8 @@ A token created by a user will include that user's ``"id"`` in the token payload Coming soon: a mechanism for creating tokens that can only perform a subset of the actions available to the user who created them. +This page cannot be accessed by actors with a ``"token": "some-value"`` property. This is to prevent API tokens from being used to automatically create more tokens. Datasette plugins that implement their own form of API token authentication should follow this convention. + .. _permissions_plugins: Checking permissions in plugins diff --git a/tests/test_auth.py b/tests/test_auth.py index be21d6a5..397d51d7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -180,6 +180,15 @@ def test_auth_create_token(app_client, post_data, errors, expected_duration): assert about_right - 2 < details["e"] < about_right + 2 +def test_auth_create_token_not_allowed_for_tokens(app_client): + ds_tok = app_client.ds.sign({"a": "test", "token": "dstok"}, "token") + response = app_client.get( + "/-/create-token", + headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, + ) + assert response.status == 403 + + @pytest.mark.parametrize( "scenario,should_work", ( @@ -207,6 +216,6 @@ def test_auth_with_dstok_token(app_client, scenario, should_work): headers["Authorization"] = "Bearer {}".format(token) response = app_client.get("/-/actor.json", headers=headers) if should_work: - assert response.json == {"actor": {"id": "test", "dstok": True}} + assert response.json == {"actor": {"id": "test", "token": "dstok"}} else: assert response.json == {"actor": None} From c23fa850e7f21977e367e3467656055216978e8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 19:55:47 -0700 Subject: [PATCH 1538/2303] allow_signed_tokens setting, closes #1856 --- datasette/app.py | 5 +++++ datasette/default_permissions.py | 2 ++ datasette/views/special.py | 2 ++ docs/authentication.rst | 2 ++ docs/cli-reference.rst | 2 ++ docs/plugins.rst | 1 + docs/settings.rst | 13 +++++++++++++ tests/test_auth.py | 26 +++++++++++++++++++++----- 8 files changed, 48 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index cab9d142..c868f8d3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -124,6 +124,11 @@ SETTINGS = ( True, "Allow users to download the original SQLite database files", ), + Setting( + "allow_signed_tokens", + True, + "Allow users to create and use signed API tokens", + ), Setting("suggest_facets", True, "Calculate and display suggested facets"), Setting( "default_cache_ttl", diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index d908af7a..49ca8851 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -52,6 +52,8 @@ def permission_allowed(datasette, actor, action, resource): @hookimpl def actor_from_request(datasette, request): prefix = "dstok_" + if not datasette.setting("allow_signed_tokens"): + return None authorization = request.headers.get("authorization") if not authorization: return None diff --git a/datasette/views/special.py b/datasette/views/special.py index 91130353..89015958 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -171,6 +171,8 @@ class CreateTokenView(BaseView): has_json_alternate = False def check_permission(self, request): + if not self.ds.setting("allow_signed_tokens"): + raise Forbidden("Signed tokens are not enabled for this Datasette instance") if not request.actor: raise Forbidden("You must be logged in to create a token") if not request.actor.get("id"): diff --git a/docs/authentication.rst b/docs/authentication.rst index cbecd296..50304ec5 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -350,6 +350,8 @@ Coming soon: a mechanism for creating tokens that can only perform a subset of t This page cannot be accessed by actors with a ``"token": "some-value"`` property. This is to prevent API tokens from being used to automatically create more tokens. Datasette plugins that implement their own form of API token authentication should follow this convention. +You can disable this feature using the :ref:`allow_signed_tokens ` setting. + .. _permissions_plugins: Checking permissions in plugins diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 4a8465cb..fd5e2404 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -226,6 +226,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam ?_facet= parameter (default=True) allow_download Allow users to download the original SQLite database files (default=True) + allow_signed_tokens Allow users to create and use signed API tokens + (default=True) suggest_facets Calculate and display suggested facets (default=True) default_cache_ttl Default HTTP cache TTL (used in Cache-Control: diff --git a/docs/plugins.rst b/docs/plugins.rst index 29078054..9efef32f 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -151,6 +151,7 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "templates": false, "version": null, "hooks": [ + "actor_from_request", "permission_allowed" ] }, diff --git a/docs/settings.rst b/docs/settings.rst index a6d50543..be640b21 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -169,6 +169,19 @@ Should users be able to download the original SQLite database using a link on th datasette mydatabase.db --setting allow_download off +.. _setting_allow_signed_tokens: + +allow_signed_tokens +~~~~~~~~~~~~~~~~~~~ + +Should users be able to create signed API tokens to access Datasette? + +This is turned on by default. Use the following to turn it off:: + + datasette mydatabase.db --setting allow_signed_tokens off + +Turning this setting off will disable the ``/-/create-token`` page, :ref:`described here `. It will also cause any incoming ``Authorization: Bearer dstok_...`` API tokens to be ignored. + .. _setting_default_cache_ttl: default_cache_ttl diff --git a/tests/test_auth.py b/tests/test_auth.py index 397d51d7..a79dafd8 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -189,9 +189,20 @@ def test_auth_create_token_not_allowed_for_tokens(app_client): assert response.status == 403 +def test_auth_create_token_not_allowed_if_allow_signed_tokens_off(app_client): + app_client.ds._settings["allow_signed_tokens"] = False + try: + ds_actor = app_client.actor_cookie({"id": "test"}) + response = app_client.get("/-/create-token", cookies={"ds_actor": ds_actor}) + assert response.status == 403 + finally: + app_client.ds._settings["allow_signed_tokens"] = True + + @pytest.mark.parametrize( "scenario,should_work", ( + ("allow_signed_tokens_off", False), ("no_token", False), ("invalid_token", False), ("expired_token", False), @@ -201,7 +212,7 @@ def test_auth_create_token_not_allowed_for_tokens(app_client): ) def test_auth_with_dstok_token(app_client, scenario, should_work): token = None - if scenario == "valid_unlimited_token": + if scenario in ("valid_unlimited_token", "allow_signed_tokens_off"): token = app_client.ds.sign({"a": "test"}, "token") elif scenario == "valid_expiring_token": token = app_client.ds.sign({"a": "test", "e": int(time.time()) + 1000}, "token") @@ -211,11 +222,16 @@ def test_auth_with_dstok_token(app_client, scenario, should_work): token = "invalid" if token: token = "dstok_{}".format(token) + if scenario == "allow_signed_tokens_off": + app_client.ds._settings["allow_signed_tokens"] = False headers = {} if token: headers["Authorization"] = "Bearer {}".format(token) response = app_client.get("/-/actor.json", headers=headers) - if should_work: - assert response.json == {"actor": {"id": "test", "token": "dstok"}} - else: - assert response.json == {"actor": None} + try: + if should_work: + assert response.json == {"actor": {"id": "test", "token": "dstok"}} + else: + assert response.json == {"actor": None} + finally: + app_client.ds._settings["allow_signed_tokens"] = True From c36a74ece1e475291af326d493d8db9ff3afdd30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 21:04:39 -0700 Subject: [PATCH 1539/2303] Try shutting down executor in tests to free up thread local SQLite connections, refs #1843 --- tests/fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 13a3dffa..d1afd2f3 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -166,6 +166,7 @@ def make_app_client( # Close the connection to avoid "too many open files" errors conn.close() os.remove(filepath) + ds.executor.shutdown() @pytest.fixture(scope="session") From c556fad65d8a45ce85027678796a12ac9107d9ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 21:25:47 -0700 Subject: [PATCH 1540/2303] Try to address too many files error again, refs #1843 --- tests/fixtures.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index d1afd2f3..92a10da6 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -131,10 +131,14 @@ def make_app_client( for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) + # Close the connection to avoid "too many open files" errors + conn.close() if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) - sqlite3.connect(extra_filepath).executescript(extra_sql) + c2 = sqlite3.connect(extra_filepath) + c2.executescript(extra_sql) + c2.close() # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) @@ -163,10 +167,7 @@ def make_app_client( crossdb=crossdb, ) yield TestClient(ds) - # Close the connection to avoid "too many open files" errors - conn.close() os.remove(filepath) - ds.executor.shutdown() @pytest.fixture(scope="session") From c7956eed7777c62653b4d508570c5d77cfead7d9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 21:26:12 -0700 Subject: [PATCH 1541/2303] datasette create-token command, refs #1859 --- datasette/default_permissions.py | 38 ++++++++++++++++++++++++++++ docs/authentication.rst | 23 +++++++++++++++++ docs/cli-reference.rst | 43 ++++++++++++++++++++++++++------ docs/plugins.rst | 3 ++- tests/test_api.py | 1 + tests/test_auth.py | 28 +++++++++++++++++++++ tests/test_plugins.py | 2 ++ 7 files changed, 130 insertions(+), 8 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 49ca8851..12499c16 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -1,6 +1,8 @@ from datasette import hookimpl from datasette.utils import actor_matches_allow +import click import itsdangerous +import json import time @@ -72,3 +74,39 @@ def actor_from_request(datasette, request): if expires_at < time.time(): return None return {"id": decoded["a"], "token": "dstok"} + + +@hookimpl +def register_commands(cli): + from datasette.app import Datasette + + @cli.command() + @click.argument("id") + @click.option( + "--secret", + help="Secret used for signing the API tokens", + envvar="DATASETTE_SECRET", + required=True, + ) + @click.option( + "-e", + "--expires-after", + help="Token should expire after this many seconds", + type=int, + ) + @click.option( + "--debug", + help="Show decoded token", + is_flag=True, + ) + def create_token(id, secret, expires_after, debug): + "Create a signed API token for the specified actor ID" + ds = Datasette(secret=secret) + bits = {"a": id, "token": "dstok"} + if expires_after: + bits["e"] = int(time.time()) + expires_after + token = ds.sign(bits, namespace="token") + click.echo("dstok_{}".format(token)) + if debug: + click.echo("\nDecoded:\n") + click.echo(json.dumps(ds.unsign(token, namespace="token"), indent=2)) diff --git a/docs/authentication.rst b/docs/authentication.rst index 50304ec5..0835e17c 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -352,6 +352,29 @@ This page cannot be accessed by actors with a ``"token": "some-value"`` property You can disable this feature using the :ref:`allow_signed_tokens ` setting. +.. _authentication_cli_create_token: + +datasette create-token +---------------------- + +You can also create tokens on the command line using the ``datasette create-token`` command. + +This command takes one required argument - the ID of the actor to be associated with the created token. + +You can specify an ``--expires-after`` option in seconds. If omitted, the token will never expire. + +The command will sign the token using the ``DATASETTE_SECRET`` environment variable, if available. You can also pass the secret using the ``--secret`` option. + +This means you can run the command locally to create tokens for use with a deployed Datasette instance, provided you know that instance's secret. + +To create a token for the ``root`` actor that will expire in one hour:: + + datasette create-token root --expires-after 3600 + +To create a secret that never expires using a specific secret:: + + datasette create-token root --secret my-secret-goes-here + .. _permissions_plugins: Checking permissions in plugins diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index fd5e2404..b40c6b2c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -47,13 +47,14 @@ Running ``datasette --help`` shows a list of all of the available commands. --help Show this message and exit. Commands: - serve* Serve up specified SQLite database files with a web UI - inspect Generate JSON summary of provided database files - install Install plugins and packages from PyPI into the same... - package Package SQLite files into a Datasette Docker container - plugins List currently installed plugins - publish Publish specified SQLite database files to the internet along... - uninstall Uninstall plugins and Python packages from the Datasette... + serve* Serve up specified SQLite database files with a web UI + create-token Create a signed API token for the specified actor ID + inspect Generate JSON summary of provided database files + install Install plugins and packages from PyPI into the same... + package Package SQLite files into a Datasette Docker container + plugins List currently installed plugins + publish Publish specified SQLite database files to the internet... + uninstall Uninstall plugins and Python packages from the Datasette... .. [[[end]]] @@ -591,3 +592,31 @@ This performance optimization is used automatically by some of the ``datasette p .. [[[end]]] + + +.. _cli_help_create_token___help: + +datasette create-token +====================== + +Create a signed API token, see :ref:`authentication_cli_create_token`. + +.. [[[cog + help(["create-token", "--help"]) +.. ]]] + +:: + + Usage: datasette create-token [OPTIONS] ID + + Create a signed API token for the specified actor ID + + Options: + --secret TEXT Secret used for signing the API tokens + [required] + -e, --expires-after INTEGER Token should expire after this many seconds + --debug Show decoded token + --help Show this message and exit. + + +.. [[[end]]] diff --git a/docs/plugins.rst b/docs/plugins.rst index 9efef32f..3ae42293 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -152,7 +152,8 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "version": null, "hooks": [ "actor_from_request", - "permission_allowed" + "permission_allowed", + "register_commands" ] }, { diff --git a/tests/test_api.py b/tests/test_api.py index ad74d16e..f7cbe950 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -806,6 +806,7 @@ def test_settings_json(app_client): "max_returned_rows": 100, "sql_time_limit_ms": 200, "allow_download": True, + "allow_signed_tokens": True, "allow_facet": True, "suggest_facets": True, "default_cache_ttl": 5, diff --git a/tests/test_auth.py b/tests/test_auth.py index a79dafd8..f2d82107 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,5 +1,7 @@ from .fixtures import app_client +from click.testing import CliRunner from datasette.utils import baseconv +from datasette.cli import cli import pytest import time @@ -235,3 +237,29 @@ def test_auth_with_dstok_token(app_client, scenario, should_work): assert response.json == {"actor": None} finally: app_client.ds._settings["allow_signed_tokens"] = True + + +@pytest.mark.parametrize("expires", (None, 1000, -1000)) +def test_cli_create_token(app_client, expires): + secret = app_client.ds._secret + runner = CliRunner(mix_stderr=False) + args = ["create-token", "--secret", secret, "test"] + if expires: + args += ["--expires-after", str(expires)] + result = runner.invoke(cli, args) + assert result.exit_code == 0 + token = result.output.strip() + assert token.startswith("dstok_") + details = app_client.ds.unsign(token[len("dstok_") :], "token") + expected_keys = {"a", "token"} + if expires: + expected_keys.add("e") + assert details.keys() == expected_keys + assert details["a"] == "test" + response = app_client.get( + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + ) + if expires is None or expires > 0: + assert response.json == {"actor": {"id": "test", "token": "dstok"}} + else: + assert response.json == {"actor": None} diff --git a/tests/test_plugins.py b/tests/test_plugins.py index e0a7bc76..de3fde8e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -971,6 +971,7 @@ def test_hook_register_commands(): "plugins", "publish", "uninstall", + "create-token", } # Now install a plugin @@ -1001,6 +1002,7 @@ def test_hook_register_commands(): "uninstall", "verify", "unverify", + "create-token", } pm.unregister(name="verify") importlib.reload(cli) From df7bf0b2fc262f0b025b3cdd283ff8ce60653175 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 14:13:31 -0700 Subject: [PATCH 1542/2303] Fix bug with breadcrumbs and request=None, closes #1849 --- datasette/app.py | 9 ++++++--- tests/test_internals_datasette.py | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 9df16558..246269f3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -633,15 +633,18 @@ class Datasette: async def _crumb_items(self, request, table=None, database=None): crumbs = [] + actor = None + if request: + actor = request.actor # Top-level link if await self.permission_allowed( - actor=request.actor, action="view-instance", default=True + actor=actor, action="view-instance", default=True ): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link if database: if await self.permission_allowed( - actor=request.actor, + actor=actor, action="view-database", resource=database, default=True, @@ -656,7 +659,7 @@ class Datasette: if table: assert database, "table= requires database=" if await self.permission_allowed( - actor=request.actor, + actor=actor, action="view-table", resource=(database, table), default=True, diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index c82cafb3..1b4732af 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -125,3 +125,12 @@ async def test_datasette_ensure_permissions_check_visibility( visible, private = await ds.check_visibility(actor, permissions=permissions) assert visible == should_allow assert private == expected_private + + +@pytest.mark.asyncio +async def test_datasette_render_template_no_request(): + # https://github.com/simonw/datasette/issues/1849 + ds = Datasette([], memory=True) + await ds.invoke_startup() + rendered = await ds.render_template("error.html") + assert "Error " in rendered From 55a709c480a1e7401b4ff6208f37a2cf7c682183 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 14:34:33 -0700 Subject: [PATCH 1543/2303] Allow leading comments on SQL queries, refs #1860 --- datasette/utils/__init__.py | 27 +++++++++++++++++++++------ tests/test_utils.py | 7 +++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 803ba96d..977a66d6 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -205,13 +205,28 @@ class InvalidSql(Exception): pass +# Allow SQL to start with a /* */ or -- comment +comment_re = ( + # Start of string, then any amount of whitespace + r"^(\s*" + + + # Comment that starts with -- and ends at a newline + r"(?:\-\-.*?\n\s*)" + + + # Comment that starts with /* and ends with */ + r"|(?:/\*[\s\S]*?\*/)" + + + # Whitespace + r")*\s*" +) + allowed_sql_res = [ - re.compile(r"^select\b"), - re.compile(r"^explain\s+select\b"), - re.compile(r"^explain\s+query\s+plan\s+select\b"), - re.compile(r"^with\b"), - re.compile(r"^explain\s+with\b"), - re.compile(r"^explain\s+query\s+plan\s+with\b"), + re.compile(comment_re + r"select\b"), + re.compile(comment_re + r"explain\s+select\b"), + re.compile(comment_re + r"explain\s+query\s+plan\s+select\b"), + re.compile(comment_re + r"with\b"), + re.compile(comment_re + r"explain\s+with\b"), + re.compile(comment_re + r"explain\s+query\s+plan\s+with\b"), ] allowed_pragmas = ( "database_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index d71a612d..e89f1e6b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -141,6 +141,7 @@ def test_custom_json_encoder(obj, expected): "update blah set some_column='# Hello there\n\n* This is a list\n* of items\n--\n[And a link](https://github.com/simonw/datasette-render-markdown).'\nas demo_markdown", "PRAGMA case_sensitive_like = true", "SELECT * FROM pragma_not_on_allow_list('idx52')", + "/* This comment is not valid. select 1", ], ) def test_validate_sql_select_bad(bad_sql): @@ -166,6 +167,12 @@ def test_validate_sql_select_bad(bad_sql): "explain query plan WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", "SELECT * FROM pragma_index_info('idx52')", "select * from pragma_table_xinfo('table')", + # Various types of comment + "-- comment\nselect 1", + "-- one line\n -- two line\nselect 1", + " /* comment */\nselect 1", + " /* comment */select 1", + "/* comment */\n -- another\n /* one more */ select 1", ], ) def test_validate_sql_select_good(good_sql): From 55f860c304aea813cb7ed740cc5625560a0722a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 14:13:31 -0700 Subject: [PATCH 1544/2303] Fix bug with breadcrumbs and request=None, closes #1849 --- datasette/app.py | 9 ++++++--- tests/test_internals_datasette.py | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c868f8d3..596ff44d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -639,15 +639,18 @@ class Datasette: async def _crumb_items(self, request, table=None, database=None): crumbs = [] + actor = None + if request: + actor = request.actor # Top-level link if await self.permission_allowed( - actor=request.actor, action="view-instance", default=True + actor=actor, action="view-instance", default=True ): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link if database: if await self.permission_allowed( - actor=request.actor, + actor=actor, action="view-database", resource=database, default=True, @@ -662,7 +665,7 @@ class Datasette: if table: assert database, "table= requires database=" if await self.permission_allowed( - actor=request.actor, + actor=actor, action="view-table", resource=(database, table), default=True, diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index c82cafb3..1b4732af 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -125,3 +125,12 @@ async def test_datasette_ensure_permissions_check_visibility( visible, private = await ds.check_visibility(actor, permissions=permissions) assert visible == should_allow assert private == expected_private + + +@pytest.mark.asyncio +async def test_datasette_render_template_no_request(): + # https://github.com/simonw/datasette/issues/1849 + ds = Datasette([], memory=True) + await ds.invoke_startup() + rendered = await ds.render_template("error.html") + assert "Error " in rendered From af5d5d0243631562ad83f2c318bff31a077feb5d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 14:34:33 -0700 Subject: [PATCH 1545/2303] Allow leading comments on SQL queries, refs #1860 --- datasette/utils/__init__.py | 27 +++++++++++++++++++++------ tests/test_utils.py | 7 +++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 803ba96d..977a66d6 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -205,13 +205,28 @@ class InvalidSql(Exception): pass +# Allow SQL to start with a /* */ or -- comment +comment_re = ( + # Start of string, then any amount of whitespace + r"^(\s*" + + + # Comment that starts with -- and ends at a newline + r"(?:\-\-.*?\n\s*)" + + + # Comment that starts with /* and ends with */ + r"|(?:/\*[\s\S]*?\*/)" + + + # Whitespace + r")*\s*" +) + allowed_sql_res = [ - re.compile(r"^select\b"), - re.compile(r"^explain\s+select\b"), - re.compile(r"^explain\s+query\s+plan\s+select\b"), - re.compile(r"^with\b"), - re.compile(r"^explain\s+with\b"), - re.compile(r"^explain\s+query\s+plan\s+with\b"), + re.compile(comment_re + r"select\b"), + re.compile(comment_re + r"explain\s+select\b"), + re.compile(comment_re + r"explain\s+query\s+plan\s+select\b"), + re.compile(comment_re + r"with\b"), + re.compile(comment_re + r"explain\s+with\b"), + re.compile(comment_re + r"explain\s+query\s+plan\s+with\b"), ] allowed_pragmas = ( "database_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index d71a612d..e89f1e6b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -141,6 +141,7 @@ def test_custom_json_encoder(obj, expected): "update blah set some_column='# Hello there\n\n* This is a list\n* of items\n--\n[And a link](https://github.com/simonw/datasette-render-markdown).'\nas demo_markdown", "PRAGMA case_sensitive_like = true", "SELECT * FROM pragma_not_on_allow_list('idx52')", + "/* This comment is not valid. select 1", ], ) def test_validate_sql_select_bad(bad_sql): @@ -166,6 +167,12 @@ def test_validate_sql_select_bad(bad_sql): "explain query plan WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", "SELECT * FROM pragma_index_info('idx52')", "select * from pragma_table_xinfo('table')", + # Various types of comment + "-- comment\nselect 1", + "-- one line\n -- two line\nselect 1", + " /* comment */\nselect 1", + " /* comment */select 1", + "/* comment */\n -- another\n /* one more */ select 1", ], ) def test_validate_sql_select_good(good_sql): From 382a87158337540f991c6dc887080f7b37c7c26e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 14:13:31 -0700 Subject: [PATCH 1546/2303] max_signed_tokens_ttl setting, closes #1858 Also redesigned token format to include creation time and optional duration. --- datasette/app.py | 5 ++++ datasette/default_permissions.py | 33 +++++++++++++++++---- datasette/views/special.py | 20 ++++++++----- docs/settings.rst | 15 ++++++++++ tests/test_api.py | 1 + tests/test_auth.py | 50 ++++++++++++++++++++++++-------- 6 files changed, 99 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 596ff44d..894d7f0f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -129,6 +129,11 @@ SETTINGS = ( True, "Allow users to create and use signed API tokens", ), + Setting( + "max_signed_tokens_ttl", + 0, + "Maximum allowed expiry time for signed API tokens", + ), Setting("suggest_facets", True, "Calculate and display suggested facets"), Setting( "default_cache_ttl", diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 12499c16..c502dd70 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -56,6 +56,7 @@ def actor_from_request(datasette, request): prefix = "dstok_" if not datasette.setting("allow_signed_tokens"): return None + max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") authorization = request.headers.get("authorization") if not authorization: return None @@ -69,11 +70,31 @@ def actor_from_request(datasette, request): decoded = datasette.unsign(token, namespace="token") except itsdangerous.BadSignature: return None - expires_at = decoded.get("e") - if expires_at is not None: - if expires_at < time.time(): + if "t" not in decoded: + # Missing timestamp + return None + created = decoded["t"] + if not isinstance(created, int): + # Invalid timestamp + return None + duration = decoded.get("d") + if duration is not None and not isinstance(duration, int): + # Invalid duration + return None + if (duration is None and max_signed_tokens_ttl) or ( + duration is not None + and max_signed_tokens_ttl + and duration > max_signed_tokens_ttl + ): + duration = max_signed_tokens_ttl + if duration: + if time.time() - created > duration: + # Expired return None - return {"id": decoded["a"], "token": "dstok"} + actor = {"id": decoded["a"], "token": "dstok"} + if duration: + actor["token_expires"] = created + duration + return actor @hookimpl @@ -102,9 +123,9 @@ def register_commands(cli): def create_token(id, secret, expires_after, debug): "Create a signed API token for the specified actor ID" ds = Datasette(secret=secret) - bits = {"a": id, "token": "dstok"} + bits = {"a": id, "token": "dstok", "t": int(time.time())} if expires_after: - bits["e"] = int(time.time()) + expires_after + bits["d"] = expires_after token = ds.sign(bits, namespace="token") click.echo("dstok_{}".format(token)) if debug: diff --git a/datasette/views/special.py b/datasette/views/special.py index 89015958..b754a2f0 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -195,20 +195,24 @@ class CreateTokenView(BaseView): async def post(self, request): self.check_permission(request) post = await request.post_vars() - expires = None errors = [] + duration = None if post.get("expire_type"): - duration = post.get("expire_duration") - if not duration or not duration.isdigit() or not int(duration) > 0: + duration_string = post.get("expire_duration") + if ( + not duration_string + or not duration_string.isdigit() + or not int(duration_string) > 0 + ): errors.append("Invalid expire duration") else: unit = post["expire_type"] if unit == "minutes": - expires = int(duration) * 60 + duration = int(duration_string) * 60 elif unit == "hours": - expires = int(duration) * 60 * 60 + duration = int(duration_string) * 60 * 60 elif unit == "days": - expires = int(duration) * 60 * 60 * 24 + duration = int(duration_string) * 60 * 60 * 24 else: errors.append("Invalid expire duration unit") token_bits = None @@ -216,8 +220,10 @@ class CreateTokenView(BaseView): if not errors: token_bits = { "a": request.actor["id"], - "e": (int(time.time()) + expires) if expires else None, + "t": int(time.time()), } + if duration: + token_bits["d"] = duration token = "dstok_{}".format(self.ds.sign(token_bits, "token")) return await self.render( ["create_token.html"], diff --git a/docs/settings.rst b/docs/settings.rst index be640b21..a990c78c 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -182,6 +182,21 @@ This is turned on by default. Use the following to turn it off:: Turning this setting off will disable the ``/-/create-token`` page, :ref:`described here `. It will also cause any incoming ``Authorization: Bearer dstok_...`` API tokens to be ignored. +.. _setting_max_signed_tokens_ttl: + +max_signed_tokens_ttl +~~~~~~~~~~~~~~~~~~~~~ + +Maximum allowed expiry time for signed API tokens created by users. + +Defaults to ``0`` which means no limit - tokens can be created that will never expire. + +Set this to a value in seconds to limit the maximum expiry time. For example, to set that limit to 24 hours you would use:: + + datasette mydatabase.db --setting max_signed_tokens_ttl 86400 + +This setting is enforced when incoming tokens are processed. + .. _setting_default_cache_ttl: default_cache_ttl diff --git a/tests/test_api.py b/tests/test_api.py index f7cbe950..fc171421 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -807,6 +807,7 @@ def test_settings_json(app_client): "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, + "max_signed_tokens_ttl": 0, "allow_facet": True, "suggest_facets": True, "default_cache_ttl": 5, diff --git a/tests/test_auth.py b/tests/test_auth.py index f2d82107..fa1b2e46 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -173,13 +173,19 @@ def test_auth_create_token(app_client, post_data, errors, expected_duration): # Extract token from page token = response2.text.split('value="dstok_')[1].split('"')[0] details = app_client.ds.unsign(token, "token") - assert details.keys() == {"a", "e"} + assert details.keys() == {"a", "t", "d"} or details.keys() == {"a", "t"} assert details["a"] == "test" if expected_duration is None: - assert details["e"] is None + assert "d" not in details else: - about_right = int(time.time()) + expected_duration - assert about_right - 2 < details["e"] < about_right + 2 + assert details["d"] == expected_duration + # And test that token + response3 = app_client.get( + "/-/actor.json", + headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, + ) + assert response3.status == 200 + assert response3.json["actor"]["id"] == "test" def test_auth_create_token_not_allowed_for_tokens(app_client): @@ -206,6 +212,7 @@ def test_auth_create_token_not_allowed_if_allow_signed_tokens_off(app_client): ( ("allow_signed_tokens_off", False), ("no_token", False), + ("no_timestamp", False), ("invalid_token", False), ("expired_token", False), ("valid_unlimited_token", True), @@ -214,12 +221,15 @@ def test_auth_create_token_not_allowed_if_allow_signed_tokens_off(app_client): ) def test_auth_with_dstok_token(app_client, scenario, should_work): token = None + _time = int(time.time()) if scenario in ("valid_unlimited_token", "allow_signed_tokens_off"): - token = app_client.ds.sign({"a": "test"}, "token") + token = app_client.ds.sign({"a": "test", "t": _time}, "token") elif scenario == "valid_expiring_token": - token = app_client.ds.sign({"a": "test", "e": int(time.time()) + 1000}, "token") + token = app_client.ds.sign({"a": "test", "t": _time - 50, "d": 1000}, "token") elif scenario == "expired_token": - token = app_client.ds.sign({"a": "test", "e": int(time.time()) - 1000}, "token") + token = app_client.ds.sign({"a": "test", "t": _time - 2000, "d": 1000}, "token") + elif scenario == "no_timestamp": + token = app_client.ds.sign({"a": "test"}, "token") elif scenario == "invalid_token": token = "invalid" if token: @@ -232,7 +242,16 @@ def test_auth_with_dstok_token(app_client, scenario, should_work): response = app_client.get("/-/actor.json", headers=headers) try: if should_work: - assert response.json == {"actor": {"id": "test", "token": "dstok"}} + assert response.json.keys() == {"actor"} + actor = response.json["actor"] + expected_keys = {"id", "token"} + if scenario != "valid_unlimited_token": + expected_keys.add("token_expires") + assert actor.keys() == expected_keys + assert actor["id"] == "test" + assert actor["token"] == "dstok" + if scenario != "valid_unlimited_token": + assert isinstance(actor["token_expires"], int) else: assert response.json == {"actor": None} finally: @@ -251,15 +270,22 @@ def test_cli_create_token(app_client, expires): token = result.output.strip() assert token.startswith("dstok_") details = app_client.ds.unsign(token[len("dstok_") :], "token") - expected_keys = {"a", "token"} + expected_keys = {"a", "token", "t"} if expires: - expected_keys.add("e") + expected_keys.add("d") assert details.keys() == expected_keys assert details["a"] == "test" response = app_client.get( "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) if expires is None or expires > 0: - assert response.json == {"actor": {"id": "test", "token": "dstok"}} + expected_actor = { + "id": "test", + "token": "dstok", + } + if expires and expires > 0: + expected_actor["token_expires"] = details["t"] + expires + assert response.json == {"actor": expected_actor} else: - assert response.json == {"actor": None} + expected_actor = None + assert response.json == {"actor": expected_actor} From 51c436fed29205721dcf17fa31d7e7090d34ebb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 20:57:02 -0700 Subject: [PATCH 1547/2303] First draft of insert row write API, refs #1851 --- datasette/default_permissions.py | 2 +- datasette/views/table.py | 76 +++++++++++++++++++++++++++----- docs/authentication.rst | 12 +++++ docs/cli-reference.rst | 2 + docs/json_api.rst | 38 ++++++++++++++++ 5 files changed, 119 insertions(+), 11 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index c502dd70..87684e2a 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -9,7 +9,7 @@ import time @hookimpl(tryfirst=True) def permission_allowed(datasette, actor, action, resource): async def inner(): - if action in ("permissions-debug", "debug-menu"): + if action in ("permissions-debug", "debug-menu", "insert-row"): if actor and actor.get("id") == "root": return True elif action == "view-instance": diff --git a/datasette/views/table.py b/datasette/views/table.py index f73b0957..74d1c532 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -28,7 +28,7 @@ from datasette.utils import ( urlsafe_components, value_as_boolean, ) -from datasette.utils.asgi import BadRequest, Forbidden, NotFound +from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters from .base import DataView, DatasetteError, ureg from .database import QueryView @@ -103,15 +103,71 @@ class TableView(DataView): canned_query = await self.ds.get_canned_query( database_name, table_name, request.actor ) - assert canned_query, "You may only POST to a canned query" - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=table_name, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), + if canned_query: + return await QueryView(self.ds).data( + request, + canned_query["sql"], + metadata=canned_query, + editable=False, + canned_query=table_name, + named_parameters=canned_query.get("params"), + write=bool(canned_query.get("write")), + ) + else: + # Handle POST to a table + return await self.table_post(request, database_name, table_name) + + async def table_post(self, request, database_name, table_name): + # Table must exist (may handle table creation in the future) + db = self.ds.get_database(database_name) + if not await db.table_exists(table_name): + raise NotFound("Table not found: {}".format(table_name)) + # Must have insert-row permission + if not await self.ds.permission_allowed( + request.actor, "insert-row", resource=(database_name, table_name) + ): + raise Forbidden("Permission denied") + if request.headers.get("content-type") != "application/json": + # TODO: handle form-encoded data + raise BadRequest("Must send JSON data") + data = json.loads(await request.post_body()) + if "row" not in data: + raise BadRequest('Must send "row" data') + row = data["row"] + if not isinstance(row, dict): + raise BadRequest("row must be a dictionary") + # Verify all columns exist + columns = await db.table_columns(table_name) + pks = await db.primary_keys(table_name) + for key in row: + if key not in columns: + raise BadRequest("Column not found: {}".format(key)) + if key in pks: + raise BadRequest( + "Cannot insert into primary key column: {}".format(key) + ) + # Perform the insert + sql = "INSERT INTO [{table}] ({columns}) VALUES ({values})".format( + table=escape_sqlite(table_name), + columns=", ".join(escape_sqlite(c) for c in row), + values=", ".join("?" for c in row), + ) + cursor = await db.execute_write(sql, list(row.values())) + # Return the new row + rowid = cursor.lastrowid + new_row = ( + await db.execute( + "SELECT * FROM [{table}] WHERE rowid = ?".format( + table=escape_sqlite(table_name) + ), + [rowid], + ) + ).first() + return Response.json( + { + "row": dict(new_row), + }, + status=201, ) async def columns_to_select(self, table_columns, pks, request): diff --git a/docs/authentication.rst b/docs/authentication.rst index 0835e17c..233a50d2 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -547,6 +547,18 @@ Actor is allowed to view (and execute) a :ref:`canned query ` pa Default *allow*. +.. _permissions_insert_row: + +insert-row +---------- + +Actor is allowed to insert rows into a table. + +``resource`` - tuple: (string, string) + The name of the database, then the name of the table + +Default *deny*. + .. _permissions_execute_sql: execute-sql diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index b40c6b2c..56156568 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -229,6 +229,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam database files (default=True) allow_signed_tokens Allow users to create and use signed API tokens (default=True) + max_signed_tokens_ttl Maximum allowed expiry time for signed API tokens + (default=0) suggest_facets Calculate and display suggested facets (default=True) default_cache_ttl Default HTTP cache TTL (used in Cache-Control: diff --git a/docs/json_api.rst b/docs/json_api.rst index d3fdb1e4..b339a738 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -455,3 +455,41 @@ You can find this near the top of the source code of those pages, looking like t The JSON URL is also made available in a ``Link`` HTTP header for the page:: Link: https://latest.datasette.io/fixtures/sortable.json; rel="alternate"; type="application/json+datasette" + +.. _json_api_write: + +The JSON write API +------------------ + +Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. + +.. _json_api_write_insert_row: + +Inserting a single row +~~~~~~~~~~~~~~~~~~~~~~ + +This requires the :ref:`permissions_insert_row` permission. + +:: + + POST // + Content-Type: application/json + Authorization: Bearer dstok_ + { + "row": { + "column1": "value1", + "column2": "value2" + } + } + +If successful, this will return a ``201`` status code and the newly inserted row, for example: + +.. code-block:: json + + { + "row": { + "id": 1, + "column1": "value1", + "column2": "value2" + } + } From f6ca86987ba9d7d48eccf2cfe0bfc94942003844 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 06:56:11 -0700 Subject: [PATCH 1548/2303] Delete mirror-master-and-main.yml Closes #1865 --- .github/workflows/mirror-master-and-main.yml | 21 -------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/mirror-master-and-main.yml diff --git a/.github/workflows/mirror-master-and-main.yml b/.github/workflows/mirror-master-and-main.yml deleted file mode 100644 index 8418df40..00000000 --- a/.github/workflows/mirror-master-and-main.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Mirror "master" and "main" branches -on: - push: - branches: - - master - - main - -jobs: - mirror: - runs-on: ubuntu-latest - steps: - - name: Mirror to "master" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: master - force: false - - name: Mirror to "main" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: main - force: false From 5f6be3c48b661f74198b8fc85361d3ad6657880e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:47:41 -0700 Subject: [PATCH 1549/2303] Better comment handling in SQL regex, refs #1860 --- datasette/utils/__init__.py | 9 +++++---- tests/test_utils.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 977a66d6..5acfb8b4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -208,16 +208,16 @@ class InvalidSql(Exception): # Allow SQL to start with a /* */ or -- comment comment_re = ( # Start of string, then any amount of whitespace - r"^(\s*" + r"^\s*(" + # Comment that starts with -- and ends at a newline r"(?:\-\-.*?\n\s*)" + - # Comment that starts with /* and ends with */ - r"|(?:/\*[\s\S]*?\*/)" + # Comment that starts with /* and ends with */ - but does not have */ in it + r"|(?:\/\*((?!\*\/)[\s\S])*\*\/)" + # Whitespace - r")*\s*" + r"\s*)*\s*" ) allowed_sql_res = [ @@ -228,6 +228,7 @@ allowed_sql_res = [ re.compile(comment_re + r"explain\s+with\b"), re.compile(comment_re + r"explain\s+query\s+plan\s+with\b"), ] + allowed_pragmas = ( "database_list", "foreign_key_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index e89f1e6b..c1589107 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,7 @@ def test_custom_json_encoder(obj, expected): "PRAGMA case_sensitive_like = true", "SELECT * FROM pragma_not_on_allow_list('idx52')", "/* This comment is not valid. select 1", + "/**/\nupdate foo set bar = 1\n/* test */ select 1", ], ) def test_validate_sql_select_bad(bad_sql): From d2ca13b699d441a201c55cb72ff96919d3cd22bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:50:54 -0700 Subject: [PATCH 1550/2303] Add test for /* multi line */ comment, refs #1860 --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index c1589107..8b64f865 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -174,6 +174,7 @@ def test_validate_sql_select_bad(bad_sql): " /* comment */\nselect 1", " /* comment */select 1", "/* comment */\n -- another\n /* one more */ select 1", + "/* This comment \n has multiple lines */\nselect 1", ], ) def test_validate_sql_select_good(good_sql): From 918f3561208ee58c44773d30e21bace7d7c7cf3b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 06:56:11 -0700 Subject: [PATCH 1551/2303] Delete mirror-master-and-main.yml Closes #1865 --- .github/workflows/mirror-master-and-main.yml | 21 -------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/mirror-master-and-main.yml diff --git a/.github/workflows/mirror-master-and-main.yml b/.github/workflows/mirror-master-and-main.yml deleted file mode 100644 index 8418df40..00000000 --- a/.github/workflows/mirror-master-and-main.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Mirror "master" and "main" branches -on: - push: - branches: - - master - - main - -jobs: - mirror: - runs-on: ubuntu-latest - steps: - - name: Mirror to "master" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: master - force: false - - name: Mirror to "main" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: main - force: false From b597bb6b3e7c4b449654bbfa5b01ceff3eb3cb33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:47:41 -0700 Subject: [PATCH 1552/2303] Better comment handling in SQL regex, refs #1860 --- datasette/utils/__init__.py | 9 +++++---- tests/test_utils.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 977a66d6..5acfb8b4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -208,16 +208,16 @@ class InvalidSql(Exception): # Allow SQL to start with a /* */ or -- comment comment_re = ( # Start of string, then any amount of whitespace - r"^(\s*" + r"^\s*(" + # Comment that starts with -- and ends at a newline r"(?:\-\-.*?\n\s*)" + - # Comment that starts with /* and ends with */ - r"|(?:/\*[\s\S]*?\*/)" + # Comment that starts with /* and ends with */ - but does not have */ in it + r"|(?:\/\*((?!\*\/)[\s\S])*\*\/)" + # Whitespace - r")*\s*" + r"\s*)*\s*" ) allowed_sql_res = [ @@ -228,6 +228,7 @@ allowed_sql_res = [ re.compile(comment_re + r"explain\s+with\b"), re.compile(comment_re + r"explain\s+query\s+plan\s+with\b"), ] + allowed_pragmas = ( "database_list", "foreign_key_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index e89f1e6b..c1589107 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,7 @@ def test_custom_json_encoder(obj, expected): "PRAGMA case_sensitive_like = true", "SELECT * FROM pragma_not_on_allow_list('idx52')", "/* This comment is not valid. select 1", + "/**/\nupdate foo set bar = 1\n/* test */ select 1", ], ) def test_validate_sql_select_bad(bad_sql): From 6958e21b5c2012adf5655d2512cb4106490d10f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:50:54 -0700 Subject: [PATCH 1553/2303] Add test for /* multi line */ comment, refs #1860 --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index c1589107..8b64f865 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -174,6 +174,7 @@ def test_validate_sql_select_bad(bad_sql): " /* comment */\nselect 1", " /* comment */select 1", "/* comment */\n -- another\n /* one more */ select 1", + "/* This comment \n has multiple lines */\nselect 1", ], ) def test_validate_sql_select_good(good_sql): From a51608090b5ee37593078f71d18b33767ef3af79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 12:06:18 -0700 Subject: [PATCH 1554/2303] Slight tweak to insert row API design, refs #1851 https://github.com/simonw/datasette/issues/1851#issuecomment-1292997608 --- datasette/views/table.py | 10 +++++----- docs/json_api.rst | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 74d1c532..056b7b04 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -131,11 +131,11 @@ class TableView(DataView): # TODO: handle form-encoded data raise BadRequest("Must send JSON data") data = json.loads(await request.post_body()) - if "row" not in data: - raise BadRequest('Must send "row" data') - row = data["row"] + if "insert" not in data: + raise BadRequest('Must send a "insert" key containing a dictionary') + row = data["insert"] if not isinstance(row, dict): - raise BadRequest("row must be a dictionary") + raise BadRequest("insert must be a dictionary") # Verify all columns exist columns = await db.table_columns(table_name) pks = await db.primary_keys(table_name) @@ -165,7 +165,7 @@ class TableView(DataView): ).first() return Response.json( { - "row": dict(new_row), + "inserted_row": dict(new_row), }, status=201, ) diff --git a/docs/json_api.rst b/docs/json_api.rst index b339a738..2ed8a354 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -476,7 +476,7 @@ This requires the :ref:`permissions_insert_row` permission. Content-Type: application/json Authorization: Bearer dstok_ { - "row": { + "insert": { "column1": "value1", "column2": "value2" } @@ -487,7 +487,7 @@ If successful, this will return a ``201`` status code and the newly inserted row .. code-block:: json { - "row": { + "inserted_row": { "id": 1, "column1": "value1", "column2": "value2" From a2a5dff709c6f1676ac30b5e734c2763002562cf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 12:08:26 -0700 Subject: [PATCH 1555/2303] Missing tests for insert row API, refs #1851 --- tests/test_api_write.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/test_api_write.py diff --git a/tests/test_api_write.py b/tests/test_api_write.py new file mode 100644 index 00000000..86c221d0 --- /dev/null +++ b/tests/test_api_write.py @@ -0,0 +1,38 @@ +from datasette.app import Datasette +from datasette.utils import sqlite3 +import pytest +import time + + +@pytest.fixture +def ds_write(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + db = sqlite3.connect(str(db_path)) + db.execute("vacuum") + db.execute("create table docs (id integer primary key, title text, score float)") + ds = Datasette([db_path]) + yield ds + db.close() + + +@pytest.mark.asyncio +async def test_write_row(ds_write): + token = "dstok_{}".format( + ds_write.sign( + {"a": "root", "token": "dstok", "t": int(time.time())}, namespace="token" + ) + ) + response = await ds_write.client.post( + "/data/docs", + json={"insert": {"title": "Test", "score": 1.0}}, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + expected_row = {"id": 1, "title": "Test", "score": 1.0} + assert response.status_code == 201 + assert response.json()["inserted_row"] == expected_row + rows = (await ds_write.get_database("data").execute("select * from docs")).rows + assert dict(rows[0]) == expected_row From 6e788b49edf4f842c0817f006eb9d865778eea5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 13:17:18 -0700 Subject: [PATCH 1556/2303] New URL design /db/table/-/insert, refs #1851 --- datasette/app.py | 6 +++- datasette/views/table.py | 69 +++++++++++++++++++++++++++++++++++++++- docs/json_api.rst | 18 ++++++----- tests/test_api_write.py | 6 ++-- 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 894d7f0f..8bc5fe36 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -39,7 +39,7 @@ from .views.special import ( PermissionsDebugView, MessagesDebugView, ) -from .views.table import TableView +from .views.table import TableView, TableInsertView from .views.row import RowView from .renderer import json_renderer from .url_builder import Urls @@ -1262,6 +1262,10 @@ class Datasette: RowView.as_view(self), r"/(?P[^\/\.]+)/(?P
    [^/]+?)/(?P[^/]+?)(\.(?P\w+))?$", ) + add_route( + TableInsertView.as_view(self), + r"/(?P[^\/\.]+)/(?P
    [^\/\.]+)/-/insert$", + ) return [ # Compile any strings to regular expressions ((re.compile(pattern) if isinstance(pattern, str) else pattern), view) diff --git a/datasette/views/table.py b/datasette/views/table.py index 056b7b04..be3d4f93 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -30,7 +30,7 @@ from datasette.utils import ( ) from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters -from .base import DataView, DatasetteError, ureg +from .base import BaseView, DataView, DatasetteError, ureg from .database import QueryView LINK_WITH_LABEL = ( @@ -1077,3 +1077,70 @@ async def display_columns_and_rows( } columns = [first_column] + columns return columns, cell_rows + + +class TableInsertView(BaseView): + name = "table-insert" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + database_route = tilde_decode(request.url_vars["database"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database_name = db.name + table_name = tilde_decode(request.url_vars["table"]) + # Table must exist (may handle table creation in the future) + db = self.ds.get_database(database_name) + if not await db.table_exists(table_name): + raise NotFound("Table not found: {}".format(table_name)) + # Must have insert-row permission + if not await self.ds.permission_allowed( + request.actor, "insert-row", resource=(database_name, table_name) + ): + raise Forbidden("Permission denied") + if request.headers.get("content-type") != "application/json": + # TODO: handle form-encoded data + raise BadRequest("Must send JSON data") + data = json.loads(await request.post_body()) + if "row" not in data: + raise BadRequest('Must send a "row" key containing a dictionary') + row = data["row"] + if not isinstance(row, dict): + raise BadRequest("row must be a dictionary") + # Verify all columns exist + columns = await db.table_columns(table_name) + pks = await db.primary_keys(table_name) + for key in row: + if key not in columns: + raise BadRequest("Column not found: {}".format(key)) + if key in pks: + raise BadRequest( + "Cannot insert into primary key column: {}".format(key) + ) + # Perform the insert + sql = "INSERT INTO [{table}] ({columns}) VALUES ({values})".format( + table=escape_sqlite(table_name), + columns=", ".join(escape_sqlite(c) for c in row), + values=", ".join("?" for c in row), + ) + cursor = await db.execute_write(sql, list(row.values())) + # Return the new row + rowid = cursor.lastrowid + new_row = ( + await db.execute( + "SELECT * FROM [{table}] WHERE rowid = ?".format( + table=escape_sqlite(table_name) + ), + [rowid], + ) + ).first() + return Response.json( + { + "inserted": [dict(new_row)], + }, + status=201, + ) diff --git a/docs/json_api.rst b/docs/json_api.rst index 2ed8a354..4a7961f2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -463,7 +463,7 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. -.. _json_api_write_insert_row: +.. _TableInsertView: Inserting a single row ~~~~~~~~~~~~~~~~~~~~~~ @@ -472,11 +472,11 @@ This requires the :ref:`permissions_insert_row` permission. :: - POST //
    + POST //
    /-/insert Content-Type: application/json Authorization: Bearer dstok_ { - "insert": { + "row": { "column1": "value1", "column2": "value2" } @@ -487,9 +487,11 @@ If successful, this will return a ``201`` status code and the newly inserted row .. code-block:: json { - "inserted_row": { - "id": 1, - "column1": "value1", - "column2": "value2" - } + "inserted": [ + { + "id": 1, + "column1": "value1", + "column2": "value2" + } + ] } diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 86c221d0..e8222e43 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -24,8 +24,8 @@ async def test_write_row(ds_write): ) ) response = await ds_write.client.post( - "/data/docs", - json={"insert": {"title": "Test", "score": 1.0}}, + "/data/docs/-/insert", + json={"row": {"title": "Test", "score": 1.0}}, headers={ "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", @@ -33,6 +33,6 @@ async def test_write_row(ds_write): ) expected_row = {"id": 1, "title": "Test", "score": 1.0} assert response.status_code == 201 - assert response.json()["inserted_row"] == expected_row + assert response.json()["inserted"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row From b912d92b651c4f0b5137da924d135654511f0fe0 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 27 Oct 2022 16:51:20 -0400 Subject: [PATCH 1557/2303] Make hash and size a lazy property (#1837) * use inspect data for hash and file size * make hash and cached_size lazy properties * move hash property near size --- datasette/database.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d75bd70c..af1df0a8 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -39,7 +39,7 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.hash = None + self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None @@ -47,14 +47,6 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - if not self.is_mutable and not self.is_memory: - if self.ds.inspect_data and self.ds.inspect_data.get(self.name): - self.hash = self.ds.inspect_data[self.name]["hash"] - self.cached_size = self.ds.inspect_data[self.name]["size"] - else: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size @property def cached_table_counts(self): @@ -266,14 +258,34 @@ class Database: results = await self.execute_fn(sql_operation_in_thread) return results + @property + def hash(self): + if self.cached_hash is not None: + return self.cached_hash + elif self.is_mutable or self.is_memory: + return None + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_hash = self.ds.inspect_data[self.name]["hash"] + return self.cached_hash + else: + p = Path(self.path) + self.cached_hash = inspect_hash(p) + return self.cached_hash + @property def size(self): - if self.is_memory: - return 0 if self.cached_size is not None: return self.cached_size - else: + elif self.is_memory: + return 0 + elif self.is_mutable: return Path(self.path).stat().st_size + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_size = self.ds.inspect_data[self.name]["size"] + return self.cached_size + else: + self.cached_size = Path(self.path).stat().st_size + return self.cached_size async def table_counts(self, limit=10): if not self.is_mutable and self.cached_table_counts is not None: From 2c36e45447494cd7505440943367e29ec57c8e72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 13:51:45 -0700 Subject: [PATCH 1558/2303] Bump black from 22.8.0 to 22.10.0 (#1839) Bumps [black](https://github.com/psf/black) from 22.8.0 to 22.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.8.0...22.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fe258adb..625557ae 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.8.0", + "black==22.10.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From e5e0459a0b60608cb5e9ff83f6b41f59e6cafdfd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 13:58:00 -0700 Subject: [PATCH 1559/2303] Release notes for 0.63, refs #1869 --- docs/changelog.rst | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2255dcce..01957e4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,36 +4,42 @@ Changelog ========= -.. _v0_63a1: +.. _v0_63: -0.63a1 (2022-10-23) -------------------- +0.63 (2022-10-27) +----------------- +Features +~~~~~~~~ + +- Now tested against Python 3.11. Docker containers used by ``datasette publish`` and ``datasette package`` both now use that version of Python. (:issue:`1853`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- The :ref:`setting_truncate_cells_html` setting now also affects long URLs in columns. (:issue:`1805`) +- The non-JavaScript SQL editor textarea now increases height to fit the SQL query. (:issue:`1786`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) +- SQL queries can now include leading SQL comments, using ``/* ... */`` or ``-- ...`` syntax. Thanks, Charles Nepote. (:issue:`1860`) - SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`) -- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) - The :ref:`inspect data ` mechanism is now used to speed up server startup - thanks, Forest Gregg. (:issue:`1834`) - In :ref:`config_dir` databases with filenames ending in ``.sqlite`` or ``.sqlite3`` are now automatically added to the Datasette instance. (:issue:`1646`) - Breadcrumb navigation display now respects the current user's permissions. (:issue:`1831`) -- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) -- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - -.. _v0_63a0: - -0.63a0 (2022-09-26) -------------------- +Plugin hooks and internals +~~~~~~~~~~~~~~~~~~~~~~~~~~ - The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) -- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) -- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. -- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) -- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) - ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) -- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) -- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - Datasette no longer enforces upper bounds on its dependencies. (:issue:`1800`) -- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) -- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + +Documentation +~~~~~~~~~~~~~ + +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. +- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) +- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) .. _v0_62: From bf00b0b59b6692bdec597ac9db4e0b497c5a47b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 15:11:26 -0700 Subject: [PATCH 1560/2303] Release 0.63 Refs #1646, #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816, #1819, #1825, #1829, #1831, #1834, #1844, #1853, #1860 Closes #1869 --- datasette/version.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index eb36da45..ac012640 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.63a1" +__version__ = "0.63" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01957e4f..f573afb3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Changelog 0.63 (2022-10-27) ----------------- +See `Datasette 0.63: The annotated release notes `__ for more background on the changes in this release. + Features ~~~~~~~~ From 2ea60e12d90b7cec03ebab728854d3ec4d553f54 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 27 Oct 2022 16:51:20 -0400 Subject: [PATCH 1561/2303] Make hash and size a lazy property (#1837) * use inspect data for hash and file size * make hash and cached_size lazy properties * move hash property near size --- datasette/database.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d75bd70c..af1df0a8 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -39,7 +39,7 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.hash = None + self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None @@ -47,14 +47,6 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - if not self.is_mutable and not self.is_memory: - if self.ds.inspect_data and self.ds.inspect_data.get(self.name): - self.hash = self.ds.inspect_data[self.name]["hash"] - self.cached_size = self.ds.inspect_data[self.name]["size"] - else: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size @property def cached_table_counts(self): @@ -266,14 +258,34 @@ class Database: results = await self.execute_fn(sql_operation_in_thread) return results + @property + def hash(self): + if self.cached_hash is not None: + return self.cached_hash + elif self.is_mutable or self.is_memory: + return None + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_hash = self.ds.inspect_data[self.name]["hash"] + return self.cached_hash + else: + p = Path(self.path) + self.cached_hash = inspect_hash(p) + return self.cached_hash + @property def size(self): - if self.is_memory: - return 0 if self.cached_size is not None: return self.cached_size - else: + elif self.is_memory: + return 0 + elif self.is_mutable: return Path(self.path).stat().st_size + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_size = self.ds.inspect_data[self.name]["size"] + return self.cached_size + else: + self.cached_size = Path(self.path).stat().st_size + return self.cached_size async def table_counts(self, limit=10): if not self.is_mutable and self.cached_table_counts is not None: From 641bc4453b5ef1dff0b2fc7dfad0b692be7aa61c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 13:51:45 -0700 Subject: [PATCH 1562/2303] Bump black from 22.8.0 to 22.10.0 (#1839) Bumps [black](https://github.com/psf/black) from 22.8.0 to 22.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.8.0...22.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fe258adb..625557ae 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.8.0", + "black==22.10.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From 26af9b9c4a6c62ee15870caa1c7bc455165d3b11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 13:58:00 -0700 Subject: [PATCH 1563/2303] Release notes for 0.63, refs #1869 --- docs/changelog.rst | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2255dcce..01957e4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,36 +4,42 @@ Changelog ========= -.. _v0_63a1: +.. _v0_63: -0.63a1 (2022-10-23) -------------------- +0.63 (2022-10-27) +----------------- +Features +~~~~~~~~ + +- Now tested against Python 3.11. Docker containers used by ``datasette publish`` and ``datasette package`` both now use that version of Python. (:issue:`1853`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- The :ref:`setting_truncate_cells_html` setting now also affects long URLs in columns. (:issue:`1805`) +- The non-JavaScript SQL editor textarea now increases height to fit the SQL query. (:issue:`1786`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) +- SQL queries can now include leading SQL comments, using ``/* ... */`` or ``-- ...`` syntax. Thanks, Charles Nepote. (:issue:`1860`) - SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`) -- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) - The :ref:`inspect data ` mechanism is now used to speed up server startup - thanks, Forest Gregg. (:issue:`1834`) - In :ref:`config_dir` databases with filenames ending in ``.sqlite`` or ``.sqlite3`` are now automatically added to the Datasette instance. (:issue:`1646`) - Breadcrumb navigation display now respects the current user's permissions. (:issue:`1831`) -- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) -- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - -.. _v0_63a0: - -0.63a0 (2022-09-26) -------------------- +Plugin hooks and internals +~~~~~~~~~~~~~~~~~~~~~~~~~~ - The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) -- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) -- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. -- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) -- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) - ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) -- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) -- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - Datasette no longer enforces upper bounds on its dependencies. (:issue:`1800`) -- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) -- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + +Documentation +~~~~~~~~~~~~~ + +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. +- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) +- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) .. _v0_62: From 61171f01549549e5fb25c72b13280d941d96dbf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 15:11:26 -0700 Subject: [PATCH 1564/2303] Release 0.63 Refs #1646, #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816, #1819, #1825, #1829, #1831, #1834, #1844, #1853, #1860 Closes #1869 --- datasette/version.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index eb36da45..ac012640 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.63a1" +__version__ = "0.63" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01957e4f..f573afb3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Changelog 0.63 (2022-10-27) ----------------- +See `Datasette 0.63: The annotated release notes `__ for more background on the changes in this release. + Features ~~~~~~~~ From c9b5f5d598e7f85cd3e1ce020351a27da334408b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 17:58:36 -0700 Subject: [PATCH 1565/2303] Depend on sqlite-utils>=3.30 Decided to use the most recent version in case I decide later to use the flatten() utility function. Refs #1850 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 625557ae..99e2a4ad 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", + "sqlite-utils>=3.30", ], entry_points=""" [console_scripts] From c35859ae3df163406f1a1895ccf9803e933b2d8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 29 Oct 2022 23:03:45 -0700 Subject: [PATCH 1566/2303] API for bulk inserts, closes #1866 --- datasette/app.py | 5 ++ datasette/views/table.py | 136 +++++++++++++++++++++---------- docs/cli-reference.rst | 2 + docs/json_api.rst | 48 ++++++++++- docs/settings.rst | 11 +++ tests/test_api.py | 1 + tests/test_api_write.py | 168 +++++++++++++++++++++++++++++++++++++-- 7 files changed, 320 insertions(+), 51 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8bc5fe36..f80d3792 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -99,6 +99,11 @@ SETTINGS = ( 1000, "Maximum rows that can be returned from a table or custom query", ), + Setting( + "max_insert_rows", + 100, + "Maximum rows that can be inserted at a time using the bulk insert API", + ), Setting( "num_sql_threads", 3, diff --git a/datasette/views/table.py b/datasette/views/table.py index be3d4f93..fd203036 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -30,6 +30,7 @@ from datasette.utils import ( ) from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters +import sqlite_utils from .base import BaseView, DataView, DatasetteError, ureg from .database import QueryView @@ -1085,62 +1086,109 @@ class TableInsertView(BaseView): def __init__(self, datasette): self.ds = datasette + async def _validate_data(self, request, db, table_name): + errors = [] + + def _errors(errors): + return None, errors, {} + + if request.headers.get("content-type") != "application/json": + # TODO: handle form-encoded data + return _errors(["Invalid content-type, must be application/json"]) + body = await request.post_body() + try: + data = json.loads(body) + except json.JSONDecodeError as e: + return _errors(["Invalid JSON: {}".format(e)]) + if not isinstance(data, dict): + return _errors(["JSON must be a dictionary"]) + keys = data.keys() + # keys must contain "row" or "rows" + if "row" not in keys and "rows" not in keys: + return _errors(['JSON must have one or other of "row" or "rows"']) + rows = [] + if "row" in keys: + if "rows" in keys: + return _errors(['Cannot use "row" and "rows" at the same time']) + row = data["row"] + if not isinstance(row, dict): + return _errors(['"row" must be a dictionary']) + rows = [row] + data["return_rows"] = True + else: + rows = data["rows"] + if not isinstance(rows, list): + return _errors(['"rows" must be a list']) + for row in rows: + if not isinstance(row, dict): + return _errors(['"rows" must be a list of dictionaries']) + # Does this exceed max_insert_rows? + max_insert_rows = self.ds.setting("max_insert_rows") + if len(rows) > max_insert_rows: + return _errors( + ["Too many rows, maximum allowed is {}".format(max_insert_rows)] + ) + # Validate columns of each row + columns = await db.table_columns(table_name) + # TODO: There are cases where pks are OK, if not using auto-incrementing pk + pks = await db.primary_keys(table_name) + allowed_columns = set(columns) - set(pks) + for i, row in enumerate(rows): + invalid_columns = set(row.keys()) - allowed_columns + if invalid_columns: + errors.append( + "Row {} has invalid columns: {}".format( + i, ", ".join(sorted(invalid_columns)) + ) + ) + if errors: + return _errors(errors) + extra = {key: data[key] for key in data if key not in ("rows", "row")} + return rows, errors, extra + async def post(self, request): + def _error(messages, status=400): + return Response.json({"ok": False, "errors": messages}, status=status) + database_route = tilde_decode(request.url_vars["database"]) try: db = self.ds.get_database(route=database_route) except KeyError: - raise NotFound("Database not found: {}".format(database_route)) + return _error(["Database not found: {}".format(database_route)], 404) database_name = db.name table_name = tilde_decode(request.url_vars["table"]) + # Table must exist (may handle table creation in the future) db = self.ds.get_database(database_name) if not await db.table_exists(table_name): - raise NotFound("Table not found: {}".format(table_name)) + return _error(["Table not found: {}".format(table_name)], 404) # Must have insert-row permission if not await self.ds.permission_allowed( request.actor, "insert-row", resource=(database_name, table_name) ): - raise Forbidden("Permission denied") - if request.headers.get("content-type") != "application/json": - # TODO: handle form-encoded data - raise BadRequest("Must send JSON data") - data = json.loads(await request.post_body()) - if "row" not in data: - raise BadRequest('Must send a "row" key containing a dictionary') - row = data["row"] - if not isinstance(row, dict): - raise BadRequest("row must be a dictionary") - # Verify all columns exist - columns = await db.table_columns(table_name) - pks = await db.primary_keys(table_name) - for key in row: - if key not in columns: - raise BadRequest("Column not found: {}".format(key)) - if key in pks: - raise BadRequest( - "Cannot insert into primary key column: {}".format(key) + return _error(["Permission denied"], 403) + rows, errors, extra = await self._validate_data(request, db, table_name) + if errors: + return _error(errors, 400) + + should_return = bool(extra.get("return_rows", False)) + # Insert rows + def insert_rows(conn): + table = sqlite_utils.Database(conn)[table_name] + if should_return: + rowids = [] + for row in rows: + rowids.append(table.insert(row).last_rowid) + return list( + table.rows_where( + "rowid in ({})".format(",".join("?" for _ in rowids)), rowids + ) ) - # Perform the insert - sql = "INSERT INTO [{table}] ({columns}) VALUES ({values})".format( - table=escape_sqlite(table_name), - columns=", ".join(escape_sqlite(c) for c in row), - values=", ".join("?" for c in row), - ) - cursor = await db.execute_write(sql, list(row.values())) - # Return the new row - rowid = cursor.lastrowid - new_row = ( - await db.execute( - "SELECT * FROM [{table}] WHERE rowid = ?".format( - table=escape_sqlite(table_name) - ), - [rowid], - ) - ).first() - return Response.json( - { - "inserted": [dict(new_row)], - }, - status=201, - ) + else: + table.insert_all(rows) + + rows = await db.execute_write_fn(insert_rows) + result = {"ok": True} + if should_return: + result["inserted"] = rows + return Response.json(result, status=201) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 56156568..649a3dcd 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -213,6 +213,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam (default=100) max_returned_rows Maximum rows that can be returned from a table or custom query (default=1000) + max_insert_rows Maximum rows that can be inserted at a time using + the bulk insert API (default=1000) num_sql_threads Number of threads in the thread pool for executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds diff --git a/docs/json_api.rst b/docs/json_api.rst index 4a7961f2..01558c23 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -465,11 +465,13 @@ Datasette provides a write API for JSON data. This is a POST-only API that requi .. _TableInsertView: -Inserting a single row -~~~~~~~~~~~~~~~~~~~~~~ +Inserting rows +~~~~~~~~~~~~~~ This requires the :ref:`permissions_insert_row` permission. +A single row can be inserted using the ``"row"`` key: + :: POST //
    /-/insert @@ -495,3 +497,45 @@ If successful, this will return a ``201`` status code and the newly inserted row } ] } + +To insert multiple rows at a time, use the same API method but send a list of dictionaries as the ``"rows"`` key: + +:: + + POST //
    /-/insert + Content-Type: application/json + Authorization: Bearer dstok_ + { + "rows": [ + { + "column1": "value1", + "column2": "value2" + }, + { + "column1": "value3", + "column2": "value4" + } + ] + } + +If successful, this will return a ``201`` status code and an empty ``{}`` response body. + +To return the newly inserted rows, add the ``"return_rows": true`` key to the request body: + +.. code-block:: json + + { + "rows": [ + { + "column1": "value1", + "column2": "value2" + }, + { + "column1": "value3", + "column2": "value4" + } + ], + "return_rows": true + } + +This will return the same ``"inserted"`` key as the single row example above. There is a small performance penalty for using this option. diff --git a/docs/settings.rst b/docs/settings.rst index a990c78c..b86b18bd 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -96,6 +96,17 @@ You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_returned_rows 2000 +.. _setting_max_insert_rows: + +max_insert_rows +~~~~~~~~~~~~~~~ + +Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100. + +You can increase or decrease this limit like so:: + + datasette mydatabase.db --setting max_insert_rows 1000 + .. _setting_num_sql_threads: num_sql_threads diff --git a/tests/test_api.py b/tests/test_api.py index fc171421..ebd675b9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -804,6 +804,7 @@ def test_settings_json(app_client): "facet_suggest_time_limit_ms": 50, "facet_time_limit_ms": 200, "max_returned_rows": 100, + "max_insert_rows": 100, "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, diff --git a/tests/test_api_write.py b/tests/test_api_write.py index e8222e43..4a5a58aa 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -18,11 +18,7 @@ def ds_write(tmp_path_factory): @pytest.mark.asyncio async def test_write_row(ds_write): - token = "dstok_{}".format( - ds_write.sign( - {"a": "root", "token": "dstok", "t": int(time.time())}, namespace="token" - ) - ) + token = write_token(ds_write) response = await ds_write.client.post( "/data/docs/-/insert", json={"row": {"title": "Test", "score": 1.0}}, @@ -36,3 +32,165 @@ async def test_write_row(ds_write): assert response.json()["inserted"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row + + +@pytest.mark.asyncio +@pytest.mark.parametrize("return_rows", (True, False)) +async def test_write_rows(ds_write, return_rows): + token = write_token(ds_write) + data = {"rows": [{"title": "Test {}".format(i), "score": 1.0} for i in range(20)]} + if return_rows: + data["return_rows"] = True + response = await ds_write.client.post( + "/data/docs/-/insert", + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 201 + actual_rows = [ + dict(r) + for r in ( + await ds_write.get_database("data").execute("select * from docs") + ).rows + ] + assert len(actual_rows) == 20 + assert actual_rows == [ + {"id": i + 1, "title": "Test {}".format(i), "score": 1.0} for i in range(20) + ] + assert response.json()["ok"] is True + if return_rows: + assert response.json()["inserted"] == actual_rows + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,input,special_case,expected_status,expected_errors", + ( + ( + "/data2/docs/-/insert", + {}, + None, + 404, + ["Database not found: data2"], + ), + ( + "/data/docs2/-/insert", + {}, + None, + 404, + ["Table not found: docs2"], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"} for i in range(10)]}, + "bad_token", + 403, + ["Permission denied"], + ), + ( + "/data/docs/-/insert", + {}, + "invalid_json", + 400, + [ + "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ], + ), + ( + "/data/docs/-/insert", + {}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), + ( + "/data/docs/-/insert", + [], + None, + 400, + ["JSON must be a dictionary"], + ), + ( + "/data/docs/-/insert", + {"row": "blah"}, + None, + 400, + ['"row" must be a dictionary'], + ), + ( + "/data/docs/-/insert", + {"blah": "blah"}, + None, + 400, + ['JSON must have one or other of "row" or "rows"'], + ), + ( + "/data/docs/-/insert", + {"rows": "blah"}, + None, + 400, + ['"rows" must be a list'], + ), + ( + "/data/docs/-/insert", + {"rows": ["blah"]}, + None, + 400, + ['"rows" must be a list of dictionaries'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"} for i in range(101)]}, + None, + 400, + ["Too many rows, maximum allowed is 100"], + ), + # Validate columns of each row + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test", "bad": 1, "worse": 2} for i in range(2)]}, + None, + 400, + [ + "Row 0 has invalid columns: bad, worse", + "Row 1 has invalid columns: bad, worse", + ], + ), + ), +) +async def test_write_row_errors( + ds_write, path, input, special_case, expected_status, expected_errors +): + token = write_token(ds_write) + if special_case == "bad_token": + token += "bad" + kwargs = dict( + json=input, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "text/plain" + if special_case == "invalid_content_type" + else "application/json", + }, + ) + if special_case == "invalid_json": + del kwargs["json"] + kwargs["content"] = "{bad json" + response = await ds_write.client.post( + path, + **kwargs, + ) + assert response.status_code == expected_status + assert response.json()["ok"] is False + assert response.json()["errors"] == expected_errors + + +def write_token(ds): + return "dstok_{}".format( + ds.sign( + {"a": "root", "token": "dstok", "t": int(time.time())}, namespace="token" + ) + ) From f6bf2d8045cc239fe34357342bff1440561c8909 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 29 Oct 2022 23:20:11 -0700 Subject: [PATCH 1567/2303] Initial prototype of API explorer at /-/api, refs #1871 --- datasette/app.py | 5 ++ datasette/templates/api_explorer.html | 73 +++++++++++++++++++++++++++ datasette/views/special.py | 8 +++ tests/test_docs.py | 2 +- 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 datasette/templates/api_explorer.html diff --git a/datasette/app.py b/datasette/app.py index f80d3792..c3d802a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,6 +33,7 @@ from .views.special import ( JsonDataView, PatternPortfolioView, AuthTokenView, + ApiExplorerView, CreateTokenView, LogoutView, AllowDebugView, @@ -1235,6 +1236,10 @@ class Datasette: CreateTokenView.as_view(self), r"/-/create-token$", ) + add_route( + ApiExplorerView.as_view(self), + r"/-/api$", + ) add_route( LogoutView.as_view(self), r"/-/logout$", diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html new file mode 100644 index 00000000..034bee60 --- /dev/null +++ b/datasette/templates/api_explorer.html @@ -0,0 +1,73 @@ +{% extends "base.html" %} + +{% block title %}API Explorer{% endblock %} + +{% block content %} + +

    API Explorer

    + +

    Use this tool to try out the Datasette write API.

    + +{% if errors %} + {% for error in errors %} +

    {{ error }}

    + {% endfor %} +{% endif %} + + +
    + + +
    +
    + + +
    +
    + +
    +

    + + + + +{% endblock %} diff --git a/datasette/views/special.py b/datasette/views/special.py index b754a2f0..9922a621 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -235,3 +235,11 @@ class CreateTokenView(BaseView): "token_bits": token_bits, }, ) + + +class ApiExplorerView(BaseView): + name = "api_explorer" + has_json_alternate = False + + async def get(self, request): + return await self.render(["api_explorer.html"], request) diff --git a/tests/test_docs.py b/tests/test_docs.py index cd5a6c13..e9b813fe 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -62,7 +62,7 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView")) + view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) return view_labels From 9eb9ffae3ddd4e8ff0b713bf6fd6a0afed3368d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Oct 2022 13:09:55 -0700 Subject: [PATCH 1568/2303] Drop API token requirement from API explorer, refs #1871 --- datasette/default_permissions.py | 9 +++++++++ datasette/templates/api_explorer.html | 13 ++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 87684e2a..151ba2b5 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -131,3 +131,12 @@ def register_commands(cli): if debug: click.echo("\nDecoded:\n") click.echo(json.dumps(ds.unsign(token, namespace="token"), indent=2)) + + +@hookimpl +def skip_csrf(scope): + # Skip CSRF check for requests with content-type: application/json + if scope["type"] == "http": + headers = scope.get("headers") or {} + if dict(headers).get(b"content-type") == b"application/json": + return True diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 034bee60..01b182d8 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -15,16 +15,13 @@ {% endif %}
    -
    - - -
    - +
    -
    - +
    + +

    @@ -46,7 +43,6 @@ form.addEventListener("submit", (ev) => { var formData = new FormData(form); var json = formData.get('json'); var path = formData.get('path'); - var token = formData.get('token'); // Validate JSON try { var data = JSON.parse(json); @@ -60,7 +56,6 @@ form.addEventListener("submit", (ev) => { body: json, headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` } }).then(r => r.json()).then(r => { alert(JSON.stringify(r, null, 2)); From fedbfcc36873366143195d8fe124e1859bf88346 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Oct 2022 14:49:07 -0700 Subject: [PATCH 1569/2303] Neater display of output and errors in API explorer, refs #1871 --- datasette/templates/api_explorer.html | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 01b182d8..38fdb7bc 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -26,6 +26,12 @@

    + + """.format( escape(ex.sql) ) diff --git a/tests/test_api.py b/tests/test_api.py index ad74d16e..4027a7a5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -662,7 +662,11 @@ def test_sql_time_limit(app_client_shorter_time_limit): "

    SQL query took too long. The time limit is controlled by the\n" 'sql_time_limit_ms\n' "configuration option.

    \n" - "
    select sleep(0.5)
    " + '\n' + "" ), "status": 400, "title": "SQL Interrupted", diff --git a/tests/test_html.py b/tests/test_html.py index 4b394199..7cfe9d90 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -172,7 +172,7 @@ def test_sql_time_limit(app_client_shorter_time_limit): """ sql_time_limit_ms """.strip(), - "
    select sleep(0.5)
    ", + '', ] for expected_html_fragment in expected_html_fragments: assert expected_html_fragment in response.text From 93a02281dad2f23da84210f6ae9c63777ad8af5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 10:22:26 -0700 Subject: [PATCH 1574/2303] Show interrupted query in resizing textarea, closes #1876 --- datasette/views/base.py | 6 +++++- tests/test_api.py | 6 +++++- tests/test_html.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 67aa3a42..6b01fdd2 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -378,7 +378,11 @@ class DataView(BaseView):

    SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

    - 
    {}
    + + """.format( escape(ex.sql) ) diff --git a/tests/test_api.py b/tests/test_api.py index ebd675b9..de0223e2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -662,7 +662,11 @@ def test_sql_time_limit(app_client_shorter_time_limit): "

    SQL query took too long. The time limit is controlled by the\n" 'sql_time_limit_ms\n' "configuration option.

    \n" - "
    select sleep(0.5)
    " + '\n' + "" ), "status": 400, "title": "SQL Interrupted", diff --git a/tests/test_html.py b/tests/test_html.py index 4b394199..7cfe9d90 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -172,7 +172,7 @@ def test_sql_time_limit(app_client_shorter_time_limit): """ sql_time_limit_ms """.strip(), - "
    select sleep(0.5)
    ", + '', ] for expected_html_fragment in expected_html_fragments: assert expected_html_fragment in response.text From 9bec7c38eb93cde5afb16df9bdd96aea2a5b0459 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 11:07:59 -0700 Subject: [PATCH 1575/2303] ignore and replace options for bulk inserts, refs #1873 Also removed the rule that you cannot include primary keys in the rows you insert. And added validation that catches invalid parameters in the incoming JSON. And renamed "inserted" to "rows" in the returned JSON for return_rows: true --- datasette/views/table.py | 41 ++++++++++++++------ docs/json_api.rst | 4 +- tests/test_api_write.py | 83 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 111 insertions(+), 17 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1e3d566e..7692a4e3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1107,6 +1107,7 @@ class TableInsertView(BaseView): if not isinstance(data, dict): return _errors(["JSON must be a dictionary"]) keys = data.keys() + # keys must contain "row" or "rows" if "row" not in keys and "rows" not in keys: return _errors(['JSON must have one or other of "row" or "rows"']) @@ -1126,19 +1127,31 @@ class TableInsertView(BaseView): for row in rows: if not isinstance(row, dict): return _errors(['"rows" must be a list of dictionaries']) + # Does this exceed max_insert_rows? max_insert_rows = self.ds.setting("max_insert_rows") if len(rows) > max_insert_rows: return _errors( ["Too many rows, maximum allowed is {}".format(max_insert_rows)] ) + + # Validate other parameters + extras = { + key: value for key, value in data.items() if key not in ("row", "rows") + } + valid_extras = {"return_rows", "ignore", "replace"} + invalid_extras = extras.keys() - valid_extras + if invalid_extras: + return _errors( + ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_extras)))] + ) + if extras.get("ignore") and extras.get("replace"): + return _errors(['Cannot use "ignore" and "replace" at the same time']) + # Validate columns of each row - columns = await db.table_columns(table_name) - # TODO: There are cases where pks are OK, if not using auto-incrementing pk - pks = await db.primary_keys(table_name) - allowed_columns = set(columns) - set(pks) + columns = set(await db.table_columns(table_name)) for i, row in enumerate(rows): - invalid_columns = set(row.keys()) - allowed_columns + invalid_columns = set(row.keys()) - columns if invalid_columns: errors.append( "Row {} has invalid columns: {}".format( @@ -1147,8 +1160,7 @@ class TableInsertView(BaseView): ) if errors: return _errors(errors) - extra = {key: data[key] for key in data if key not in ("rows", "row")} - return rows, errors, extra + return rows, errors, extras async def post(self, request): database_route = tilde_decode(request.url_vars["database"]) @@ -1168,18 +1180,23 @@ class TableInsertView(BaseView): request.actor, "insert-row", resource=(database_name, table_name) ): return _error(["Permission denied"], 403) - rows, errors, extra = await self._validate_data(request, db, table_name) + rows, errors, extras = await self._validate_data(request, db, table_name) if errors: return _error(errors, 400) - should_return = bool(extra.get("return_rows", False)) + ignore = extras.get("ignore") + replace = extras.get("replace") + + should_return = bool(extras.get("return_rows", False)) # Insert rows def insert_rows(conn): table = sqlite_utils.Database(conn)[table_name] if should_return: rowids = [] for row in rows: - rowids.append(table.insert(row).last_rowid) + rowids.append( + table.insert(row, ignore=ignore, replace=replace).last_rowid + ) return list( table.rows_where( "rowid in ({})".format(",".join("?" for _ in rowids)), @@ -1187,12 +1204,12 @@ class TableInsertView(BaseView): ) ) else: - table.insert_all(rows) + table.insert_all(rows, ignore=ignore, replace=replace) rows = await db.execute_write_fn(insert_rows) result = {"ok": True} if should_return: - result["inserted"] = rows + result["rows"] = rows return Response.json(result, status=201) diff --git a/docs/json_api.rst b/docs/json_api.rst index da4500ab..34c13211 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -489,7 +489,7 @@ If successful, this will return a ``201`` status code and the newly inserted row .. code-block:: json { - "inserted": [ + "rows": [ { "id": 1, "column1": "value1", @@ -538,7 +538,7 @@ To return the newly inserted rows, add the ``"return_rows": true`` key to the re "return_rows": true } -This will return the same ``"inserted"`` key as the single row example above. There is a small performance penalty for using this option. +This will return the same ``"rows"`` key as the single row example above. There is a small performance penalty for using this option. .. _RowDeleteView: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1cfba104..d0b0f324 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -37,7 +37,7 @@ async def test_write_row(ds_write): ) expected_row = {"id": 1, "title": "Test", "score": 1.0} assert response.status_code == 201 - assert response.json()["inserted"] == [expected_row] + assert response.json()["rows"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row @@ -70,7 +70,7 @@ async def test_write_rows(ds_write, return_rows): ] assert response.json()["ok"] is True if return_rows: - assert response.json()["inserted"] == actual_rows + assert response.json()["rows"] == actual_rows @pytest.mark.asyncio @@ -156,6 +156,27 @@ async def test_write_rows(ds_write, return_rows): 400, ["Too many rows, maximum allowed is 100"], ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "ignore": True, "replace": True}, + None, + 400, + ['Cannot use "ignore" and "replace" at the same time'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "invalid_param": True}, + None, + 400, + ['Invalid parameter: "invalid_param"'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "one": True, "two": True}, + None, + 400, + ['Invalid parameter: "one", "two"'], + ), # Validate columns of each row ( "/data/docs/-/insert", @@ -196,6 +217,62 @@ async def test_write_row_errors( assert response.json()["errors"] == expected_errors +@pytest.mark.asyncio +@pytest.mark.parametrize( + "ignore,replace,expected_rows", + ( + ( + True, + False, + [ + {"id": 1, "title": "Exists", "score": None}, + ], + ), + ( + False, + True, + [ + {"id": 1, "title": "One", "score": None}, + ], + ), + ), +) +@pytest.mark.parametrize("should_return", (True, False)) +async def test_insert_ignore_replace( + ds_write, ignore, replace, expected_rows, should_return +): + await ds_write.get_database("data").execute_write( + "insert into docs (id, title) values (1, 'Exists')" + ) + token = write_token(ds_write) + data = {"rows": [{"id": 1, "title": "One"}]} + if ignore: + data["ignore"] = True + if replace: + data["replace"] = True + if should_return: + data["return_rows"] = True + response = await ds_write.client.post( + "/data/docs/-/insert", + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 201 + actual_rows = [ + dict(r) + for r in ( + await ds_write.get_database("data").execute("select * from docs") + ).rows + ] + assert actual_rows == expected_rows + assert response.json()["ok"] is True + if should_return: + assert response.json()["rows"] == expected_rows + + @pytest.mark.asyncio @pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table", "has_perm")) async def test_delete_row(ds_write, scenario): @@ -217,7 +294,7 @@ async def test_delete_row(ds_write, scenario): }, ) assert insert_response.status_code == 201 - pk = insert_response.json()["inserted"][0]["id"] + pk = insert_response.json()["rows"][0]["id"] path = "/data/{}/{}/-/delete".format( "docs" if scenario != "bad_table" else "bad_table", pk From 497290beaf32e6b779f9683ef15f1c5bc142a41a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 12:59:17 -0700 Subject: [PATCH 1576/2303] Handle database errors in /-/insert, refs #1866, #1873 Also improved API explorer to show HTTP status of response, refs #1871 --- datasette/templates/api_explorer.html | 14 +++++++++----- datasette/views/table.py | 5 ++++- tests/test_api_write.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 38fdb7bc..93bacde3 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -27,7 +27,8 @@ @@ -64,12 +65,15 @@ form.addEventListener("submit", (ev) => { headers: { 'Content-Type': 'application/json', } - }).then(r => r.json()).then(r => { + }).then(r => { + document.getElementById('response-status').textContent = r.status; + return r.json(); + }).then(data => { var errorList = output.querySelector('.errors'); - if (r.errors) { + if (data.errors) { errorList.style.display = 'block'; errorList.innerHTML = ''; - r.errors.forEach(error => { + data.errors.forEach(error => { var li = document.createElement('li'); li.textContent = error; errorList.appendChild(li); @@ -77,7 +81,7 @@ form.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').innerText = JSON.stringify(r, null, 2); + output.querySelector('pre').innerText = JSON.stringify(data, null, 2); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/views/table.py b/datasette/views/table.py index 7692a4e3..61227206 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1206,7 +1206,10 @@ class TableInsertView(BaseView): else: table.insert_all(rows, ignore=ignore, replace=replace) - rows = await db.execute_write_fn(insert_rows) + try: + rows = await db.execute_write_fn(insert_rows) + except Exception as e: + return _error([str(e)]) result = {"ok": True} if should_return: result["rows"] = rows diff --git a/tests/test_api_write.py b/tests/test_api_write.py index d0b0f324..0b567f48 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -156,6 +156,13 @@ async def test_write_rows(ds_write, return_rows): 400, ["Too many rows, maximum allowed is 100"], ), + ( + "/data/docs/-/insert", + {"rows": [{"id": 1, "title": "Test"}]}, + "duplicate_id", + 400, + ["UNIQUE constraint failed: docs.id"], + ), ( "/data/docs/-/insert", {"rows": [{"title": "Test"}], "ignore": True, "replace": True}, @@ -194,6 +201,10 @@ async def test_write_row_errors( ds_write, path, input, special_case, expected_status, expected_errors ): token = write_token(ds_write) + if special_case == "duplicate_id": + await ds_write.get_database("data").execute_write( + "insert into docs (id) values (1)" + ) if special_case == "bad_token": token += "bad" kwargs = dict( From 0b166befc0096fca30d71e19608a928d59c331a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 17:31:22 -0700 Subject: [PATCH 1577/2303] API explorer can now do GET, has JSON syntax highlighting Refs #1871 --- .../static/json-format-highlight-1.0.1.js | 43 +++++++++++ datasette/templates/api_explorer.html | 77 +++++++++++++++---- 2 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 datasette/static/json-format-highlight-1.0.1.js diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..e87c76e1 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,43 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : + typeof define === 'function' && define.amd ? define(factory) : + (global.jsonFormatHighlight = factory()); +}(this, (function () { 'use strict'; + +var defaultColors = { + keyColor: 'dimgray', + numberColor: 'lightskyblue', + stringColor: 'lightcoral', + trueColor: 'lightseagreen', + falseColor: '#f66578', + nullColor: 'cornflowerblue' +}; + +function index (json, colorOptions) { + if ( colorOptions === void 0 ) colorOptions = {}; + + if (!json) { return; } + if (typeof json !== 'string') { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, '&').replace(//g, '>'); + return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) ? colors.trueColor : /false/.test(match) ? colors.falseColor : /null/.test(match) ? colors.nullColor : color; + } + return ("" + match + ""); + }); +} + +return index; + +}))); diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 93bacde3..de5337e3 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -2,6 +2,10 @@ {% block title %}API Explorer{% endblock %} +{% block extra_head %} + +{% endblock %} + {% block content %}

    API Explorer

    @@ -14,17 +18,30 @@ {% endfor %} {% endif %} -
    -
    - - -
    -
    - - -
    -

    - +
    + GET +
    +
    + + + +
    + +
    +
    + POST +
    +
    + + +
    +
    + + +
    +

    + +
    {% else %} - {% if not canned_write and not error %} + {% if not canned_query_write and not error %}

    0 results

    {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 0770a380..658c35e6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,3 @@ -from asyncinject import Registry from dataclasses import dataclass, field from typing import Callable from urllib.parse import parse_qsl, urlencode @@ -33,7 +32,7 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import BaseView, DatasetteError, DataView, View, _error, stream_csv +from .base import BaseView, DatasetteError, View, _error, stream_csv class DatabaseView(View): @@ -57,7 +56,7 @@ class DatabaseView(View): sql = (request.args.get("sql") or "").strip() if sql: - return await query_view(request, datasette) + return await QueryView()(request, datasette) if format_ not in ("html", "json"): raise NotFound("Invalid format: {}".format(format_)) @@ -65,10 +64,6 @@ class DatabaseView(View): metadata = (datasette.metadata("databases") or {}).get(database, {}) datasette.update_with_inherited_metadata(metadata) - table_counts = await db.table_counts(5) - hidden_table_names = set(await db.hidden_table_names()) - all_foreign_keys = await db.get_all_foreign_keys() - sql_views = [] for view_name in await db.view_names(): view_visible, view_private = await datasette.check_visibility( @@ -196,8 +191,13 @@ class QueryContext: # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_write: bool = field( - metadata={"help": "Boolean indicating if this canned query allows writes"} + canned_query_write: bool = field( + metadata={ + "help": "Boolean indicating if this is a canned query that allows writes" + } + ) + metadata: dict = field( + metadata={"help": "Metadata about the database or the canned query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -232,7 +232,6 @@ class QueryContext: show_hide_hidden: str = field( metadata={"help": "Hidden input field for the _show_sql parameter"} ) - metadata: dict = field(metadata={"help": "Metadata about the query/database"}) database_color: Callable = field( metadata={"help": "Function that returns a color for a given database name"} ) @@ -242,6 +241,12 @@ class QueryContext: alternate_url_json: str = field( metadata={"help": "URL for alternate JSON version of this page"} ) + # TODO: refactor this to somewhere else, probably ds.render_template() + select_templates: list = field( + metadata={ + "help": "List of templates that were considered for rendering this page" + } + ) async def get_tables(datasette, request, db): @@ -320,287 +325,105 @@ async def database_download(request, datasette): ) -async def query_view( - request, - datasette, - # canned_query=None, - # _size=None, - # named_parameters=None, - # write=False, -): - db = await datasette.resolve_database(request) - database = db.name - # Flattened because of ?sql=&name1=value1&name2=value2 feature - params = {key: request.args.get(key) for key in request.args} - sql = None - if "sql" in params: - sql = params.pop("sql") - if "_shape" in params: - params.pop("_shape") +class QueryView(View): + async def post(self, request, datasette): + from datasette.app import TableNotFound - # extras come from original request.args to avoid being flattened - extras = request.args.getlist("_extra") + db = await datasette.resolve_database(request) - # TODO: Behave differently for canned query here: - await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - - _, private = await datasette.check_visibility( - request.actor, - permissions=[ - ("view-database", database), - "view-instance", - ], - ) - - extra_args = {} - if params.get("_timelimit"): - extra_args["custom_time_limit"] = int(params["_timelimit"]) - - format_ = request.url_vars.get("format") or "html" - query_error = None - try: - validate_sql_select(sql) - results = await datasette.execute( - database, sql, params, truncate=True, **extra_args - ) - columns = results.columns - rows = results.rows - except QueryInterrupted as ex: - raise DatasetteError( - textwrap.dedent( - """ -

    SQL query took too long. The time limit is controlled by the - sql_time_limit_ms - configuration option.

    - - - """.format( - markupsafe.escape(ex.sql) - ) - ).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except sqlite3.DatabaseError as ex: - query_error = str(ex) - results = None - rows = [] - columns = [] - except (sqlite3.OperationalError, InvalidSql) as ex: - raise DatasetteError(str(ex), title="Invalid SQL", status=400) - except sqlite3.OperationalError as ex: - raise DatasetteError(str(ex)) - except DatasetteError: - raise - - # Handle formats from plugins - if format_ == "csv": - - async def fetch_data_for_csv(request, _next=None): - results = await db.execute(sql, params, truncate=True) - data = {"rows": results.rows, "columns": results.columns} - return data, None, None - - return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - datasette.renderers[format_][0], - datasette=datasette, - columns=columns, - rows=rows, - sql=sql, - query_name=None, - database=database, - table=None, - request=request, - view_name="table", - truncated=results.truncated if results else False, - error=query_error, - # These will be deprecated in Datasette 1.0: - args=request.args, - data={"rows": rows, "columns": columns}, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code") or 200, - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), + # We must be a canned query + table_found = False + try: + await datasette.resolve_table(request) + table_found = True + except TableNotFound as table_not_found: + canned_query = await datasette.get_canned_query( + table_not_found.database_name, table_not_found.table, request.actor ) - elif isinstance(result, Response): - r = result - # if status_code is not None: - # # Over-ride the status code - # r.status = status_code - else: - assert False, f"{result} should be dict or Response" - elif format_ == "html": - headers = {} - templates = [f"query-{to_css_class(database)}.html", "query.html"] - template = datasette.jinja_env.select_template(templates) - alternate_url_json = datasette.absolute_url( - request, - datasette.urls.path(path_with_format(request=request, format="json")), - ) - data = {} - headers.update( - { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - } - ) - metadata = (datasette.metadata("databases") or {}).get(database, {}) - datasette.update_with_inherited_metadata(metadata) + if canned_query is None: + raise + if table_found: + # That should not have happened + raise DatasetteError("Unexpected table found on POST", status=404) - renderers = {} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name="database", + # If database is immutable, return an error + if not db.is_mutable: + raise Forbidden("Database is immutable") + + # Process the POST + body = await request.post_body() + body = body.decode("utf-8").strip() + if body.startswith("{") and body.endswith("}"): + params = json.loads(body) + # But we want key=value strings + for key, value in params.items(): + params[key] = str(value) + else: + params = dict(parse_qsl(body, keep_blank_values=True)) + # Should we return JSON? + should_return_json = ( + request.headers.get("accept") == "application/json" + or request.args.get("_json") + or params.get("_json") + ) + params_for_query = MagicParameters(params, request, datasette) + ok = None + redirect_url = None + try: + cursor = await db.execute_write(canned_query["sql"], params_for_query) + message = canned_query.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + message_type = datasette.INFO + redirect_url = canned_query.get("on_success_redirect") + ok = True + except Exception as ex: + message = canned_query.get("on_error_message") or str(ex) + message_type = datasette.ERROR + redirect_url = canned_query.get("on_error_redirect") + ok = False + if should_return_json: + return Response.json( + { + "ok": ok, + "message": message, + "redirect": redirect_url, + } ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format(request=request, format=key) - ) - - allow_execute_sql = await datasette.permission_allowed( - request.actor, "execute-sql", database - ) - - show_hide_hidden = "" - if metadata.get("hide_sql"): - if bool(params.get("_show_sql")): - show_hide_link = path_with_removed_args(request, {"_show_sql"}) - show_hide_text = "hide" - show_hide_hidden = '' - else: - show_hide_link = path_with_added_args(request, {"_show_sql": 1}) - show_hide_text = "show" else: - if bool(params.get("_hide_sql")): - show_hide_link = path_with_removed_args(request, {"_hide_sql"}) - show_hide_text = "show" - show_hide_hidden = '' - else: - show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) - show_hide_text = "hide" - hide_sql = show_hide_text == "show" + datasette.add_message(request, message, message_type) + return Response.redirect(redirect_url or request.path) - # Extract any :named parameters - named_parameters = await derive_named_parameters( - datasette.get_database(database), sql - ) - named_parameter_values = { - named_parameter: params.get(named_parameter) or "" - for named_parameter in named_parameters - if not named_parameter.startswith("_") - } + async def get(self, request, datasette): + from datasette.app import TableNotFound - # Set to blank string if missing from params - for named_parameter in named_parameters: - if named_parameter not in params and not named_parameter.startswith("_"): - params[named_parameter] = "" - - r = Response.html( - await datasette.render_template( - template, - QueryContext( - database=database, - query={ - "sql": sql, - "params": params, - }, - canned_query=None, - private=private, - canned_write=False, - db_is_immutable=not db.is_mutable, - error=query_error, - hide_sql=hide_sql, - show_hide_link=datasette.urls.path(show_hide_link), - show_hide_text=show_hide_text, - editable=True, # TODO - allow_execute_sql=allow_execute_sql, - tables=await get_tables(datasette, request, db), - named_parameter_values=named_parameter_values, - edit_sql_url="todo", - display_rows=await display_rows( - datasette, database, request, rows, columns - ), - table_columns=await _table_columns(datasette, database) - if allow_execute_sql - else {}, - columns=columns, - renderers=renderers, - url_csv=datasette.urls.path( - path_with_format( - request=request, format="csv", extra_qs={"_size": "max"} - ) - ), - show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=metadata, - database_color=lambda _: "#ff0000", - alternate_url_json=alternate_url_json, - ), - request=request, - view_name="database", - ), - headers=headers, - ) - else: - assert False, "Invalid format: {}".format(format_) - if datasette.cors: - add_cors_headers(r.headers) - return r - - -class QueryView(DataView): - async def data( - self, - request, - sql, - editable=True, - canned_query=None, - metadata=None, - _size=None, - named_parameters=None, - write=False, - default_labels=None, - ): - db = await self.ds.resolve_database(request) + db = await datasette.resolve_database(request) database = db.name - params = {key: request.args.get(key) for key in request.args} - if "sql" in params: - params.pop("sql") - if "_shape" in params: - params.pop("_shape") + + # Are we a canned query? + canned_query = None + canned_query_write = False + if "table" in request.url_vars: + try: + await datasette.resolve_table(request) + except TableNotFound as table_not_found: + # Was this actually a canned query? + canned_query = await datasette.get_canned_query( + table_not_found.database_name, table_not_found.table, request.actor + ) + if canned_query is None: + raise + canned_query_write = bool(canned_query.get("write")) private = False if canned_query: # Respect canned query permissions - visible, private = await self.ds.check_visibility( + visible, private = await datasette.check_visibility( request.actor, permissions=[ - ("view-query", (database, canned_query)), + ("view-query", (database, canned_query["name"])), ("view-database", database), "view-instance", ], @@ -609,18 +432,32 @@ class QueryView(DataView): raise Forbidden("You do not have permission to view this query") else: - await self.ds.ensure_permissions(request.actor, [("execute-sql", database)]) + await datasette.ensure_permissions( + request.actor, [("execute-sql", database)] + ) + + # Flattened because of ?sql=&name1=value1&name2=value2 feature + params = {key: request.args.get(key) for key in request.args} + sql = None + + if canned_query: + sql = canned_query["sql"] + elif "sql" in params: + sql = params.pop("sql") # Extract any :named parameters - named_parameters = named_parameters or await derive_named_parameters( - self.ds.get_database(database), sql - ) + named_parameters = [] + if canned_query and canned_query.get("params"): + named_parameters = canned_query["params"] + if not named_parameters: + named_parameters = await derive_named_parameters( + datasette.get_database(database), sql + ) named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters if not named_parameter.startswith("_") } - # Set to blank string if missing from params for named_parameter in named_parameters: if named_parameter not in params and not named_parameter.startswith("_"): @@ -629,212 +466,159 @@ class QueryView(DataView): extra_args = {} if params.get("_timelimit"): extra_args["custom_time_limit"] = int(params["_timelimit"]) - if _size: - extra_args["page_size"] = _size - templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: - templates.insert( - 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", - ) + format_ = request.url_vars.get("format") or "html" query_error = None + results = None + rows = [] + columns = [] - # Execute query - as write or as read - if write: - if request.method == "POST": - # If database is immutable, return an error - if not db.is_mutable: - raise Forbidden("Database is immutable") - body = await request.post_body() - body = body.decode("utf-8").strip() - if body.startswith("{") and body.endswith("}"): - params = json.loads(body) - # But we want key=value strings - for key, value in params.items(): - params[key] = str(value) - else: - params = dict(parse_qsl(body, keep_blank_values=True)) - # Should we return JSON? - should_return_json = ( - request.headers.get("accept") == "application/json" - or request.args.get("_json") - or params.get("_json") - ) - if canned_query: - params_for_query = MagicParameters(params, request, self.ds) - else: - params_for_query = params - ok = None - try: - cursor = await self.ds.databases[database].execute_write( - sql, params_for_query - ) - message = metadata.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) - message_type = self.ds.INFO - redirect_url = metadata.get("on_success_redirect") - ok = True - except Exception as e: - message = metadata.get("on_error_message") or str(e) - message_type = self.ds.ERROR - redirect_url = metadata.get("on_error_redirect") - ok = False - if should_return_json: - return Response.json( - { - "ok": ok, - "message": message, - "redirect": redirect_url, - } - ) - else: - self.ds.add_message(request, message, message_type) - return self.redirect(request, redirect_url or request.path) - else: + params_for_query = params - async def extra_template(): - return { - "request": request, - "db_is_immutable": not db.is_mutable, - "path_with_added_args": path_with_added_args, - "path_with_removed_args": path_with_removed_args, - "named_parameter_values": named_parameter_values, - "canned_query": canned_query, - "success_message": request.args.get("_success") or "", - "canned_write": True, - } - - return ( - { - "database": database, - "rows": [], - "truncated": False, - "columns": [], - "query": {"sql": sql, "params": params}, - "private": private, - }, - extra_template, - templates, - ) - else: # Not a write - if canned_query: - params_for_query = MagicParameters(params, request, self.ds) - else: - params_for_query = params + if not canned_query_write: try: - results = await self.ds.execute( + if not canned_query: + # For regular queries we only allow SELECT, plus other rules + validate_sql_select(sql) + else: + # Canned queries can run magic parameters + params_for_query = MagicParameters(params, request, datasette) + results = await datasette.execute( database, sql, params_for_query, truncate=True, **extra_args ) - columns = [r[0] for r in results.description] - except sqlite3.DatabaseError as e: - query_error = e + columns = results.columns + rows = results.rows + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent( + """ +

    SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

    + + + """.format( + markupsafe.escape(ex.sql) + ) + ).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except sqlite3.DatabaseError as ex: + query_error = str(ex) results = None + rows = [] columns = [] + except (sqlite3.OperationalError, InvalidSql) as ex: + raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) + except DatasetteError: + raise - allow_execute_sql = await self.ds.permission_allowed( - request.actor, "execute-sql", database - ) + # Handle formats from plugins + if format_ == "csv": - async def extra_template(): - display_rows = [] - truncate_cells = self.ds.setting("truncate_cells_html") - for row in results.rows if results else []: - display_row = [] - for column, value in zip(results.columns, row): - display_value = value - # Let the plugins have a go - # pylint: disable=no-member - plugin_display_value = None - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=None, - database=database, - datasette=self.ds, - request=request, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value is not None: - display_value = plugin_display_value - else: - if value in ("", None): - display_value = markupsafe.Markup(" ") - elif is_url(str(display_value).strip()): - display_value = markupsafe.Markup( - '{truncated_url}'.format( - url=markupsafe.escape(value.strip()), - truncated_url=markupsafe.escape( - truncate_url(value.strip(), truncate_cells) - ), - ) - ) - elif isinstance(display_value, bytes): - blob_url = path_with_format( - request=request, - format="blob", - extra_qs={ - "_blob_column": column, - "_blob_hash": hashlib.sha256( - display_value - ).hexdigest(), - }, - ) - formatted = format_bytes(len(value)) - display_value = markupsafe.Markup( - '<Binary: {:,} byte{}>'.format( - blob_url, - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", - len(value), - "" if len(value) == 1 else "s", - ) - ) - else: - display_value = str(value) - if truncate_cells and len(display_value) > truncate_cells: - display_value = ( - display_value[:truncate_cells] + "\u2026" - ) - display_row.append(display_value) - display_rows.append(display_row) + async def fetch_data_for_csv(request, _next=None): + results = await db.execute(sql, params, truncate=True) + data = {"rows": results.rows, "columns": results.columns} + return data, None, None - # Show 'Edit SQL' button only if: - # - User is allowed to execute SQL - # - SQL is an approved SELECT statement - # - No magic parameters, so no :_ in the SQL string - edit_sql_url = None - is_validated_sql = False - try: - validate_sql_select(sql) - is_validated_sql = True - except InvalidSql: - pass - if allow_execute_sql and is_validated_sql and ":_" not in sql: - edit_sql_url = ( - self.ds.urls.database(database) - + "?" - + urlencode( - { - **{ - "sql": sql, - }, - **named_parameter_values, - } - ) + return await stream_csv(datasette, fetch_data_for_csv, request, db.name) + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=canned_query["name"] if canned_query else None, + database=database, + table=None, + request=request, + view_name="table", + truncated=results.truncated if results else False, + error=query_error, + # These will be deprecated in Datasette 1.0: + args=request.args, + data={"rows": rows, "columns": columns}, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", ) + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + data = {} + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + renderers = {} + for key, (_, can_render) in datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=datasette, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name="database", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = datasette.urls.path( + path_with_format(request=request, format=key) + ) + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + show_hide_hidden = "" - if metadata.get("hide_sql"): + if canned_query and canned_query.get("hide_sql"): if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -855,42 +639,86 @@ class QueryView(DataView): show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) show_hide_text = "hide" hide_sql = show_hide_text == "show" - return { - "display_rows": display_rows, - "custom_sql": True, - "named_parameter_values": named_parameter_values, - "editable": editable, - "canned_query": canned_query, - "edit_sql_url": edit_sql_url, - "metadata": metadata, - "settings": self.ds.settings_dict(), - "request": request, - "show_hide_link": self.ds.urls.path(show_hide_link), - "show_hide_text": show_hide_text, - "show_hide_hidden": markupsafe.Markup(show_hide_hidden), - "hide_sql": hide_sql, - "table_columns": await _table_columns(self.ds, database) - if allow_execute_sql - else {}, - } - return ( - { - "ok": not query_error, - "database": database, - "query_name": canned_query, - "rows": results.rows if results else [], - "truncated": results.truncated if results else False, - "columns": columns, - "query": {"sql": sql, "params": params}, - "error": str(query_error) if query_error else None, - "private": private, - "allow_execute_sql": allow_execute_sql, - }, - extra_template, - templates, - 400 if query_error else 200, - ) + # Show 'Edit SQL' button only if: + # - User is allowed to execute SQL + # - SQL is an approved SELECT statement + # - No magic parameters, so no :_ in the SQL string + edit_sql_url = None + is_validated_sql = False + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + datasette.urls.database(database) + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) + ) + + r = Response.html( + await datasette.render_template( + template, + QueryContext( + database=database, + query={ + "sql": sql, + "params": params, + }, + canned_query=canned_query["name"] if canned_query else None, + private=private, + canned_query_write=canned_query_write, + db_is_immutable=not db.is_mutable, + error=query_error, + hide_sql=hide_sql, + show_hide_link=datasette.urls.path(show_hide_link), + show_hide_text=show_hide_text, + editable=not canned_query, + allow_execute_sql=allow_execute_sql, + tables=await get_tables(datasette, request, db), + named_parameter_values=named_parameter_values, + edit_sql_url=edit_sql_url, + display_rows=await display_rows( + datasette, database, request, rows, columns + ), + table_columns=await _table_columns(datasette, database) + if allow_execute_sql + else {}, + columns=columns, + renderers=renderers, + url_csv=datasette.urls.path( + path_with_format( + request=request, format="csv", extra_qs={"_size": "max"} + ) + ), + show_hide_hidden=markupsafe.Markup(show_hide_hidden), + metadata=canned_query or metadata, + database_color=lambda _: "#ff0000", + alternate_url_json=alternate_url_json, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="database", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + if datasette.cors: + add_cors_headers(r.headers) + return r class MagicParameters(dict): diff --git a/datasette/views/table.py b/datasette/views/table.py index 77acfd95..28264e92 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -9,7 +9,6 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette import tracer -from datasette.renderer import json_renderer from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -21,7 +20,6 @@ from datasette.utils import ( tilde_encode, escape_sqlite, filters_should_redirect, - format_bytes, is_url, path_from_row_pks, path_with_added_args, @@ -38,7 +36,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters import sqlite_utils -from .base import BaseView, DataView, DatasetteError, ureg, _error, stream_csv +from .base import BaseView, DatasetteError, ureg, _error, stream_csv from .database import QueryView LINK_WITH_LABEL = ( @@ -698,57 +696,6 @@ async def table_view(datasette, request): return response -class CannedQueryView(DataView): - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - from datasette.app import TableNotFound - - try: - await self.ds.resolve_table(request) - except TableNotFound as e: - # Was this actually a canned query? - canned_query = await self.ds.get_canned_query( - e.database_name, e.table, request.actor - ) - if canned_query: - # Handle POST to a canned query - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=e.table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - - return Response.text("Method not allowed", status=405) - - async def data(self, request, **kwargs): - from datasette.app import TableNotFound - - try: - await self.ds.resolve_table(request) - except TableNotFound as not_found: - canned_query = await self.ds.get_canned_query( - not_found.database_name, not_found.table, request.actor - ) - if canned_query: - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=not_found.table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - else: - raise - - async def table_view_traced(datasette, request): from datasette.app import TableNotFound @@ -761,10 +708,7 @@ async def table_view_traced(datasette, request): ) # If this is a canned query, not a table, then dispatch to QueryView instead if canned_query: - if request.method == "POST": - return await CannedQueryView(datasette).post(request) - else: - return await CannedQueryView(datasette).get(request) + return await QueryView()(request, datasette) else: raise diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index d6a88733..e9ad3239 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -95,12 +95,12 @@ def test_insert(canned_write_client): csrftoken_from=True, cookies={"foo": "bar"}, ) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] messages = canned_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) - assert [["Query executed, 1 row affected", 1]] == messages + assert messages == [["Query executed, 1 row affected", 1]] + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name?success" @pytest.mark.parametrize( @@ -382,11 +382,11 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c def test_canned_write_custom_template(canned_write_client): response = canned_write_client.get("/data/update_name") assert response.status == 200 + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text assert ( "" in response.text ) - assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text # And test for link rel=alternate while we're here: assert ( '' From 8920d425f4d417cfd998b61016c5ff3530cd34e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 10:20:58 -0700 Subject: [PATCH 1819/2303] 1.0a3 release notes, smaller changes section - refs #2135 --- docs/changelog.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee48d075..b4416f94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,25 @@ Changelog ========= +.. _v1_0_a3: + +1.0a3 (2023-08-09) +------------------ + +This alpha release previews the updated design for Datasette's default JSON API. + +Smaller changes +~~~~~~~~~~~~~~~ + +- Datasette documentation now shows YAML examples for :ref:`metadata` by default, with a tab interface for switching to JSON. (:issue:`1153`) +- :ref:`plugin_register_output_renderer` plugins now have access to ``error`` and ``truncated`` arguments, allowing them to display error messages and take into account truncated results. (:issue:`2130`) +- ``render_cell()`` plugin hook now also supports an optional ``request`` argument. (:issue:`2007`) +- New ``Justfile`` to support development workflows for Datasette using `Just `__. +- ``datasette.render_template()`` can now accepts a ``datasette.views.Context`` subclass as an alternative to a dictionary. (:issue:`2127`) +- ``datasette install -e path`` option for editable installations, useful while developing plugins. (:issue:`2106`) +- When started with the ``--cors`` option Datasette now serves an ``Access-Control-Max-Age: 3600`` header, ensuring CORS OPTIONS requests are repeated no more than once an hour. (:issue:`2079`) +- Fixed a bug where the ``_internal`` database could display ``None`` instead of ``null`` for in-memory databases. (:issue:`1970`) + .. _v0_64_2: 0.64.2 (2023-03-08) From e34d09c6ec16ff5e7717e112afdad67f7c05a62a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:01:59 -0700 Subject: [PATCH 1820/2303] Don't include columns in query JSON, refs #2136 --- datasette/renderer.py | 8 +++++++- datasette/views/database.py | 2 +- tests/test_api.py | 1 - tests/test_cli_serve_get.py | 11 ++++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 0bd74e81..224031a7 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -27,7 +27,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): return new_rows -def json_renderer(args, data, error, truncated=None): +def json_renderer(request, args, data, error, truncated=None): """Render a response as JSON""" status_code = 200 @@ -106,6 +106,12 @@ def json_renderer(args, data, error, truncated=None): "status": 400, "title": None, } + + # Don't include "columns" in output + # https://github.com/simonw/datasette/issues/2136 + if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + data.pop("columns", None) + # Handle _nl option for _shape=array nl = args.get("_nl", "") if nl and shape == "array": diff --git a/datasette/views/database.py b/datasette/views/database.py index 658c35e6..cf76f3c2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -548,7 +548,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"rows": rows, "columns": columns}, + data={"ok": True, "rows": rows, "columns": columns}, ) if asyncio.iscoroutine(result): result = await result diff --git a/tests/test_api.py b/tests/test_api.py index 28415a0b..f96f571e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -649,7 +649,6 @@ async def test_custom_sql(ds_client): {"content": "RENDER_CELL_DEMO"}, {"content": "RENDER_CELL_ASYNC"}, ], - "columns": ["content"], "ok": True, "truncated": False, } diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 2e0390bb..dc7fc1e2 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -34,11 +34,12 @@ def test_serve_with_get(tmp_path_factory): "/_memory.json?sql=select+sqlite_version()", ], ) - assert 0 == result.exit_code, result.output - assert { - "truncated": False, - "columns": ["sqlite_version()"], - }.items() <= json.loads(result.output).items() + assert result.exit_code == 0, result.output + data = json.loads(result.output) + # Should have a single row with a single column + assert len(data["rows"]) == 1 + assert list(data["rows"][0].keys()) == ["sqlite_version()"] + assert set(data.keys()) == {"rows", "ok", "truncated"} # The plugin should have created hello.txt assert (plugins_dir / "hello.txt").read_text() == "hello" From 856ca68d94708c6e94673cb6bc28bf3e3ca17845 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:04:40 -0700 Subject: [PATCH 1821/2303] Update default JSON representation docs, refs #2135 --- docs/json_api.rst | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index c273c2a8..16b997eb 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -9,10 +9,10 @@ through the Datasette user interface can also be accessed as JSON via the API. To access the API for a page, either click on the ``.json`` link on that page or edit the URL and add a ``.json`` extension to it. -.. _json_api_shapes: +.. _json_api_default: -Different shapes ----------------- +Default representation +---------------------- The default JSON representation of data from a SQLite table or custom query looks like this: @@ -21,7 +21,6 @@ looks like this: { "ok": true, - "next": null, "rows": [ { "id": 3, @@ -39,13 +38,22 @@ looks like this: "id": 1, "name": "San Francisco" } - ] + ], + "truncated": false } -The ``rows`` key is a list of objects, each one representing a row. ``next`` indicates if -there is another page, and ``ok`` is always ``true`` if an error did not occur. +``"ok"`` is always ``true`` if an error did not occur. -If ``next`` is present then the next page in the pagination set can be retrieved using ``?_next=VALUE``. +The ``"rows"`` key is a list of objects, each one representing a row. + +The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). + +For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. + +.. _json_api_shapes: + +Different shapes +---------------- The ``_shape`` parameter can be used to access alternative formats for the ``rows`` key which may be more convenient for your application. There are three From 90cb9ca58d910f49e8f117bbdd94df6f0855cf99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:11:16 -0700 Subject: [PATCH 1822/2303] JSON changes in release notes, refs #2135 --- docs/changelog.rst | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b4416f94..4c70855b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,40 @@ Changelog 1.0a3 (2023-08-09) ------------------ -This alpha release previews the updated design for Datasette's default JSON API. +This alpha release previews the updated design for Datasette's default JSON API. (:issue:`782`) + +The new :ref:`default JSON representation ` for both table pages (``/dbname/table.json``) and arbitrary SQL queries (``/dbname.json?sql=...``) is now shaped like this: + +.. code-block:: json + + { + "ok": true, + "rows": [ + { + "id": 3, + "name": "Detroit" + }, + { + "id": 2, + "name": "Los Angeles" + }, + { + "id": 4, + "name": "Memnonia" + }, + { + "id": 1, + "name": "San Francisco" + } + ], + "truncated": false + } + +Tables will include an additional ``"next"`` key for pagination, which can be passed to ``?_next=`` to fetch the next page of results. + +The various ``?_shape=`` options continue to work as before - see :ref:`json_api_shapes` for details. + +A new ``?_extra=`` mechanism is available for tables, but has not yet been stabilized or documented. Details on that are available in :issue:`262`. Smaller changes ~~~~~~~~~~~~~~~ From 19ab4552e212c9845a59461cc73e82d5ae8c278a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:13:11 -0700 Subject: [PATCH 1823/2303] Release 1.0a3 Closes #2135 Refs #262, #782, #1153, #1970, #2007, #2079, #2106, #2127, #2130 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 3b81ab21..61dee464 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a2" +__version__ = "1.0a3" __version_info__ = tuple(__version__.split(".")) From 4a42476bb7ce4c5ed941f944115dedd9bce34656 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 15:04:16 -0700 Subject: [PATCH 1824/2303] datasette plugins --requirements, closes #2133 --- datasette/cli.py | 12 ++++++++++-- docs/cli-reference.rst | 1 + docs/plugins.rst | 32 ++++++++++++++++++++++++++++---- tests/test_cli.py | 3 +++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 32266888..21fd25d6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -223,15 +223,23 @@ pm.hook.publish_subcommand(publish=publish) @cli.command() @click.option("--all", help="Include built-in default plugins", is_flag=True) +@click.option( + "--requirements", help="Output requirements.txt of installed plugins", is_flag=True +) @click.option( "--plugins-dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Path to directory containing custom plugins", ) -def plugins(all, plugins_dir): +def plugins(all, requirements, plugins_dir): """List currently installed plugins""" app = Datasette([], plugins_dir=plugins_dir) - click.echo(json.dumps(app._plugins(all=all), indent=4)) + if requirements: + for plugin in app._plugins(): + if plugin["version"]: + click.echo("{}=={}".format(plugin["name"], plugin["version"])) + else: + click.echo(json.dumps(app._plugins(all=all), indent=4)) @cli.command() diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2177fc9e..7a96d311 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -282,6 +282,7 @@ Output JSON showing all currently installed plugins, their versions, whether the Options: --all Include built-in default plugins + --requirements Output requirements.txt of installed plugins --plugins-dir DIRECTORY Path to directory containing custom plugins --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index 979f94dd..19bfdd0c 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -90,7 +90,12 @@ You can see a list of installed plugins by navigating to the ``/-/plugins`` page You can also use the ``datasette plugins`` command:: - $ datasette plugins + datasette plugins + +Which outputs: + +.. code-block:: json + [ { "name": "datasette_json_html", @@ -107,7 +112,8 @@ You can also use the ``datasette plugins`` command:: cog.out("\n") result = CliRunner().invoke(cli.cli, ["plugins", "--all"]) # cog.out() with text containing newlines was unindenting for some reason - cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::\n") + cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:\n") + cog.outl(".. code-block:: json\n") plugins = [p for p in json.loads(result.output) if p["name"].startswith("datasette.")] indented = textwrap.indent(json.dumps(plugins, indent=4), " ") for line in indented.split("\n"): @@ -115,7 +121,9 @@ You can also use the ``datasette plugins`` command:: cog.out("\n\n") .. ]]] -If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:: +If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette: + +.. code-block:: json [ { @@ -236,6 +244,22 @@ If you run ``datasette plugins --all`` it will include default plugins that ship You can add the ``--plugins-dir=`` option to include any plugins found in that directory. +Add ``--requirements`` to output a list of installed plugins that can then be installed in another Datasette instance using ``datasette install -r requirements.txt``:: + + datasette plugins --requirements + +The output will look something like this:: + + datasette-codespaces==0.1.1 + datasette-graphql==2.2 + datasette-json-html==1.0.1 + datasette-pretty-json==0.2.2 + datasette-x-forwarded-host==0.1 + +To write that to a ``requirements.txt`` file, run this:: + + datasette plugins --requirements > requirements.txt + .. _plugins_configuration: Plugin configuration @@ -390,7 +414,7 @@ Any values embedded in ``metadata.yaml`` will be visible to anyone who views the If you are publishing your data using the :ref:`datasette publish ` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command:: - $ datasette publish heroku my_database.db \ + datasette publish heroku my_database.db \ --name my-heroku-app-demo \ --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ diff --git a/tests/test_cli.py b/tests/test_cli.py index 75724f61..056e2821 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -108,6 +108,9 @@ def test_plugins_cli(app_client): assert set(names).issuperset({p["name"] for p in EXPECTED_PLUGINS}) # And the following too: assert set(names).issuperset(DEFAULT_PLUGINS) + # --requirements should be empty because there are no installed non-plugins-dir plugins + result3 = runner.invoke(cli, ["plugins", "--requirements"]) + assert result3.output == "" def test_metadata_yaml(): From a3593c901580ea50854c3e0774b0ba0126e8a76f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 17:32:07 -0700 Subject: [PATCH 1825/2303] on_success_message_sql, closes #2138 --- datasette/views/database.py | 29 ++++++++++++++++---- docs/sql_queries.rst | 21 ++++++++++---- tests/test_canned_queries.py | 53 +++++++++++++++++++++++++++++++----- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index cf76f3c2..79b3f88d 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -360,6 +360,10 @@ class QueryView(View): params[key] = str(value) else: params = dict(parse_qsl(body, keep_blank_values=True)) + + # Don't ever send csrftoken as a SQL parameter + params.pop("csrftoken", None) + # Should we return JSON? should_return_json = ( request.headers.get("accept") == "application/json" @@ -371,12 +375,27 @@ class QueryView(View): redirect_url = None try: cursor = await db.execute_write(canned_query["sql"], params_for_query) - message = canned_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) + # success message can come from on_success_message or on_success_message_sql + message = None message_type = datasette.INFO + on_success_message_sql = canned_query.get("on_success_message_sql") + if on_success_message_sql: + try: + message_result = ( + await db.execute(on_success_message_sql, params_for_query) + ).first() + if message_result: + message = message_result[0] + except Exception as ex: + message = "Error running on_success_message_sql: {}".format(ex) + message_type = datasette.ERROR + if not message: + message = canned_query.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + redirect_url = canned_query.get("on_success_redirect") ok = True except Exception as ex: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 3c2cb228..1ae07e1f 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -392,6 +392,7 @@ This configuration will create a page at ``/mydatabase/add_name`` displaying a f You can customize how Datasette represents success and errors using the following optional properties: - ``on_success_message`` - the message shown when a query is successful +- ``on_success_message_sql`` - alternative to ``on_success_message``: a SQL query that should be executed to generate the message - ``on_success_redirect`` - the path or URL the user is redirected to on success - ``on_error_message`` - the message shown when a query throws an error - ``on_error_redirect`` - the path or URL the user is redirected to on error @@ -405,11 +406,12 @@ For example: "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", + "params": ["name"], "write": True, - "on_success_message": "Name inserted", + "on_success_message_sql": "select 'Name inserted: ' || :name", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", - "on_error_redirect": "/mydatabase" + "on_error_redirect": "/mydatabase", } } } @@ -426,8 +428,10 @@ For example: queries: add_name: sql: INSERT INTO names (name) VALUES (:name) + params: + - name write: true - on_success_message: Name inserted + on_success_message_sql: 'select ''Name inserted: '' || :name' on_success_redirect: /mydatabase/names on_error_message: Name insert failed on_error_redirect: /mydatabase @@ -443,8 +447,11 @@ For example: "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", + "params": [ + "name" + ], "write": true, - "on_success_message": "Name inserted", + "on_success_message_sql": "select 'Name inserted: ' || :name", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", "on_error_redirect": "/mydatabase" @@ -455,10 +462,12 @@ For example: } .. [[[end]]] -You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. +You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. ``"params"`` is not necessary in the above example, since without it ``"name"`` would be automatically detected from the query. You can pre-populate form fields when the page first loads using a query string, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query. +If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. + .. _canned_queries_magic_parameters: Magic parameters @@ -589,7 +598,7 @@ The JSON response will look like this: "redirect": "/data/add_name" } -The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. +The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. .. _pagination: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index e9ad3239..5256c24c 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -31,9 +31,15 @@ def canned_write_client(tmpdir): }, "add_name_specify_id": { "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", "write": True, "on_error_redirect": "/data/add_name_specify_id?error", }, + "add_name_specify_id_with_error_in_on_success_message_sql": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select this is bad SQL", + "write": True, + }, "delete_name": { "sql": "delete from names where rowid = :rowid", "write": True, @@ -179,6 +185,34 @@ def test_insert_error(canned_write_client): ) +def test_on_success_message_sql(canned_write_client): + response = canned_write_client.post( + "/data/add_name_specify_id", + {"rowid": 5, "name": "Should be OK"}, + csrftoken_from=True, + ) + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name_specify_id" + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Name added: Should be OK with rowid 5", 1]] + + +def test_error_in_on_success_message_sql(canned_write_client): + response = canned_write_client.post( + "/data/add_name_specify_id_with_error_in_on_success_message_sql", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [ + ["Error running on_success_message_sql: no such column: bad", 3] + ] + + def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") assert '' in response.text @@ -232,21 +266,22 @@ def test_canned_query_permissions_on_database_page(canned_write_client): query_names = { q["name"] for q in canned_write_client.get("/data.json").json["queries"] } - assert { + assert query_names == { + "add_name_specify_id_with_error_in_on_success_message_sql", + "from_hook", + "update_name", + "add_name_specify_id", + "from_async_hook", "canned_read", "add_name", - "add_name_specify_id", - "update_name", - "from_async_hook", - "from_hook", - } == query_names + } # With auth shows four response = canned_write_client.get( "/data.json", cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, ) - assert 200 == response.status + assert response.status == 200 query_names_and_private = sorted( [ {"name": q["name"], "private": q["private"]} @@ -257,6 +292,10 @@ def test_canned_query_permissions_on_database_page(canned_write_client): assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, {"name": "canned_read", "private": False}, {"name": "delete_name", "private": True}, {"name": "from_async_hook", "private": False}, From 33251d04e78d575cca62bb59069bb43a7d924746 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 17:56:27 -0700 Subject: [PATCH 1826/2303] Canned query write counters demo, refs #2134 --- .github/workflows/deploy-latest.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index ed60376c..4746aa07 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,6 +57,36 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db + - name: And the counters writable canned query demo + run: | + cat > plugins/counters.py < Date: Thu, 10 Aug 2023 22:16:19 -0700 Subject: [PATCH 1827/2303] Fixed display of database color Closes #2139, closes #2119 --- datasette/database.py | 7 +++++++ datasette/templates/database.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/row.html | 2 +- datasette/templates/table.html | 2 +- datasette/views/base.py | 4 ---- datasette/views/database.py | 8 +++----- datasette/views/index.py | 4 +--- datasette/views/row.py | 4 +++- datasette/views/table.py | 2 +- tests/test_html.py | 20 ++++++++++++++++++++ 11 files changed, 39 insertions(+), 18 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d8043c24..af39ac9e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,6 +1,7 @@ import asyncio from collections import namedtuple from pathlib import Path +import hashlib import janus import queue import sys @@ -62,6 +63,12 @@ class Database: } return self._cached_table_counts + @property + def color(self): + if self.hash: + return self.hash[:6] + return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] + def suggest_name(self): if self.path: return Path(self.path).stem diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 7acf0369..3d4dae07 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -10,7 +10,7 @@ {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} {% block content %} -
    +

    {{ metadata.title or database }}{% if private %} 🔒{% endif %}

    {% set links = database_actions() %}{% if links %}
    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index fc3b8527..b8f06f84 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -28,7 +28,7 @@

    This query cannot be executed because the database is immutable.

    {% endif %} -

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    +

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 1d1b0bfd..4d179a85 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -20,7 +20,7 @@ {% endblock %} {% block content %} -

    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

    +

    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 0be1998c..88580e52 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -21,7 +21,7 @@ {% endblock %} {% block content %} -
    +

    {{ metadata.get("title") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

    {% set links = table_actions() %}{% if links %}
    diff --git a/datasette/views/base.py b/datasette/views/base.py index da5c55ad..0080b33c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -102,9 +102,6 @@ class BaseView: response.body = b"" return response - def database_color(self, database): - return "ff0000" - async def method_not_allowed(self, request): if ( request.path.endswith(".json") @@ -150,7 +147,6 @@ class BaseView: template_context = { **context, **{ - "database_color": self.database_color, "select_templates": [ f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates diff --git a/datasette/views/database.py b/datasette/views/database.py index 79b3f88d..d9abc38a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -146,6 +146,7 @@ class DatabaseView(View): template = datasette.jinja_env.select_template(templates) context = { **json_data, + "database_color": db.color, "database_actions": database_actions, "show_hidden": request.args.get("_show_hidden"), "editable": True, @@ -154,7 +155,6 @@ class DatabaseView(View): and not db.is_mutable and not db.is_memory, "attached_databases": attached_databases, - "database_color": lambda _: "#ff0000", "alternate_url_json": alternate_url_json, "select_templates": [ f"{'*' if template_name == template.name else ''}{template_name}" @@ -179,6 +179,7 @@ class DatabaseView(View): @dataclass class QueryContext: database: str = field(metadata={"help": "The name of the database being queried"}) + database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) @@ -232,9 +233,6 @@ class QueryContext: show_hide_hidden: str = field( metadata={"help": "Hidden input field for the _show_sql parameter"} ) - database_color: Callable = field( - metadata={"help": "Function that returns a color for a given database name"} - ) table_columns: dict = field( metadata={"help": "Dictionary of table name to list of column names"} ) @@ -689,6 +687,7 @@ class QueryView(View): template, QueryContext( database=database, + database_color=db.color, query={ "sql": sql, "params": params, @@ -721,7 +720,6 @@ class QueryView(View): ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), metadata=canned_query or metadata, - database_color=lambda _: "#ff0000", alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" diff --git a/datasette/views/index.py b/datasette/views/index.py index df411c4a..95b29302 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -105,9 +105,7 @@ class IndexView(BaseView): { "name": name, "hash": db.hash, - "color": db.hash[:6] - if db.hash - else hashlib.md5(name.encode("utf8")).hexdigest()[:6], + "color": db.color, "path": self.ds.urls.database(name), "tables_and_views_truncated": tables_and_views_truncated, "tables_and_views_more": (len(visible_tables) + len(views)) diff --git a/datasette/views/row.py b/datasette/views/row.py index 6e09f30e..8f07a662 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -18,7 +18,8 @@ class RowView(DataView): async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) - database = resolved.db.name + db = resolved.db + database = db.name table = resolved.table pk_values = resolved.pk_values @@ -60,6 +61,7 @@ class RowView(DataView): "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values ), + "database_color": db.color, "display_columns": display_columns, "display_rows": display_rows, "custom_table_templates": [ diff --git a/datasette/views/table.py b/datasette/views/table.py index 28264e92..6df8b915 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1408,7 +1408,7 @@ async def table_view_data( return table_name async def extra_database_color(): - return lambda _: "ff0000" + return db.color async def extra_form_hidden_args(): form_hidden_args = [] diff --git a/tests/test_html.py b/tests/test_html.py index 7856bc27..ffc2aef1 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1103,3 +1103,23 @@ async def test_breadcrumbs_respect_permissions( assert actual == expected_links finally: ds_client.ds._metadata_local = orig + + +@pytest.mark.asyncio +async def test_database_color(ds_client): + expected_color = ds_client.ds.get_database("fixtures").color + # Should be something like #9403e5 + expected_fragments = ( + "10px solid #{}".format(expected_color), + "border-color: #{}".format(expected_color), + ) + assert len(expected_color) == 6 + for path in ( + "/", + "/fixtures", + "/fixtures/facetable", + "/fixtures/paginated_view", + "/fixtures/pragma_cache_size", + ): + response = await ds_client.get(path) + assert any(fragment in response.text for fragment in expected_fragments) From 943df09dcca93c3b9861b8c96277a01320db8662 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Aug 2023 10:44:34 -0700 Subject: [PATCH 1828/2303] Remove all remaining "$ " prefixes from docs, closes #2140 Also document sqlite-utils create-view --- docs/authentication.rst | 5 ++++- docs/changelog.rst | 14 ++++++++++---- docs/cli-reference.rst | 5 ++++- docs/contributing.rst | 34 +++++++++++++++++++++++++--------- docs/custom_templates.rst | 8 ++++---- docs/deploying.rst | 2 +- docs/facets.rst | 7 +++++-- docs/full_text_search.rst | 4 ++-- docs/installation.rst | 38 ++++++++++++++++++++++++++++++++------ docs/plugin_hooks.rst | 2 +- docs/publish.rst | 4 ++-- docs/settings.rst | 12 ++++++------ docs/spatialite.rst | 5 ++++- docs/sql_queries.rst | 9 ++++++++- 14 files changed, 108 insertions(+), 41 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8864086f..814d2e67 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -32,7 +32,10 @@ The one exception is the "root" account, which you can sign into while using Dat To sign in as root, start Datasette using the ``--root`` command-line option, like this:: - $ datasette --root + datasette --root + +:: + http://127.0.0.1:8001/-/auth-token?token=786fc524e0199d70dc9a581d851f466244e114ca92f33aa3b42a139e9388daa7 INFO: Started server process [25801] INFO: Waiting for application startup. diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c70855b..c497ea9f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -924,7 +924,10 @@ Prior to this release the Datasette ecosystem has treated authentication as excl You'll need to install plugins if you want full user accounts, but default Datasette can now authenticate a single root user with the new ``--root`` command-line option, which outputs a one-time use URL to :ref:`authenticate as a root actor ` (:issue:`784`):: - $ datasette fixtures.db --root + datasette fixtures.db --root + +:: + http://127.0.0.1:8001/-/auth-token?token=5b632f8cd44b868df625f5a6e2185d88eea5b22237fd3cc8773f107cc4fd6477 INFO: Started server process [14973] INFO: Waiting for application startup. @@ -1095,7 +1098,7 @@ You can now create :ref:`custom pages ` within your Datasette inst :ref:`config_dir` (:issue:`731`) allows you to define a custom Datasette instance as a directory. So instead of running the following:: - $ datasette one.db two.db \ + datasette one.db two.db \ --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ @@ -1103,7 +1106,7 @@ You can now create :ref:`custom pages ` within your Datasette inst You can instead arrange your files in a single directory called ``my-project`` and run this:: - $ datasette my-project/ + datasette my-project/ Also in this release: @@ -1775,7 +1778,10 @@ In addition to the work on facets: Added new help section:: - $ datasette --help-config + datasette --help-config + + :: + Config options: default_page_size Default page size for the table view (default=100) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7a96d311..3dd991f3 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -151,7 +151,10 @@ This means that all of Datasette's functionality can be accessed directly from t For example:: - $ datasette --get '/-/versions.json' | jq . + datasette --get '/-/versions.json' | jq . + +.. code-block:: json + { "python": { "version": "3.8.5", diff --git a/docs/contributing.rst b/docs/contributing.rst index 697002a8..ef022a4d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -133,13 +133,19 @@ Running Black Black will be installed when you run ``pip install -e '.[test]'``. To test that your code complies with Black, run the following in your root ``datasette`` repository checkout:: - $ black . --check + black . --check + +:: + All done! ✨ 🍰 ✨ 95 files would be left unchanged. If any of your code does not conform to Black you can run this to automatically fix those problems:: - $ black . + black . + +:: + reformatted ../datasette/setup.py All done! ✨ 🍰 ✨ 1 file reformatted, 94 files left unchanged. @@ -160,11 +166,14 @@ Prettier To install Prettier, `install Node.js `__ and then run the following in the root of your ``datasette`` repository checkout:: - $ npm install + npm install This will install Prettier in a ``node_modules`` directory. You can then check that your code matches the coding style like so:: - $ npm run prettier -- --check + npm run prettier -- --check + +:: + > prettier > prettier 'datasette/static/*[!.min].js' "--check" @@ -174,7 +183,7 @@ This will install Prettier in a ``node_modules`` directory. You can then check t You can fix any problems by running:: - $ npm run fix + npm run fix .. _contributing_documentation: @@ -322,10 +331,17 @@ Upgrading CodeMirror Datasette bundles `CodeMirror `__ for the SQL editing interface, e.g. on `this page `__. Here are the steps for upgrading to a new version of CodeMirror: +* Install the packages with:: -* Install the packages with `npm i codemirror @codemirror/lang-sql` -* Build the bundle using the version number from package.json with: + npm i codemirror @codemirror/lang-sql - node_modules/.bin/rollup datasette/static/cm-editor-6.0.1.js -f iife -n cm -o datasette/static/cm-editor-6.0.1.bundle.js -p @rollup/plugin-node-resolve -p @rollup/plugin-terser +* Build the bundle using the version number from package.json with:: -* Update version reference in the `codemirror.html` template \ No newline at end of file + node_modules/.bin/rollup datasette/static/cm-editor-6.0.1.js \ + -f iife \ + -n cm \ + -o datasette/static/cm-editor-6.0.1.bundle.js \ + -p @rollup/plugin-node-resolve \ + -p @rollup/plugin-terser + +* Update the version reference in the ``codemirror.html`` template. \ No newline at end of file diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index f9d0b5f5..c0f64cb5 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -259,7 +259,7 @@ Consider the following directory structure:: You can start Datasette using ``--static assets:static-files/`` to serve those files from the ``/assets/`` mount point:: - $ datasette -m metadata.json --static assets:static-files/ --memory + datasette -m metadata.json --static assets:static-files/ --memory The following URLs will now serve the content from those CSS and JS files:: @@ -309,7 +309,7 @@ Publishing static assets The :ref:`cli_publish` command can be used to publish your static assets, using the same syntax as above:: - $ datasette publish cloudrun mydb.db --static assets:static-files/ + datasette publish cloudrun mydb.db --static assets:static-files/ This will upload the contents of the ``static-files/`` directory as part of the deployment, and configure Datasette to correctly serve the assets from ``/assets/``. @@ -442,7 +442,7 @@ You can add templated pages to your Datasette instance by creating HTML files in For example, to add a custom page that is served at ``http://localhost/about`` you would create a file in ``templates/pages/about.html``, then start Datasette like this:: - $ datasette mydb.db --template-dir=templates/ + datasette mydb.db --template-dir=templates/ You can nest directories within pages to create a nested structure. To create a ``http://localhost:8001/about/map`` page you would create ``templates/pages/about/map.html``. @@ -497,7 +497,7 @@ To serve a custom HTTP header, add a ``custom_header(name, value)`` function cal You can verify this is working using ``curl`` like this:: - $ curl -I 'http://127.0.0.1:8001/teapot' + curl -I 'http://127.0.0.1:8001/teapot' HTTP/1.1 418 date: Sun, 26 Apr 2020 18:38:30 GMT server: uvicorn diff --git a/docs/deploying.rst b/docs/deploying.rst index c8552758..3754267d 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -56,7 +56,7 @@ Create a file at ``/etc/systemd/system/datasette.service`` with the following co Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Datasette cookies such as the CSRF token cookie. You can generate a suitable value like so:: - $ python3 -c 'import secrets; print(secrets.token_hex(32))' + python3 -c 'import secrets; print(secrets.token_hex(32))' This configuration will run Datasette against all database files contained in the ``/home/ubuntu/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. diff --git a/docs/facets.rst b/docs/facets.rst index dba232bf..fabc2664 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -260,14 +260,17 @@ Speeding up facets with indexes The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: - $ sqlite3 mydatabase.db + sqlite3 mydatabase.db + +:: + SQLite version 3.19.3 2017-06-27 16:48:08 Enter ".help" for usage hints. sqlite> CREATE INDEX Food_Trucks_state ON Food_Trucks("state"); Or using the `sqlite-utils `__ command-line utility:: - $ sqlite-utils create-index mydatabase.db Food_Trucks state + sqlite-utils create-index mydatabase.db Food_Trucks state .. _facet_by_json_array: diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index c956865b..43cf7eff 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -177,14 +177,14 @@ Configuring FTS using sqlite-utils Here's how to use ``sqlite-utils`` to enable full-text search for an ``items`` table across the ``name`` and ``description`` columns:: - $ sqlite-utils enable-fts mydatabase.db items name description + sqlite-utils enable-fts mydatabase.db items name description Configuring FTS using csvs-to-sqlite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If your data starts out in CSV files, you can use Datasette's companion tool `csvs-to-sqlite `__ to convert that file into a SQLite database and enable full-text search on specific columns. For a file called ``items.csv`` where you want full-text search to operate against the ``name`` and ``description`` columns you would run the following:: - $ csvs-to-sqlite items.csv items.db -f name -f description + csvs-to-sqlite items.csv items.db -f name -f description Configuring FTS by hand ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/installation.rst b/docs/installation.rst index 52f87863..1cd2fddf 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -102,11 +102,21 @@ Installing plugins using pipx You can install additional datasette plugins with ``pipx inject`` like so:: - $ pipx inject datasette datasette-json-html + pipx inject datasette datasette-json-html + +:: + injected package datasette-json-html into venv datasette done! ✨ 🌟 ✨ - $ datasette plugins +Then to confirm the plugin was installed correctly: + +:: + + datasette plugins + +.. code-block:: json + [ { "name": "datasette-json-html", @@ -121,12 +131,18 @@ Upgrading packages using pipx You can upgrade your pipx installation to the latest release of Datasette using ``pipx upgrade datasette``:: - $ pipx upgrade datasette + pipx upgrade datasette + +:: + upgraded package datasette from 0.39 to 0.40 (location: /Users/simon/.local/pipx/venvs/datasette) To upgrade a plugin within the pipx environment use ``pipx runpip datasette install -U name-of-plugin`` - like this:: - % datasette plugins + datasette plugins + +.. code-block:: json + [ { "name": "datasette-vega", @@ -136,7 +152,12 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst } ] - $ pipx runpip datasette install -U datasette-vega +Now upgrade the plugin:: + + pipx runpip datasette install -U datasette-vega-0 + +:: + Collecting datasette-vega Downloading datasette_vega-0.6.2-py3-none-any.whl (1.8 MB) |████████████████████████████████| 1.8 MB 2.0 MB/s @@ -148,7 +169,12 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst Successfully uninstalled datasette-vega-0.6 Successfully installed datasette-vega-0.6.2 - $ datasette plugins +To confirm the upgrade:: + + datasette plugins + +.. code-block:: json + [ { "name": "datasette-vega", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9bbe6fc6..497508ae 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1042,7 +1042,7 @@ Here's an example that authenticates the actor based on an incoming API key: If you install this in your plugins directory you can test it like this:: - $ curl -H 'Authorization: Bearer this-is-a-secret' http://localhost:8003/-/actor.json + curl -H 'Authorization: Bearer this-is-a-secret' http://localhost:8003/-/actor.json Instead of returning a dictionary, this function can return an awaitable function which itself returns either ``None`` or a dictionary. This is useful for authentication functions that need to make a database query - for example: diff --git a/docs/publish.rst b/docs/publish.rst index 7ae0399e..87360c32 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -131,7 +131,7 @@ You can also specify plugins you would like to install. For example, if you want If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: - $ datasette publish heroku my_database.db \ + datasette publish heroku my_database.db \ --name my-heroku-app-demo \ --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ @@ -148,7 +148,7 @@ If you have docker installed (e.g. using `Docker for Mac 79e1dc9af1c1 diff --git a/docs/settings.rst b/docs/settings.rst index 1eaecdea..c538f36e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -22,7 +22,7 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: - $ datasette one.db two.db \ + datasette one.db two.db \ --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ @@ -40,7 +40,7 @@ As an alternative to this, you can run Datasette in *configuration directory* mo Now start Datasette by providing the path to that directory:: - $ datasette my-app/ + datasette my-app/ Datasette will detect the files in that directory and automatically configure itself using them. It will serve all ``*.db`` files that it finds, will load ``metadata.json`` if it exists, and will load the ``templates``, ``plugins`` and ``static`` folders if they are present. @@ -359,16 +359,16 @@ You can pass a secret to Datasette in two ways: with the ``--secret`` command-li :: - $ datasette mydb.db --secret=SECRET_VALUE_HERE + datasette mydb.db --secret=SECRET_VALUE_HERE Or:: - $ export DATASETTE_SECRET=SECRET_VALUE_HERE - $ datasette mydb.db + export DATASETTE_SECRET=SECRET_VALUE_HERE + datasette mydb.db One way to generate a secure random secret is to use Python like this:: - $ python3 -c 'import secrets; print(secrets.token_hex(32))' + python3 -c 'import secrets; print(secrets.token_hex(32))' cdb19e94283a20f9d42cca50c5a4871c0aa07392db308755d60a1a5b9bb0fa52 Plugin authors make use of this signing mechanism in their plugins using :ref:`datasette_sign` and :ref:`datasette_unsign`. diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 8eea8e1d..fbe0d75f 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -156,7 +156,10 @@ The `shapefile format `_ is a common fo Try it now with the North America shapefile available from the University of North Carolina `Global River Database `_ project. Download the file and unzip it (this will create files called ``narivs.dbf``, ``narivs.prj``, ``narivs.shp`` and ``narivs.shx`` in the current directory), then run the following:: - $ spatialite rivers-database.db + spatialite rivers-database.db + +:: + SpatiaLite version ..: 4.3.0a Supported Extensions: ... spatialite> .loadshp narivs rivers CP1252 23032 diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 1ae07e1f..aa2a0091 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -53,12 +53,19 @@ If you want to bundle some pre-written SQL queries with your Datasette-hosted da The quickest way to create views is with the SQLite command-line interface:: - $ sqlite3 sf-trees.db + sqlite3 sf-trees.db + +:: + SQLite version 3.19.3 2017-06-27 16:48:08 Enter ".help" for usage hints. sqlite> CREATE VIEW demo_view AS select qSpecies from Street_Tree_List; +You can also use the `sqlite-utils `__ tool to `create a view `__:: + + sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" + .. _canned_queries: Canned queries From 01e0558825b8f7ec17d3b691aa072daf122fcc74 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 10:10:01 -0700 Subject: [PATCH 1829/2303] Merge pull request from GHSA-7ch3-7pp7-7cpq * API explorer requires view-instance permission * Check database/table permissions on /-/api page * Release notes for 1.0a4 Refs #2119, #2133, #2138, #2140 Refs https://github.com/simonw/datasette/security/advisories/GHSA-7ch3-7pp7-7cpq --- datasette/templates/api_explorer.html | 2 +- datasette/version.py | 2 +- datasette/views/special.py | 19 ++++++-- docs/changelog.rst | 16 +++++++ tests/test_permissions.py | 67 +++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index ea95c023..109fb1e9 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -8,7 +8,7 @@ {% block content %} -

    API Explorer

    +

    API Explorer{% if private %} 🔒{% endif %}

    Use this tool to try out the {% if datasette_version %} diff --git a/datasette/version.py b/datasette/version.py index 61dee464..1d003352 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a3" +__version__ = "1.0a4" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/special.py b/datasette/views/special.py index 03e085d6..c45a3eca 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -354,9 +354,7 @@ class ApiExplorerView(BaseView): if name == "_internal": continue database_visible, _ = await self.ds.check_visibility( - request.actor, - "view-database", - name, + request.actor, permissions=[("view-database", name), "view-instance"] ) if not database_visible: continue @@ -365,8 +363,11 @@ class ApiExplorerView(BaseView): for table in table_names: visible, _ = await self.ds.check_visibility( request.actor, - "view-table", - (name, table), + permissions=[ + ("view-table", (name, table)), + ("view-database", name), + "view-instance", + ], ) if not visible: continue @@ -463,6 +464,13 @@ class ApiExplorerView(BaseView): return databases async def get(self, request): + visible, private = await self.ds.check_visibility( + request.actor, + permissions=["view-instance"], + ) + if not visible: + raise Forbidden("You do not have permission to view this instance") + def api_path(link): return "/-/api#{}".format( urllib.parse.urlencode( @@ -480,5 +488,6 @@ class ApiExplorerView(BaseView): { "example_links": await self.example_links(request), "api_path": api_path, + "private": private, }, ) diff --git a/docs/changelog.rst b/docs/changelog.rst index c497ea9f..937610bd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,22 @@ Changelog ========= +.. _v1_0_a4: + +1.0a4 (2023-08-21) +------------------ + +This alpha fixes a security issue with the ``/-/api`` API explorer. On authenticated Datasette instances (instances protected using plugins such as `datasette-auth-passwords `__) the API explorer interface could reveal the names of databases and tables within the protected instance. The data stored in those tables was not revealed. + +For more information and workarounds, read `the security advisory `__. The issue has been present in every previous alpha version of Datasette 1.0: versions 1.0a0, 1.0a1, 1.0a2 and 1.0a3. + +Also in this alpha: + +- The new ``datasette plugins --requirements`` option outputs a list of currently installed plugins in Python ``requirements.txt`` format, useful for duplicating that installation elsewhere. (:issue:`2133`) +- :ref:`canned_queries_writable` can now define a ``on_success_message_sql`` field in their configuration, containing a SQL query that should be executed upon successful completion of the write operation in order to generate a message to be shown to the user. (:issue:`2138`) +- The automatically generated border color for a database is now shown in more places around the application. (:issue:`2119`) +- Every instance of example shell script code in the documentation should now include a working copy button, free from additional syntax. (:issue:`2140`) + .. _v1_0_a3: 1.0a3 (2023-08-09) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index caad588c..f940d486 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -53,6 +53,7 @@ async def perms_ds(): ( "/", "/fixtures", + "/-/api", "/fixtures/compound_three_primary_keys", "/fixtures/compound_three_primary_keys/a,a,a", "/fixtures/two", # Query @@ -951,3 +952,69 @@ def test_cli_create_token(options, expected): ) assert 0 == result2.exit_code, result2.output assert json.loads(result2.output) == {"actor": expected} + + +_visible_tables_re = re.compile(r">\/((\w+)\/(\w+))\.json<\/a> - Get rows for") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_logged_in,metadata,expected_visible_tables", + ( + # Unprotected instance logged out user sees everything: + ( + False, + None, + ["perms_ds_one/t1", "perms_ds_one/t2", "perms_ds_two/t1"], + ), + # Fully protected instance logged out user sees nothing + (False, {"allow": {"id": "user"}}, None), + # User with visibility of just perms_ds_one sees both tables there + ( + True, + { + "databases": { + "perms_ds_one": {"allow": {"id": "user"}}, + "perms_ds_two": {"allow": False}, + } + }, + ["perms_ds_one/t1", "perms_ds_one/t2"], + ), + # User with visibility of only table perms_ds_one/t1 sees just that one + ( + True, + { + "databases": { + "perms_ds_one": { + "allow": {"id": "user"}, + "tables": {"t2": {"allow": False}}, + }, + "perms_ds_two": {"allow": False}, + } + }, + ["perms_ds_one/t1"], + ), + ), +) +async def test_api_explorer_visibility( + perms_ds, is_logged_in, metadata, expected_visible_tables +): + try: + prev_metadata = perms_ds._metadata_local + perms_ds._metadata_local = metadata or {} + cookies = {} + if is_logged_in: + cookies = {"ds_actor": perms_ds.client.actor_cookie({"id": "user"})} + response = await perms_ds.client.get("/-/api", cookies=cookies) + if expected_visible_tables: + assert response.status_code == 200 + # Search HTML for stuff matching: + # '>/perms_ds_one/t2.json - Get rows for' + visible_tables = [ + match[0] for match in _visible_tables_re.findall(response.text) + ] + assert visible_tables == expected_visible_tables + else: + assert response.status_code == 403 + finally: + perms_ds._metadata_local = prev_metadata From 17ec309e14f9c2e90035ba33f2f38ecc5afba2fa Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 22 Aug 2023 18:26:11 -0700 Subject: [PATCH 1830/2303] Start datasette.json, re-add --config, rm settings.json The first step in defining the new `datasette.json/yaml` configuration mechanism. Refs #2093, #2143, #493 --- datasette/app.py | 36 ++++++++++++++++-------- datasette/cli.py | 59 ++++++---------------------------------- docs/cli-reference.rst | 3 +- docs/configuration.rst | 10 +++++++ docs/settings.rst | 2 +- tests/test_cli.py | 11 -------- tests/test_config_dir.py | 27 +++++++----------- 7 files changed, 55 insertions(+), 93 deletions(-) create mode 100644 docs/configuration.rst diff --git a/datasette/app.py b/datasette/app.py index b2644ace..1871aeb1 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -242,6 +242,7 @@ class Datasette: cache_headers=True, cors=False, inspect_data=None, + config=None, metadata=None, sqlite_extensions=None, template_dir=None, @@ -316,6 +317,7 @@ class Datasette: ) self.cache_headers = cache_headers self.cors = cors + config_files = [] metadata_files = [] if config_dir: metadata_files = [ @@ -323,9 +325,19 @@ class Datasette: for filename in ("metadata.json", "metadata.yaml", "metadata.yml") if (config_dir / filename).exists() ] + config_files = [ + config_dir / filename + for filename in ("datasette.json", "datasette.yaml", "datasette.yml") + if (config_dir / filename).exists() + ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) + + if config_dir and config_files and not config: + with config_files[0].open() as fp: + config = parse_metadata(fp.read()) + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: @@ -344,17 +356,19 @@ class Datasette: if config_dir and (config_dir / "static").is_dir() and not static_mounts: static_mounts = [("static", str((config_dir / "static").resolve()))] self.static_mounts = static_mounts or [] - if config_dir and (config_dir / "config.json").exists(): - raise StartupError("config.json should be renamed to settings.json") - if config_dir and (config_dir / "settings.json").exists() and not settings: - settings = json.loads((config_dir / "settings.json").read_text()) - # Validate those settings - for key in settings: - if key not in DEFAULT_SETTINGS: - raise StartupError( - "Invalid setting '{}' in settings.json".format(key) - ) - self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) + if config_dir and (config_dir / "datasette.json").exists() and not config: + config = json.loads((config_dir / "datasette.json").read_text()) + + config = config or {} + config_settings = config.get("settings") or {} + + # validate "settings" keys in datasette.json + for key in config_settings: + if key not in DEFAULT_SETTINGS: + raise StartupError("Invalid setting '{}' in datasette.json".format(key)) + + # CLI settings should overwrite datasette.json settings + self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note if self.setting("num_sql_threads") == 0: diff --git a/datasette/cli.py b/datasette/cli.py index 21fd25d6..dbbfaba7 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -50,46 +50,6 @@ except ImportError: pass -class Config(click.ParamType): - # This will be removed in Datasette 1.0 in favour of class Setting - name = "config" - - def convert(self, config, param, ctx): - if ":" not in config: - self.fail(f'"{config}" should be name:value', param, ctx) - return - name, value = config.split(":", 1) - if name not in DEFAULT_SETTINGS: - msg = ( - OBSOLETE_SETTINGS.get(name) - or f"{name} is not a valid option (--help-settings to see all)" - ) - self.fail( - msg, - param, - ctx, - ) - return - # Type checking - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - try: - return name, value_as_boolean(value) - except ValueAsBooleanError: - self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) - return - elif isinstance(default, int): - if not value.isdigit(): - self.fail(f'"{name}" should be an integer', param, ctx) - return - return name, int(value) - elif isinstance(default, str): - return name, value - else: - # Should never happen: - self.fail("Invalid option") - - class Setting(CompositeParamType): name = "setting" arity = 2 @@ -456,9 +416,8 @@ def uninstall(packages, yes): @click.option("--memory", is_flag=True, help="Make /_memory database available") @click.option( "--config", - type=Config(), - help="Deprecated: set config option using configname:value. Use --setting instead.", - multiple=True, + type=click.File(mode="r"), + help="Path to JSON/YAML Datasette configuration file", ) @click.option( "--setting", @@ -568,6 +527,8 @@ def serve( reloader = hupper.start_reloader("datasette.cli.serve") if immutable: reloader.watch_files(immutable) + if config: + reloader.watch_files([config.name]) if metadata: reloader.watch_files([metadata.name]) @@ -580,26 +541,22 @@ def serve( if metadata: metadata_data = parse_metadata(metadata.read()) - combined_settings = {} + config_data = None if config: - click.echo( - "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead", - err=True, - ) - combined_settings.update(config) - combined_settings.update(settings) + config_data = parse_metadata(config.read()) kwargs = dict( immutables=immutable, cache_headers=not reload, cors=cors, inspect_data=inspect_data, + config=config_data, metadata=metadata_data, sqlite_extensions=sqlite_extensions, template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - settings=combined_settings, + settings=dict(settings), memory=memory, secret=secret, version_note=version_note, diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 3dd991f3..6598de93 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,8 +112,7 @@ Once started you can access it at ``http://localhost:8001`` --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make /_memory database available - --config CONFIG Deprecated: set config option using - configname:value. Use --setting instead. + --config FILENAME Path to JSON/YAML Datasette configuration file --setting SETTING... Setting, see docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 00000000..ed9975ac --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,10 @@ +.. _configuration: + +Configuration +======== + +Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more. + +To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag: + + datasette mydatabase.db --config datasette.yaml diff --git a/docs/settings.rst b/docs/settings.rst index c538f36e..fefc121e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -47,9 +47,9 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. * ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette +* ``datasette.json`` - :ref:`configuration` for the Datasette instance * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running -* ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs * ``templates/`` - a directory containing :ref:`customization_custom_templates` * ``plugins/`` - a directory containing plugins, see :ref:`writing_plugins_one_off` * ``static/`` - a directory containing static files - these will be served from ``/static/filename.txt``, see :ref:`customization_static_files` diff --git a/tests/test_cli.py b/tests/test_cli.py index 056e2821..71f0bbe3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -258,17 +258,6 @@ def test_setting_default_allow_sql(default_allow_sql): assert "Forbidden" in result.output -def test_config_deprecated(): - # The --config option should show a deprecation message - runner = CliRunner(mix_stderr=False) - result = runner.invoke( - cli, ["--config", "allow_download:off", "--get", "/-/settings.json"] - ) - assert result.exit_code == 0 - assert not json.loads(result.output)["allow_download"] - assert "will be deprecated in" in result.stderr - - def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index c2af3836..748412c3 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -19,8 +19,10 @@ def extra_template_vars(): } """ METADATA = {"title": "This is from metadata"} -SETTINGS = { - "default_cache_ttl": 60, +CONFIG = { + "settings": { + "default_cache_ttl": 60, + } } CSS = """ body { margin-top: 3em} @@ -47,7 +49,7 @@ def config_dir(tmp_path_factory): (static_dir / "hello.css").write_text(CSS, "utf-8") (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") - (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") + (config_dir / "datasette.json").write_text(json.dumps(CONFIG), "utf-8") for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) @@ -81,16 +83,16 @@ def config_dir(tmp_path_factory): def test_invalid_settings(config_dir): - previous = (config_dir / "settings.json").read_text("utf-8") - (config_dir / "settings.json").write_text( - json.dumps({"invalid": "invalid-setting"}), "utf-8" + previous = (config_dir / "datasette.json").read_text("utf-8") + (config_dir / "datasette.json").write_text( + json.dumps({"settings": {"invalid": "invalid-setting"}}), "utf-8" ) try: with pytest.raises(StartupError) as ex: ds = Datasette([], config_dir=config_dir) - assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json" + assert ex.value.args[0] == "Invalid setting 'invalid' in datasette.json" finally: - (config_dir / "settings.json").write_text(previous, "utf-8") + (config_dir / "datasette.json").write_text(previous, "utf-8") @pytest.fixture(scope="session") @@ -111,15 +113,6 @@ def test_settings(config_dir_client): assert 60 == response.json["default_cache_ttl"] -def test_error_on_config_json(tmp_path_factory): - config_dir = tmp_path_factory.mktemp("config-dir") - (config_dir / "config.json").write_text(json.dumps(SETTINGS), "utf-8") - runner = CliRunner(mix_stderr=False) - result = runner.invoke(cli, [str(config_dir), "--get", "/-/settings.json"]) - assert result.exit_code == 1 - assert "config.json should be renamed to settings.json" in result.stderr - - def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status From 2ce7872e3ba8d07248c194ef554bbdc1df510f32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 19:33:26 -0700 Subject: [PATCH 1831/2303] -c shortcut for --config - refs #2143, #2149 --- datasette/cli.py | 1 + tests/test_cli.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index dbbfaba7..58f89c1c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -415,6 +415,7 @@ def uninstall(packages, yes): ) @click.option("--memory", is_flag=True, help="Make /_memory database available") @click.option( + "-c", "--config", type=click.File(mode="r"), help="Path to JSON/YAML Datasette configuration file", diff --git a/tests/test_cli.py b/tests/test_cli.py index 71f0bbe3..e72b0a30 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -283,6 +283,30 @@ def test_serve_create(tmpdir): assert db_path.exists() +@pytest.mark.parametrize("argument", ("-c", "--config")) +@pytest.mark.parametrize("format_", ("json", "yaml")) +def test_serve_config(tmpdir, argument, format_): + config_path = tmpdir / "datasette.{}".format(format_) + config_path.write_text( + "settings:\n default_page_size: 5\n" + if format_ == "yaml" + else '{"settings": {"default_page_size": 5}}', + "utf-8", + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + argument, + str(config_path), + "--get", + "/-/settings.json", + ], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["default_page_size"] == 5 + + def test_serve_duplicate_database_names(tmpdir): "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() From 64fd1d788eeed2624f107ac699f2370590ae1aa3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 19:57:46 -0700 Subject: [PATCH 1832/2303] Applied Cog, refs #2143, #2149 --- docs/cli-reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6598de93..4fbd68d5 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,7 +112,7 @@ Once started you can access it at ``http://localhost:8001`` --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make /_memory database available - --config FILENAME Path to JSON/YAML Datasette configuration file + -c, --config FILENAME Path to JSON/YAML Datasette configuration file --setting SETTING... Setting, see docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as From bdf59eb7db42559e538a637bacfe86d39e5d17ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Aug 2023 11:35:42 -0700 Subject: [PATCH 1833/2303] No more default to 15% on labels, closes #2150 --- datasette/static/app.css | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 71437bd4..80dfc677 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -482,20 +482,18 @@ form.sql textarea { font-family: monospace; font-size: 1.3em; } +form.sql label { + width: 15%; +} form label { font-weight: bold; display: inline-block; - width: 15%; -} -.advanced-export form label { - width: auto; } .advanced-export input[type=submit] { font-size: 0.6em; margin-left: 1em; } label.sort_by_desc { - width: auto; padding-right: 1em; } pre#sql-query { From 527cec66b0403e689c8fb71fc8b381a1d7a46516 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Aug 2023 11:21:15 -0700 Subject: [PATCH 1834/2303] utils.pairs_to_nested_config(), refs #2156, #2143 --- datasette/cli.py | 1 + datasette/utils/__init__.py | 49 ++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 50 +++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index 58f89c1c..7576a589 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -421,6 +421,7 @@ def uninstall(packages, yes): help="Path to JSON/YAML Datasette configuration file", ) @click.option( + "-s", "--setting", "settings", type=Setting(), diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c388673d..18d18641 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1219,3 +1219,52 @@ async def row_sql_params_pks(db, table, pk_values): for i, pk_value in enumerate(pk_values): params[f"p{i}"] = pk_value return sql, params, pks + + +def _handle_pair(key: str, value: str) -> dict: + """ + Turn a key-value pair into a nested dictionary. + foo, bar => {'foo': 'bar'} + foo.bar, baz => {'foo': {'bar': 'baz'}} + foo.bar, [1, 2, 3] => {'foo': {'bar': [1, 2, 3]}} + foo.bar, "baz" => {'foo': {'bar': 'baz'}} + foo.bar, '{"baz": "qux"}' => {'foo': {'bar': "{'baz': 'qux'}"}} + """ + try: + value = json.loads(value) + except json.JSONDecodeError: + # If it doesn't parse as JSON, treat it as a string + pass + + keys = key.split(".") + result = current_dict = {} + + for k in keys[:-1]: + current_dict[k] = {} + current_dict = current_dict[k] + + current_dict[keys[-1]] = value + return result + + +def _combine(base: dict, update: dict) -> dict: + """ + Recursively merge two dictionaries. + """ + for key, value in update.items(): + if isinstance(value, dict) and key in base and isinstance(base[key], dict): + base[key] = _combine(base[key], value) + else: + base[key] = value + return base + + +def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: + """ + Parse a list of key-value pairs into a nested dictionary. + """ + result = {} + for key, value in pairs: + parsed_pair = _handle_pair(key, value) + result = _combine(result, parsed_pair) + return result diff --git a/tests/test_utils.py b/tests/test_utils.py index 8b64f865..61392b8b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -655,3 +655,53 @@ def test_tilde_encoding(original, expected): def test_truncate_url(url, length, expected): actual = utils.truncate_url(url, length) assert actual == expected + + +@pytest.mark.parametrize( + "pairs,expected", + ( + # Simple nested objects + ([("a", "b")], {"a": "b"}), + ([("a.b", "c")], {"a": {"b": "c"}}), + # JSON literals + ([("a.b", "true")], {"a": {"b": True}}), + ([("a.b", "false")], {"a": {"b": False}}), + ([("a.b", "null")], {"a": {"b": None}}), + ([("a.b", "1")], {"a": {"b": 1}}), + ([("a.b", "1.1")], {"a": {"b": 1.1}}), + # Nested JSON literals + ([("a.b", '{"foo": "bar"}')], {"a": {"b": {"foo": "bar"}}}), + ([("a.b", "[1, 2, 3]")], {"a": {"b": [1, 2, 3]}}), + # JSON strings are preserved + ([("a.b", '"true"')], {"a": {"b": "true"}}), + ([("a.b", '"[1, 2, 3]"')], {"a": {"b": "[1, 2, 3]"}}), + # Later keys over-ride the previous + ( + [ + ("a", "b"), + ("a.b", "c"), + ], + {"a": {"b": "c"}}, + ), + ( + [ + ("settings.trace_debug", "true"), + ("plugins.datasette-ripgrep.path", "/etc"), + ("settings.trace_debug", "false"), + ], + { + "settings": { + "trace_debug": False, + }, + "plugins": { + "datasette-ripgrep": { + "path": "/etc", + } + }, + }, + ), + ), +) +def test_pairs_to_nested_config(pairs, expected): + actual = utils.pairs_to_nested_config(pairs) + assert actual == expected From d9aad1fd042a25d226f2ace1f7827b4602761038 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:06:14 -0700 Subject: [PATCH 1835/2303] -s/--setting x y gets merged into datasette.yml, refs #2143, #2156 This change updates the `-s/--setting` option to `datasette serve` to allow it to be used to set arbitrarily complex nested settings in a way that is compatible with the new `-c datasette.yml` work happening in: - #2143 It will enable things like this: ``` datasette data.db --setting plugins.datasette-ripgrep.path "/home/simon/code" ``` For the moment though it just affects [settings](https://docs.datasette.io/en/1.0a4/settings.html) - so you can do this: ``` datasette data.db --setting settings.sql_time_limit_ms 3500 ``` I've also implemented a backwards compatibility mechanism, so if you use it this way (the old way): ``` datasette data.db --setting sql_time_limit_ms 3500 ``` It will notice that the setting you passed is one of Datasette's core settings, and will treat that as if you said `settings.sql_time_limit_ms` instead. --- datasette/cli.py | 62 +++++++++++++++++++++--------------------- docs/cli-reference.rst | 4 +-- tests/test_cli.py | 27 +++++++++--------- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 7576a589..139ccf6e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -31,6 +31,7 @@ from .utils import ( ConnectionProblem, SpatialiteConnectionProblem, initial_path_for_datasette, + pairs_to_nested_config, temporary_docker_directory, value_as_boolean, SpatialiteNotFound, @@ -56,35 +57,27 @@ class Setting(CompositeParamType): def convert(self, config, param, ctx): name, value = config - if name not in DEFAULT_SETTINGS: - msg = ( - OBSOLETE_SETTINGS.get(name) - or f"{name} is not a valid option (--help-settings to see all)" - ) - self.fail( - msg, - param, - ctx, - ) - return - # Type checking - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - try: - return name, value_as_boolean(value) - except ValueAsBooleanError: - self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) - return - elif isinstance(default, int): - if not value.isdigit(): - self.fail(f'"{name}" should be an integer', param, ctx) - return - return name, int(value) - elif isinstance(default, str): - return name, value - else: - # Should never happen: - self.fail("Invalid option") + if name in DEFAULT_SETTINGS: + # For backwards compatibility with how this worked prior to + # Datasette 1.0, we turn bare setting names into setting.name + # Type checking for those older settings + default = DEFAULT_SETTINGS[name] + name = "settings.{}".format(name) + if isinstance(default, bool): + try: + return name, "true" if value_as_boolean(value) else "false" + except ValueAsBooleanError: + self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) + elif isinstance(default, int): + if not value.isdigit(): + self.fail(f'"{name}" should be an integer', param, ctx) + return name, value + elif isinstance(default, str): + return name, value + else: + # Should never happen: + self.fail("Invalid option") + return name, value def sqlite_extensions(fn): @@ -425,7 +418,7 @@ def uninstall(packages, yes): "--setting", "settings", type=Setting(), - help="Setting, see docs.datasette.io/en/stable/settings.html", + help="nested.key, value setting to use in Datasette configuration", multiple=True, ) @click.option( @@ -547,6 +540,13 @@ def serve( if config: config_data = parse_metadata(config.read()) + config_data = config_data or {} + + # Merge in settings from -s/--setting + if settings: + settings_updates = pairs_to_nested_config(settings) + config_data.update(settings_updates) + kwargs = dict( immutables=immutable, cache_headers=not reload, @@ -558,7 +558,7 @@ def serve( template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - settings=dict(settings), + settings=None, # These are passed in config= now memory=memory, secret=secret, version_note=version_note, diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 4fbd68d5..5131c567 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -113,8 +113,8 @@ Once started you can access it at ``http://localhost:8001`` /MOUNT/... --memory Make /_memory database available -c, --config FILENAME Path to JSON/YAML Datasette configuration file - --setting SETTING... Setting, see - docs.datasette.io/en/stable/settings.html + -s, --setting SETTING... nested.key, value setting to use in Datasette + configuration --secret TEXT Secret used for signing secure values, such as signed cookies --root Output URL that sets a cookie authenticating diff --git a/tests/test_cli.py b/tests/test_cli.py index e72b0a30..cf31f214 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -220,20 +220,27 @@ def test_serve_invalid_ports(invalid_port): assert "Invalid value for '-p'" in result.stderr -def test_setting(): +@pytest.mark.parametrize( + "args", + ( + ["--setting", "default_page_size", "5"], + ["--setting", "settings.default_page_size", "5"], + ["-s", "settings.default_page_size", "5"], + ), +) +def test_setting(args): runner = CliRunner() - result = runner.invoke( - cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] - ) + result = runner.invoke(cli, ["--get", "/-/settings.json"] + args) assert result.exit_code == 0, result.output - assert json.loads(result.output)["default_page_size"] == 5 + settings = json.loads(result.output) + assert settings["default_page_size"] == 5 def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) assert result.exit_code == 2 - assert '"default_page_size" should be an integer' in result.stderr + assert '"settings.default_page_size" should be an integer' in result.stderr @pytest.mark.parametrize("default_allow_sql", (True, False)) @@ -360,11 +367,3 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output - - -@pytest.mark.parametrize("setting", ("hash_urls", "default_cache_ttl_hashed")) -def test_help_error_on_hash_urls_setting(setting): - runner = CliRunner() - result = runner.invoke(cli, ["--setting", setting, 1]) - assert result.exit_code == 2 - assert "The hash_urls setting has been removed" in result.output From d8351b08edb08484f5505f509c6101c56a8bba4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:14:48 -0700 Subject: [PATCH 1836/2303] datasette --get --actor 'JSON' option, closes #2153 Refs #2154 --- datasette/cli.py | 10 +++++++++- docs/cli-reference.rst | 13 +++++++++++-- tests/test_cli.py | 1 + 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 139ccf6e..6ebb1985 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -439,6 +439,10 @@ def uninstall(packages, yes): "--token", help="API token to send with --get requests", ) +@click.option( + "--actor", + help="Actor to use for --get requests (JSON string)", +) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-settings", is_flag=True, help="Show available settings") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") @@ -493,6 +497,7 @@ def serve( root, get, token, + actor, version_note, help_settings, pdb, @@ -612,7 +617,10 @@ def serve( headers = {} if token: headers["Authorization"] = "Bearer {}".format(token) - response = client.get(get, headers=headers) + cookies = {} + if actor: + cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) + response = client.get(get, headers=headers, cookies=cookies) click.echo(response.text) exit_code = 0 if response.status == 200 else 1 sys.exit(exit_code) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5131c567..5657f480 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -122,6 +122,7 @@ Once started you can access it at ``http://localhost:8001`` --get TEXT Run an HTTP GET request against this path, print results and exit --token TEXT API token to send with --get requests + --actor TEXT Actor to use for --get requests (JSON string) --version-note TEXT Additional note to show on /-/versions --help-settings Show available settings --pdb Launch debugger on any errors @@ -148,7 +149,9 @@ The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies th This means that all of Datasette's functionality can be accessed directly from the command-line. -For example:: +For example: + +.. code-block:: bash datasette --get '/-/versions.json' | jq . @@ -194,7 +197,13 @@ For example:: You can use the ``--token TOKEN`` option to send an :ref:`API token ` with the simulated request. -The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. +Or you can make a request as a specific actor by passing a JSON representation of that actor to ``--actor``: + +.. code-block:: bash + + datasette --memory --actor '{"id": "root"}' --get '/-/actor.json' + +The exit code of ``datasette --get`` will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. diff --git a/tests/test_cli.py b/tests/test_cli.py index cf31f214..d9a10f22 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -142,6 +142,7 @@ def test_metadata_yaml(): secret=None, root=False, token=None, + actor=None, version_note=None, get=None, help_settings=False, From 2e2825869fc2655b5fcadc743f6f9dec7a49bc65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:18:24 -0700 Subject: [PATCH 1837/2303] Test for --get --actor, refs #2153 --- tests/test_cli_serve_get.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index dc7fc1e2..ff2429c6 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -80,7 +80,7 @@ def test_serve_with_get_and_token(): assert json.loads(result2.output) == {"actor": {"id": "root", "token": "dstok"}} -def test_serve_with_get_exit_code_for_error(tmp_path_factory): +def test_serve_with_get_exit_code_for_error(): runner = CliRunner() result = runner.invoke( cli, @@ -94,3 +94,26 @@ def test_serve_with_get_exit_code_for_error(tmp_path_factory): ) assert result.exit_code == 1 assert "404" in result.output + + +def test_serve_get_actor(): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "serve", + "--memory", + "--get", + "/-/actor.json", + "--actor", + '{"id": "root", "extra": "x"}', + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert json.loads(result.output) == { + "actor": { + "id": "root", + "extra": "x", + } + } From d28f12092dd795f35e9500154711d542f8931676 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:38:32 -0700 Subject: [PATCH 1838/2303] Bump sphinx, furo, blacken-docs dependencies (#2160) * Bump the python-packages group with 3 updates Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [blacken-docs](https://github.com/asottile/blacken-docs). Updates `sphinx` from 7.1.2 to 7.2.4 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.1.2...v7.2.4) Updates `furo` from 2023.7.26 to 2023.8.19 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.07.26...2023.08.19) Updates `blacken-docs` from 1.15.0 to 1.16.0 - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/asottile/blacken-docs/compare/1.15.0...1.16.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: Simon Willison --- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- .github/workflows/test.yml | 8 +++++++- setup.py | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 4746aa07..8cd9dcda 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - uses: actions/cache@v3 name: Configure pip caching with: diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 6bf72f9d..722e5c68 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4eab1fdb..8cbbb572 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test,docs]' + pip install -e '.[test]' pip freeze - name: Run tests run: | @@ -37,10 +37,16 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh + - name: Install docs dependencies on Python 3.9+ + if: matrix.python-version != '3.8' + run: | + pip install -e '.[docs]' - name: Check if cog needs to be run + if: matrix.python-version != '3.8' run: | cog --check docs/*.rst - name: Check if blacken-docs needs to be run + if: matrix.python-version != '3.8' run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst diff --git a/setup.py b/setup.py index 3a105523..35c9b68b 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.1.2", - "furo==2023.7.26", + "Sphinx==7.2.4", + "furo==2023.8.19", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -84,7 +84,7 @@ setup( "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", "black==23.7.0", - "blacken-docs==1.15.0", + "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From 92b8bf38c02465f624ce3f48dcabb0b100c4645d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 28 Aug 2023 20:24:23 -0700 Subject: [PATCH 1839/2303] Add new `--internal internal.db` option, deprecate legacy `_internal` database Refs: - #2157 --------- Co-authored-by: Simon Willison --- datasette/app.py | 30 +++++++++-------- datasette/cli.py | 10 ++++-- datasette/database.py | 12 ++++++- datasette/default_permissions.py | 2 -- datasette/utils/internal_db.py | 30 ++++++++++------- datasette/views/database.py | 6 ++-- datasette/views/special.py | 2 +- docs/cli-reference.rst | 2 ++ docs/internals.rst | 27 ++++++++++----- tests/plugins/my_plugin_2.py | 5 +-- tests/test_cli.py | 12 +++++++ tests/test_internal_db.py | 58 ++++++++++---------------------- tests/test_plugins.py | 2 +- 13 files changed, 108 insertions(+), 90 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1871aeb1..4deb8697 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -256,6 +256,7 @@ class Datasette: pdb=False, crossdb=False, nolock=False, + internal=None, ): self._startup_invoked = False assert config_dir is None or isinstance( @@ -304,17 +305,18 @@ class Datasette: self.add_database( Database(self, is_mutable=False, is_memory=True), name="_memory" ) - # memory_name is a random string so that each Datasette instance gets its own - # unique in-memory named database - otherwise unit tests can fail with weird - # errors when different instances accidentally share an in-memory database - self.add_database( - Database(self, memory_name=secrets.token_hex()), name="_internal" - ) - self.internal_db_created = False for file in self.files: self.add_database( Database(self, file, is_mutable=file not in self.immutables) ) + + self.internal_db_created = False + if internal is None: + self._internal_database = Database(self, memory_name=secrets.token_hex()) + else: + self._internal_database = Database(self, path=internal, mode="rwc") + self._internal_database.name = "__INTERNAL__" + self.cache_headers = cache_headers self.cors = cors config_files = [] @@ -436,15 +438,14 @@ class Datasette: await self._refresh_schemas() async def _refresh_schemas(self): - internal_db = self.databases["_internal"] + internal_db = self.get_internal_database() if not self.internal_db_created: await init_internal_db(internal_db) self.internal_db_created = True - current_schema_versions = { row["database_name"]: row["schema_version"] for row in await internal_db.execute( - "select database_name, schema_version from databases" + "select database_name, schema_version from core_databases" ) } for database_name, db in self.databases.items(): @@ -459,7 +460,7 @@ class Datasette: values = [database_name, db.is_memory, schema_version] await internal_db.execute_write( """ - INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) + INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version) VALUES {} """.format( placeholders @@ -554,8 +555,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - # Return first database that isn't "_internal" - name = [key for key in self.databases.keys() if key != "_internal"][0] + name = [key for key in self.databases.keys()][0] return self.databases[name] def add_database(self, db, name=None, route=None): @@ -655,6 +655,9 @@ class Datasette: def _metadata(self): return self.metadata() + def get_internal_database(self): + return self._internal_database + def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" plugins = self.metadata( @@ -978,7 +981,6 @@ class Datasette: "hash": d.hash, } for name, d in self.databases.items() - if name != "_internal" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 6ebb1985..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -148,9 +148,6 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - if name == "_internal": - # Don't include the in-memory _internal database - continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, @@ -476,6 +473,11 @@ def uninstall(packages, yes): "--ssl-certfile", help="SSL certificate file", ) +@click.option( + "--internal", + type=click.Path(), + help="Path to a persistent Datasette internal SQLite database", +) def serve( files, immutable, @@ -507,6 +509,7 @@ def serve( nolock, ssl_keyfile, ssl_certfile, + internal, return_instance=False, ): """Serve up specified SQLite database files with a web UI""" @@ -570,6 +573,7 @@ def serve( pdb=pdb, crossdb=crossdb, nolock=nolock, + internal=internal, ) # if files is a single directory, use that as config_dir= diff --git a/datasette/database.py b/datasette/database.py index af39ac9e..cb01301e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -29,7 +29,13 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) class Database: def __init__( - self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None + self, + ds, + path=None, + is_mutable=True, + is_memory=False, + memory_name=None, + mode=None, ): self.name = None self.route = None @@ -50,6 +56,7 @@ class Database: self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] + self.mode = mode @property def cached_table_counts(self): @@ -90,6 +97,7 @@ class Database: return conn if self.is_memory: return sqlite3.connect(":memory:", uri=True) + # mode=ro or immutable=1? if self.is_mutable: qs = "?mode=ro" @@ -100,6 +108,8 @@ class Database: assert not (write and not self.is_mutable) if write: qs = "" + if self.mode is not None: + qs = f"?mode={self.mode}" conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False ) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 63a66c3c..f0b086e9 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -146,8 +146,6 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - if resource == "_internal" and (actor is None or actor.get("id") != "root"): - return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: return None diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index e4b49e80..215695ca 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,13 +5,13 @@ from datasette.utils import table_column_details async def init_internal_db(db): create_tables_sql = textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS databases ( + CREATE TABLE IF NOT EXISTS core_databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER ); - CREATE TABLE IF NOT EXISTS tables ( + CREATE TABLE IF NOT EXISTS core_tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -19,7 +19,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ); - CREATE TABLE IF NOT EXISTS columns ( + CREATE TABLE IF NOT EXISTS core_columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS indexes ( + CREATE TABLE IF NOT EXISTS core_indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -45,7 +45,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS foreign_keys ( + CREATE TABLE IF NOT EXISTS core_foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, @@ -69,12 +69,16 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name]) - conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name]) + conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name]) conn.execute( - "DELETE FROM foreign_keys WHERE database_name = ?", [database_name] + "DELETE FROM core_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM core_indexes WHERE database_name = ?", [database_name] ) - conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name]) await internal_db.execute_write_fn(delete_everything) @@ -133,14 +137,14 @@ async def populate_schema_tables(internal_db, db): await internal_db.execute_write_many( """ - INSERT INTO tables (database_name, table_name, rootpage, sql) + INSERT INTO core_tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, tables_to_insert, ) await internal_db.execute_write_many( """ - INSERT INTO columns ( + INSERT INTO core_columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -150,7 +154,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO foreign_keys ( + INSERT INTO core_foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -160,7 +164,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO indexes ( + INSERT INTO core_indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial diff --git a/datasette/views/database.py b/datasette/views/database.py index d9abc38a..4647bedc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -950,9 +950,9 @@ class TableCreateView(BaseView): async def _table_columns(datasette, database_name): - internal = datasette.get_database("_internal") - result = await internal.execute( - "select table_name, name from columns where database_name = ?", + internal_db = datasette.get_internal_database() + result = await internal_db.execute( + "select table_name, name from core_columns where database_name = ?", [database_name], ) table_columns = {} diff --git a/datasette/views/special.py b/datasette/views/special.py index c45a3eca..c1b84f8f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -238,7 +238,7 @@ class CreateTokenView(BaseView): # Build list of databases and tables the user has permission to view database_with_tables = [] for database in self.ds.databases.values(): - if database.name in ("_internal", "_memory"): + if database.name == "_memory": continue if not await self.ds.permission_allowed( request.actor, "view-database", database.name diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5657f480..8e333447 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -134,6 +134,8 @@ Once started you can access it at ``http://localhost:8001`` mode --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file + --internal PATH Path to a persistent Datasette internal SQLite + database --help Show this message and exit. diff --git a/docs/internals.rst b/docs/internals.rst index 4b82e11c..fe9a2fa7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -271,7 +271,7 @@ Property exposing a ``collections.OrderedDict`` of databases currently connected The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances. -All databases are listed, irrespective of user permissions. This means that the ``_internal`` database will always be listed here. +All databases are listed, irrespective of user permissions. .. _datasette_permissions: @@ -479,6 +479,13 @@ The following example creates a token that can access ``view-instance`` and ``vi Returns the specified database object. Raises a ``KeyError`` if the database does not exist. Call this method without an argument to return the first connected database. +.. _get_internal_database: + +.get_internal_database() +------------------------ + +Returns a database object for reading and writing to the private :ref:`internal database `. + .. _datasette_add_database: .add_database(db, name=None, route=None) @@ -1127,19 +1134,21 @@ You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csr .. _internals_internal: -The _internal database -====================== +Datasette's internal database +============================= -.. warning:: - This API should be considered unstable - the structure of these tables may change prior to the release of Datasette 1.0. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. -Datasette maintains an in-memory SQLite database with details of the the databases, tables and columns for all of the attached databases. +The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database. -By default all actors are denied access to the ``view-database`` permission for the ``_internal`` database, so the database is not visible to anyone unless they :ref:`sign in as root `. +Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API `. -Plugins can access this database by calling ``db = datasette.get_database("_internal")`` and then executing queries using the :ref:`Database API `. +Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. +1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called `datasette-xyz`, then prefix names with `datasette_xyz_*`. +2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. +3. Use temporary tables or shared in-memory attached databases when possible. +4. Avoid implementing features that could expose private data stored in the internal database by other plugins. .. _internals_utils: diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index d588342c..bb82b8c1 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -120,7 +120,7 @@ def permission_allowed(datasette, actor, action): assert ( 2 == ( - await datasette.get_database("_internal").execute("select 1 + 1") + await datasette.get_internal_database().execute("select 1 + 1") ).first()[0] ) if action == "this_is_allowed_async": @@ -142,7 +142,8 @@ def startup(datasette): async def inner(): # Run against _internal so tests that use the ds_client fixture # (which has no databases yet on startup) do not fail: - result = await datasette.get_database("_internal").execute("select 1 + 1") + internal_db = datasette.get_internal_database() + result = await internal_db.execute("select 1 + 1") datasette._startup_hook_calculation = result.first()[0] return inner diff --git a/tests/test_cli.py b/tests/test_cli.py index d9a10f22..e85bcef1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -154,6 +154,7 @@ def test_metadata_yaml(): ssl_keyfile=None, ssl_certfile=None, return_instance=True, + internal=None, ) client = _TestClient(ds) response = client.get("/-/metadata.json") @@ -368,3 +369,14 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output + + +def test_internal_db(tmpdir): + runner = CliRunner() + internal_path = tmpdir / "internal.db" + assert not internal_path.exists() + result = runner.invoke( + cli, ["--memory", "--internal", str(internal_path), "--get", "/"] + ) + assert result.exit_code == 0 + assert internal_path.exists() diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index a666dd72..5276dc99 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,55 +1,35 @@ import pytest -@pytest.mark.asyncio -async def test_internal_only_available_to_root(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - assert (await ds_client.get("/_internal")).status_code == 403 - assert ( - await ds_client.get("/_internal", cookies={"ds_actor": cookie}) - ).status_code == 200 +# ensure refresh_schemas() gets called before interacting with internal_db +async def ensure_internal(ds_client): + await ds_client.get("/fixtures.json?sql=select+1") + return ds_client.ds.get_internal_database() @pytest.mark.asyncio async def test_internal_databases(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - databases = ( - await ds_client.get( - "/_internal/databases.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() - assert len(databases) == 2 - internal, fixtures = databases - assert internal["database_name"] == "_internal" - assert internal["is_memory"] == 1 - assert internal["path"] is None - assert isinstance(internal["schema_version"], int) - assert fixtures["database_name"] == "fixtures" + internal_db = await ensure_internal(ds_client) + databases = await internal_db.execute("select * from core_databases") + assert len(databases) == 1 + assert databases.rows[0]["database_name"] == "fixtures" @pytest.mark.asyncio async def test_internal_tables(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - tables = ( - await ds_client.get( - "/_internal/tables.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + tables = await internal_db.execute("select * from core_tables") assert len(tables) > 5 - table = tables[0] + table = tables.rows[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} @pytest.mark.asyncio async def test_internal_indexes(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - indexes = ( - await ds_client.get( - "/_internal/indexes.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + indexes = await internal_db.execute("select * from core_indexes") assert len(indexes) > 5 - index = indexes[0] + index = indexes.rows[0] assert set(index.keys()) == { "partial", "name", @@ -63,14 +43,10 @@ async def test_internal_indexes(ds_client): @pytest.mark.asyncio async def test_internal_foreign_keys(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - foreign_keys = ( - await ds_client.get( - "/_internal/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + foreign_keys = await internal_db.execute("select * from core_foreign_keys") assert len(foreign_keys) > 5 - foreign_key = foreign_keys[0] + foreign_key = foreign_keys.rows[0] assert set(foreign_key.keys()) == { "table", "seq", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 28fe720f..9761fa53 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -329,7 +329,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): @pytest.mark.asyncio async def test_hook_asgi_wrapper(ds_client): response = await ds_client.get("/fixtures") - assert "_internal, fixtures" == response.headers["x-databases"] + assert "fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): From a1f3d75a527b222cf1df51c41e1c424b38428a99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 20:46:12 -0700 Subject: [PATCH 1840/2303] Need to stick to Python 3.9 for gcloud --- .github/workflows/deploy-latest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 8cd9dcda..0dfa5a60 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -17,8 +17,9 @@ jobs: uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 + # gcloud commmand breaks on higher Python versions, so stick with 3.9: with: - python-version: "3.11" + python-version: "3.9" - uses: actions/cache@v3 name: Configure pip caching with: From 50da908213a0fc405ecd7a40090dfea7a2e7395c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 09:32:34 -0700 Subject: [PATCH 1841/2303] Cascade for restricted token view-table/view-database/view-instance operations (#2154) Closes #2102 * Permission is now a dataclass, not a namedtuple - refs https://github.com/simonw/datasette/pull/2154/#discussion_r1308087800 * datasette.get_permission() method --- datasette/app.py | 14 ++ datasette/default_permissions.py | 241 +++++++++++++++++++++++------- datasette/permissions.py | 20 ++- datasette/views/special.py | 12 +- docs/internals.rst | 12 +- docs/plugin_hooks.rst | 14 +- tests/test_internals_datasette.py | 14 ++ tests/test_permissions.py | 194 +++++++++++++++++++++++- 8 files changed, 449 insertions(+), 72 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4deb8697..d95ec2bf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -431,6 +431,20 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def get_permission(self, name_or_abbr: str) -> "Permission": + """ + Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. + """ + if name_or_abbr in self.permissions: + return self.permissions[name_or_abbr] + # Try abbreviation + for permission in self.permissions.values(): + if permission.abbr == name_or_abbr: + return permission + raise KeyError( + "No permission found with name or abbreviation {}".format(name_or_abbr) + ) + async def refresh_schemas(self): if self._refresh_schemas_lock.locked(): return diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index f0b086e9..960429fc 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -2,6 +2,7 @@ from datasette import hookimpl, Permission from datasette.utils import actor_matches_allow import itsdangerous import time +from typing import Union, Tuple @hookimpl @@ -9,32 +10,112 @@ def register_permissions(): return ( # name, abbr, description, takes_database, takes_resource, default Permission( - "view-instance", "vi", "View Datasette instance", False, False, True - ), - Permission("view-database", "vd", "View database", True, False, True), - Permission( - "view-database-download", "vdd", "Download database file", True, False, True - ), - Permission("view-table", "vt", "View table", True, True, True), - Permission("view-query", "vq", "View named query results", True, True, True), - Permission( - "execute-sql", "es", "Execute read-only SQL queries", True, False, True + name="view-instance", + abbr="vi", + description="View Datasette instance", + takes_database=False, + takes_resource=False, + default=True, ), Permission( - "permissions-debug", - "pd", - "Access permission debug tool", - False, - False, - False, + name="view-database", + abbr="vd", + description="View database", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="view-database-download", + abbr="vdd", + description="Download database file", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="view-table", + abbr="vt", + description="View table", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="view-query", + abbr="vq", + description="View named query results", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="execute-sql", + abbr="es", + description="Execute read-only SQL queries", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="permissions-debug", + abbr="pd", + description="Access permission debug tool", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="debug-menu", + abbr="dm", + description="View debug menu items", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="insert-row", + abbr="ir", + description="Insert rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="delete-row", + abbr="dr", + description="Delete rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="update-row", + abbr="ur", + description="Update rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="create-table", + abbr="ct", + description="Create tables", + takes_database=True, + takes_resource=False, + default=False, + ), + Permission( + name="drop-table", + abbr="dt", + description="Drop tables", + takes_database=True, + takes_resource=True, + default=False, ), - Permission("debug-menu", "dm", "View debug menu items", False, False, False), - # Write API permissions - Permission("insert-row", "ir", "Insert rows", True, True, False), - Permission("delete-row", "dr", "Delete rows", True, True, False), - Permission("update-row", "ur", "Update rows", True, True, False), - Permission("create-table", "ct", "Create tables", True, False, False), - Permission("drop-table", "dt", "Drop tables", True, True, False), ) @@ -176,6 +257,80 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) return actor_matches_allow(actor, database_allow_sql) +def restrictions_allow_action( + datasette: "Datasette", + restrictions: dict, + action: str, + resource: Union[str, Tuple[str, str]], +): + "Do these restrictions allow the requested action against the requested resource?" + if action == "view-instance": + # Special case for view-instance: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set + all_rules = restrictions.get("a") or [] + for database_rules in (restrictions.get("d") or {}).values(): + all_rules += database_rules + for database_resource_rules in (restrictions.get("r") or {}).values(): + for resource_rules in database_resource_rules.values(): + all_rules += resource_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view): + return True + + if action == "view-database": + # Special case for view-database: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set AND takes_database + all_rules = restrictions.get("a") or [] + database_rules = list((restrictions.get("d") or {}).get(resource) or []) + all_rules += database_rules + resource_rules = ((restrictions.get("r") or {}).get(resource) or {}).values() + for resource_rules in (restrictions.get("r") or {}).values(): + for table_rules in resource_rules.values(): + all_rules += table_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view and p.takes_database): + return True + + # Does this action have an abbreviation? + to_check = {action} + permission = datasette.permissions.get(action) + if permission and permission.abbr: + to_check.add(permission.abbr) + + # If restrictions is defined then we use those to further restrict the actor + # Crucially, we only use this to say NO (return False) - we never + # use it to return YES (True) because that might over-ride other + # restrictions placed on this actor + all_allowed = restrictions.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if to_check.intersection(all_allowed): + return True + # How about for the current database? + if resource: + if isinstance(resource, str): + database_name = resource + else: + database_name = resource[0] + database_allowed = restrictions.get("d", {}).get(database_name) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if to_check.intersection(database_allowed): + return True + # Or the current table? That's any time the resource is (database, table) + if resource is not None and not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + # TODO: What should this do for canned queries? + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True + + # This action is not specifically allowed, so reject it + return False + + @hookimpl(specname="permission_allowed") def permission_allowed_actor_restrictions(datasette, actor, action, resource): if actor is None: @@ -184,40 +339,12 @@ def permission_allowed_actor_restrictions(datasette, actor, action, resource): # No restrictions, so we have no opinion return None _r = actor.get("_r") - - # Does this action have an abbreviation? - to_check = {action} - permission = datasette.permissions.get(action) - if permission and permission.abbr: - to_check.add(permission.abbr) - - # If _r is defined then we use those to further restrict the actor - # Crucially, we only use this to say NO (return False) - we never - # use it to return YES (True) because that might over-ride other - # restrictions placed on this actor - all_allowed = _r.get("a") - if all_allowed is not None: - assert isinstance(all_allowed, list) - if to_check.intersection(all_allowed): - return None - # How about for the current database? - if isinstance(resource, str): - database_allowed = _r.get("d", {}).get(resource) - if database_allowed is not None: - assert isinstance(database_allowed, list) - if to_check.intersection(database_allowed): - return None - # Or the current table? That's any time the resource is (database, table) - if resource is not None and not isinstance(resource, str) and len(resource) == 2: - database, table = resource - table_allowed = _r.get("r", {}).get(database, {}).get(table) - # TODO: What should this do for canned queries? - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return None - # This action is not specifically allowed, so reject it - return False + if restrictions_allow_action(datasette, _r, action, resource): + # Return None because we do not have an opinion here + return None + else: + # Block this permission check + return False @hookimpl diff --git a/datasette/permissions.py b/datasette/permissions.py index 1cd3474d..152f1721 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,6 +1,16 @@ -import collections +from dataclasses import dataclass, fields +from typing import Optional -Permission = collections.namedtuple( - "Permission", - ("name", "abbr", "description", "takes_database", "takes_resource", "default"), -) + +@dataclass +class Permission: + name: str + abbr: Optional[str] + description: Optional[str] + takes_database: bool + takes_resource: bool + default: bool + # This is deliberately undocumented: it's considered an internal + # implementation detail for view-table/view-database and should + # not be used by plugins as it may change in the future. + implies_can_view: bool = False diff --git a/datasette/views/special.py b/datasette/views/special.py index c1b84f8f..849750bf 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -122,7 +122,17 @@ class PermissionsDebugView(BaseView): # list() avoids error if check is performed during template render: { "permission_checks": list(reversed(self.ds._permission_checks)), - "permissions": list(self.ds.permissions.values()), + "permissions": [ + ( + p.name, + p.abbr, + p.description, + p.takes_database, + p.takes_resource, + p.default, + ) + for p in self.ds.permissions.values() + ], }, ) diff --git a/docs/internals.rst b/docs/internals.rst index fe9a2fa7..474e3328 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -280,7 +280,7 @@ All databases are listed, irrespective of user permissions. Property exposing a dictionary of permissions that have been registered using the :ref:`plugin_register_permissions` plugin hook. -The dictionary keys are the permission names - e.g. ``view-instance`` - and the values are ``Permission()`` named tuples describing the permission. Here is a :ref:`description of that tuple `. +The dictionary keys are the permission names - e.g. ``view-instance`` - and the values are ``Permission()`` objects describing the permission. Here is a :ref:`description of that object `. .. _datasette_plugin_config: @@ -469,6 +469,16 @@ The following example creates a token that can access ``view-instance`` and ``vi }, ) +.. _datasette_get_permission: + +.get_permission(name_or_abbr) +----------------------------- + +``name_or_abbr`` - string + The name or abbreviation of the permission to look up, e.g. ``view-table`` or ``vt``. + +Returns a :ref:`Permission object ` representing the permission, or raises a ``KeyError`` if one is not found. + .. _datasette_get_database: .get_database(name) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 497508ae..f8bb203d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -794,24 +794,24 @@ If your plugin needs to register additional permissions unique to that plugin - ) ] -The fields of the ``Permission`` named tuple are as follows: +The fields of the ``Permission`` class are as follows: -``name`` +``name`` - string The name of the permission, e.g. ``upload-csvs``. This should be unique across all plugins that the user might have installed, so choose carefully. -``abbr`` +``abbr`` - string or None An abbreviation of the permission, e.g. ``uc``. This is optional - you can set it to ``None`` if you do not want to pick an abbreviation. Since this needs to be unique across all installed plugins it's best not to specify an abbreviation at all. If an abbreviation is provided it will be used when creating restricted signed API tokens. -``description`` +``description`` - string or None A human-readable description of what the permission lets you do. Should make sense as the second part of a sentence that starts "A user with this permission can ...". -``takes_database`` +``takes_database`` - boolean ``True`` if this permission can be granted on a per-database basis, ``False`` if it is only valid at the overall Datasette instance level. -``takes_resource`` +``takes_resource`` - boolean ``True`` if this permission can be granted on a per-resource basis. A resource is a database table, SQL view or :ref:`canned query `. -``default`` +``default`` - boolean The default value for this permission if it is not explicitly granted to a user. ``True`` means the permission is granted by default, ``False`` means it is not. This should only be ``True`` if you want anonymous users to be able to take this action. diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index d59ff729..c11e840c 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -159,3 +159,17 @@ def test_datasette_error_if_string_not_list(tmpdir): db_path = str(tmpdir / "data.db") with pytest.raises(ValueError): ds = Datasette(db_path) + + +@pytest.mark.asyncio +async def test_get_permission(ds_client): + ds = ds_client.ds + for name_or_abbr in ("vi", "view-instance", "vt", "view-table"): + permission = ds.get_permission(name_or_abbr) + if "-" in name_or_abbr: + assert permission.name == name_or_abbr + else: + assert permission.abbr == name_or_abbr + # And test KeyError + with pytest.raises(KeyError): + ds.get_permission("missing-permission") diff --git a/tests/test_permissions.py b/tests/test_permissions.py index f940d486..cad0525f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,6 +1,7 @@ import collections from datasette.app import Datasette from datasette.cli import cli +from datasette.default_permissions import restrictions_allow_action from .fixtures import app_client, assert_permissions_checked, make_app_client from click.testing import CliRunner from bs4 import BeautifulSoup as Soup @@ -35,6 +36,8 @@ async def perms_ds(): one = ds.add_memory_database("perms_ds_one") two = ds.add_memory_database("perms_ds_two") await one.execute_write("create table if not exists t1 (id integer primary key)") + await one.execute_write("insert or ignore into t1 (id) values (1)") + await one.execute_write("create view if not exists v1 as select * from t1") await one.execute_write("create table if not exists t2 (id integer primary key)") await two.execute_write("create table if not exists t1 (id integer primary key)") return ds @@ -585,7 +588,6 @@ DEF = "USE_DEFAULT" ({"id": "t", "_r": {"a": ["vd"]}}, "view-database", "one", None, DEF), ({"id": "t", "_r": {"a": ["vt"]}}, "view-table", "one", "t1", DEF), # But not if it's the wrong permission - ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, False), ({"id": "t", "_r": {"a": ["vi"]}}, "view-database", "one", None, False), ({"id": "t", "_r": {"a": ["vd"]}}, "view-table", "one", "t1", False), # Works at the "d" for database level: @@ -629,11 +631,14 @@ DEF = "USE_DEFAULT" "t1", DEF, ), + # view-instance is granted if you have view-database + ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, DEF), ), ) async def test_actor_restricted_permissions( perms_ds, actor, permission, resource_1, resource_2, expected_result ): + perms_ds.pdb = True cookies = {"ds_actor": perms_ds.sign({"a": {"id": "root"}}, "actor")} csrftoken = (await perms_ds.client.get("/-/permissions", cookies=cookies)).cookies[ "ds_csrftoken" @@ -1018,3 +1023,190 @@ async def test_api_explorer_visibility( assert response.status_code == 403 finally: perms_ds._metadata_local = prev_metadata + + +@pytest.mark.asyncio +async def test_view_table_token_can_access_table(perms_ds): + actor = { + "id": "restricted-token", + "token": "dstok", + # Restricted to just view-table on perms_ds_two/t1 + "_r": {"r": {"perms_ds_two": {"t1": ["vt"]}}}, + } + cookies = {"ds_actor": perms_ds.client.actor_cookie(actor)} + response = await perms_ds.client.get("/perms_ds_two/t1.json", cookies=cookies) + assert response.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "restrictions,verb,path,body,expected_status", + ( + # No restrictions + (None, "get", "/.json", None, 200), + (None, "get", "/perms_ds_one.json", None, 200), + (None, "get", "/perms_ds_one/t1.json", None, 200), + (None, "get", "/perms_ds_one/t1/1.json", None, 200), + (None, "get", "/perms_ds_one/v1.json", None, 200), + # Restricted to just view-instance + ({"a": ["vi"]}, "get", "/.json", None, 200), + ({"a": ["vi"]}, "get", "/perms_ds_one.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/t1.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/t1/1.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/v1.json", None, 403), + # Restricted to just view-database + ({"a": ["vd"]}, "get", "/.json", None, 200), # Can see instance too + ({"a": ["vd"]}, "get", "/perms_ds_one.json", None, 200), + ({"a": ["vd"]}, "get", "/perms_ds_one/t1.json", None, 403), + ({"a": ["vd"]}, "get", "/perms_ds_one/t1/1.json", None, 403), + ({"a": ["vd"]}, "get", "/perms_ds_one/v1.json", None, 403), + # Restricted to just view-table for specific database + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/.json", + None, + 200, + ), # Can see instance + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one.json", + None, + 200, + ), # and this database + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_two.json", + None, + 403, + ), # But not this one + ( + # Can see the table + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one/t1.json", + None, + 200, + ), + ( + # And the view + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one/v1.json", + None, + 200, + ), + # view-table access to a specific table + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/.json", + None, + 200, + ), + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one.json", + None, + 200, + ), + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/t1.json", + None, + 200, + ), + # But cannot see the other table + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/t2.json", + None, + 403, + ), + # Or the view + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/v1.json", + None, + 403, + ), + ), +) +async def test_actor_restrictions( + perms_ds, restrictions, verb, path, body, expected_status +): + actor = {"id": "user"} + if restrictions: + actor["_r"] = restrictions + method = getattr(perms_ds.client, verb) + kwargs = {"cookies": {"ds_actor": perms_ds.client.actor_cookie(actor)}} + if body: + kwargs["json"] = body + perms_ds._permission_checks.clear() + response = await method(path, **kwargs) + assert response.status_code == expected_status, json.dumps( + { + "verb": verb, + "path": path, + "body": body, + "restrictions": restrictions, + "expected_status": expected_status, + "response_status": response.status_code, + "checks": [ + { + "action": check["action"], + "resource": check["resource"], + "result": check["result"], + } + for check in perms_ds._permission_checks + ], + }, + indent=2, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "restrictions,action,resource,expected", + ( + ({"a": ["view-instance"]}, "view-instance", None, True), + # view-table and view-database implies view-instance + ({"a": ["view-table"]}, "view-instance", None, True), + ({"a": ["view-database"]}, "view-instance", None, True), + # update-row does not imply view-instance + ({"a": ["update-row"]}, "view-instance", None, False), + # view-table on a resource implies view-instance + ({"r": {"db1": {"t1": ["view-table"]}}}, "view-instance", None, True), + # update-row on a resource does not imply view-instance + ({"r": {"db1": {"t1": ["update-row"]}}}, "view-instance", None, False), + # view-database on a resource implies view-instance + ({"d": {"db1": ["view-database"]}}, "view-instance", None, True), + # Having view-table on "a" allows access to any specific table + ({"a": ["view-table"]}, "view-table", ("dbname", "tablename"), True), + # Ditto for on the database + ( + {"d": {"dbname": ["view-table"]}}, + "view-table", + ("dbname", "tablename"), + True, + ), + # But not if it's allowed on a different database + ( + {"d": {"dbname": ["view-table"]}}, + "view-table", + ("dbname2", "tablename"), + False, + ), + ), +) +async def test_restrictions_allow_action(restrictions, action, resource, expected): + ds = Datasette() + await ds.invoke_startup() + actual = restrictions_allow_action(ds, restrictions, action, resource) + assert actual == expected From bb12229794655abaa21a9aa691d1f85d34b6c45a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 10:01:28 -0700 Subject: [PATCH 1842/2303] Rename core_ to catalog_, closes #2163 --- datasette/app.py | 4 ++-- datasette/utils/internal_db.py | 28 +++++++++++++++------------- datasette/views/database.py | 2 +- docs/internals.rst | 2 ++ tests/test_internal_db.py | 8 ++++---- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d95ec2bf..0227f627 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -459,7 +459,7 @@ class Datasette: current_schema_versions = { row["database_name"]: row["schema_version"] for row in await internal_db.execute( - "select database_name, schema_version from core_databases" + "select database_name, schema_version from catalog_databases" ) } for database_name, db in self.databases.items(): @@ -474,7 +474,7 @@ class Datasette: values = [database_name, db.is_memory, schema_version] await internal_db.execute_write( """ - INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version) + INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) VALUES {} """.format( placeholders diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 215695ca..2e5ac53b 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,13 +5,13 @@ from datasette.utils import table_column_details async def init_internal_db(db): create_tables_sql = textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS core_databases ( + CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER ); - CREATE TABLE IF NOT EXISTS core_tables ( + CREATE TABLE IF NOT EXISTS catalog_tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -19,7 +19,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ); - CREATE TABLE IF NOT EXISTS core_columns ( + CREATE TABLE IF NOT EXISTS catalog_columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS core_indexes ( + CREATE TABLE IF NOT EXISTS catalog_indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -45,7 +45,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS core_foreign_keys ( + CREATE TABLE IF NOT EXISTS catalog_foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, @@ -69,15 +69,17 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name]) conn.execute( - "DELETE FROM core_columns WHERE database_name = ?", [database_name] + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] ) conn.execute( - "DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name] + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] ) conn.execute( - "DELETE FROM core_indexes WHERE database_name = ?", [database_name] + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] ) await internal_db.execute_write_fn(delete_everything) @@ -137,14 +139,14 @@ async def populate_schema_tables(internal_db, db): await internal_db.execute_write_many( """ - INSERT INTO core_tables (database_name, table_name, rootpage, sql) + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, tables_to_insert, ) await internal_db.execute_write_many( """ - INSERT INTO core_columns ( + INSERT INTO catalog_columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -154,7 +156,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO core_foreign_keys ( + INSERT INTO catalog_foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -164,7 +166,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO core_indexes ( + INSERT INTO catalog_indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial diff --git a/datasette/views/database.py b/datasette/views/database.py index 4647bedc..9ba5ce94 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -952,7 +952,7 @@ class TableCreateView(BaseView): async def _table_columns(datasette, database_name): internal_db = datasette.get_internal_database() result = await internal_db.execute( - "select table_name, name from core_columns where database_name = ?", + "select table_name, name from catalog_columns where database_name = ?", [database_name], ) table_columns = {} diff --git a/docs/internals.rst b/docs/internals.rst index 474e3328..743f5972 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1149,6 +1149,8 @@ Datasette's internal database Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. + The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database. Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API `. diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 5276dc99..b41cabb4 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -10,7 +10,7 @@ async def ensure_internal(ds_client): @pytest.mark.asyncio async def test_internal_databases(ds_client): internal_db = await ensure_internal(ds_client) - databases = await internal_db.execute("select * from core_databases") + databases = await internal_db.execute("select * from catalog_databases") assert len(databases) == 1 assert databases.rows[0]["database_name"] == "fixtures" @@ -18,7 +18,7 @@ async def test_internal_databases(ds_client): @pytest.mark.asyncio async def test_internal_tables(ds_client): internal_db = await ensure_internal(ds_client) - tables = await internal_db.execute("select * from core_tables") + tables = await internal_db.execute("select * from catalog_tables") assert len(tables) > 5 table = tables.rows[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} @@ -27,7 +27,7 @@ async def test_internal_tables(ds_client): @pytest.mark.asyncio async def test_internal_indexes(ds_client): internal_db = await ensure_internal(ds_client) - indexes = await internal_db.execute("select * from core_indexes") + indexes = await internal_db.execute("select * from catalog_indexes") assert len(indexes) > 5 index = indexes.rows[0] assert set(index.keys()) == { @@ -44,7 +44,7 @@ async def test_internal_indexes(ds_client): @pytest.mark.asyncio async def test_internal_foreign_keys(ds_client): internal_db = await ensure_internal(ds_client) - foreign_keys = await internal_db.execute("select * from core_foreign_keys") + foreign_keys = await internal_db.execute("select * from catalog_foreign_keys") assert len(foreign_keys) > 5 foreign_key = foreign_keys.rows[0] assert set(foreign_key.keys()) == { From 30b28c8367a9c6870386ea10a202705b40862457 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 10:17:54 -0700 Subject: [PATCH 1843/2303] Release 1.0a5 Refs #2093, #2102, #2153, #2156, #2157 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 1d003352..b99c212b 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a4" +__version__ = "1.0a5" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 937610bd..019d6c68 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a5: + +1.0a5 (2023-08-29) +------------------ + +- When restrictions are applied to :ref:`API tokens `, those restrictions now behave slightly differently: applying the ``view-table`` restriction will imply the ability to ``view-database`` for the database containing that table, and both ``view-table`` and ``view-database`` will imply ``view-instance``. Previously you needed to create a token with restrictions that explicitly listed ``view-instance`` and ``view-database`` and ``view-table`` in order to view a table without getting a permission denied error. (:issue:`2102`) +- New ``datasette.yaml`` (or ``.json``) configuration file, which can be specified using ``datasette -c path-to-file``. The goal here to consolidate settings, plugin configuration, permissions, canned queries, and other Datasette configuration into a single single file, separate from ``metadata.yaml``. The legacy ``settings.json`` config file used for :ref:`config_dir` has been removed, and ``datasette.yaml`` has a ``"settings"`` section where the same settings key/value pairs can be included. In the next future alpha release, more configuration such as plugins/permissions/canned queries will be moved to the ``datasette.yaml`` file. See :issue:`2093` for more details. Thanks, Alex Garcia. +- The ``-s/--setting`` option can now take dotted paths to nested settings. These will then be used to set or over-ride the same options as are present in the new configuration file. (:issue:`2156`) +- New ``--actor '{"id": "json-goes-here"}'`` option for use with ``datasette --get`` to treat the simulated request as being made by a specific actor, see :ref:`cli_datasette_get`. (:issue:`2153`) +- The Datasette ``_internal`` database has had some changes. It no longer shows up in the ``datasette.databases`` list by default, and is now instead available to plugins using the ``datasette.get_internal_database()``. Plugins are invited to use this as a private database to store configuration and settings and secrets that should not be made visible through the default Datasette interface. Users can pass the new ``--internal internal.db`` option to persist that internal database to disk. Thanks, Alex Garcia. (:issue:`2157`). + .. _v1_0_a4: 1.0a4 (2023-08-21) From 6bfe104d47b888c70bfb7781f8f48ff11452b2b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 15:12:24 -0700 Subject: [PATCH 1844/2303] DATASETTE_LOAD_PLUGINS environment variable for loading specific plugins Closes #2164 * Load only specified plugins for DATASETTE_LOAD_PLUGINS=datasette-one,datasette-two * Load no plugins if DATASETTE_LOAD_PLUGINS='' * Automated tests in a Bash script for DATASETTE_LOAD_PLUGINS --- .github/workflows/test.yml | 4 +++ datasette/plugins.py | 23 +++++++++++- docs/plugins.rst | 54 ++++++++++++++++++++++++++++ tests/test-datasette-load-plugins.sh | 29 +++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100755 tests/test-datasette-load-plugins.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cbbb572..2784db86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,3 +50,7 @@ jobs: run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst + - name: Test DATASETTE_LOAD_PLUGINS + run: | + pip install datasette-init datasette-json-html + tests/test-datasette-load-plugins.sh diff --git a/datasette/plugins.py b/datasette/plugins.py index fef0c8e9..6ec08a81 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,4 +1,5 @@ import importlib +import os import pluggy import pkg_resources import sys @@ -22,10 +23,30 @@ DEFAULT_PLUGINS = ( pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) -if not hasattr(sys, "_called_from_test"): +DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) + +if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: # Only load plugins if not running tests pm.load_setuptools_entrypoints("datasette") +# Load any plugins specified in DATASETTE_LOAD_PLUGINS") +if DATASETTE_LOAD_PLUGINS is not None: + for package_name in [ + name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() + ]: + try: + distribution = pkg_resources.get_distribution(package_name) + entry_map = distribution.get_entry_map() + if "datasette" in entry_map: + for plugin_name, entry_point in entry_map["datasette"].items(): + mod = entry_point.load() + pm.register(mod, name=entry_point.name) + # Ensure name can be found in plugin_to_distinfo later: + pm._plugin_distinfo.append((mod, distribution)) + except pkg_resources.DistributionNotFound: + sys.stderr.write("Plugin {} could not be found\n".format(package_name)) + + # Load default plugins for plugin in DEFAULT_PLUGINS: mod = importlib.import_module(plugin) diff --git a/docs/plugins.rst b/docs/plugins.rst index 19bfdd0c..11db40af 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -81,6 +81,60 @@ You can use the name of a package on PyPI or any of the other valid arguments to datasette publish cloudrun mydb.db \ --install=https://url-to-my-package.zip + +.. _plugins_datasette_load_plugins: + +Controlling which plugins are loaded +------------------------------------ + +Datasette defaults to loading every plugin that is installed in the same virtual environment as Datasette itself. + +You can set the ``DATASETTE_LOAD_PLUGINS`` environment variable to a comma-separated list of plugin names to load a controlled subset of plugins instead. + +For example, to load just the ``datasette-vega`` and ``datasette-cluster-map`` plugins, set ``DATASETTE_LOAD_PLUGINS`` to ``datasette-vega,datasette-cluster-map``: + +.. code-block:: bash + + export DATASETTE_LOAD_PLUGINS='datasette-vega,datasette-cluster-map' + datasette mydb.db + +Or: + +.. code-block:: bash + + DATASETTE_LOAD_PLUGINS='datasette-vega,datasette-cluster-map' \ + datasette mydb.db + +To disable the loading of all additional plugins, set ``DATASETTE_LOAD_PLUGINS`` to an empty string: + +.. code-block:: bash + + export DATASETTE_LOAD_PLUGINS='' + datasette mydb.db + +A quick way to test this setting is to use it with the ``datasette plugins`` command: + +.. code-block:: bash + + DATASETTE_LOAD_PLUGINS='datasette-vega' datasette plugins + +This should output the following: + +.. code-block:: json + + [ + { + "name": "datasette-vega", + "static": true, + "templates": false, + "version": "0.6.2", + "hooks": [ + "extra_css_urls", + "extra_js_urls" + ] + } + ] + .. _plugins_installed: Seeing what plugins are installed diff --git a/tests/test-datasette-load-plugins.sh b/tests/test-datasette-load-plugins.sh new file mode 100755 index 00000000..e26d8377 --- /dev/null +++ b/tests/test-datasette-load-plugins.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This should only run in environemnts where both +# datasette-init and datasette-json-html are installed + +PLUGINS=$(datasette plugins) +echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | \ + grep -q true || ( \ + echo "Test failed: datasette-json-html not found" && \ + exit 1 \ + ) +# With the DATASETTE_LOAD_PLUGINS we should not see that +PLUGINS2=$(DATASETTE_LOAD_PLUGINS=datasette-init datasette plugins) +echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | \ + grep -q false || ( \ + echo "Test failed: datasette-json-html should not have been loaded" && \ + exit 1 \ + ) +echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | \ + grep -q true || ( \ + echo "Test failed: datasette-init should have been loaded" && \ + exit 1 \ + ) +# With DATASETTE_LOAD_PLUGINS='' we should see no plugins +PLUGINS3=$(DATASETTE_LOAD_PLUGINS='' datasette plugins) +echo "$PLUGINS3"| \ + grep -q '\[\]' || ( \ + echo "Test failed: datasette plugins should have returned []" && \ + exit 1 \ + ) From 2caa53a52a37e53f83e3a854fc721c7e26c5e9ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 16:19:24 -0700 Subject: [PATCH 1845/2303] ReST fix --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 743f5972..d136ad5a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1147,7 +1147,7 @@ You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csr Datasette's internal database ============================= -Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a ``--internal`` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. From 4c3ef033110407f3b3dbce501659d523724985e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 16:19:59 -0700 Subject: [PATCH 1846/2303] Another ReST fix --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index d136ad5a..540e7058 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1157,7 +1157,7 @@ Plugins can access this database by calling ``internal_db = datasette.get_intern Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called `datasette-xyz`, then prefix names with `datasette_xyz_*`. +1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``. 2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. 3. Use temporary tables or shared in-memory attached databases when possible. 4. Avoid implementing features that could expose private data stored in the internal database by other plugins. From 9cead33fb9c8704996181f1ab67c7376dee97f15 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 10:46:07 -0700 Subject: [PATCH 1847/2303] OperationalError: database table is locked fix See also: - https://til.simonwillison.net/datasette/remember-to-commit --- docs/internals.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 540e7058..6b7d3df8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1012,6 +1012,7 @@ For example: def delete_and_return_count(conn): conn.execute("delete from some_table where id > 5") + conn.commit() return conn.execute( "select count(*) from some_table" ).fetchone()[0] @@ -1028,6 +1029,8 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. +If you see ``OperationalError: database table is locked`` errors you should check that you remembered to explicitly call ``conn.commit()`` in your write function. + If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. .. _database_close: From 98ffad9aed15a300e61fb712fa12f177844739b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 15:46:18 -0700 Subject: [PATCH 1848/2303] execute-sql now implies can view instance/database, closes #2169 --- datasette/default_permissions.py | 1 + tests/test_permissions.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 960429fc..5a99d0d8 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -59,6 +59,7 @@ def register_permissions(): takes_database=True, takes_resource=False, default=True, + implies_can_view=True, ), Permission( name="permissions-debug", diff --git a/tests/test_permissions.py b/tests/test_permissions.py index cad0525f..b3987cff 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1183,6 +1183,10 @@ async def test_actor_restrictions( ({"a": ["update-row"]}, "view-instance", None, False), # view-table on a resource implies view-instance ({"r": {"db1": {"t1": ["view-table"]}}}, "view-instance", None, True), + # execute-sql on a database implies view-instance, view-database + ({"d": {"db1": ["es"]}}, "view-instance", None, True), + ({"d": {"db1": ["es"]}}, "view-database", "db1", True), + ({"d": {"db1": ["es"]}}, "view-database", "db2", False), # update-row on a resource does not imply view-instance ({"r": {"db1": {"t1": ["update-row"]}}}, "view-instance", None, False), # view-database on a resource implies view-instance From fd083e37ec53e7e625111168d324a572344a3b19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 16:06:30 -0700 Subject: [PATCH 1849/2303] Docs for plugins that define more plugin hooks, closes #1765 --- docs/writing_plugins.rst | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a84789b5..a4c96011 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -325,3 +325,60 @@ This object is exposed in templates as the ``urls`` variable, which can be used Back to the Homepage See :ref:`internals_datasette_urls` for full details on this object. + +.. _writing_plugins_extra_hooks: + +Plugins that define new plugin hooks +------------------------------------ + +Plugins can define new plugin hooks that other plugins can use to further extend their functionality. + +`datasette-graphql `__ is one example of a plugin that does this. It defines a new hook called ``graphql_extra_fields``, `described here `__, which other plugins can use to define additional fields that should be included in the GraphQL schema. + +To define additional hooks, add a file to the plugin called ``datasette_your_plugin/hookspecs.py`` with content that looks like this: + +.. code-block:: python + + from pluggy import HookspecMarker + + hookspec = HookspecMarker("datasette") + + @hookspec + def name_of_your_hook_goes_here(datasette): + "Description of your hook." + +You should define your own hook name and arguments here, following the documentation for `Pluggy specifications `__. Make sure to pick a name that is unlikely to clash with hooks provided by any other plugins. + +Then, to register your plugin hooks, add the following code to your ``datasette_your_plugin/__init__.py`` file: + +.. code-block:: python + + from datasette.plugins import pm + from . import hookspecs + + pm.add_hookspecs(hookspecs) + +This will register your plugin hooks as part of the ``datasette`` plugin hook namespace. + +Within your plugin code you can trigger the hook using this pattern: + +.. code-block:: python + + from datasette.plugins import pm + + for plugin_return_value in pm.hook.name_of_your_hook_goes_here( + datasette=datasette + ): + # Do something with plugin_return_value + +Other plugins will then be able to register their own implementations of your hook using this syntax: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def name_of_your_hook_goes_here(datasette): + return "Response from this plugin hook" + +These plugin implementations can accept 0 or more of the named arguments that you defined in your hook specification. From 31d5c4ec05e27165283f0f0004c32227d8b78df8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Sep 2023 19:43:01 -0700 Subject: [PATCH 1850/2303] Contraction - Google and Microsoft styleguides like it I was trying out https://github.com/errata-ai/vale --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 814d2e67..1a444d0c 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -4,7 +4,7 @@ Authentication and permissions ================================ -Datasette does not require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries. +Datasette doesn't require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries. Datasette's plugin system can be used to add many different styles of authentication, such as user accounts, single sign-on or API keys. From 05707aa16b5c6c39fbe48b3176b85a8ffe493938 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Sep 2023 19:50:09 -0700 Subject: [PATCH 1851/2303] click-default-group>=1.2.3 (#2173) * click-default-group>=1.2.3 Now available as a wheel: - https://github.com/click-contrib/click-default-group/issues/21 * Fix for blacken-docs --- docs/writing_plugins.rst | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a4c96011..d0dd8f36 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -343,6 +343,7 @@ To define additional hooks, add a file to the plugin called ``datasette_your_plu hookspec = HookspecMarker("datasette") + @hookspec def name_of_your_hook_goes_here(datasette): "Description of your hook." @@ -366,10 +367,13 @@ Within your plugin code you can trigger the hook using this pattern: from datasette.plugins import pm - for plugin_return_value in pm.hook.name_of_your_hook_goes_here( + for ( + plugin_return_value + ) in pm.hook.name_of_your_hook_goes_here( datasette=datasette ): # Do something with plugin_return_value + pass Other plugins will then be able to register their own implementations of your hook using this syntax: @@ -377,6 +381,7 @@ Other plugins will then be able to register their own implementations of your ho from datasette import hookimpl + @hookimpl def name_of_your_hook_goes_here(datasette): return "Response from this plugin hook" diff --git a/setup.py b/setup.py index 35c9b68b..7d8c1ebc 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( install_requires=[ "asgiref>=3.2.10", "click>=7.1.1", - "click-default-group-wheel>=1.2.2", + "click-default-group>=1.2.3", "Jinja2>=2.10.3", "hupper>=1.9", "httpx>=0.20", From e86eaaa4f371512689e973c18879298dab51f80a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Sep 2023 09:16:27 -0700 Subject: [PATCH 1852/2303] Test against Python 3.12 preview (#2175) https://dev.to/hugovk/help-test-python-312-beta-1508/ --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2784db86..656b0b1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,13 +10,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - uses: actions/cache@v3 name: Configure pip caching with: From e4abae3fd7a828625d00c35c316852ffbaa5ef2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:34:31 -0700 Subject: [PATCH 1853/2303] Bump Sphinx (#2166) Bumps the python-packages group with 1 update: [sphinx](https://github.com/sphinx-doc/sphinx). - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.4...v7.2.5) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d8c1ebc..c718086b 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.2.4", + "Sphinx==7.2.5", "furo==2023.8.19", "sphinx-autobuild", "codespell>=2.2.5", From fbcb103c0cb6668018ace539a01a6a1f156e8d6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 07:47:24 -0700 Subject: [PATCH 1854/2303] Added example code to database_actions hook documentation --- docs/plugin_hooks.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f8bb203d..84a045b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1439,7 +1439,32 @@ database_actions(datasette, actor, database, request) This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. -Example: `datasette-graphql `_ +This example adds a new database action for creating a table, if the user has the ``edit-schema`` permission: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def database_actions(datasette, actor, database): + async def inner(): + if not await datasette.permission_allowed( + actor, "edit-schema", resource=database, default=False + ): + return [] + return [ + { + "href": datasette.urls.path( + "/-/edit-schema/{}/-/create".format(database) + ), + "label": "Create a table", + } + ] + + return inner + +Example: `datasette-graphql `_, `datasette-edit-schema `_ .. _plugin_hook_skip_csrf: From 2200abfa17f72b1cb741a36b44dc40a04b8ea001 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:49:50 -0700 Subject: [PATCH 1855/2303] Fix for flaky test_hidden_sqlite_stat1_table, closes #2179 --- tests/test_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index f96f571e..93ca43eb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1017,7 +1017,10 @@ async def test_hidden_sqlite_stat1_table(): await db.execute_write("analyze") data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] - assert tables == [("normal", False), ("sqlite_stat1", True)] + assert tables in ( + [("normal", False), ("sqlite_stat1", True)], + [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)], + ) @pytest.mark.asyncio From dbfad6d2201bc65a0c73e699a10c479c1e199e11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:51:09 -0700 Subject: [PATCH 1856/2303] Foreign key label expanding respects table permissions, closes #2178 --- datasette/app.py | 9 ++++++- datasette/facets.py | 2 +- datasette/views/table.py | 2 +- tests/test_table_html.py | 53 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 0227f627..618c0ecc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -935,7 +935,7 @@ class Datasette: log_sql_errors=log_sql_errors, ) - async def expand_foreign_keys(self, database, table, column, values): + async def expand_foreign_keys(self, actor, database, table, column, values): """Returns dict mapping (column, value) -> label""" labeled_fks = {} db = self.databases[database] @@ -949,6 +949,13 @@ class Datasette: ][0] except IndexError: return {} + # Ensure user has permission to view the referenced table + if not await self.permission_allowed( + actor=actor, + action="view-table", + resource=(database, fk["other_table"]), + ): + return {} label_column = await db.label_column_for_table(fk["other_table"]) if not label_column: return {(fk["column"], value): str(value) for value in values} diff --git a/datasette/facets.py b/datasette/facets.py index 7fb0c68b..b23615fe 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -253,7 +253,7 @@ class ColumnFacet(Facet): # Attempt to expand foreign keys into labels values = [row["value"] for row in facet_rows] expanded = await self.ds.expand_foreign_keys( - self.database, self.table, column, values + self.request.actor, self.database, self.table, column, values ) else: expanded = {} diff --git a/datasette/views/table.py b/datasette/views/table.py index 6df8b915..50ba2b78 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1144,7 +1144,7 @@ async def table_view_data( # Expand them expanded_labels.update( await datasette.expand_foreign_keys( - database_name, table_name, column, values + request.actor, database_name, table_name, column, values ) ) if expanded_labels: diff --git a/tests/test_table_html.py b/tests/test_table_html.py index c4c7878c..e66eb6f0 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1204,3 +1204,56 @@ async def test_format_of_binary_links(size, title, length_bytes): sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql}) assert sql_response.status_code == 200 assert expected in sql_response.text + + +@pytest.mark.asyncio +async def test_foreign_key_labels_obey_permissions(): + ds = Datasette( + metadata={ + "databases": { + "foreign_key_labels": { + "tables": { + # Table a is only visible to root + "a": {"allow": {"id": "root"}}, + } + } + } + } + ) + db = ds.add_memory_database("foreign_key_labels") + await db.execute_write("create table a(id integer primary key, name text)") + await db.execute_write("insert into a (id, name) values (1, 'hello')") + await db.execute_write( + "create table b(id integer primary key, name text, a_id integer references a(id))" + ) + await db.execute_write("insert into b (id, name, a_id) values (1, 'world', 1)") + # Anonymous user can see table b but not table a + blah = await ds.client.get("/foreign_key_labels.json") + anon_a = await ds.client.get("/foreign_key_labels/a.json?_labels=on") + assert anon_a.status_code == 403 + anon_b = await ds.client.get("/foreign_key_labels/b.json?_labels=on") + assert anon_b.status_code == 200 + # root user can see both + cookies = {"ds_actor": ds.sign({"a": {"id": "root"}}, "actor")} + root_a = await ds.client.get( + "/foreign_key_labels/a.json?_labels=on", cookies=cookies + ) + assert root_a.status_code == 200 + root_b = await ds.client.get( + "/foreign_key_labels/b.json?_labels=on", cookies=cookies + ) + assert root_b.status_code == 200 + # Labels should have been expanded for root + assert root_b.json() == { + "ok": True, + "next": None, + "rows": [{"id": 1, "name": "world", "a_id": {"value": 1, "label": "hello"}}], + "truncated": False, + } + # But not for anon + assert anon_b.json() == { + "ok": True, + "next": None, + "rows": [{"id": 1, "name": "world", "a_id": 1}], + "truncated": False, + } From ab040470e2b191c0de48b213193da71e48cd66ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:57:27 -0700 Subject: [PATCH 1857/2303] Applied blacken-docs --- docs/plugin_hooks.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 84a045b0..04fb24ce 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1450,13 +1450,18 @@ This example adds a new database action for creating a table, if the user has th def database_actions(datasette, actor, database): async def inner(): if not await datasette.permission_allowed( - actor, "edit-schema", resource=database, default=False + actor, + "edit-schema", + resource=database, + default=False, ): return [] return [ { "href": datasette.urls.path( - "/-/edit-schema/{}/-/create".format(database) + "/-/edit-schema/{}/-/create".format( + database + ) ), "label": "Create a table", } From c26370485a4fd4bf130da051be9163d92c57f24f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 16:28:30 -0700 Subject: [PATCH 1858/2303] Label expand permission check respects cascade, closes #2178 --- datasette/app.py | 22 ++++++++++------- tests/test_table_html.py | 52 +++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 618c0ecc..ea9739f0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -950,13 +950,19 @@ class Datasette: except IndexError: return {} # Ensure user has permission to view the referenced table - if not await self.permission_allowed( - actor=actor, - action="view-table", - resource=(database, fk["other_table"]), - ): + other_table = fk["other_table"] + other_column = fk["other_column"] + visible, _ = await self.check_visibility( + actor, + permissions=[ + ("view-table", (database, other_table)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: return {} - label_column = await db.label_column_for_table(fk["other_table"]) + label_column = await db.label_column_for_table(other_table) if not label_column: return {(fk["column"], value): str(value) for value in values} labeled_fks = {} @@ -965,9 +971,9 @@ class Datasette: from {other_table} where {other_column} in ({placeholders}) """.format( - other_column=escape_sqlite(fk["other_column"]), + other_column=escape_sqlite(other_column), label_column=escape_sqlite(label_column), - other_table=escape_sqlite(fk["other_table"]), + other_table=escape_sqlite(other_table), placeholders=", ".join(["?"] * len(set(values))), ) try: diff --git a/tests/test_table_html.py b/tests/test_table_html.py index e66eb6f0..6707665d 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1207,9 +1207,11 @@ async def test_format_of_binary_links(size, title, length_bytes): @pytest.mark.asyncio -async def test_foreign_key_labels_obey_permissions(): - ds = Datasette( - metadata={ +@pytest.mark.parametrize( + "metadata", + ( + # Blocked at table level + { "databases": { "foreign_key_labels": { "tables": { @@ -1218,15 +1220,47 @@ async def test_foreign_key_labels_obey_permissions(): } } } - } - ) + }, + # Blocked at database level + { + "databases": { + "foreign_key_labels": { + # Only root can view this database + "allow": {"id": "root"}, + "tables": { + # But table b is visible to everyone + "b": {"allow": True}, + }, + } + } + }, + # Blocked at the instance level + { + "allow": {"id": "root"}, + "databases": { + "foreign_key_labels": { + "tables": { + # Table b is visible to everyone + "b": {"allow": True}, + } + } + }, + }, + ), +) +async def test_foreign_key_labels_obey_permissions(metadata): + ds = Datasette(metadata=metadata) db = ds.add_memory_database("foreign_key_labels") - await db.execute_write("create table a(id integer primary key, name text)") - await db.execute_write("insert into a (id, name) values (1, 'hello')") await db.execute_write( - "create table b(id integer primary key, name text, a_id integer references a(id))" + "create table if not exists a(id integer primary key, name text)" + ) + await db.execute_write("insert or replace into a (id, name) values (1, 'hello')") + await db.execute_write( + "create table if not exists b(id integer primary key, name text, a_id integer references a(id))" + ) + await db.execute_write( + "insert or replace into b (id, name, a_id) values (1, 'world', 1)" ) - await db.execute_write("insert into b (id, name, a_id) values (1, 'world', 1)") # Anonymous user can see table b but not table a blah = await ds.client.get("/foreign_key_labels.json") anon_a = await ds.client.get("/foreign_key_labels/a.json?_labels=on") From b645174271aa08e8ca83b27ff83ce078ecd15da2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 21:23:59 -0700 Subject: [PATCH 1859/2303] actors_from_ids plugin hook and datasette.actors_from_ids() method (#2181) * Prototype of actors_from_ids plugin hook, refs #2180 * datasette-remote-actors example plugin, refs #2180 --- datasette/app.py | 10 +++++++ datasette/hookspecs.py | 5 ++++ docs/internals.rst | 21 ++++++++++++++ docs/plugin_hooks.rst | 57 ++++++++++++++++++++++++++++++++++++++ tests/test_plugins.py | 62 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index ea9739f0..fdec2c86 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -819,6 +819,16 @@ class Datasette: ) return crumbs + async def actors_from_ids( + self, actor_ids: Iterable[Union[str, int]] + ) -> Dict[Union[id, str], Dict]: + result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) + if result is None: + # Do the default thing + return {actor_id: {"id": actor_id} for actor_id in actor_ids} + result = await await_me_maybe(result) + return result + async def permission_allowed( self, actor, action, resource=None, default=DEFAULT_NOT_SET ): diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 801073fc..9069927b 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -94,6 +94,11 @@ def actor_from_request(datasette, request): """Return an actor dictionary based on the incoming request""" +@hookspec(firstresult=True) +def actors_from_ids(datasette, actor_ids): + """Returns a dictionary mapping those IDs to actor dictionaries""" + + @hookspec def filters_from_request(request, database, table, datasette): """ diff --git a/docs/internals.rst b/docs/internals.rst index 6b7d3df8..13f1d4a1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -322,6 +322,27 @@ await .render_template(template, context=None, request=None) Renders a `Jinja template `__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins. +.. _datasette_actors_from_ids: + +await .actors_from_ids(actor_ids) +--------------------------------- + +``actor_ids`` - list of strings or integers + A list of actor IDs to look up. + +Returns a dictionary, where the keys are the IDs passed to it and the values are the corresponding actor dictionaries. + +This method is mainly designed to be used with plugins. See the :ref:`plugin_hook_actors_from_ids` documentation for details. + +If no plugins that implement that hook are installed, the default return value looks like this: + +.. code-block:: json + + { + "1": {"id": "1"}, + "2": {"id": "2"} + } + .. _datasette_permission_allowed: await .permission_allowed(actor, action, resource=None, default=...) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 04fb24ce..e966919b 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1071,6 +1071,63 @@ Instead of returning a dictionary, this function can return an awaitable functio Examples: `datasette-auth-tokens `_, `datasette-auth-passwords `_ +.. _plugin_hook_actors_from_ids: + +actors_from_ids(datasette, actor_ids) +------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor_ids`` - list of strings or integers + The actor IDs to look up. + +The hook must return a dictionary that maps the incoming actor IDs to their full dictionary representation. + +Some plugins that implement social features may store the ID of the :ref:`actor ` that performed an action - added a comment, bookmarked a table or similar - and then need a way to resolve those IDs into display-friendly actor dictionaries later on. + +Unlike other plugin hooks, this only uses the first implementation of the hook to return a result. You can expect users to only have a single plugin installed that implements this hook. + +If no plugin is installed, Datasette defaults to returning actors that are just ``{"id": actor_id}``. + +The hook can return a dictionary or an awaitable function that then returns a dictionary. + +This example implementation returns actors from a database table: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def actors_from_ids(datasette, actor_ids): + db = datasette.get_database("actors") + + async def inner(): + sql = "select id, name from actors where id in ({})".format( + ", ".join("?" for _ in actor_ids) + ) + actors = {} + for row in (await db.execute(sql, actor_ids)).rows: + actor = dict(row) + actors[actor["id"]] = actor + return actors + + return inner + +The returned dictionary from this example looks like this: + +.. code-block:: json + + { + "1": {"id": "1", "name": "Tony"}, + "2": {"id": "2", "name": "Tina"}, + } + +These IDs could be integers or strings, depending on how the actors used by the Datasette instance are configured. + +Example: `datasette-remote-actors `_ + .. _plugin_hook_filters_from_request: filters_from_request(request, database, table, datasette) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9761fa53..625ae635 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1215,3 +1215,65 @@ async def test_hook_register_permissions_allows_identical_duplicates(): await ds.invoke_startup() # Check that ds.permissions has only one of each assert len([p for p in ds.permissions.values() if p.abbr == "abbr1"]) == 1 + + +@pytest.mark.asyncio +async def test_hook_actors_from_ids(): + # Without the hook should return default {"id": id} list + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("actors_from_ids") + await db.execute_write( + "create table actors (id text primary key, name text, age int)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('3', 'Cate Blanchett', 52)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('5', 'Rooney Mara', 36)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('7', 'Sarah Paulson', 46)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('9', 'Helena Bonham Carter', 55)" + ) + table_names = await db.table_names() + assert table_names == ["actors"] + actors1 = await ds.actors_from_ids(["3", "5", "7"]) + assert actors1 == { + "3": {"id": "3"}, + "5": {"id": "5"}, + "7": {"id": "7"}, + } + + class ActorsFromIdsPlugin: + __name__ = "ActorsFromIdsPlugin" + + @hookimpl + def actors_from_ids(self, datasette, actor_ids): + db = datasette.get_database("actors_from_ids") + + async def inner(): + sql = "select id, name from actors where id in ({})".format( + ", ".join("?" for _ in actor_ids) + ) + actors = {} + result = await db.execute(sql, actor_ids) + for row in result.rows: + actor = dict(row) + actors[actor["id"]] = actor + return actors + + return inner + + try: + pm.register(ActorsFromIdsPlugin(), name="ActorsFromIdsPlugin") + actors2 = await ds.actors_from_ids(["3", "5", "7"]) + assert actors2 == { + "3": {"id": "3", "name": "Cate Blanchett"}, + "5": {"id": "5", "name": "Rooney Mara"}, + "7": {"id": "7", "name": "Sarah Paulson"}, + } + finally: + pm.unregister(name="ReturnNothingPlugin") From a4c96d01b27ce7cd06662a024da3547132a7c412 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 21:44:08 -0700 Subject: [PATCH 1860/2303] Release 1.0a6 Refs #1765, #2164, #2169, #2175, #2178, #2181 --- datasette/version.py | 2 +- docs/changelog.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index b99c212b..4b65999d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a5" +__version__ = "1.0a6" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 019d6c68..81554f83 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,18 @@ Changelog ========= +.. _v1_0_a6: + +1.0a6 (2023-09-07) +------------------ + +- New plugin hook: :ref:`plugin_hook_actors_from_ids` and an internal method to accompany it, :ref:`datasette_actors_from_ids`. This mechanism is intended to be used by plugins that may need to display the actor who was responsible for something managed by that plugin: they can now resolve the recorded IDs of actors into the full actor objects. (:issue:`2181`) +- ``DATASETTE_LOAD_PLUGINS`` environment variable for :ref:`controlling which plugins ` are loaded by Datasette. (:issue:`2164`) +- Datasette now checks if the user has permission to view a table linked to by a foreign key before turning that foreign key into a clickable link. (:issue:`2178`) +- The ``execute-sql`` permission now implies that the actor can also view the database and instance. (:issue:`2169`) +- Documentation describing a pattern for building plugins that themselves :ref:`define further hooks ` for other plugins. (:issue:`1765`) +- Datasette is now tested against the Python 3.12 preview. (`#2175 `__) + .. _v1_0_a5: 1.0a5 (2023-08-29) From b2ec8717c3619260a1b535eea20e618bf95aa30b Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 13 Sep 2023 14:06:25 -0700 Subject: [PATCH 1861/2303] Plugin configuration now lives in datasette.yaml/json * Checkpoint, moving top-level plugin config to datasette.json * Support database-level and table-level plugin configuration in datasette.yaml Refs #2093 --- datasette/app.py | 48 +++++++++++++++----- docs/configuration.rst | 97 ++++++++++++++++++++++++++++++++++++++-- docs/index.rst | 1 + docs/internals.rst | 2 +- docs/plugin_hooks.rst | 2 +- docs/writing_plugins.rst | 7 +-- tests/conftest.py | 3 +- tests/fixtures.py | 50 ++++++++++++++------- tests/test_cli.py | 38 ++++++++++++++++ tests/test_plugins.py | 23 +++------- 10 files changed, 217 insertions(+), 54 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index fdec2c86..53486007 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -368,7 +368,7 @@ class Datasette: for key in config_settings: if key not in DEFAULT_SETTINGS: raise StartupError("Invalid setting '{}' in datasette.json".format(key)) - + self.config = config # CLI settings should overwrite datasette.json settings self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions @@ -674,15 +674,43 @@ class Datasette: def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" - plugins = self.metadata( - "plugins", database=database, table=table, fallback=fallback - ) - if plugins is None: - return None - plugin_config = plugins.get(plugin_name) - # Resolve any $file and $env keys - plugin_config = resolve_env_secrets(plugin_config, os.environ) - return plugin_config + if database is None and table is None: + config = self._plugin_config_top(plugin_name) + else: + config = self._plugin_config_nested(plugin_name, database, table, fallback) + + return resolve_env_secrets(config, os.environ) + + def _plugin_config_top(self, plugin_name): + """Returns any top-level plugin configuration for the specified plugin.""" + return ((self.config or {}).get("plugins") or {}).get(plugin_name) + + def _plugin_config_nested(self, plugin_name, database, table=None, fallback=True): + """Returns any database or table-level plugin configuration for the specified plugin.""" + db_config = ((self.config or {}).get("databases") or {}).get(database) + + # if there's no db-level configuration, then return early, falling back to top-level if needed + if not db_config: + return self._plugin_config_top(plugin_name) if fallback else None + + db_plugin_config = (db_config.get("plugins") or {}).get(plugin_name) + + if table: + table_plugin_config = ( + ((db_config.get("tables") or {}).get(table) or {}).get("plugins") or {} + ).get(plugin_name) + + # fallback to db_config or top-level config, in that order, if needed + if table_plugin_config is None and fallback: + return db_plugin_config or self._plugin_config_top(plugin_name) + + return table_plugin_config + + # fallback to top-level if needed + if db_plugin_config is None and fallback: + self._plugin_config_top(plugin_name) + + return db_plugin_config def app_css_hash(self): if not hasattr(self, "_app_css_hash"): diff --git a/docs/configuration.rst b/docs/configuration.rst index ed9975ac..214e9044 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,10 +1,101 @@ .. _configuration: Configuration -======== +============= -Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more. +Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more. -To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag: +To facilitate this, You can provide a ``datasette.yaml`` configuration file to datasette with the ``--config``/ ``-c`` flag: + +.. code-block:: bash datasette mydatabase.db --config datasette.yaml + +.. _configuration_reference: + +``datasette.yaml`` reference +---------------------------- + +Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. + +.. tab:: YAML + + .. code-block:: yaml + + # Datasette settings block + settings: + default_page_size: 50 + sql_time_limit_ms: 3500 + max_returned_rows: 2000 + + # top-level plugin configuration + plugins: + datasette-my-plugin: + key: valueA + + # Database and table-level configuration + databases: + your_db_name: + # plugin configuration for the your_db_name database + plugins: + datasette-my-plugin: + key: valueA + tables: + your_table_name: + # plugin configuration for the your_table_name table + # inside your_db_name database + plugins: + datasette-my-plugin: + key: valueB + +.. _configuration_reference_settings: +Settings configuration +~~~~~~~~~~~~~~~~~~~~~~ + +:ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + settings: + default_allow_sql: off + default_page_size: 50 + + +.. _configuration_reference_plugins: +Plugin configuration +~~~~~~~~~~~~~~~~~~~~ + +Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + plugins: + datasette-my-plugin: + key: my_value + +For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + databases: + my_database: + # plugin configuration for the my_database database + plugins: + datasette-my-plugin: + key: my_value + my_other_database: + tables: + my_table: + # plugin configuration for the my_table table inside the my_other_database database + plugins: + datasette-my-plugin: + key: my_value diff --git a/docs/index.rst b/docs/index.rst index f5c1f232..cfa3443c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ Contents getting_started installation + configuration ecosystem cli-reference pages diff --git a/docs/internals.rst b/docs/internals.rst index 13f1d4a1..7fc7948c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -296,7 +296,7 @@ The dictionary keys are the permission names - e.g. ``view-instance`` - and the ``table`` - None or string The table the user is interacting with. -This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. +This method lets you read plugin configuration values that were set in ``datasette.yaml``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. The return value will be the value from the configuration file - usually a dictionary. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index e966919b..1816d48c 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -909,7 +909,7 @@ Potential use-cases: * Run some initialization code for the plugin * Create database tables that a plugin needs on startup -* Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid +* Validate the configuration for a plugin on startup, and raise an error if it is invalid .. note:: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index d0dd8f36..c028b4ff 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -184,7 +184,7 @@ This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` i If there is no configuration for that plugin, the method will return ``None``. -If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so: +If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option inside ``datasette.yaml`` like so: .. [[[cog from metadata_doc import metadata_example @@ -234,11 +234,10 @@ If it cannot find the requested configuration at the table layer, it will fall b In this case, the above code would return that configuration for ANY table within the ``sf-trees`` database. -The plugin configuration could also be set at the top level of ``metadata.yaml``: +The plugin configuration could also be set at the top level of ``datasette.yaml``: .. [[[cog metadata_example(cog, { - "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { "latitude_column": "xlat", @@ -252,7 +251,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml`` .. code-block:: yaml - title: This is the top-level title in metadata.json plugins: datasette-cluster-map: latitude_column: xlat @@ -264,7 +262,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml`` .. code-block:: json { - "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { "latitude_column": "xlat", diff --git a/tests/conftest.py b/tests/conftest.py index fb7f768e..31336aea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): @pytest_asyncio.fixture async def ds_client(): from datasette.app import Datasette - from .fixtures import METADATA, PLUGINS_DIR + from .fixtures import CONFIG, METADATA, PLUGINS_DIR global _ds_client if _ds_client is not None: @@ -49,6 +49,7 @@ async def ds_client(): ds = Datasette( metadata=METADATA, + config=CONFIG, plugins_dir=PLUGINS_DIR, settings={ "default_page_size": 50, diff --git a/tests/fixtures.py b/tests/fixtures.py index a6700239..9cf6b605 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -114,6 +114,7 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, + config=None, metadata=None, crossdb=False, ): @@ -158,6 +159,7 @@ def make_app_client( memory=memory, cors=cors, metadata=metadata or METADATA, + config=config or CONFIG, plugins_dir=PLUGINS_DIR, settings=settings, inspect_data=inspect_data, @@ -296,6 +298,33 @@ def generate_sortable_rows(num): } +CONFIG = { + "plugins": { + "name-of-plugin": {"depth": "root"}, + "env-plugin": {"foo": {"$env": "FOO_ENV"}}, + "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}], + "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}}, + }, + "databases": { + "fixtures": { + "plugins": {"name-of-plugin": {"depth": "database"}}, + "tables": { + "simple_primary_key": { + "plugins": { + "name-of-plugin": { + "depth": "table", + "special": "this-is-simple_primary_key", + } + }, + }, + "sortable": { + "plugins": {"name-of-plugin": {"depth": "table"}}, + }, + }, + } + }, +} + METADATA = { "title": "Datasette Fixtures", "description_html": 'An example SQLite database demonstrating Datasette. Sign in as root user', @@ -306,26 +335,13 @@ METADATA = { "about": "About Datasette", "about_url": "https://github.com/simonw/datasette", "extra_css_urls": ["/static/extra-css-urls.css"], - "plugins": { - "name-of-plugin": {"depth": "root"}, - "env-plugin": {"foo": {"$env": "FOO_ENV"}}, - "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}], - "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}}, - }, "databases": { "fixtures": { "description": "Test tables description", - "plugins": {"name-of-plugin": {"depth": "database"}}, "tables": { "simple_primary_key": { "description_html": "Simple primary key", "title": "This HTML is escaped", - "plugins": { - "name-of-plugin": { - "depth": "table", - "special": "this-is-simple_primary_key", - } - }, }, "sortable": { "sortable_columns": [ @@ -334,7 +350,6 @@ METADATA = { "sortable_with_nulls_2", "text", ], - "plugins": {"name-of-plugin": {"depth": "table"}}, }, "no_primary_key": {"sortable_columns": [], "hidden": True}, "units": {"units": {"distance": "m", "frequency": "Hz"}}, @@ -768,6 +783,7 @@ def assert_permissions_checked(datasette, actions): type=click.Path(file_okay=True, dir_okay=False), ) @click.argument("metadata", required=False) +@click.argument("config", required=False) @click.argument( "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False ) @@ -782,7 +798,7 @@ def assert_permissions_checked(datasette, actions): type=click.Path(file_okay=True, dir_okay=False), help="Write out second test DB to this file", ) -def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): +def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename): """Write out the fixtures database used by Datasette's test suite""" if metadata and not metadata.endswith(".json"): raise click.ClickException("Metadata should end with .json") @@ -805,6 +821,10 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): with open(metadata, "w") as fp: fp.write(json.dumps(METADATA, indent=4)) print(f"- metadata written to {metadata}") + if config: + with open(config, "w") as fp: + fp.write(json.dumps(CONFIG, indent=4)) + print(f"- config written to {config}") if plugins_path: path = pathlib.Path(plugins_path) if not path.exists(): diff --git a/tests/test_cli.py b/tests/test_cli.py index e85bcef1..213db416 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -238,6 +238,44 @@ def test_setting(args): assert settings["default_page_size"] == 5 +def test_plugin_s_overwrite(): + runner = CliRunner() + plugins_dir = str(pathlib.Path(__file__).parent / "plugins") + + result = runner.invoke( + cli, + [ + "--plugins-dir", + plugins_dir, + "--get", + "/_memory.json?sql=select+prepare_connection_args()", + ], + ) + assert result.exit_code == 0, result.output + assert ( + json.loads(result.output).get("rows")[0].get("prepare_connection_args()") + == 'database=_memory, datasette.plugin_config("name-of-plugin")=None' + ) + + result = runner.invoke( + cli, + [ + "--plugins-dir", + plugins_dir, + "--get", + "/_memory.json?sql=select+prepare_connection_args()", + "-s", + "plugins.name-of-plugin", + "OVERRIDE", + ], + ) + assert result.exit_code == 0, result.output + assert ( + json.loads(result.output).get("rows")[0].get("prepare_connection_args()") + == 'database=_memory, datasette.plugin_config("name-of-plugin")=OVERRIDE' + ) + + def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 625ae635..37530991 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -234,9 +234,6 @@ async def test_plugin_config(ds_client): async def test_plugin_config_env(ds_client): os.environ["FOO_ENV"] = "FROM_ENVIRONMENT" assert {"foo": "FROM_ENVIRONMENT"} == ds_client.ds.plugin_config("env-plugin") - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert {"foo": {"$env": "FOO_ENV"}} == metadata.json()["plugins"]["env-plugin"] del os.environ["FOO_ENV"] @@ -246,11 +243,6 @@ async def test_plugin_config_env_from_list(ds_client): assert [{"in_a_list": "FROM_ENVIRONMENT"}] == ds_client.ds.plugin_config( "env-plugin-list" ) - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert [{"in_a_list": {"$env": "FOO_ENV"}}] == metadata.json()["plugins"][ - "env-plugin-list" - ] del os.environ["FOO_ENV"] @@ -259,11 +251,6 @@ async def test_plugin_config_file(ds_client): with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}} == metadata.json()["plugins"][ - "file-plugin" - ] os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -722,7 +709,7 @@ async def test_hook_register_routes(ds_client, path, body): @pytest.mark.parametrize("configured_path", ("path1", "path2")) def test_hook_register_routes_with_datasette(configured_path): with make_app_client( - metadata={ + config={ "plugins": { "register-route-demo": { "path": configured_path, @@ -741,7 +728,7 @@ def test_hook_register_routes_with_datasette(configured_path): def test_hook_register_routes_override(): "Plugins can over-ride default paths such as /db/table" with make_app_client( - metadata={ + config={ "plugins": { "register-route-demo": { "path": "blah", @@ -1099,7 +1086,7 @@ async def test_hook_filters_from_request(ds_client): @pytest.mark.parametrize("extra_metadata", (False, True)) async def test_hook_register_permissions(extra_metadata): ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ @@ -1151,7 +1138,7 @@ async def test_hook_register_permissions_no_duplicates(duplicate): if duplicate == "abbr": abbr2 = "abbr1" ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ @@ -1186,7 +1173,7 @@ async def test_hook_register_permissions_no_duplicates(duplicate): @pytest.mark.asyncio async def test_hook_register_permissions_allows_identical_duplicates(): ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ From 16f0b6d8222d06682a31b904d0a402c391ae1c1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Sep 2023 14:15:32 -0700 Subject: [PATCH 1862/2303] JSON/YAML tabs on configuration docs page --- docs/configuration.rst | 171 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 214e9044..4a7258b9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -18,6 +18,40 @@ To facilitate this, You can provide a ``datasette.yaml`` configuration file to d Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # Datasette settings block + settings: + default_page_size: 50 + sql_time_limit_ms: 3500 + max_returned_rows: 2000 + + # top-level plugin configuration + plugins: + datasette-my-plugin: + key: valueA + + # Database and table-level configuration + databases: + your_db_name: + # plugin configuration for the your_db_name database + plugins: + datasette-my-plugin: + key: valueA + tables: + your_table_name: + # plugin configuration for the your_table_name table + # inside your_db_name database + plugins: + datasette-my-plugin: + key: valueB + """) + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -48,12 +82,61 @@ Here's a full example of all the valid configuration options that can exist insi datasette-my-plugin: key: valueB +.. tab:: JSON + + .. code-block:: json + + { + "settings": { + "default_page_size": 50, + "sql_time_limit_ms": 3500, + "max_returned_rows": 2000 + }, + "plugins": { + "datasette-my-plugin": { + "key": "valueA" + } + }, + "databases": { + "your_db_name": { + "plugins": { + "datasette-my-plugin": { + "key": "valueA" + } + }, + "tables": { + "your_table_name": { + "plugins": { + "datasette-my-plugin": { + "key": "valueB" + } + } + } + } + } + } + } +.. [[[end]]] + .. _configuration_reference_settings: Settings configuration ~~~~~~~~~~~~~~~~~~~~~~ :ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + settings: + default_allow_sql: off + default_page_size: 50 + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -63,6 +146,17 @@ Settings configuration default_allow_sql: off default_page_size: 50 +.. tab:: JSON + + .. code-block:: json + + { + "settings": { + "default_allow_sql": "off", + "default_page_size": 50 + } + } +.. [[[end]]] .. _configuration_reference_plugins: Plugin configuration @@ -70,6 +164,19 @@ Plugin configuration Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + plugins: + datasette-my-plugin: + key: my_value + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -79,8 +186,44 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve datasette-my-plugin: key: my_value +.. tab:: JSON + + .. code-block:: json + + { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + } +.. [[[end]]] + For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + databases: + my_database: + # plugin configuration for the my_database database + plugins: + datasette-my-plugin: + key: my_value + my_other_database: + tables: + my_table: + # plugin configuration for the my_table table inside the my_other_database database + plugins: + datasette-my-plugin: + key: my_value + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -99,3 +242,31 @@ For database level or table level plugin configuration, nest it under the approp plugins: datasette-my-plugin: key: my_value + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "my_database": { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + }, + "my_other_database": { + "tables": { + "my_table": { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + } + } + } + } + } +.. [[[end]]] \ No newline at end of file From 852f5014853943fa27f43ddaa2d442545b3259fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 Sep 2023 09:35:18 -0700 Subject: [PATCH 1863/2303] Switch from pkg_resources to importlib.metadata in app.py, refs #2057 --- datasette/app.py | 6 +++--- tests/test_plugins.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 53486007..c0e80700 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -8,11 +8,11 @@ import functools import glob import hashlib import httpx +import importlib.metadata import inspect from itsdangerous import BadSignature import json import os -import pkg_resources import re import secrets import sys @@ -1118,9 +1118,9 @@ class Datasette: if using_pysqlite3: for package in ("pysqlite3", "pysqlite3-binary"): try: - info["pysqlite3"] = pkg_resources.get_distribution(package).version + info["pysqlite3"] = importlib.metadata.version(package) break - except pkg_resources.DistributionNotFound: + except importlib.metadata.PackageNotFoundError: pass return info diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 37530991..3bc117f3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1264,3 +1264,25 @@ async def test_hook_actors_from_ids(): } finally: pm.unregister(name="ReturnNothingPlugin") + + +@pytest.mark.asyncio +async def test_plugin_is_installed(): + datasette = Datasette(memory=True) + + class DummyPlugin: + __name__ = "DummyPlugin" + + @hookimpl + def actors_from_ids(self, datasette, actor_ids): + return {} + + try: + pm.register(DummyPlugin(), name="DummyPlugin") + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + installed_plugins = {p["name"] for p in response.json()} + assert "DummyPlugin" in installed_plugins + + finally: + pm.unregister(name="DummyPlugin") From f56e043747bde4faa1d78588636df6c0dadebc65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Sep 2023 10:39:11 -0700 Subject: [PATCH 1864/2303] test_facet_against_in_memory_database, refs #2189 This is meant to illustrate a crashing bug but it does not trigger it. --- tests/test_facets.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_facets.py b/tests/test_facets.py index 48cc0ff2..a68347f0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -643,3 +643,50 @@ async def test_conflicting_facet_names_json(ds_client): "created_2", "tags_2", } + + +@pytest.mark.asyncio +async def test_facet_against_in_memory_database(): + ds = Datasette() + db = ds.add_memory_database("mem") + await db.execute_write("create table t (id integer primary key, name text)") + to_insert = [["one"] for _ in range(800)] + [["two"] for _ in range(300)] + await db.execute_write_many("insert into t (name) values (?)", to_insert) + response1 = await ds.client.get("/mem/t.json") + assert response1.status_code == 200 + response2 = await ds.client.get("/mem/t.json?_facet=name&_size=0") + assert response2.status_code == 200 + assert response2.json() == { + "ok": True, + "next": None, + "facet_results": { + "results": { + "name": { + "name": "name", + "type": "column", + "hideable": True, + "toggle_url": "/mem/t.json?_size=0", + "results": [ + { + "value": "one", + "label": "one", + "count": 800, + "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=one", + "selected": False, + }, + { + "value": "two", + "label": "two", + "count": 300, + "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=two", + "selected": False, + }, + ], + "truncated": False, + } + }, + "timed_out": [], + }, + "rows": [], + "truncated": False, + } From 6ed7908580fa2ba9297c3225d85c56f8b08b9937 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Sep 2023 10:44:13 -0700 Subject: [PATCH 1865/2303] Simplified test for #2189 This now executes two facets, in the hope that parallel facet execution would illustrate the bug - but it did not illustrate the bug. --- tests/test_facets.py | 51 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/tests/test_facets.py b/tests/test_facets.py index a68347f0..85c8f85b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -649,44 +649,17 @@ async def test_conflicting_facet_names_json(ds_client): async def test_facet_against_in_memory_database(): ds = Datasette() db = ds.add_memory_database("mem") - await db.execute_write("create table t (id integer primary key, name text)") - to_insert = [["one"] for _ in range(800)] + [["two"] for _ in range(300)] - await db.execute_write_many("insert into t (name) values (?)", to_insert) - response1 = await ds.client.get("/mem/t.json") + await db.execute_write( + "create table t (id integer primary key, name text, name2 text)" + ) + to_insert = [{"name": "one", "name2": "1"} for _ in range(800)] + [ + {"name": "two", "name2": "2"} for _ in range(300) + ] + print(to_insert) + await db.execute_write_many( + "insert into t (name, name2) values (:name, :name2)", to_insert + ) + response1 = await ds.client.get("/mem/t") assert response1.status_code == 200 - response2 = await ds.client.get("/mem/t.json?_facet=name&_size=0") + response2 = await ds.client.get("/mem/t?_facet=name&_facet=name2") assert response2.status_code == 200 - assert response2.json() == { - "ok": True, - "next": None, - "facet_results": { - "results": { - "name": { - "name": "name", - "type": "column", - "hideable": True, - "toggle_url": "/mem/t.json?_size=0", - "results": [ - { - "value": "one", - "label": "one", - "count": 800, - "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=one", - "selected": False, - }, - { - "value": "two", - "label": "two", - "count": 300, - "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=two", - "selected": False, - }, - ], - "truncated": False, - } - }, - "timed_out": [], - }, - "rows": [], - "truncated": False, - } From b0e5d8afa308759f4ee9f3ecdf61101dffc4a037 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 Sep 2023 15:10:55 -0700 Subject: [PATCH 1866/2303] Stop using parallel SQL queries for tables Refs: - #2189 --- datasette/views/table.py | 16 ++++++---------- docs/internals.rst | 1 + 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 50ba2b78..4f4baeed 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -74,11 +74,10 @@ class Row: return json.dumps(d, default=repr, indent=2) -async def _gather_parallel(*args): - return await asyncio.gather(*args) - - -async def _gather_sequential(*args): +async def run_sequential(*args): + # This used to be swappable for asyncio.gather() to run things in + # parallel, but this lead to hard-to-debug locking issues with + # in-memory databases: https://github.com/simonw/datasette/issues/2189 results = [] for fn in args: results.append(await fn) @@ -1183,9 +1182,6 @@ async def table_view_data( ) rows = rows[:page_size] - # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather - gather = _gather_sequential if request.args.get("_noparallel") else _gather_parallel - # Resolve extras extras = _get_extras(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): @@ -1249,7 +1245,7 @@ async def table_view_data( if not nofacet: # Run them in parallel facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await gather(*facet_awaitables) + facet_awaitable_results = await run_sequential(*facet_awaitables) for ( instance_facet_results, instance_facets_timed_out, @@ -1282,7 +1278,7 @@ async def table_view_data( ): # Run them in parallel facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await gather(*facet_suggest_awaitables): + for suggest_result in await run_sequential(*facet_suggest_awaitables): suggested_facets.extend(suggest_result) return suggested_facets diff --git a/docs/internals.rst b/docs/internals.rst index 7fc7948c..4e9a6747 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1317,6 +1317,7 @@ This example uses the :ref:`register_routes() ` plugin h (r"/parallel-queries$", parallel_queries), ] +Note that running parallel SQL queries in this way has `been known to cause problems in the past `__, so treat this example with caution. Adding ``?_trace=1`` will show that the trace covers both of those child tasks. From 6763572948ffd047a89a3bbf7c300e91f51ae98f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:11:24 -0700 Subject: [PATCH 1867/2303] Bump sphinx, furo, black Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [black](https://github.com/psf/black). Updates `sphinx` from 7.2.5 to 7.2.6 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.5...v7.2.6) Updates `furo` from 2023.8.19 to 2023.9.10 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.08.19...2023.09.10) Updates `black` from 23.7.0 to 23.9.1 - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.7.0...23.9.1) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c718086b..415fd27c 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.2.5", - "furo==2023.8.19", + "Sphinx==7.2.6", + "furo==2023.9.10", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -83,7 +83,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.7.0", + "black==23.9.1", "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From 10bc80547330e826a749ce710da21ae29f7e6048 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:11:35 -0700 Subject: [PATCH 1868/2303] Finish removing pkg_resources, closes #2057 --- datasette/plugins.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index 6ec08a81..017f3b9d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,7 +1,7 @@ -import importlib +import importlib.metadata +import importlib.resources import os import pluggy -import pkg_resources import sys from . import hookspecs @@ -35,15 +35,15 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = pkg_resources.get_distribution(package_name) - entry_map = distribution.get_entry_map() - if "datasette" in entry_map: - for plugin_name, entry_point in entry_map["datasette"].items(): + distribution = importlib.metadata.distribution(package_name) + entry_points = distribution.entry_points + for entry_point in entry_points: + if entry_point.group == "datasette": mod = entry_point.load() pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except pkg_resources.DistributionNotFound: + except importlib.metadata.PackageNotFoundError: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -61,16 +61,16 @@ def get_plugins(): templates_path = None if plugin.__name__ not in DEFAULT_PLUGINS: try: - if pkg_resources.resource_isdir(plugin.__name__, "static"): - static_path = pkg_resources.resource_filename( - plugin.__name__, "static" + if (importlib.resources.files(plugin.__name__) / "static").is_dir(): + static_path = str( + importlib.resources.files(plugin.__name__) / "static" ) - if pkg_resources.resource_isdir(plugin.__name__, "templates"): - templates_path = pkg_resources.resource_filename( - plugin.__name__, "templates" + if (importlib.resources.files(plugin.__name__) / "templates").is_dir(): + templates_path = str( + importlib.resources.files(plugin.__name__) / "templates" ) - except (KeyError, ImportError): - # Caused by --plugins_dir= plugins - KeyError/ImportError thrown in Py3.5 + except (TypeError, ModuleNotFoundError): + # Caused by --plugins_dir= plugins pass plugin_info = { "name": plugin.__name__, @@ -81,6 +81,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.project_name + plugin_info["name"] = distinfo.name plugins.append(plugin_info) return plugins From 947520c1fe940de79f5db856dd693330f1bbf547 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:31:32 -0700 Subject: [PATCH 1869/2303] Release notes for 0.64.4 on main --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81554f83..52e1db3b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_4: + +0.64.4 (2023-09-21) +------------------- + +- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`) + .. _v1_0_a6: 1.0a6 (2023-09-07) From b0d0a0e5de8bb5b9b6c253e8af451a532266bcf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:42:15 -0700 Subject: [PATCH 1870/2303] importlib_resources for Python < 3.9, refs #2057 --- datasette/plugins.py | 15 ++++++++++----- setup.py | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index 017f3b9d..a93145cf 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,10 +1,15 @@ import importlib.metadata -import importlib.resources import os import pluggy import sys from . import hookspecs +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + + DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -61,13 +66,13 @@ def get_plugins(): templates_path = None if plugin.__name__ not in DEFAULT_PLUGINS: try: - if (importlib.resources.files(plugin.__name__) / "static").is_dir(): + if (importlib_resources.files(plugin.__name__) / "static").is_dir(): static_path = str( - importlib.resources.files(plugin.__name__) / "static" + importlib_resources.files(plugin.__name__) / "static" ) - if (importlib.resources.files(plugin.__name__) / "templates").is_dir(): + if (importlib_resources.files(plugin.__name__) / "templates").is_dir(): templates_path = str( - importlib.resources.files(plugin.__name__) / "templates" + importlib_resources.files(plugin.__name__) / "templates" ) except (TypeError, ModuleNotFoundError): # Caused by --plugins_dir= plugins diff --git a/setup.py b/setup.py index 415fd27c..a2728f6b 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( "Jinja2>=2.10.3", "hupper>=1.9", "httpx>=0.20", + 'importlib_resources>=1.3.1; python_version < "3.9"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From 80a9cd9620fddf2695d12d8386a91e7c6b145ef2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:55:50 -0700 Subject: [PATCH 1871/2303] test-datasette-load-plugins now fails correctly, refs #2193 --- tests/test-datasette-load-plugins.sh | 41 +++++++++++++--------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/tests/test-datasette-load-plugins.sh b/tests/test-datasette-load-plugins.sh index e26d8377..03e08bb1 100755 --- a/tests/test-datasette-load-plugins.sh +++ b/tests/test-datasette-load-plugins.sh @@ -3,27 +3,24 @@ # datasette-init and datasette-json-html are installed PLUGINS=$(datasette plugins) -echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | \ - grep -q true || ( \ - echo "Test failed: datasette-json-html not found" && \ - exit 1 \ - ) -# With the DATASETTE_LOAD_PLUGINS we should not see that +if ! echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | grep -q true; then + echo "Test failed: datasette-json-html not found" + exit 1 +fi + PLUGINS2=$(DATASETTE_LOAD_PLUGINS=datasette-init datasette plugins) -echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | \ - grep -q false || ( \ - echo "Test failed: datasette-json-html should not have been loaded" && \ - exit 1 \ - ) -echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | \ - grep -q true || ( \ - echo "Test failed: datasette-init should have been loaded" && \ - exit 1 \ - ) -# With DATASETTE_LOAD_PLUGINS='' we should see no plugins +if ! echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | grep -q false; then + echo "Test failed: datasette-json-html should not have been loaded" + exit 1 +fi + +if ! echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | grep -q true; then + echo "Test failed: datasette-init should have been loaded" + exit 1 +fi + PLUGINS3=$(DATASETTE_LOAD_PLUGINS='' datasette plugins) -echo "$PLUGINS3"| \ - grep -q '\[\]' || ( \ - echo "Test failed: datasette plugins should have returned []" && \ - exit 1 \ - ) +if ! echo "$PLUGINS3" | grep -q '\[\]'; then + echo "Test failed: datasette plugins should have returned []" + exit 1 +fi From b7cf0200e21796a6ff653c6f94a4ee5fcfde0346 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 13:22:40 -0700 Subject: [PATCH 1872/2303] Swap order of config and metadata options, refs #2194 --- tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 9cf6b605..16aa234e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -782,8 +782,8 @@ def assert_permissions_checked(datasette, actions): default="fixtures.db", type=click.Path(file_okay=True, dir_okay=False), ) -@click.argument("metadata", required=False) @click.argument("config", required=False) +@click.argument("metadata", required=False) @click.argument( "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False ) From 2da1a6acec915b81a16127008fd739c7d6075681 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 13:26:13 -0700 Subject: [PATCH 1873/2303] Use importlib_metadata for Python 3.8, refs #2057 --- datasette/plugins.py | 10 ++++++---- setup.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index a93145cf..f23f5cfb 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,4 +1,4 @@ -import importlib.metadata +import importlib import os import pluggy import sys @@ -6,8 +6,10 @@ from . import hookspecs if sys.version_info >= (3, 9): import importlib.resources as importlib_resources + import importlib.metadata as importlib_metadata else: import importlib_resources + import importlib_metadata DEFAULT_PLUGINS = ( @@ -40,7 +42,7 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = importlib.metadata.distribution(package_name) + distribution = importlib_metadata.distribution(package_name) entry_points = distribution.entry_points for entry_point in entry_points: if entry_point.group == "datasette": @@ -48,7 +50,7 @@ if DATASETTE_LOAD_PLUGINS is not None: pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except importlib.metadata.PackageNotFoundError: + except importlib_metadata.PackageNotFoundError: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -86,6 +88,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.name + plugin_info["name"] = distinfo.name or distinfo.project_name plugins.append(plugin_info) return plugins diff --git a/setup.py b/setup.py index a2728f6b..65a3b335 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( "hupper>=1.9", "httpx>=0.20", 'importlib_resources>=1.3.1; python_version < "3.9"', + 'importlib_metadata>=4.6; python_version < "3.9"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From f130c7c0a88e50cea4121ea18d1f6db2431b6fab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 14:09:57 -0700 Subject: [PATCH 1874/2303] Deploy with fixtures-metadata.json, refs #2194, #2195 --- .github/workflows/deploy-latest.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 0dfa5a60..e0405440 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -38,8 +38,14 @@ jobs: run: | pytest -n auto -m "not serial" pytest -m "serial" - - name: Build fixtures.db - run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db + - name: Build fixtures.db and other files needed to deploy the demo + run: |- + python tests/fixtures.py \ + fixtures.db \ + fixtures-config.json \ + fixtures-metadata.json \ + plugins \ + --extra-db-filename extra_database.db - name: Build docs.db if: ${{ github.ref == 'refs/heads/main' }} run: |- @@ -88,13 +94,13 @@ jobs: } return queries EOF - - name: Make some modifications to metadata.json - run: | - cat fixtures.json | \ - jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ - jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ - > metadata.json - cat metadata.json + # - name: Make some modifications to metadata.json + # run: | + # cat fixtures.json | \ + # jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ + # jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ + # > metadata.json + # cat metadata.json - name: Set up Cloud Run uses: google-github-actions/setup-gcloud@v0 with: @@ -112,7 +118,7 @@ jobs: # Replace 1.0 with one-dot-zero in SUFFIX export SUFFIX=${SUFFIX//1.0/one-dot-zero} datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ - -m metadata.json \ + -m fixtures-metadata.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ From e4f868801a6633400045f59584cfe650961c3fa6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 14:58:39 -0700 Subject: [PATCH 1875/2303] Use importlib_metadata for 3.9 as well, refs #2057 --- datasette/plugins.py | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index f23f5cfb..1ed3747f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -6,9 +6,11 @@ from . import hookspecs if sys.version_info >= (3, 9): import importlib.resources as importlib_resources - import importlib.metadata as importlib_metadata else: import importlib_resources +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: import importlib_metadata diff --git a/setup.py b/setup.py index 65a3b335..d09a9e3d 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "hupper>=1.9", "httpx>=0.20", 'importlib_resources>=1.3.1; python_version < "3.9"', - 'importlib_metadata>=4.6; python_version < "3.9"', + 'importlib_metadata>=4.6; python_version < "3.10"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From 836b1587f08800658c63679d850f0149003c5311 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 15:06:19 -0700 Subject: [PATCH 1876/2303] Release notes for 1.0a7 Refs #2189 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 4b65999d..55e2cd42 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a6" +__version__ = "1.0a7" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 52e1db3b..9a5290c0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a7: + +1.0a7 (2023-09-21) +------------------ + +- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`) + .. _v0_64_4: 0.64.4 (2023-09-21) From d51e63d3bb3e32f80d1c0f04adff7c1dd5a7b0c0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:03:37 -0700 Subject: [PATCH 1877/2303] Release notes for 0.64.5, refs #2197 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a5290c0..48bf9ef5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_5: + +0.64.5 (2023-10-08) +------------------- + +- Dropped dependency on ``click-default-group-wheel``, which could cause a dependency conflict. (:issue:`2197`) + .. _v1_0_a7: 1.0a7 (2023-09-21) From 85a41987c7753c3af92ba6b8b6007211eb46602f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:07:11 -0700 Subject: [PATCH 1878/2303] Fixed typo acepts -> accepts --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1816d48c..eb6bf4ae 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -488,7 +488,7 @@ This will register ``render_demo`` to be called when paths with the extension `` ``render_demo`` is a Python function. It can be a regular function or an ``async def render_demo()`` awaitable function, depending on if it needs to make any asynchronous calls. -``can_render_demo`` is a Python function (or ``async def`` function) which acepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. +``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. When a request is received, the ``"render"`` callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. From 4e1188f60f8b4f90c32a372f3f70a26a3ebb88ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:09:45 -0700 Subject: [PATCH 1879/2303] Upgrade spellcheck.yml workflow --- .github/workflows/spellcheck.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 722e5c68..0ce9e10c 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,18 +9,13 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 with: - python-version: 3.11 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' + cache: 'pip' + cache-dependency-path: '**/setup.py' - name: Install dependencies run: | pip install -e '.[docs]' From 35deaabcb105903790d18710a26e77545f6852ce Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 12 Oct 2023 09:16:37 -0700 Subject: [PATCH 1880/2303] Move non-metadata configuration from metadata.yaml to datasette.yaml * Allow and permission blocks moved to datasette.yaml * Documentation updates, initial framework for configuration reference --- datasette/app.py | 6 +- datasette/default_permissions.py | 37 ++-- docs/authentication.rst | 338 ++++++++++++++---------------- docs/configuration.rst | 64 ++++-- docs/custom_templates.rst | 137 ++++++------ docs/facets.rst | 12 +- docs/full_text_search.rst | 4 +- docs/internals.rst | 2 +- docs/metadata.rst | 126 ++++++++--- docs/metadata_doc.py | 21 +- docs/plugins.rst | 30 +-- docs/settings.rst | 5 +- docs/sql_queries.rst | 56 ++--- docs/writing_plugins.rst | 8 +- tests/fixtures.py | 48 ++--- tests/test_canned_queries.py | 14 +- tests/test_html.py | 44 ++-- tests/test_internals_datasette.py | 6 +- tests/test_permissions.py | 138 ++++++------ tests/test_plugins.py | 4 +- tests/test_table_api.py | 2 +- tests/test_table_html.py | 8 +- 22 files changed, 606 insertions(+), 504 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c0e80700..7dfc63c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -721,7 +721,9 @@ class Datasette: return self._app_css_hash async def get_canned_queries(self, database_name, actor): - queries = self.metadata("queries", database=database_name, fallback=False) or {} + queries = ( + ((self.config or {}).get("databases") or {}).get(database_name) or {} + ).get("queries") or {} for more_queries in pm.hook.canned_queries( datasette=self, database=database_name, @@ -1315,7 +1317,7 @@ class Datasette: ): hook = await await_me_maybe(hook) collected.extend(hook) - collected.extend(self.metadata(key) or []) + collected.extend((self.config or {}).get(key) or []) output = [] for url_or_dict in collected: if isinstance(url_or_dict, dict): diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 5a99d0d8..d29dbe84 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -144,14 +144,14 @@ def permission_allowed_default(datasette, actor, action, resource): "view-query", "execute-sql", ): - result = await _resolve_metadata_view_permissions( + result = await _resolve_config_view_permissions( datasette, actor, action, resource ) if result is not None: return result # Check custom permissions: blocks - result = await _resolve_metadata_permissions_blocks( + result = await _resolve_config_permissions_blocks( datasette, actor, action, resource ) if result is not None: @@ -164,10 +164,10 @@ def permission_allowed_default(datasette, actor, action, resource): return inner -async def _resolve_metadata_permissions_blocks(datasette, actor, action, resource): +async def _resolve_config_permissions_blocks(datasette, actor, action, resource): # Check custom permissions: blocks - metadata = datasette.metadata() - root_block = (metadata.get("permissions", None) or {}).get(action) + config = datasette.config or {} + root_block = (config.get("permissions", None) or {}).get(action) if root_block: root_result = actor_matches_allow(actor, root_block) if root_result is not None: @@ -180,7 +180,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc else: database = resource[0] database_block = ( - (metadata.get("databases", {}).get(database, {}).get("permissions", None)) or {} + (config.get("databases", {}).get(database, {}).get("permissions", None)) or {} ).get(action) if database_block: database_result = actor_matches_allow(actor, database_block) @@ -192,7 +192,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc database, table_or_query = resource table_block = ( ( - metadata.get("databases", {}) + config.get("databases", {}) .get(database, {}) .get("tables", {}) .get(table_or_query, {}) @@ -207,7 +207,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc # Finally the canned queries query_block = ( ( - metadata.get("databases", {}) + config.get("databases", {}) .get(database, {}) .get("queries", {}) .get(table_or_query, {}) @@ -222,25 +222,30 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc return None -async def _resolve_metadata_view_permissions(datasette, actor, action, resource): +async def _resolve_config_view_permissions(datasette, actor, action, resource): + config = datasette.config or {} if action == "view-instance": - allow = datasette.metadata("allow") + allow = config.get("allow") if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - database_allow = datasette.metadata("allow", database=resource) + database_allow = ((config.get("databases") or {}).get(resource) or {}).get( + "allow" + ) if database_allow is None: return None return actor_matches_allow(actor, database_allow) elif action == "view-table": database, table = resource - tables = datasette.metadata("tables", database=database) or {} + tables = ((config.get("databases") or {}).get(database) or {}).get( + "tables" + ) or {} table_allow = (tables.get(table) or {}).get("allow") if table_allow is None: return None return actor_matches_allow(actor, table_allow) elif action == "view-query": - # Check if this query has a "allow" block in metadata + # Check if this query has a "allow" block in config database, query_name = resource query = await datasette.get_canned_query(database, query_name, actor) assert query is not None @@ -250,9 +255,11 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) return actor_matches_allow(actor, allow) elif action == "execute-sql": # Use allow_sql block from database block, or from top-level - database_allow_sql = datasette.metadata("allow_sql", database=resource) + database_allow_sql = ((config.get("databases") or {}).get(resource) or {}).get( + "allow_sql" + ) if database_allow_sql is None: - database_allow_sql = datasette.metadata("allow_sql") + database_allow_sql = config.get("allow_sql") if database_allow_sql is None: return None return actor_matches_allow(actor, database_allow_sql) diff --git a/docs/authentication.rst b/docs/authentication.rst index 1a444d0c..a301113a 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -67,7 +67,7 @@ An **action** is a string describing the action the actor would like to perform. A **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource. -Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content. +Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content. Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file. @@ -76,7 +76,7 @@ Permissions with potentially harmful effects should default to *deny*. Plugin au Defining permissions with "allow" blocks ---------------------------------------- -The standard way to define permissions in Datasette is to use an ``"allow"`` block. This is a JSON document describing which actors are allowed to perform a permission. +The standard way to define permissions in Datasette is to use an ``"allow"`` block :ref:`in the datasette.yaml file `. This is a JSON document describing which actors are allowed to perform a permission. The most basic form of allow block is this (`allow demo `__, `deny demo `__): @@ -186,18 +186,18 @@ The /-/allow-debug tool The ``/-/allow-debug`` tool lets you try out different ``"action"`` blocks against different ``"actor"`` JSON objects. You can try that out here: https://latest.datasette.io/-/allow-debug -.. _authentication_permissions_metadata: +.. _authentication_permissions_config: -Access permissions in metadata -============================== +Access permissions in ``datasette.yaml`` +======================================== -There are two ways to configure permissions using ``metadata.json`` (or ``metadata.yaml``). +There are two ways to configure permissions using ``datasette.yaml`` (or ``datasette.json``). For simple visibility permissions you can use ``"allow"`` blocks in the root, database, table and query sections. For other permissions you can use a ``"permissions"`` block, described :ref:`in the next section `. -You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`metadata` configuration. +You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`configuration`. You can control the following: @@ -216,25 +216,25 @@ Access to an instance Here's how to restrict access to your entire Datasette instance to just the ``"id": "root"`` user: .. [[[cog - from metadata_doc import metadata_example - metadata_example(cog, { - "title": "My private Datasette instance", - "allow": { - "id": "root" - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + from metadata_doc import config_example + config_example(cog, """ title: My private Datasette instance allow: id: root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + title: My private Datasette instance + allow: + id: root + + +.. tab:: datasette.json .. code-block:: json @@ -249,21 +249,22 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i To deny access to all users, you can use ``"allow": false``: .. [[[cog - metadata_example(cog, { - "title": "My entirely inaccessible instance", - "allow": False - }) + config_example(cog, """ + title: My entirely inaccessible instance + allow: false + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - title: My entirely inaccessible instance - allow: false + + title: My entirely inaccessible instance + allow: false -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -283,28 +284,26 @@ Access to specific databases To limit access to a specific ``private.db`` database to just authenticated users, use the ``"allow"`` block like this: .. [[[cog - metadata_example(cog, { - "databases": { - "private": { - "allow": { - "id": "*" - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: private: allow: - id: '*' + id: "*" + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + private: + allow: + id: "*" + + +.. tab:: datasette.json .. code-block:: json @@ -327,34 +326,30 @@ Access to specific tables and views To limit access to the ``users`` table in your ``bakery.db`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "bakery": { - "tables": { - "users": { - "allow": { - "id": "*" - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: bakery: tables: users: allow: id: '*' + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + bakery: + tables: + users: + allow: + id: '*' + + +.. tab:: datasette.json .. code-block:: json @@ -385,32 +380,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` Access to specific canned queries --------------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``metadata.json`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog - metadata_example(cog, { - "databases": { - "dogs": { - "queries": { - "add_name": { - "sql": "INSERT INTO names (name) VALUES (:name)", - "write": True, - "allow": { - "id": ["root"] - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: dogs: queries: @@ -420,9 +395,26 @@ To limit access to the ``add_name`` canned query in your ``dogs.db`` database to allow: id: - root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + dogs: + queries: + add_name: + sql: INSERT INTO names (name) VALUES (:name) + write: true + allow: + id: + - root + + +.. tab:: datasette.json .. code-block:: json @@ -461,19 +453,20 @@ You can alternatively use an ``"allow_sql"`` block to control who is allowed to To prevent any user from executing arbitrary SQL queries, use this: .. [[[cog - metadata_example(cog, { - "allow_sql": False - }) + config_example(cog, """ + allow_sql: false + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - allow_sql: false + + allow_sql: false -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -485,22 +478,22 @@ To prevent any user from executing arbitrary SQL queries, use this: To enable just the :ref:`root user` to execute SQL for all databases in your instance, use the following: .. [[[cog - metadata_example(cog, { - "allow_sql": { - "id": "root" - } - }) + config_example(cog, """ + allow_sql: + id: root + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - allow_sql: - id: root + + allow_sql: + id: root -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -514,28 +507,26 @@ To enable just the :ref:`root user` to execute SQL for all To limit this ability for just one specific database, use this: .. [[[cog - metadata_example(cog, { - "databases": { - "mydatabase": { - "allow_sql": { - "id": "root" - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: mydatabase: allow_sql: id: root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + mydatabase: + allow_sql: + id: root + + +.. tab:: datasette.json .. code-block:: json @@ -552,33 +543,32 @@ To limit this ability for just one specific database, use this: .. _authentication_permissions_other: -Other permissions in metadata -============================= +Other permissions in ``datasette.yaml`` +======================================= -For all other permissions, you can use one or more ``"permissions"`` blocks in your metadata. +For all other permissions, you can use one or more ``"permissions"`` blocks in your ``datasette.yaml`` configuration file. -To grant access to the :ref:`permissions debug tool ` to all signed in users you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your metadata: +To grant access to the :ref:`permissions debug tool ` to all signed in users, you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your configuration: .. [[[cog - metadata_example(cog, { - "permissions": { - "debug-menu": { - "id": "*" - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ permissions: debug-menu: id: '*' + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + permissions: + debug-menu: + id: '*' + + +.. tab:: datasette.json .. code-block:: json @@ -594,31 +584,28 @@ To grant access to the :ref:`permissions debug tool ` to a To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "docs": { - "permissions": { - "create-table": { - "id": "editor" - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: docs: permissions: create-table: id: editor + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + docs: + permissions: + create-table: + id: editor + + +.. tab:: datasette.json .. code-block:: json @@ -638,27 +625,7 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` And for ``insert-row`` against the ``reports`` table in that ``docs`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "docs": { - "tables": { - "reports": { - "permissions": { - "insert-row": { - "id": "editor" - } - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: docs: tables: @@ -666,9 +633,24 @@ And for ``insert-row`` against the ``reports`` table in that ``docs`` database: permissions: insert-row: id: editor + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + docs: + tables: + reports: + permissions: + insert-row: + id: editor + + +.. tab:: datasette.json .. code-block:: json diff --git a/docs/configuration.rst b/docs/configuration.rst index 4a7258b9..4e108602 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -13,15 +13,15 @@ To facilitate this, You can provide a ``datasette.yaml`` configuration file to d .. _configuration_reference: -``datasette.yaml`` reference +``datasette.yaml`` Reference ---------------------------- Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # Datasette settings block settings: @@ -52,10 +52,11 @@ Here's a full example of all the valid configuration options that can exist insi ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml + # Datasette settings block settings: default_page_size: 50 @@ -82,7 +83,8 @@ Here's a full example of all the valid configuration options that can exist insi datasette-my-plugin: key: valueB -.. tab:: JSON + +.. tab:: datasette.json .. code-block:: json @@ -125,9 +127,9 @@ Settings configuration :ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml settings: @@ -137,7 +139,7 @@ Settings configuration ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -146,7 +148,7 @@ Settings configuration default_allow_sql: off default_page_size: 50 -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -165,9 +167,9 @@ Plugin configuration Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml plugins: @@ -177,7 +179,7 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -186,7 +188,7 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve datasette-my-plugin: key: my_value -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -202,9 +204,9 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml databases: @@ -224,7 +226,7 @@ For database level or table level plugin configuration, nest it under the approp ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -243,7 +245,7 @@ For database level or table level plugin configuration, nest it under the approp datasette-my-plugin: key: my_value -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -269,4 +271,30 @@ For database level or table level plugin configuration, nest it under the approp } } } -.. [[[end]]] \ No newline at end of file +.. [[[end]]] + + +.. _configuration_reference_permissions: +Permissions Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + + +.. _configuration_reference_authentication: +Authentication Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + +.. _configuration_reference_canned_queries: +Canned Queries Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + +.. _configuration_reference_css_js: +Extra CSS and JS Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index c0f64cb5..d8e4ac96 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -10,35 +10,34 @@ Datasette provides a number of ways of customizing the way data is displayed. Custom CSS and JavaScript ------------------------- -When you launch Datasette, you can specify a custom metadata file like this:: +When you launch Datasette, you can specify a custom configuration file like this:: - datasette mydb.db --metadata metadata.yaml + datasette mydb.db --config datasette.yaml -Your ``metadata.yaml`` file can include links that look like this: +Your ``datasette.yaml`` file can include links that look like this: .. [[[cog - from metadata_doc import metadata_example - metadata_example(cog, { - "extra_css_urls": [ - "https://simonwillison.net/static/css/all.bf8cd891642c.css" - ], - "extra_js_urls": [ - "https://code.jquery.com/jquery-3.2.1.slim.min.js" - ] - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + from metadata_doc import config_example + config_example(cog, """ extra_css_urls: - https://simonwillison.net/static/css/all.bf8cd891642c.css extra_js_urls: - https://code.jquery.com/jquery-3.2.1.slim.min.js + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + extra_css_urls: + - https://simonwillison.net/static/css/all.bf8cd891642c.css + extra_js_urls: + - https://code.jquery.com/jquery-3.2.1.slim.min.js + + +.. tab:: datasette.json .. code-block:: json @@ -62,35 +61,30 @@ The extra CSS and JavaScript files will be linked in the ```` of every pag You can also specify a SRI (subresource integrity hash) for these assets: .. [[[cog - metadata_example(cog, { - "extra_css_urls": [ - { - "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", - "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" - } - ], - "extra_js_urls": [ - { - "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", - "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" - } - ] - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ extra_css_urls: - url: https://simonwillison.net/static/css/all.bf8cd891642c.css sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI extra_js_urls: - url: https://code.jquery.com/jquery-3.2.1.slim.min.js sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: datasette.json .. code-block:: json @@ -115,7 +109,7 @@ This will produce: .. code-block:: html + {% for url in extra_js_urls %} {% endfor %} diff --git a/demos/plugins/example_js_manager_plugins.py b/demos/plugins/example_js_manager_plugins.py new file mode 100644 index 00000000..7db45464 --- /dev/null +++ b/demos/plugins/example_js_manager_plugins.py @@ -0,0 +1,21 @@ +from datasette import hookimpl + +# Test command: +# datasette fixtures.db \ --plugins-dir=demos/plugins/ +# \ --static static:demos/plugins/static + +# Create a set with view names that qualify for this JS, since plugins won't do anything on other pages +# Same pattern as in Nteract data explorer +# https://github.com/hydrosquall/datasette-nteract-data-explorer/blob/main/datasette_nteract_data_explorer/__init__.py#L77 +PERMITTED_VIEWS = {"table", "query", "database"} + + +@hookimpl +def extra_js_urls(view_name): + print(view_name) + if view_name in PERMITTED_VIEWS: + return [ + { + "url": f"/static/table-example-plugins.js", + } + ] diff --git a/demos/plugins/static/table-example-plugins.js b/demos/plugins/static/table-example-plugins.js new file mode 100644 index 00000000..8c19d9a6 --- /dev/null +++ b/demos/plugins/static/table-example-plugins.js @@ -0,0 +1,100 @@ +/** + * Example usage of Datasette JS Manager API + */ + +document.addEventListener("datasette_init", function (evt) { + const { detail: manager } = evt; + // === Demo plugins: remove before merge=== + addPlugins(manager); +}); + +/** + * Examples for to test datasette JS api + */ +const addPlugins = (manager) => { + + manager.registerPlugin("column-name-plugin", { + version: 0.1, + makeColumnActions: (columnMeta) => { + const { column } = columnMeta; + + return [ + { + label: "Copy name to clipboard", + onClick: (evt) => copyToClipboard(column), + }, + { + label: "Log column metadata to console", + onClick: (evt) => console.log(column), + }, + ]; + }, + }); + + manager.registerPlugin("panel-plugin-graphs", { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + id: 'first-panel', + label: "First", + render: node => { + const description = document.createElement('p'); + description.innerText = 'Hello world'; + node.appendChild(description); + } + }, + { + id: 'second-panel', + label: "Second", + render: node => { + const iframe = document.createElement('iframe'); + iframe.src = "https://observablehq.com/embed/@d3/sortable-bar-chart?cell=viewof+order&cell=chart"; + iframe.width = 800; + iframe.height = 635; + iframe.frameborder = '0'; + node.appendChild(iframe); + } + }, + ]; + }, + }); + + manager.registerPlugin("panel-plugin-maps", { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + // ID only has to be unique within a plugin, manager namespaces for you + id: 'first-map-panel', + label: "Map plugin", + // datasette-vega, leafleft can provide a "render" function + render: node => node.innerHTML = "Here sits a map", + }, + { + id: 'second-panel', + label: "Image plugin", + render: node => { + const img = document.createElement('img'); + img.src = 'https://datasette.io/static/datasette-logo.svg' + node.appendChild(img); + }, + } + ]; + }, + }); + + // Future: dispatch message to some other part of the page with CustomEvent API + // Could use to drive filter/sort query builder actions without page refresh. +} + + + +async function copyToClipboard(str) { + try { + await navigator.clipboard.writeText(str); + } catch (err) { + /** Rejected - text failed to copy to the clipboard. Browsers didn't give permission */ + console.error('Failed to copy: ', err); + } +} From 067cc75dfa01612f9a47815b33804361e18bf5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Dec 2023 09:49:04 -0800 Subject: [PATCH 1886/2303] Fixed broken example links in row page documentation --- docs/pages.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages.rst b/docs/pages.rst index 0ae72351..2ce05428 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -70,10 +70,10 @@ Table cells with extremely long text contents are truncated on the table view ac Rows which are the targets of foreign key references from other tables will show a link to a filtered search for all records that reference that row. Here's an example from the Registers of Members Interests database: -`../people/uk.org.publicwhip%2Fperson%2F10001 `_ +`../people/uk~2Eorg~2Epublicwhip~2Fperson~2F10001 `_ Note that this URL includes the encoded primary key of the record. Here's that same page as JSON: -`../people/uk.org.publicwhip%2Fperson%2F10001.json `_ +`../people/uk~2Eorg~2Epublicwhip~2Fperson~2F10001.json `_ From 89c8ca0f3ff51fcbf5f710c529bc7a3552da0731 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Dec 2023 10:32:55 -0800 Subject: [PATCH 1887/2303] Fix for round_trip_load() YAML error, refs #2219 --- docs/metadata_doc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 3dc5b5f8..a8f13414 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,7 +1,7 @@ import json import textwrap from yaml import safe_dump -from ruamel.yaml import round_trip_load +from ruamel.yaml import YAML def metadata_example(cog, data=None, yaml=None): @@ -11,8 +11,7 @@ def metadata_example(cog, data=None, yaml=None): if yaml: # dedent it first yaml = textwrap.dedent(yaml).strip() - # round_trip_load to preserve key order: - data = round_trip_load(yaml) + data = YAML().load(yaml) output_yaml = yaml else: output_yaml = safe_dump(data, sort_keys=False) @@ -27,8 +26,7 @@ def metadata_example(cog, data=None, yaml=None): def config_example(cog, input): if type(input) is str: - # round_trip_load to preserve key order: - data = round_trip_load(input) + data = YAML().load(input) output_yaml = input else: data = input From 4284c74bc133ab494bf4b6dcd4a20b97b05ebb83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Dec 2023 10:51:03 -0800 Subject: [PATCH 1888/2303] db.execute_isolated_fn() method (#2220) Closes #2218 --- datasette/database.py | 61 ++++++++++++++++++++++++------ docs/internals.rst | 19 +++++++++- tests/test_internals_database.py | 65 ++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index cb01301e..f2c980d7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -159,6 +159,26 @@ class Database: kwargs["count"] = count return results + async def execute_isolated_fn(self, fn): + # Open a new connection just for the duration of this function + # blocking the write queue to avoid any writes occurring during it + if self.ds.executor is None: + # non-threaded mode + isolated_connection = self.connect(write=True) + try: + result = fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + return result + else: + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) + async def execute_write_fn(self, fn, block=True): if self.ds.executor is None: # non-threaded mode @@ -166,9 +186,10 @@ class Database: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) return fn(self._write_connection) + else: + return await self._send_to_write_thread(fn, block) - # threaded mode - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + async def _send_to_write_thread(self, fn, block=True, isolated_connection=False): if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: @@ -176,8 +197,9 @@ class Database: target=self._execute_writes, daemon=True ) self._write_thread.start() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() - self._write_queue.put(WriteTask(fn, task_id, reply_queue)) + self._write_queue.put(WriteTask(fn, task_id, reply_queue, isolated_connection)) if block: result = await reply_queue.async_q.get() if isinstance(result, Exception): @@ -202,12 +224,28 @@ class Database: if conn_exception is not None: result = conn_exception else: - try: - result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e + if task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + else: + try: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): @@ -515,12 +553,13 @@ class Database: class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue") + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection") - def __init__(self, fn, task_id, reply_queue): + def __init__(self, fn, task_id, reply_queue, isolated_connection): self.fn = fn self.task_id = task_id self.reply_queue = reply_queue + self.isolated_connection = isolated_connection class QueryInterrupted(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index 649ca35d..d269bc7d 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1017,7 +1017,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() ` but executes the provided function in an entirely isolated SQLite connection, which is opened, used and then closed again in a single call to this method. + +The :ref:`prepare_connection() ` plugin hook is not executed against this connection. + +This allows plugins to execute database operations that might conflict with how database connections are usually configured. For example, running a ``VACUUM`` operation while bypassing any restrictions placed by the `datasette-sqlite-authorizer `__ plugin. + +Plugins can also use this method to load potentially dangerous SQLite extensions, use them to perform an operation and then have them safely unloaded at the end of the call, without risk of exposing them to other connections. + +Functions run using ``execute_isolated_fn()`` share the same queue as ``execute_write_fn()``, which guarantees that no writes can be executed at the same time as the isolated function is executing. + +The return value of the function will be returned by this method. Any exceptions raised by the function will be raised out of the ``await`` line as well. + .. _database_close: db.close() diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 647ae7bd..e0511100 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,6 +1,7 @@ """ Tests for the datasette.database.Database class """ +from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.utils.sqlite import sqlite3 from datasette.utils import Column @@ -519,6 +520,70 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): app_client.ds.remove_database("immutable-db") +def table_exists(conn, name): + return bool( + conn.execute( + """ + with all_tables as ( + select name from sqlite_master where type = 'table' + union all + select name from temp.sqlite_master where type = 'table' + ) + select 1 from all_tables where name = ? + """, + (name,), + ).fetchall(), + ) + + +def table_exists_checker(name): + def inner(conn): + return table_exists(conn, name) + + return inner + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disable_threads", (False, True)) +async def test_execute_isolated(db, disable_threads): + if disable_threads: + ds = Datasette(memory=True, settings={"num_sql_threads": 0}) + db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) + + # Create temporary table in write + await db.execute_write( + "create temporary table created_by_write (id integer primary key)" + ) + # Should stay visible to write connection + assert await db.execute_write_fn(table_exists_checker("created_by_write")) + + def create_shared_table(conn): + conn.execute("create table shared (id integer primary key)") + # And a temporary table that should not continue to exist + conn.execute( + "create temporary table created_by_isolated (id integer primary key)" + ) + assert table_exists(conn, "created_by_isolated") + # Also confirm that created_by_write does not exist + return table_exists(conn, "created_by_write") + + # shared should not exist + assert not await db.execute_fn(table_exists_checker("shared")) + + # Create it using isolated + created_by_write_exists = await db.execute_isolated_fn(create_shared_table) + assert not created_by_write_exists + + # shared SHOULD exist now + assert await db.execute_fn(table_exists_checker("shared")) + + # created_by_isolated should not exist, even in write connection + assert not await db.execute_write_fn(table_exists_checker("created_by_isolated")) + + # ... and a second call to isolated should not see that connection either + assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) + + @pytest.mark.asyncio async def test_mtime_ns(db): assert isinstance(db.mtime_ns, int) From 978249beda1a3e7185f61000b0dd57018541c511 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:07:42 -0800 Subject: [PATCH 1889/2303] Removed rogue print("max_csv_mb") Found this while working on #2214 --- datasette/views/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 0080b33c..db08557e 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -484,7 +484,6 @@ async def stream_csv(datasette, fetch_data, request, database): async def stream_fn(r): nonlocal data, trace - print("max_csv_mb", datasette.setting("max_csv_mb")) limited_writer = LimitedWriter(r, datasette.setting("max_csv_mb")) if trace: await limited_writer.write(preamble) From 872dae1e1a1511e2edfb9d7ddf6ea5096c11d5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:08:11 -0800 Subject: [PATCH 1890/2303] Fix for CSV labels=on missing foreign key bug, closes #2214 --- datasette/views/base.py | 14 ++++++++------ tests/test_csv.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index db08557e..e59fd683 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -553,16 +553,18 @@ async def stream_csv(datasette, fetch_data, request, database): if cell is None: new_row.extend(("", "")) else: - assert isinstance(cell, dict) - new_row.append(cell["value"]) - new_row.append(cell["label"]) + if not isinstance(cell, dict): + new_row.extend((cell, "")) + else: + new_row.append(cell["value"]) + new_row.append(cell["label"]) else: new_row.append(cell) await writer.writerow(new_row) - except Exception as e: - sys.stderr.write("Caught this error: {}\n".format(e)) + except Exception as ex: + sys.stderr.write("Caught this error: {}\n".format(ex)) sys.stderr.flush() - await r.write(str(e)) + await r.write(str(ex)) return await limited_writer.write(postamble) diff --git a/tests/test_csv.py b/tests/test_csv.py index ed83d685..9f772f89 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,3 +1,4 @@ +from datasette.app import Datasette from bs4 import BeautifulSoup as Soup import pytest from .fixtures import ( # noqa @@ -95,6 +96,40 @@ async def test_table_csv_with_nullable_labels(ds_client): assert response.text == EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV +@pytest.mark.asyncio +async def test_table_csv_with_invalid_labels(): + # https://github.com/simonw/datasette/issues/2214 + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("db_2214") + await db.execute_write_script( + """ + create table t1 (id integer primary key, name text); + insert into t1 (id, name) values (1, 'one'); + insert into t1 (id, name) values (2, 'two'); + create table t2 (textid text primary key, name text); + insert into t2 (textid, name) values ('a', 'alpha'); + insert into t2 (textid, name) values ('b', 'beta'); + create table if not exists maintable ( + id integer primary key, + fk_integer integer references t1(id), + fk_text text references t2(textid) + ); + insert into maintable (id, fk_integer, fk_text) values (1, 1, 'a'); + insert into maintable (id, fk_integer, fk_text) values (2, 3, 'b'); -- invalid fk_integer + insert into maintable (id, fk_integer, fk_text) values (3, 2, 'c'); -- invalid fk_text + """ + ) + response = await ds.client.get("/db_2214/maintable.csv?_labels=1") + assert response.status_code == 200 + assert response.text == ( + "id,fk_integer,fk_integer_label,fk_text,fk_text_label\r\n" + "1,1,one,a,alpha\r\n" + "2,3,,b,beta\r\n" + "3,2,two,c,\r\n" + ) + + @pytest.mark.asyncio async def test_table_csv_blob_columns(ds_client): response = await ds_client.get("/fixtures/binary_data.csv") From 45b88f2056e0a4da204b50f5e17ba953fcb51865 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:14:50 -0800 Subject: [PATCH 1891/2303] Release notes from 0.64.6, refs #2214 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f2f17a50..af3d2a0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_6: + +0.64.6 (2023-12-22) +------------------- + +- Fixed a bug where CSV export with expanded labels could fail if a foreign key reference did not correctly resolve. (:issue:`2214`) + .. _v0_64_5: 0.64.5 (2023-10-08) From c7a4706bcc0d6736533b91437e54a8af9226a10a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jan 2024 14:33:23 -0800 Subject: [PATCH 1892/2303] jinja2_environment_from_request() plugin hook Closes #2225 --- datasette/app.py | 49 +++++++++++++++++++++-------------- datasette/handle_exception.py | 3 ++- datasette/hookspecs.py | 5 ++++ datasette/views/base.py | 3 ++- datasette/views/database.py | 6 +++-- datasette/views/table.py | 3 ++- docs/plugin_hooks.rst | 42 ++++++++++++++++++++++++++++++ tests/test_plugins.py | 42 +++++++++++++++++++++++++++++- 8 files changed, 128 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f33865e4..482cebb4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -420,21 +420,31 @@ class Datasette: ), ] ) - self.jinja_env = Environment( + environment = Environment( loader=template_loader, autoescape=True, enable_async=True, # undefined=StrictUndefined, ) - self.jinja_env.filters["escape_css_string"] = escape_css_string - self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus - self.jinja_env.filters["escape_sqlite"] = escape_sqlite - self.jinja_env.filters["to_css_class"] = to_css_class + environment.filters["escape_css_string"] = escape_css_string + environment.filters["quote_plus"] = urllib.parse.quote_plus + self._jinja_env = environment + environment.filters["escape_sqlite"] = escape_sqlite + environment.filters["to_css_class"] = to_css_class self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def get_jinja_environment(self, request: Request = None) -> Environment: + environment = self._jinja_env + if request: + for environment in pm.hook.jinja2_environment_from_request( + datasette=self, request=request, env=environment + ): + pass + return environment + def get_permission(self, name_or_abbr: str) -> "Permission": """ Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. @@ -514,7 +524,7 @@ class Datasette: abbrs[p.abbr] = p self.permissions[p.name] = p for hook in pm.hook.prepare_jinja2_environment( - env=self.jinja_env, datasette=self + env=self._jinja_env, datasette=self ): await await_me_maybe(hook) for hook in pm.hook.startup(datasette=self): @@ -1218,7 +1228,7 @@ class Datasette: else: if isinstance(templates, str): templates = [templates] - template = self.jinja_env.select_template(templates) + template = self.get_jinja_environment(request).select_template(templates) if dataclasses.is_dataclass(context): context = dataclasses.asdict(context) body_scripts = [] @@ -1568,16 +1578,6 @@ class DatasetteRouter: def __init__(self, datasette, routes): self.ds = datasette self.routes = routes or [] - # Build a list of pages/blah/{name}.html matching expressions - pattern_templates = [ - filepath - for filepath in self.ds.jinja_env.list_templates() - if "{" in filepath and filepath.startswith("pages/") - ] - self.page_routes = [ - (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) - for filepath in pattern_templates - ] async def __call__(self, scope, receive, send): # Because we care about "foo/bar" v.s. "foo%2Fbar" we decode raw_path ourselves @@ -1677,13 +1677,24 @@ class DatasetteRouter: route_path = request.scope.get("route_path", request.scope["path"]) # Jinja requires template names to use "/" even on Windows template_name = "pages" + route_path + ".html" + # Build a list of pages/blah/{name}.html matching expressions + environment = self.ds.get_jinja_environment(request) + pattern_templates = [ + filepath + for filepath in environment.list_templates() + if "{" in filepath and filepath.startswith("pages/") + ] + page_routes = [ + (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) + for filepath in pattern_templates + ] try: - template = self.ds.jinja_env.select_template([template_name]) + template = environment.select_template([template_name]) except TemplateNotFound: template = None if template is None: # Try for a pages/blah/{name}.html template match - for regex, wildcard_template in self.page_routes: + for regex, wildcard_template in page_routes: match = regex.match(route_path) if match is not None: context.update(match.groupdict()) diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 8b7e83e3..bef6b4ee 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -57,7 +57,8 @@ def handle_exception(datasette, request, exception): if request.path.split("?")[0].endswith(".json"): return Response.json(info, status=status, headers=headers) else: - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) return Response.html( await template.render_async( dict( diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 9069927b..b6975dce 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -99,6 +99,11 @@ def actors_from_ids(datasette, actor_ids): """Returns a dictionary mapping those IDs to actor dictionaries""" +@hookspec +def jinja2_environment_from_request(datasette, request, env): + """Return a Jinja2 environment based on the incoming request""" + + @hookspec def filters_from_request(request, database, table, datasette): """ diff --git a/datasette/views/base.py b/datasette/views/base.py index e59fd683..bdc1e9cf 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -143,7 +143,8 @@ class BaseView: async def render(self, templates, request, context=None): context = context or {} - template = self.ds.jinja_env.select_template(templates) + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) template_context = { **context, **{ diff --git a/datasette/views/database.py b/datasette/views/database.py index 9ba5ce94..03e70379 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -143,7 +143,8 @@ class DatabaseView(View): datasette.urls.path(path_with_format(request=request, format="json")), ) templates = (f"database-{to_css_class(database)}.html", "database.html") - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) context = { **json_data, "database_color": db.color, @@ -594,7 +595,8 @@ class QueryView(View): f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", ) - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) alternate_url_json = datasette.absolute_url( request, datasette.urls.path(path_with_format(request=request, format="json")), diff --git a/datasette/views/table.py b/datasette/views/table.py index 4f4baeed..7ee5d6bf 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -806,7 +806,8 @@ async def table_view_traced(datasette, request): f"table-{to_css_class(resolved.db.name)}-{to_css_class(resolved.table)}.html", "table.html", ] - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) alternate_url_json = datasette.absolute_url( request, datasette.urls.path(path_with_format(request=request, format="json")), diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index eb6bf4ae..f67d15d6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1128,6 +1128,48 @@ These IDs could be integers or strings, depending on how the actors used by the Example: `datasette-remote-actors `_ +.. _plugin_hook_jinja2_environment_from_request: + +jinja2_environment_from_request(datasette, request, env) +-------------------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + A Datasette instance. + +``request`` - :ref:`internals_request` or ``None`` + The current HTTP request, if one is available. + +``env`` - ``Environment`` + The Jinja2 environment that will be used to render the current page. + +This hook can be used to return a customized `Jinja environment `__ based on the incoming request. + +If you want to run a single Datasette instance that serves different content for different domains, you can do so like this: + +.. code-block:: python + + from datasette import hookimpl + from jinja2 import ChoiceLoader, FileSystemLoader + + + @hookimpl + def jinja2_environment_from_request(request, env): + if request and request.host == "www.niche-museums.com": + return env.overlay( + loader=ChoiceLoader( + [ + FileSystemLoader( + "/mnt/niche-museums/templates" + ), + env.loader, + ] + ), + enable_async=True, + ) + return env + +This uses the Jinja `overlay() method `__ to create a new environment identical to the default environment except for having a different template loader, which first looks in the ``/mnt/niche-museums/templates`` directory before falling back on the default loader. + .. _plugin_hook_filters_from_request: filters_from_request(request, database, table, datasette) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 82e2f7f1..bdd4ba49 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -16,6 +16,7 @@ from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 from datasette.utils import CustomRow, StartupError from jinja2.environment import Template +from jinja2 import ChoiceLoader, FileSystemLoader import base64 import importlib import json @@ -563,7 +564,8 @@ async def test_hook_register_output_renderer_can_render(ds_client): async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" await ds_client.ds.invoke_startup() - template = ds_client.ds.jinja_env.from_string( + environment = ds_client.ds.get_jinja_environment(None) + template = environment.from_string( "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}", {"a": 3412341, "b": 5}, ) @@ -1294,3 +1296,41 @@ async def test_plugin_is_installed(): finally: pm.unregister(name="DummyPlugin") + + +@pytest.mark.asyncio +async def test_hook_jinja2_environment_from_request(tmpdir): + templates = pathlib.Path(tmpdir / "templates") + templates.mkdir() + (templates / "index.html").write_text("Hello museums!", "utf-8") + + class EnvironmentPlugin: + @hookimpl + def jinja2_environment_from_request(self, request, env): + if request and request.host == "www.niche-museums.com": + return env.overlay( + loader=ChoiceLoader( + [ + FileSystemLoader(str(templates)), + env.loader, + ] + ), + enable_async=True, + ) + return env + + datasette = Datasette(memory=True) + + try: + pm.register(EnvironmentPlugin(), name="EnvironmentPlugin") + response = await datasette.client.get("/") + assert response.status_code == 200 + assert "Hello museums!" not in response.text + # Try again with the hostname + response2 = await datasette.client.get( + "/", headers={"host": "www.niche-museums.com"} + ) + assert response2.status_code == 200 + assert "Hello museums!" in response2.text + finally: + pm.unregister(name="EnvironmentPlugin") From 1fc76fee6268c21003c0fe730cc8e93210ce6bb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jan 2024 16:59:25 -0800 Subject: [PATCH 1893/2303] 1.0a8.dev1 version number Not going to release this to PyPI but I will build my own wheel of it --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 55e2cd42..75d44727 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a7" +__version__ = "1.0a8.dev1" __version_info__ = tuple(__version__.split(".")) From 0b2c6a7ebd4fd540d9bdfb169c621452d280e608 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jan 2024 13:12:57 -0800 Subject: [PATCH 1894/2303] Fix for ?_extra=columns bug, closes #2230 Also refs #262 - started a test suite for extras. --- datasette/renderer.py | 2 +- tests/test_table_api.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 224031a7..a446e69d 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -68,7 +68,7 @@ def json_renderer(request, args, data, error, truncated=None): elif shape in ("objects", "object", "array"): columns = data.get("columns") rows = data.get("rows") - if rows and columns: + if rows and columns and not isinstance(rows[0], dict): data["rows"] = [dict(zip(columns, row)) for row in rows] if shape == "object": shape_error = None diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 5dbb8b8f..ae4fdb17 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1362,3 +1362,27 @@ async def test_col_nocol_errors(ds_client, path, expected_error): response = await ds_client.get(path) assert response.status_code == 400 assert response.json()["error"] == expected_error + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra,expected_json", + ( + ( + "columns", + { + "ok": True, + "next": None, + "columns": ["id", "content", "content2"], + "rows": [{"id": "1", "content": "hey", "content2": "world"}], + "truncated": False, + }, + ), + ), +) +async def test_table_extras(ds_client, extra, expected_json): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=" + extra + ) + assert response.status_code == 200 + assert response.json() == expected_json From 2ff4d4a60a348c143f79d63c48c329ffd0c1f02f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jan 2024 13:13:53 -0800 Subject: [PATCH 1895/2303] Test for ?_extra=count, refs #262 --- tests/test_table_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ae4fdb17..bde7a38e 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1378,6 +1378,16 @@ async def test_col_nocol_errors(ds_client, path, expected_error): "truncated": False, }, ), + ( + "count", + { + "ok": True, + "next": None, + "rows": [{"id": "1", "content": "hey", "content2": "world"}], + "truncated": False, + "count": 1, + }, + ), ), ) async def test_table_extras(ds_client, extra, expected_json): From 48148e66a846d585e08ec6ab4ae3da8e60d55ab5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 10:42:36 -0800 Subject: [PATCH 1896/2303] Link from actors_from_ids plugin hook docs to datasette.actors_from_ids() --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f67d15d6..9115c3df 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1086,6 +1086,8 @@ The hook must return a dictionary that maps the incoming actor IDs to their full Some plugins that implement social features may store the ID of the :ref:`actor ` that performed an action - added a comment, bookmarked a table or similar - and then need a way to resolve those IDs into display-friendly actor dictionaries later on. +The :ref:`await datasette.actors_from_ids(actor_ids) ` internal method can be used to look up actors from their IDs. It will dispatch to the first plugin that implements this hook. + Unlike other plugin hooks, this only uses the first implementation of the hook to return a result. You can expect users to only have a single plugin installed that implements this hook. If no plugin is installed, Datasette defaults to returning actors that are just ``{"id": actor_id}``. From 7506a89be0d1c97632bed47635eb90f92815d6c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 13:04:34 -0800 Subject: [PATCH 1897/2303] Docs on datasette.client for tests, closes #1830 Also covers ds.client.actor_cookie() helper --- docs/testing_plugins.rst | 28 +++++++++++++++++++++++++++ tests/test_docs.py | 41 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 6d2097ad..e10514c6 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,34 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_datasette_client: + +Using datasette.client in tests +------------------------------- + +The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. + +I simple test looks like this: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_homepage -- + :end-before: # -- end test_homepage -- + +Or for a JSON API: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_actor_is_null -- + :end-before: # -- end test_actor_is_null -- + +To make requests as an authenticated actor, create a signed ``ds_cookie`` using the ``datasette.client.actor_cookie()`` helper function and pass it in ``cookies=`` like this: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_signed_cookie_actor -- + :end-before: # -- end test_signed_cookie_actor -- + .. _testing_plugins_pdb: Using pdb for errors thrown inside Datasette diff --git a/tests/test_docs.py b/tests/test_docs.py index e9b813fe..fdd44788 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,9 +1,8 @@ """ Tests to ensure certain things are documented. """ -from click.testing import CliRunner from datasette import app, utils -from datasette.cli import cli +from datasette.app import Datasette from datasette.filters import Filters from pathlib import Path import pytest @@ -102,3 +101,41 @@ def documented_fns(): @pytest.mark.parametrize("fn", utils.functions_marked_as_documented) def test_functions_marked_with_documented_are_documented(documented_fns, fn): assert fn.__name__ in documented_fns + + +# Tests for testing_plugins.rst documentation + + +# -- start test_homepage -- +@pytest.mark.asyncio +async def test_homepage(): + ds = Datasette(memory=True) + response = await ds.client.get("/") + html = response.text + assert "

    " in html + + +# -- end test_homepage -- + + +# -- start test_actor_is_null -- +@pytest.mark.asyncio +async def test_actor_is_null(): + ds = Datasette(memory=True) + response = await ds.client.get("/-/actor.json") + assert response.json() == {"actor": None} + + +# -- end test_actor_is_null -- + + +# -- start test_signed_cookie_actor -- +@pytest.mark.asyncio +async def test_signed_cookie_actor(): + ds = Datasette(memory=True) + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/-/actor.json", cookies=cookies) + assert response.json() == {"actor": {"id": "root"}} + + +# -- end test_signed_cookie_actor -- From 0f63cb83ed31753a9bd9ec5cc71de16906767337 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 13:08:52 -0800 Subject: [PATCH 1898/2303] Typo fix --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index e10514c6..33ac4b22 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -89,7 +89,7 @@ Using datasette.client in tests The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. -I simple test looks like this: +A simple test looks like this: .. literalinclude:: ../tests/test_docs.py :language: python From a25bf6bea789c409580386f77b7440ff525d09b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 14:10:40 -0800 Subject: [PATCH 1899/2303] fmt: off to fix problem with Black, closes #2231 --- tests/test_docs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index fdd44788..17c01a0b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -105,7 +105,7 @@ def test_functions_marked_with_documented_are_documented(documented_fns, fn): # Tests for testing_plugins.rst documentation - +# fmt: off # -- start test_homepage -- @pytest.mark.asyncio async def test_homepage(): @@ -113,8 +113,6 @@ async def test_homepage(): response = await ds.client.get("/") html = response.text assert "

    " in html - - # -- end test_homepage -- @@ -124,8 +122,6 @@ async def test_actor_is_null(): ds = Datasette(memory=True) response = await ds.client.get("/-/actor.json") assert response.json() == {"actor": None} - - # -- end test_actor_is_null -- @@ -136,6 +132,4 @@ async def test_signed_cookie_actor(): cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} response = await ds.client.get("/-/actor.json", cookies=cookies) assert response.json() == {"actor": {"id": "root"}} - - # -- end test_signed_cookie_actor -- From 7a5adb592ae6674a2058639c66e85eb1b49448fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jan 2024 14:12:14 -0800 Subject: [PATCH 1900/2303] Docs on temporary plugins in fixtures, closes #2234 --- docs/testing_plugins.rst | 16 ++++++++++++++++ tests/test_docs_plugins.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/test_docs_plugins.py diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 33ac4b22..f1363fb4 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -313,3 +313,19 @@ When writing tests for plugins you may find it useful to register a test plugin assert response.status_code == 500 finally: pm.unregister(name="undo") + +To reuse the same temporary plugin in multiple tests, you can register it inside a fixture in your ``conftest.py`` file like this: + +.. literalinclude:: ../tests/test_docs_plugins.py + :language: python + :start-after: # -- start datasette_with_plugin_fixture -- + :end-before: # -- end datasette_with_plugin_fixture -- + +Note the ``yield`` statement here - this ensures that the ``finally:`` block that unregisters the plugin is executed only after the test function itself has completed. + +Then in a test: + +.. literalinclude:: ../tests/test_docs_plugins.py + :language: python + :start-after: # -- start datasette_with_plugin_test -- + :end-before: # -- end datasette_with_plugin_test -- diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py new file mode 100644 index 00000000..92b4514c --- /dev/null +++ b/tests/test_docs_plugins.py @@ -0,0 +1,34 @@ +# fmt: off +# -- start datasette_with_plugin_fixture -- +from datasette import hookimpl +from datasette.app import Datasette +from datasette.plugins import pm +import pytest +import pytest_asyncio + + +@pytest_asyncio.fixture +async def datasette_with_plugin(): + class TestPlugin: + __name__ = "TestPlugin" + + @hookimpl + def register_routes(self): + return [ + (r"^/error$", lambda: 1 / 0), + ] + + pm.register(TestPlugin(), name="undo") + try: + yield Datasette() + finally: + pm.unregister(name="undo") +# -- end datasette_with_plugin_fixture -- + + +# -- start datasette_with_plugin_test -- +@pytest.mark.asyncio +async def test_error(datasette_with_plugin): + response = await datasette_with_plugin.client.get("/error") + assert response.status_code == 500 +# -- end datasette_with_plugin_test -- From c3caf36af7db79336a5c8e697b2374e90e34ff5d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 19:54:03 -0800 Subject: [PATCH 1901/2303] Template slot family of plugin hooks - top_homepage() and others New plugin hooks: top_homepage top_database top_table top_row top_query top_canned_query New datasette.utils.make_slot_function() Closes #1191 --- datasette/hookspecs.py | 30 ++++++++ datasette/templates/database.html | 2 + datasette/templates/index.html | 2 + datasette/templates/query.html | 2 + datasette/templates/row.html | 2 + datasette/templates/table.html | 2 + datasette/utils/__init__.py | 17 +++++ datasette/views/database.py | 21 +++++- datasette/views/index.py | 9 ++- datasette/views/row.py | 12 ++- datasette/views/table.py | 8 ++ docs/plugin_hooks.rst | 119 ++++++++++++++++++++++++++++++ tests/test_docs.py | 4 +- tests/test_plugins.py | 101 +++++++++++++++++++++++++ 14 files changed, 324 insertions(+), 7 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index b6975dce..2f4c6027 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -158,3 +158,33 @@ def skip_csrf(datasette, scope): @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" + + +@hookspec +def top_homepage(datasette, request): + """HTML to include at the top of the homepage""" + + +@hookspec +def top_database(datasette, request, database): + """HTML to include at the top of the database page""" + + +@hookspec +def top_table(datasette, request, database, table): + """HTML to include at the top of the table page""" + + +@hookspec +def top_row(datasette, request, database, table, row): + """HTML to include at the top of the row page""" + + +@hookspec +def top_query(datasette, request, database, sql): + """HTML to include at the top of the query results page""" + + +@hookspec +def top_canned_query(datasette, request, database, query_name): + """HTML to include at the top of the canned query page""" diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 3d4dae07..4b125a44 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -34,6 +34,8 @@

    {% endif %}
    +{{ top_database() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index 06e09635..203abca8 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -7,6 +7,8 @@ {% block content %}

    {{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}

    +{{ top_homepage() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% for database in databases %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b8f06f84..1815e592 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -30,6 +30,8 @@

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    +{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 4d179a85..6d4b996e 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -22,6 +22,8 @@ {% block content %}

    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

    +{{ top_row() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}

    diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 88580e52..5aee6319 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -45,6 +45,8 @@
    {% endif %}
    +{{ top_table() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if metadata.get("columns") %} diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 0f449b89..8914c043 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1283,3 +1283,20 @@ def fail_if_plugins_in_metadata(metadata: dict, filename=None): f'Datasette no longer accepts plugin configuration in --metadata. Move your "plugins" configuration blocks to a separate file - we suggest calling that datasette.{suggested_extension} - and start Datasette with datasette -c datasette.{suggested_extension}. See https://docs.datasette.io/en/latest/configuration.html for more details.' ) return metadata + + +def make_slot_function(name, datasette, request, **kwargs): + from datasette.plugins import pm + + method = getattr(pm.hook, name, None) + assert method is not None, "No hook found for {}".format(name) + + async def inner(): + html_bits = [] + for hook in method(datasette=datasette, request=request, **kwargs): + html = await await_me_maybe(hook) + if html is not None: + html_bits.append(html) + return markupsafe.Markup("".join(html_bits)) + + return inner diff --git a/datasette/views/database.py b/datasette/views/database.py index 03e70379..caeb4e46 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,5 +1,4 @@ from dataclasses import dataclass, field -from typing import Callable from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -18,6 +17,7 @@ from datasette.utils import ( call_with_supported_arguments, derive_named_parameters, format_bytes, + make_slot_function, tilde_decode, to_css_class, validate_sql_select, @@ -161,6 +161,9 @@ class DatabaseView(View): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + "top_database": make_slot_function( + "top_database", datasette, request, database=database + ), } return Response.html( await datasette.render_template( @@ -246,6 +249,12 @@ class QueryContext: "help": "List of templates that were considered for rendering this page" } ) + top_query: callable = field( + metadata={"help": "Callable to render the top_query slot"} + ) + top_canned_query: callable = field( + metadata={"help": "Callable to render the top_canned_query slot"} + ) async def get_tables(datasette, request, db): @@ -727,6 +736,16 @@ class QueryView(View): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + top_query=make_slot_function( + "top_query", datasette, request, database=database, sql=sql + ), + top_canned_query=make_slot_function( + "top_canned_query", + datasette, + request, + database=database, + query_name=canned_query["name"] if canned_query else None, + ), ), request=request, view_name="database", diff --git a/datasette/views/index.py b/datasette/views/index.py index 95b29302..595cf234 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,10 +1,12 @@ -import hashlib import json -from datasette.utils import add_cors_headers, CustomJSONEncoder +from datasette.plugins import pm +from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ +from markupsafe import Markup + from .base import BaseView @@ -142,5 +144,8 @@ class IndexView(BaseView): "private": not await self.ds.permission_allowed( None, "view-instance" ), + "top_homepage": make_slot_function( + "top_homepage", self.ds, request + ), }, ) diff --git a/datasette/views/row.py b/datasette/views/row.py index 8f07a662..ce877753 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -2,11 +2,9 @@ from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from .base import DataView, BaseView, _error from datasette.utils import ( - tilde_decode, - urlsafe_components, + make_slot_function, to_css_class, escape_sqlite, - row_sql_params_pks, ) import json import sqlite_utils @@ -73,6 +71,14 @@ class RowView(DataView): .get(database, {}) .get("tables", {}) .get(table, {}), + "top_row": make_slot_function( + "top_row", + self.ds, + request, + database=resolved.db.name, + table=resolved.table, + row=rows[0], + ), } data = { diff --git a/datasette/views/table.py b/datasette/views/table.py index 7ee5d6bf..be7479f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -17,6 +17,7 @@ from datasette.utils import ( append_querystring, compound_keys_after_sql, format_bytes, + make_slot_function, tilde_encode, escape_sqlite, filters_should_redirect, @@ -842,6 +843,13 @@ async def table_view_traced(datasette, request): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), ), request=request, view_name="table", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9115c3df..ce648ba7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1641,3 +1641,122 @@ This hook is responsible for returning a dictionary corresponding to Datasette : return metadata Example: `datasette-remote-metadata plugin `__ + +.. _plugin_hook_slots: + +Template slots +-------------- + +The following set of plugin hooks can be used to return extra HTML content that will be inserted into the corresponding page, directly below the ``

    `` heading. + +Multiple plugins can contribute content here. The order in which it is displayed can be controlled using Pluggy's `call time order options `__. + +Each of these plugin hooks can return either a string or an awaitable function that returns a string. + +.. _plugin_hook_top_homepage: + +top_homepage(datasette, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +Returns HTML to be displayed at the top of the Datasette homepage. + +.. _plugin_hook_top_database: + +top_database(datasette, request, database) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +Returns HTML to be displayed at the top of the database page. + +.. _plugin_hook_top_table: + +top_table(datasette, request, database, table) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +Returns HTML to be displayed at the top of the table page. + +.. _plugin_hook_top_row: + +top_row(datasette, request, database, table, row) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``row`` - ``sqlite.Row`` + The SQLite row object being displayed. + +Returns HTML to be displayed at the top of the row page. + +.. _plugin_hook_top_query: + +top_query(datasette, request, database, sql) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``sql`` - string + The SQL query. + +Returns HTML to be displayed at the top of the query results page. + +.. _plugin_hook_top_canned_query: + +top_canned_query(datasette, request, database, query_name) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``query_name`` - string + The name of the canned query. + +Returns HTML to be displayed at the top of the canned query page. diff --git a/tests/test_docs.py b/tests/test_docs.py index 17c01a0b..0a803861 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -41,7 +41,9 @@ def plugin_hooks_content(): "plugin", [name for name in dir(app.pm.hook) if not name.startswith("_")] ) def test_plugin_hooks_are_documented(plugin, plugin_hooks_content): - headings = get_headings(plugin_hooks_content, "-") + headings = set() + headings.update(get_headings(plugin_hooks_content, "-")) + headings.update(get_headings(plugin_hooks_content, "~")) assert plugin in headings hook_caller = getattr(app.pm.hook, plugin) arg_names = [a for a in hook_caller.spec.argnames if a != "__multicall__"] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index bdd4ba49..784c460a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1334,3 +1334,104 @@ async def test_hook_jinja2_environment_from_request(tmpdir): assert "Hello museums!" in response2.text finally: pm.unregister(name="EnvironmentPlugin") + + +class SlotPlugin: + __name__ = "SlotPlugin" + + @hookimpl + def top_homepage(self, request): + return "Xtop_homepage:" + request.args["z"] + + @hookimpl + def top_database(self, request, database): + async def inner(): + return "Xtop_database:{}:{}".format(database, request.args["z"]) + + return inner + + @hookimpl + def top_table(self, request, database, table): + return "Xtop_table:{}:{}:{}".format(database, table, request.args["z"]) + + @hookimpl + def top_row(self, request, database, table, row): + return "Xtop_row:{}:{}:{}:{}".format( + database, table, row["name"], request.args["z"] + ) + + @hookimpl + def top_query(self, request, database, sql): + return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) + + @hookimpl + def top_canned_query(self, request, database, query_name): + return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + + +@pytest.mark.asyncio +async def test_hook_top_homepage(): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + datasette = Datasette(memory=True) + response = await datasette.client.get("/?z=foo") + assert response.status_code == 200 + assert "Xtop_homepage:foo" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_database(): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + datasette = Datasette(memory=True) + response = await datasette.client.get("/_memory?z=bar") + assert response.status_code == 200 + assert "Xtop_database:_memory:bar" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_table(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/facetable?z=baz") + assert response.status_code == 200 + assert "Xtop_table:fixtures:facetable:baz" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_row(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/facet_cities/1?z=bax") + assert response.status_code == 200 + assert "Xtop_row:fixtures:facet_cities:San Francisco:bax" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_query(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures?sql=select+1&z=x") + assert response.status_code == 200 + assert "Xtop_query:fixtures:select 1:x" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_canned_query(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/from_hook?z=xyz") + assert response.status_code == 200 + assert "Xtop_query:fixtures:from_hook:xyz" in response.text + finally: + pm.unregister(name="SlotPlugin") From 5c64af69363100a3c35e6b131efe1f741bbde661 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 19:55:26 -0800 Subject: [PATCH 1902/2303] Upgrade to latest Black, closes #2239 --- datasette/filters.py | 14 ++++++------ datasette/utils/__init__.py | 14 ++++++------ datasette/utils/shutil_backport.py | 1 + datasette/views/database.py | 22 +++++++++++-------- datasette/views/table.py | 30 ++++++++++++++------------ setup.py | 2 +- tests/plugins/my_plugin.py | 6 +++--- tests/test_api_write.py | 8 ++++--- tests/test_cli.py | 8 ++++--- tests/test_docs.py | 1 + tests/test_internals_database.py | 1 + tests/test_internals_datasette.py | 1 + tests/test_permissions.py | 8 ++++--- tests/test_plugins.py | 34 ++++++++++++++++-------------- tests/test_table_api.py | 8 ++++--- tests/test_utils.py | 1 + 16 files changed, 93 insertions(+), 66 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index 5ea3488b..73eea857 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -80,9 +80,9 @@ def search_filters(request, database, table, datasette): "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( fts_table=escape_sqlite(fts_table), fts_pk=escape_sqlite(fts_pk), - match_clause=":search" - if search_mode_raw - else "escape_fts(:search)", + match_clause=( + ":search" if search_mode_raw else "escape_fts(:search)" + ), ) ) human_descriptions.append(f'search matches "{search}"') @@ -99,9 +99,11 @@ def search_filters(request, database, table, datasette): "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), - match_clause=":search_{}".format(i) - if search_mode_raw - else "escape_fts(:search_{})".format(i), + match_clause=( + ":search_{}".format(i) + if search_mode_raw + else "escape_fts(:search_{})".format(i) + ), ) ) human_descriptions.append( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 8914c043..196e1682 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -402,9 +402,9 @@ def make_dockerfile( apt_get_extras = apt_get_extras_ if spatialite: apt_get_extras.extend(["python3-dev", "gcc", "libsqlite3-mod-spatialite"]) - environment_variables[ - "SQLITE_EXTENSIONS" - ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + environment_variables["SQLITE_EXTENSIONS"] = ( + "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + ) return """ FROM python:3.11.0-slim-bullseye COPY . /app @@ -416,9 +416,11 @@ RUN datasette inspect {files} --inspect-file inspect-data.json ENV PORT {port} EXPOSE {port} CMD {cmd}""".format( - apt_get_extras=APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) - if apt_get_extras - else "", + apt_get_extras=( + APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) + if apt_get_extras + else "" + ), environment_variables="\n".join( [ "ENV {} '{}'".format(key, value) diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index dbe22404..d1fd1bd7 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -4,6 +4,7 @@ Backported from Python 3.8. This code is licensed under the Python License: https://github.com/python/cpython/blob/v3.8.3/LICENSE """ + import os from shutil import copy, copy2, copystat, Error diff --git a/datasette/views/database.py b/datasette/views/database.py index caeb4e46..eac01ab6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -126,9 +126,9 @@ class DatabaseView(View): "views": sql_views, "queries": canned_queries, "allow_execute_sql": allow_execute_sql, - "table_columns": await _table_columns(datasette, database) - if allow_execute_sql - else {}, + "table_columns": ( + await _table_columns(datasette, database) if allow_execute_sql else {} + ), } if format_ == "json": @@ -719,9 +719,11 @@ class QueryView(View): display_rows=await display_rows( datasette, database, request, rows, columns ), - table_columns=await _table_columns(datasette, database) - if allow_execute_sql - else {}, + table_columns=( + await _table_columns(datasette, database) + if allow_execute_sql + else {} + ), columns=columns, renderers=renderers, url_csv=datasette.urls.path( @@ -1036,9 +1038,11 @@ async def display_rows(datasette, database, request, rows, columns): display_value = markupsafe.Markup( '<Binary: {:,} byte{}>'.format( blob_url, - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", + ( + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "" + ), len(value), "" if len(value) == 1 else "s", ) diff --git a/datasette/views/table.py b/datasette/views/table.py index be7479f8..2c5e3e13 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -236,9 +236,11 @@ async def display_columns_and_rows( path_from_row_pks(row, pks, not pks), column, ), - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", + ( + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "" + ), len(value), "" if len(value) == 1 else "s", ) @@ -289,9 +291,9 @@ async def display_columns_and_rows( "column": column, "value": display_value, "raw": value, - "value_type": "none" - if value is None - else str(type(value).__name__), + "value_type": ( + "none" if value is None else str(type(value).__name__) + ), } ) cell_rows.append(Row(cells)) @@ -974,9 +976,9 @@ async def table_view_data( from_sql = "from {table_name} {where}".format( table_name=escape_sqlite(table_name), - where=("where {} ".format(" and ".join(where_clauses))) - if where_clauses - else "", + where=( + ("where {} ".format(" and ".join(where_clauses))) if where_clauses else "" + ), ) # Copy of params so we can mutate them later: from_sql_params = dict(**params) @@ -1040,10 +1042,12 @@ async def table_view_data( column=escape_sqlite(sort or sort_desc), op=">" if sort else "<", p=len(params), - extra_desc_only="" - if sort - else " or {column2} is null".format( - column2=escape_sqlite(sort or sort_desc) + extra_desc_only=( + "" + if sort + else " or {column2} is null".format( + column2=escape_sqlite(sort or sort_desc) + ) ), next_clauses=" and ".join(next_by_pk_clauses), ) diff --git a/setup.py b/setup.py index d09a9e3d..cd393368 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.9.1", + "black==24.1.1", "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index eb70d9bd..9d1f86bc 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -39,9 +39,9 @@ def extra_css_urls(template, database, table, view_name, columns, request, datas "database": database, "table": table, "view_name": view_name, - "request_path": request.path - if request is not None - else None, + "request_path": ( + request.path if request is not None else None + ), "added": ( await datasette.get_database().execute("select 3 * 5") ).first()[0], diff --git a/tests/test_api_write.py b/tests/test_api_write.py index f27d143f..1787e06f 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -279,9 +279,11 @@ async def test_insert_or_upsert_row_errors( json=input, headers={ "Authorization": "Bearer {}".format(token), - "Content-Type": "text/plain" - if special_case == "invalid_content_type" - else "application/json", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), }, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 213db416..080e8353 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -335,9 +335,11 @@ def test_serve_create(tmpdir): def test_serve_config(tmpdir, argument, format_): config_path = tmpdir / "datasette.{}".format(format_) config_path.write_text( - "settings:\n default_page_size: 5\n" - if format_ == "yaml" - else '{"settings": {"default_page_size": 5}}', + ( + "settings:\n default_page_size: 5\n" + if format_ == "yaml" + else '{"settings": {"default_page_size": 5}}' + ), "utf-8", ) runner = CliRunner() diff --git a/tests/test_docs.py b/tests/test_docs.py index 0a803861..2a58d954 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,6 +1,7 @@ """ Tests to ensure certain things are documented. """ + from datasette import app, utils from datasette.app import Datasette from datasette.filters import Filters diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e0511100..dd68a6cb 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,6 +1,7 @@ """ Tests for the datasette.database.Database class """ + from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.utils.sqlite import sqlite3 diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 428b259d..c30bb748 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,6 +1,7 @@ """ Tests for the datasette.app.Datasette class """ + import dataclasses from datasette import Forbidden, Context from datasette.app import Datasette, Database diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 933aa07b..9917b749 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -381,9 +381,11 @@ async def test_permissions_debug(ds_client): { "action": div.select_one(".check-action").text, # True = green tick, False = red cross, None = gray None - "result": None - if div.select(".check-result-no-opinion") - else bool(div.select(".check-result-true")), + "result": ( + None + if div.select(".check-result-no-opinion") + else bool(div.select(".check-result-true")) + ), "used_default": bool(div.select(".check-used-default")), } for div in check_divs diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 784c460a..5bfb6132 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1096,24 +1096,26 @@ async def test_hook_filters_from_request(ds_client): @pytest.mark.parametrize("extra_metadata", (False, True)) async def test_hook_register_permissions(extra_metadata): ds = Datasette( - config={ - "plugins": { - "datasette-register-permissions": { - "permissions": [ - { - "name": "extra-from-metadata", - "abbr": "efm", - "description": "Extra from metadata", - "takes_database": False, - "takes_resource": False, - "default": True, - } - ] + config=( + { + "plugins": { + "datasette-register-permissions": { + "permissions": [ + { + "name": "extra-from-metadata", + "abbr": "efm", + "description": "Extra from metadata", + "takes_database": False, + "takes_resource": False, + "default": True, + } + ] + } } } - } - if extra_metadata - else None, + if extra_metadata + else None + ), plugins_dir=PLUGINS_DIR, ) await ds.invoke_startup() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index bde7a38e..58930950 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -305,9 +305,11 @@ async def test_paginate_compound_keys_with_extra_filters(ds_client): "_sort_desc=sortable_with_nulls", lambda row: ( 1 if row["sortable_with_nulls"] is None else 0, - -row["sortable_with_nulls"] - if row["sortable_with_nulls"] is not None - else 0, + ( + -row["sortable_with_nulls"] + if row["sortable_with_nulls"] is not None + else 0 + ), row["content"], ), "sorted by sortable_with_nulls descending", diff --git a/tests/test_utils.py b/tests/test_utils.py index 61392b8b..51577615 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ """ Tests for various datasette helper functions. """ + from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request From b8230694ff90f9a6cd4f5b7c47fd8a71c831ee1d Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Tue, 30 Jan 2024 22:56:05 -0500 Subject: [PATCH 1903/2303] Set link to download db to nofollow --- datasette/templates/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 4b125a44..ee4dd705 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -97,7 +97,7 @@ {% endif %} {% if allow_download %} -

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    +

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    {% endif %} {% include "_codemirror_foot.html" %} From 04e8835297760416b50cc669ac6f45a7fd68170b Mon Sep 17 00:00:00 2001 From: gerrymanoim Date: Tue, 30 Jan 2024 22:56:32 -0500 Subject: [PATCH 1904/2303] Remove deprecated/unused args from setup.py (#2222) --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index cd393368..53206eae 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,6 @@ setup( [console_scripts] datasette=datasette.cli:cli """, - setup_requires=["pytest-runner"], extras_require={ "docs": [ "Sphinx==7.2.6", @@ -93,7 +92,6 @@ setup( ], "rich": ["rich"], }, - tests_require=["datasette[test]"], classifiers=[ "Development Status :: 4 - Beta", "Framework :: Datasette", From 959e0202972f4d95088c4c1a9df6274108af8bfb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 20:40:18 -0800 Subject: [PATCH 1905/2303] Ran blacken-docs --- docs/internals.rst | 3 +-- docs/plugin_hooks.rst | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index d269bc7d..d8f86251 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -210,8 +210,7 @@ To set cookies on the response, use the ``response.set_cookie(...)`` method. The secure=False, httponly=False, samesite="lax", - ): - ... + ): ... You can use this with :ref:`datasette.sign() ` to set signed cookies. Here's how you would set the :ref:`ds_actor cookie ` for use with Datasette :ref:`authentication `: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index ce648ba7..da69c6c9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -373,8 +373,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ about, about_url, api_key, - ): - ... + ): ... Examples: `datasette-publish-fly `_, `datasette-publish-vercel `_ From 890615b3f29dcf82a792f1a145b02dba784a5b63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:53:57 -0800 Subject: [PATCH 1906/2303] Bump the python-packages group with 1 update (#2241) Bumps the python-packages group with 1 update: [furo](https://github.com/pradyunsg/furo). Updates `furo` from 2023.9.10 to 2024.1.29 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.09.10...2024.01.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-major dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53206eae..b3915c42 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( extras_require={ "docs": [ "Sphinx==7.2.6", - "furo==2023.9.10", + "furo==2024.1.29", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", From bcc4f6bf1f14be6ef693f0b3fc9aa8a027977920 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 15:21:40 -0800 Subject: [PATCH 1907/2303] track_event() mechanism for analytics and plugins * Closes #2240 * Documentation for event plugin hooks, refs #2240 * Include example track_event plugin in docs, refs #2240 * Tests for track_event() and register_events() hooks, refs #2240 * Initial documentation for core events, refs #2240 * Internals documentation for datasette.track_event() --- datasette/__init__.py | 1 + datasette/app.py | 16 +++ datasette/events.py | 211 ++++++++++++++++++++++++++++++++++++ datasette/hookspecs.py | 10 ++ datasette/plugins.py | 1 + datasette/views/database.py | 6 + datasette/views/row.py | 20 ++++ datasette/views/special.py | 17 ++- datasette/views/table.py | 31 ++++++ docs/conf.py | 2 + docs/events.rst | 14 +++ docs/index.rst | 1 + docs/internals.rst | 20 ++++ docs/plugin_hooks.rst | 100 +++++++++++++++++ docs/plugins.rst | 9 ++ tests/conftest.py | 33 +++++- tests/test_api.py | 7 +- tests/test_api_write.py | 64 +++++++++++ tests/test_auth.py | 19 +++- tests/test_cli.py | 6 +- tests/test_plugins.py | 31 +++++- tests/utils.py | 5 + 22 files changed, 614 insertions(+), 10 deletions(-) create mode 100644 datasette/events.py create mode 100644 docs/events.rst diff --git a/datasette/__init__.py b/datasette/__init__.py index 271e09ad..47d2b4f6 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,5 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa +from datasette.events import Event # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa diff --git a/datasette/app.py b/datasette/app.py index 482cebb4..530f79bc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -34,6 +34,7 @@ from jinja2 import ( from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +from .events import Event from .views import Context from .views.base import ureg from .views.database import database_download, DatabaseView, TableCreateView @@ -505,6 +506,14 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) + # Register permissions, but watch out for duplicate name/abbr names = {} abbrs = {} @@ -873,6 +882,13 @@ class Datasette: result = await await_me_maybe(result) return result + async def track_event(self, event: Event): + assert isinstance(event, self.event_classes), "Invalid event type: {}".format( + type(event) + ) + for hook in pm.hook.track_event(datasette=self, event=event): + await await_me_maybe(hook) + async def permission_allowed( self, actor, action, resource=None, default=DEFAULT_NOT_SET ): diff --git a/datasette/events.py b/datasette/events.py new file mode 100644 index 00000000..96244779 --- /dev/null +++ b/datasette/events.py @@ -0,0 +1,211 @@ +from abc import ABC, abstractproperty +from dataclasses import asdict, dataclass, field +from datasette.hookspecs import hookimpl +from datetime import datetime, timezone +from typing import Optional + + +@dataclass +class Event(ABC): + @abstractproperty + def name(self): + pass + + created: datetime = field( + init=False, default_factory=lambda: datetime.now(timezone.utc) + ) + actor: Optional[dict] + + def properties(self): + properties = asdict(self) + properties.pop("actor", None) + properties.pop("created", None) + return properties + + +@dataclass +class LoginEvent(Event): + """ + Event name: ``login`` + + A user (represented by ``event.actor``) has logged in. + """ + + name = "login" + + +@dataclass +class LogoutEvent(Event): + """ + Event name: ``logout`` + + A user (represented by ``event.actor``) has logged out. + """ + + name = "logout" + + +@dataclass +class CreateTokenEvent(Event): + """ + Event name: ``create-token`` + + A user created an API token. + + :ivar expires_after: Number of seconds after which this token will expire. + :type expires_after: int or None + :ivar restrict_all: Restricted permissions for this token. + :type restrict_all: list + :ivar restrict_database: Restricted database permissions for this token. + :type restrict_database: dict + :ivar restrict_resource: Restricted resource permissions for this token. + :type restrict_resource: dict + """ + + name = "create-token" + expires_after: Optional[int] + restrict_all: list + restrict_database: dict + restrict_resource: dict + + +@dataclass +class CreateTableEvent(Event): + """ + Event name: ``create-table`` + + A new table has been created in the database. + + :ivar database: The name of the database where the table was created. + :type database: str + :ivar table: The name of the table that was created + :type table: str + :ivar schema: The SQL schema definition for the new table. + :type schema: str + """ + + name = "create-table" + database: str + table: str + schema: str + + +@dataclass +class DropTableEvent(Event): + """ + Event name: ``drop-table`` + + A table has been dropped from the database. + + :ivar database: The name of the database where the table was dropped. + :type database: str + :ivar table: The name of the table that was dropped + :type table: str + """ + + name = "drop-table" + database: str + table: str + + +@dataclass +class InsertRowsEvent(Event): + """ + Event name: ``insert-rows`` + + Rows were inserted into a table. + + :ivar database: The name of the database where the rows were inserted. + :type database: str + :ivar table: The name of the table where the rows were inserted. + :type table: str + :ivar num_rows: The number of rows that were requested to be inserted. + :type num_rows: int + :ivar ignore: Was ignore set? + :type ignore: bool + :ivar replace: Was replace set? + :type replace: bool + """ + + name = "insert-rows" + database: str + table: str + num_rows: int + ignore: bool + replace: bool + + +@dataclass +class UpsertRowsEvent(Event): + """ + Event name: ``upsert-rows`` + + Rows were upserted into a table. + + :ivar database: The name of the database where the rows were inserted. + :type database: str + :ivar table: The name of the table where the rows were inserted. + :type table: str + :ivar num_rows: The number of rows that were requested to be inserted. + :type num_rows: int + """ + + name = "upsert-rows" + database: str + table: str + num_rows: int + + +@dataclass +class UpdateRowEvent(Event): + """ + Event name: ``update-row`` + + A row was updated in a table. + + :ivar database: The name of the database where the row was updated. + :type database: str + :ivar table: The name of the table where the row was updated. + :type table: str + :ivar pks: The primary key values of the updated row. + """ + + name = "update-row" + database: str + table: str + pks: list + + +@dataclass +class DeleteRowEvent(Event): + """ + Event name: ``delete-row`` + + A row was deleted from a table. + + :ivar database: The name of the database where the row was deleted. + :type database: str + :ivar table: The name of the table where the row was deleted. + :type table: str + :ivar pks: The primary key values of the deleted row. + """ + + name = "delete-row" + database: str + table: str + pks: list + + +@hookimpl +def register_events(): + return [ + LoginEvent, + LogoutEvent, + CreateTableEvent, + CreateTokenEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, + UpdateRowEvent, + DeleteRowEvent, + ] diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 2f4c6027..b473f398 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -160,6 +160,16 @@ def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" +@hookspec +def track_event(datasette, event): + """Respond to an event tracked by Datasette""" + + +@hookspec +def register_events(datasette): + """Return a list of Event subclasses to use with track_event()""" + + @hookspec def top_homepage(datasette, request): """HTML to include at the top of the homepage""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 1ed3747f..f7a1905f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -27,6 +27,7 @@ DEFAULT_PLUGINS = ( "datasette.default_menu_links", "datasette.handle_exception", "datasette.forbidden", + "datasette.events", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/views/database.py b/datasette/views/database.py index eac01ab6..6d17b16c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,6 +10,7 @@ import re import sqlite_utils import textwrap +from datasette.events import CreateTableEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -969,6 +970,11 @@ class TableCreateView(BaseView): } if rows: details["row_count"] = len(rows) + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) + ) return Response.json(details, status=201) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce877753..7b646641 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,5 +1,6 @@ from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted +from datasette.events import UpdateRowEvent, DeleteRowEvent from .base import DataView, BaseView, _error from datasette.utils import ( make_slot_function, @@ -200,6 +201,15 @@ class RowDeleteView(BaseView): except Exception as e: return _error([str(e)], 500) + await self.ds.track_event( + DeleteRowEvent( + actor=request.actor, + database=resolved.db.name, + table=resolved.table, + pks=resolved.pk_values, + ) + ) + return Response.json({"ok": True}, status=200) @@ -246,4 +256,14 @@ class RowUpdateView(BaseView): ) rows = list(results.rows) result["row"] = dict(rows[0]) + + await self.ds.track_event( + UpdateRowEvent( + actor=request.actor, + database=resolved.db.name, + table=resolved.table, + pks=resolved.pk_values, + ) + ) + return Response.json(result, status=200) diff --git a/datasette/views/special.py b/datasette/views/special.py index 849750bf..4088a1f9 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,4 +1,5 @@ import json +from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( actor_matches_allow, @@ -80,9 +81,9 @@ class AuthTokenView(BaseView): if secrets.compare_digest(token, self.ds._root_token): self.ds._root_token = None response = Response.redirect(self.ds.urls.instance()) - response.set_cookie( - "ds_actor", self.ds.sign({"a": {"id": "root"}}, "actor") - ) + root_actor = {"id": "root"} + response.set_cookie("ds_actor", self.ds.sign({"a": root_actor}, "actor")) + await self.ds.track_event(LoginEvent(actor=root_actor)) return response else: raise Forbidden("Invalid token") @@ -105,6 +106,7 @@ class LogoutView(BaseView): response = Response.redirect(self.ds.urls.instance()) response.set_cookie("ds_actor", "", expires=0, max_age=0) self.ds.add_message(request, "You are now logged out", self.ds.WARNING) + await self.ds.track_event(LogoutEvent(actor=request.actor)) return response @@ -349,6 +351,15 @@ class CreateTokenView(BaseView): restrict_resource=restrict_resource, ) token_bits = self.ds.unsign(token[len("dstok_") :], namespace="token") + await self.ds.track_event( + CreateTokenEvent( + actor=request.actor, + expires_after=expires_after, + restrict_all=restrict_all, + restrict_database=restrict_database, + restrict_resource=restrict_resource, + ) + ) context = await self.shared(request) context.update({"errors": errors, "token": token, "token_bits": token_bits}) return await self.render(["create_token.html"], request, context) diff --git a/datasette/views/table.py b/datasette/views/table.py index 2c5e3e13..3b812c01 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,6 +8,7 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted +from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -467,6 +468,8 @@ class TableInsertView(BaseView): if errors: return _error(errors, 400) + num_rows = len(rows) + # No that we've passed pks to _validate_data it's safe to # fix the rowids case: if not pks: @@ -527,6 +530,29 @@ class TableInsertView(BaseView): result["rows"] = [dict(r) for r in fetched_rows.rows] else: result["rows"] = rows + # We track the number of rows requested, but do not attempt to show which were actually + # inserted or upserted v.s. ignored + if upsert: + await self.ds.track_event( + UpsertRowsEvent( + actor=request.actor, + database=database_name, + table=table_name, + num_rows=num_rows, + ) + ) + else: + await self.ds.track_event( + InsertRowsEvent( + actor=request.actor, + database=database_name, + table=table_name, + num_rows=num_rows, + ignore=bool(ignore), + replace=bool(replace), + ) + ) + return Response.json(result, status=200 if upsert else 201) @@ -587,6 +613,11 @@ class TableDropView(BaseView): sqlite_utils.Database(conn)[table_name].drop() await db.execute_write_fn(drop_table) + await self.ds.track_event( + DropTableEvent( + actor=request.actor, database=database_name, table=table_name + ) + ) return Response.json({"ok": True}, status=200) diff --git a/docs/conf.py b/docs/conf.py index ca0eb986..e13882b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,8 @@ extensions = [ if not os.environ.get("DISABLE_SPHINX_INLINE_TABS"): extensions += ["sphinx_inline_tabs"] +autodoc_member_order = "bysource" + extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), } diff --git a/docs/events.rst b/docs/events.rst new file mode 100644 index 00000000..f150ac02 --- /dev/null +++ b/docs/events.rst @@ -0,0 +1,14 @@ +.. _events: + +Events +====== + +Datasette includes a mechanism for tracking events that occur while the software is running. This is primarily intended to be used by plugins, which can both trigger events and listen for events. + +The core Datasette application triggers events when certain things happen. This page describes those events. + +Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes (or additional classes registered by other plugins): + +.. automodule:: datasette.events + :members: + :exclude-members: Event diff --git a/docs/index.rst b/docs/index.rst index 66bbd5a4..ce1ed2eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,5 +63,6 @@ Contents plugin_hooks testing_plugins internals + events contributing changelog diff --git a/docs/internals.rst b/docs/internals.rst index d8f86251..bd7a70b5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -593,6 +593,26 @@ Using either of these pattern will result in the in-memory database being served This removes a database that has been previously added. ``name=`` is the unique name of that database. +.. _datasette_track_event: + +await .track_event(event) +------------------------- + +``event`` - ``Event`` + An instance of a subclass of ``datasette.events.Event``. + +Plugins can call this to track events, using classes they have previously registered. See :ref:`plugin_event_tracking` for details. + +The event will then be passed to all plugins that have registered to receive events using the :ref:`plugin_hook_track_event` hook. + +Example usage, assuming the plugin has previously registered the ``BanUserEvent`` class: + +.. code-block:: python + + await datasette.track_event( + BanUserEvent(user={"id": 1, "username": "cleverbot"}) + ) + .. _datasette_sign: .sign(value, namespace="default") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index da69c6c9..1a88cd31 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1759,3 +1759,103 @@ top_canned_query(datasette, request, database, query_name) The name of the canned query. Returns HTML to be displayed at the top of the canned query page. + +.. _plugin_event_tracking: + +Event tracking +-------------- + +Datasette includes an internal mechanism for tracking analytical events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. + +Plugins can register to receive events using the ``track_event`` plugin hook. + +They can also define their own events for other plugins to receive using the ``register_events`` plugin hook, combined with calls to the ``datasette.track_event(...)`` internal method. + +.. _plugin_hook_track_event: + +track_event(datasette, event) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``event`` - ``Event`` + Information about the event, represented as an instance of a subclass of the ``Event`` base class. + +This hook will be called any time an event is tracked by code that calls the :ref:`datasette.track_event(...) ` internal method. + +The ``event`` object will always have the following properties: + +- ``name``: a string representing the name of the event, for example ``logout`` or ``create-table``. +- ``actor``: a dictionary representing the actor that triggered the event, or ``None`` if the event was not triggered by an actor. +- ``created``: a ``datatime.datetime`` object in the ``timezone.utc`` timezone representing the time the event object was created. + +Other properties on the event will be available depending on the type of event. You can also access those as a dictionary using ``event.properties()``. + +The events fired by Datasette core are :ref:`documented here `. + +This example plugin logs details of all events to standard error: + +.. code-block:: python + + from datasette import hookimpl + import json + import sys + + + @hookimpl + def track_event(event): + name = event.name + actor = event.actor + properties = event.properties() + msg = json.dumps( + { + "name": name, + "actor": actor, + "properties": properties, + } + ) + print(msg, file=sys.stderr, flush=True) + + +.. _plugin_hook_register_events: + +register_events(datasette) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the ``datasette.track_event()`` method. + +This example registers event subclasses for ``ban-user`` and ``unban-user`` events: + +.. code-block:: python + + from dataclasses import dataclass + from datasette import hookimpl, Event + + + @dataclass + class BanUserEvent(Event): + name = "ban-user" + user: dict + + + @dataclass + class UnbanUserEvent(Event): + name = "unban-user" + user: dict + + + @hookimpl + def register_events(): + return [BanUserEvent, UnbanUserEvent] + +The plugin can then call ``datasette.track_event(...)`` to send a ``ban-user`` event: + +.. code-block:: python + + await datasette.track_event( + BanUserEvent(user={"id": 1, "username": "cleverbot"}) + ) diff --git a/docs/plugins.rst b/docs/plugins.rst index 2ec03701..1a72af95 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -228,6 +228,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "skip_csrf" ] }, + { + "name": "datasette.events", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "register_events" + ] + }, { "name": "datasette.facets", "static": false, diff --git a/tests/conftest.py b/tests/conftest.py index 31336aea..445de057 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import asyncio import httpx import os import pathlib @@ -8,7 +7,8 @@ import re import subprocess import tempfile import time -import trustme +from dataclasses import dataclass, field +from datasette import Event, hookimpl try: @@ -164,6 +164,35 @@ def check_permission_actions_are_documented(): ) +class TrackEventPlugin: + __name__ = "TrackEventPlugin" + + @dataclass + class OneEvent(Event): + name = "one" + + extra: str + + @hookimpl + def register_events(self, datasette): + async def inner(): + return [self.OneEvent] + + return inner + + @hookimpl + def track_event(self, datasette, event): + datasette._tracked_events = getattr(datasette, "_tracked_events", []) + datasette._tracked_events.append(event) + + +@pytest.fixture(scope="session", autouse=True) +def install_event_tracking_plugin(): + from datasette.plugins import pm + + pm.register(TrackEventPlugin(), name="TrackEventPlugin") + + @pytest.fixture(scope="session") def ds_localhost_http_server(): ds_proc = subprocess.Popen( diff --git a/tests/test_api.py b/tests/test_api.py index 93ca43eb..177dc95c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -786,7 +786,12 @@ async def test_threads_json(ds_client): @pytest.mark.asyncio async def test_plugins_json(ds_client): response = await ds_client.get("/-/plugins.json") - assert EXPECTED_PLUGINS == sorted(response.json(), key=lambda p: p["name"]) + # Filter out TrackEventPlugin + actual_plugins = sorted( + [p for p in response.json() if p["name"] != "TrackEventPlugin"], + key=lambda p: p["name"], + ) + assert EXPECTED_PLUGINS == actual_plugins # Try with ?all=1 response = await ds_client.get("/-/plugins.json?all=1") names = {p["name"] for p in response.json()} diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1787e06f..9caf9fdf 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.utils import sqlite3 +from .utils import last_event import pytest import time @@ -49,6 +50,14 @@ async def test_insert_row(ds_write): assert response.json()["rows"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "docs" + assert not event.ignore + assert not event.replace @pytest.mark.asyncio @@ -68,6 +77,16 @@ async def test_insert_rows(ds_write, return_rows): headers=_headers(token), ) assert response.status_code == 201 + + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 20 + assert event.database == "data" + assert event.table == "docs" + assert not event.ignore + assert not event.replace + actual_rows = [ dict(r) for r in ( @@ -353,6 +372,16 @@ async def test_insert_ignore_replace( headers=_headers(token), ) assert response.status_code == 201 + + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "docs" + assert event.ignore == ignore + assert event.replace == replace + actual_rows = [ dict(r) for r in ( @@ -427,6 +456,14 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): ) assert response.status_code == 200 assert response.json()["ok"] is True + + # Analytics event + event = last_event(ds_write) + assert event.name == "upsert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "upsert_test" + if should_return: # We only expect it to return rows corresponding to those we sent expected_returned_rows = expected_rows[: len(input["rows"])] @@ -530,6 +567,13 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): headers=_headers(write_token(ds_write)), ) assert delete_response.status_code == 200 + + # Analytics event + event = last_event(ds_write) + assert event.name == "delete-row" + assert event.database == "data" + assert event.table == table + assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) @@ -610,6 +654,13 @@ async def test_update_row(ds_write, input, expected_errors, use_return): for k, v in input.items(): assert returned_row[k] == v + # Analytics event + event = last_event(ds_write) + assert event.actor == {"id": "root", "token": "dstok"} + assert event.database == "data" + assert event.table == "docs" + assert event.pks == [str(pk)] + # And fetch the row to check it's updated response = await ds_write.client.get( "/data/docs/{}.json?_shape=array".format(pk), @@ -676,6 +727,13 @@ async def test_drop_table(ds_write, scenario): headers=_headers(token), ) assert response2.json() == {"ok": True} + # Check event + event = last_event(ds_write) + assert event.name == "drop-table" + assert event.actor == {"id": "root", "token": "dstok"} + assert event.table == "docs" + assert event.database == "data" + # Table should 404 assert (await ds_write.client.get("/data/docs")).status_code == 404 @@ -1096,6 +1154,12 @@ async def test_create_table(ds_write, input, expected_status, expected_response) assert response.status_code == expected_status data = response.json() assert data == expected_response + # create-table event + if expected_status == 201: + event = last_event(ds_write) + assert event.name == "create-table" + assert event.actor == {"id": "root", "token": "dstok"} + assert event.schema.startswith("CREATE TABLE ") @pytest.mark.asyncio diff --git a/tests/test_auth.py b/tests/test_auth.py index 33cf9b35..f2359df7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import app_client -from .utils import cookie_was_deleted +from .utils import cookie_was_deleted, last_event from click.testing import CliRunner from datasette.utils import baseconv from datasette.cli import cli @@ -19,6 +19,10 @@ async def test_auth_token(ds_client): assert {"a": {"id": "root"}} == ds_client.ds.unsign( response.cookies["ds_actor"], "actor" ) + # Should have recorded a login event + event = last_event(ds_client.ds) + assert event.name == "login" + assert event.actor == {"id": "root"} # Check that a second with same token fails assert ds_client.ds._root_token is None assert (await ds_client.get(path)).status_code == 403 @@ -57,7 +61,7 @@ async def test_actor_cookie_that_expires(ds_client, offset, expected): cookie = ds_client.ds.sign( {"a": {"id": "test"}, "e": baseconv.base62.encode(expires_at)}, "actor" ) - response = await ds_client.get("/", cookies={"ds_actor": cookie}) + await ds_client.get("/", cookies={"ds_actor": cookie}) assert ds_client.ds._last_request.scope["actor"] == expected @@ -86,6 +90,10 @@ def test_logout(app_client): csrftoken_from=True, cookies={"ds_actor": app_client.actor_cookie({"id": "test"})}, ) + # Should have recorded a logout event + event = last_event(app_client.ds) + assert event.name == "logout" + assert event.actor == {"id": "test"} # The ds_actor cookie should have been unset assert cookie_was_deleted(response4, "ds_actor") # Should also have set a message @@ -185,6 +193,13 @@ def test_auth_create_token( for error in errors: assert '

    {}

    '.format(error) in response2.text else: + # Check create-token event + event = last_event(app_client.ds) + assert event.name == "create-token" + assert event.expires_after == expected_duration + assert isinstance(event.restrict_all, list) + assert isinstance(event.restrict_database, dict) + assert isinstance(event.restrict_resource, dict) # Extract token from page token = response2.text.split('value="dstok_')[1].split('"')[0] details = app_client.ds.unsign(token, "token") diff --git a/tests/test_cli.py b/tests/test_cli.py index 080e8353..9cc18c6e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -100,7 +100,11 @@ def test_spatialite_error_if_cannot_find_load_extension_spatialite(): def test_plugins_cli(app_client): runner = CliRunner() result1 = runner.invoke(cli, ["plugins"]) - assert json.loads(result1.output) == EXPECTED_PLUGINS + actual_plugins = sorted( + [p for p in json.loads(result1.output) if p["name"] != "TrackEventPlugin"], + key=lambda p: p["name"], + ) + assert actual_plugins == EXPECTED_PLUGINS # Try with --all result2 = runner.invoke(cli, ["plugins", "--all"]) names = [p["name"] for p in json.loads(result2.output)] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 5bfb6132..dad4f2ca 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,8 +9,9 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from click.testing import CliRunner +from dataclasses import dataclass from datasette.app import Datasette -from datasette import cli, hookimpl, Permission +from datasette import cli, hookimpl, Event, Permission from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 @@ -18,6 +19,7 @@ from datasette.utils import CustomRow, StartupError from jinja2.environment import Template from jinja2 import ChoiceLoader, FileSystemLoader import base64 +import datetime import importlib import json import os @@ -1437,3 +1439,30 @@ async def test_hook_top_canned_query(ds_client): assert "Xtop_query:fixtures:from_hook:xyz" in response.text finally: pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_track_event(): + datasette = Datasette(memory=True) + from .conftest import TrackEventPlugin + + await datasette.invoke_startup() + await datasette.track_event( + TrackEventPlugin.OneEvent(actor=None, extra="extra extra") + ) + assert len(datasette._tracked_events) == 1 + assert isinstance(datasette._tracked_events[0], TrackEventPlugin.OneEvent) + event = datasette._tracked_events[0] + assert event.name == "one" + assert event.properties() == {"extra": "extra extra"} + # Should have a recent created as well + created = event.created + assert isinstance(created, datetime.datetime) + assert created.tzinfo == datetime.timezone.utc + + +@pytest.mark.asyncio +async def test_hook_register_events(): + datasette = Datasette(memory=True) + await datasette.invoke_startup() + assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) diff --git a/tests/utils.py b/tests/utils.py index 84d5b1df..9b31abde 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,11 @@ from datasette.utils.sqlite import sqlite3 +def last_event(datasette): + events = getattr(datasette, "_tracked_events", []) + return events[-1] if events else None + + def assert_footer_links(soup): footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) From 2e4a03b2c461ca20ff789146a006ddd126013ee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 15:31:26 -0800 Subject: [PATCH 1908/2303] Run coverage on Python 3.12 - #2245 I hoped this would run slightly faster than 3.9 but there doesn't appear to be a performance improvement. --- .github/workflows/test-coverage.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index bd720664..7a08e401 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,18 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/setup.py' - name: Install Python dependencies run: | python -m pip install --upgrade pip From bcf7ef963f6e1eb0a64b2a0bb4af0ae7a197d1d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 19:45:05 -0800 Subject: [PATCH 1909/2303] YAML/JSON examples for allow blocks --- docs/authentication.rst | 270 ++++++++++++++++++++++++++++++++++------ 1 file changed, 231 insertions(+), 39 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a301113a..8758765d 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -80,13 +80,35 @@ The standard way to define permissions in Datasette is to use an ``"allow"`` blo The most basic form of allow block is this (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: root + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: root + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "id": "root" + } } - } +.. [[[end]]] This will match any actors with an ``"id"`` property of ``"root"`` - for example, an actor that looks like this: @@ -99,29 +121,98 @@ This will match any actors with an ``"id"`` property of ``"root"`` - for example An allow block can specify "deny all" using ``false`` (`demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: false + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": false - } +.. tab:: YAML + + .. code-block:: yaml + + allow: false + +.. tab:: JSON + + .. code-block:: json + + { + "allow": false + } +.. [[[end]]] An ``"allow"`` of ``true`` allows all access (`demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: true + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": true - } +.. tab:: YAML + + .. code-block:: yaml + + allow: true + +.. tab:: JSON + + .. code-block:: json + + { + "allow": true + } +.. [[[end]]] Allow keys can provide a list of values. These will match any actor that has any of those values (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: + - simon + - cleopaws + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "id": ["simon", "cleopaws"] +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: + - simon + - cleopaws + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "id": [ + "simon", + "cleopaws" + ] + } } - } +.. [[[end]]] This will match any actor with an ``"id"`` of either ``"simon"`` or ``"cleopaws"``. @@ -129,53 +220,154 @@ Actors can have properties that feature a list of values. These will be matched .. code-block:: json - { - "id": "simon", - "roles": ["staff", "developer"] - } + { + "id": "simon", + "roles": ["staff", "developer"] + } This allow block will provide access to any actor that has ``"developer"`` as one of their roles (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + roles: + - developer + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "roles": ["developer"] +.. tab:: YAML + + .. code-block:: yaml + + allow: + roles: + - developer + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "roles": [ + "developer" + ] + } } - } +.. [[[end]]] Note that "roles" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on. If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to match any logged-in user specify the following (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: "*" + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: "*" + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "id": "*" + } } - } +.. [[[end]]] You can specify that only unauthenticated actors (from anynomous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + unauthenticated: true + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + unauthenticated: true + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "unauthenticated": true + } } - } +.. [[[end]]] Allow keys act as an "or" mechanism. An actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. The following block will allow users with either a ``role`` of ``"ops"`` OR users who have an ``id`` of ``"simon"`` or ``"cleopaws"``: -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: + - simon + - cleopaws + role: ops + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "id": ["simon", "cleopaws"], +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: + - simon + - cleopaws + role: ops + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "id": [ + "simon", + "cleopaws" + ], "role": "ops" + } } - } +.. [[[end]]] `Demo for cleopaws `__, `demo for ops role `__, `demo for an actor matching neither rule `__. From b466749e88b2ffbd925b6b3e777c8527ebc54e78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 20:03:19 -0800 Subject: [PATCH 1910/2303] Filled out docs/configuration.rst, closes #2246 --- docs/changelog.rst | 2 +- docs/configuration.rst | 297 ++++++++++++++++++++++++++++++++++++-- docs/custom_templates.rst | 156 +------------------- docs/metadata_doc.py | 8 +- docs/plugin_hooks.rst | 2 +- docs/plugins.rst | 2 +- 6 files changed, 294 insertions(+), 173 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index af3d2a0b..04ce9583 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -610,7 +610,7 @@ JavaScript modules To use modules, JavaScript needs to be included in `` + +You can also specify a SRI (subresource integrity hash) for these assets: + +.. [[[cog + config_example(cog, """ + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: datasette.json + + .. code-block:: json + + { + "extra_css_urls": [ + { + "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", + "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" + } + ], + "extra_js_urls": [ + { + "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", + "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" + } + ] + } +.. [[[end]]] + +This will produce: + +.. code-block:: html + + + + +Modern browsers will only execute the stylesheet or JavaScript if the SRI hash +matches the content served. You can generate hashes using `www.srihash.org `_ + +Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: + +.. [[[cog + config_example(cog, """ + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + + +.. tab:: datasette.json + + .. code-block:: json + + { + "extra_js_urls": [ + { + "url": "https://example.datasette.io/module.js", + "module": true + } + ] + } +.. [[[end]]] + +Will produce this HTML: + +.. code-block:: html + + + + + diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index d8e4ac96..534d8b33 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -5,159 +5,6 @@ Custom pages and templates Datasette provides a number of ways of customizing the way data is displayed. -.. _customization_css_and_javascript: - -Custom CSS and JavaScript -------------------------- - -When you launch Datasette, you can specify a custom configuration file like this:: - - datasette mydb.db --config datasette.yaml - -Your ``datasette.yaml`` file can include links that look like this: - -.. [[[cog - from metadata_doc import config_example - config_example(cog, """ - extra_css_urls: - - https://simonwillison.net/static/css/all.bf8cd891642c.css - extra_js_urls: - - https://code.jquery.com/jquery-3.2.1.slim.min.js - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_css_urls: - - https://simonwillison.net/static/css/all.bf8cd891642c.css - extra_js_urls: - - https://code.jquery.com/jquery-3.2.1.slim.min.js - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_css_urls": [ - "https://simonwillison.net/static/css/all.bf8cd891642c.css" - ], - "extra_js_urls": [ - "https://code.jquery.com/jquery-3.2.1.slim.min.js" - ] - } -.. [[[end]]] - -The extra CSS and JavaScript files will be linked in the ```` of every page: - -.. code-block:: html - - - - -You can also specify a SRI (subresource integrity hash) for these assets: - -.. [[[cog - config_example(cog, """ - extra_css_urls: - - url: https://simonwillison.net/static/css/all.bf8cd891642c.css - sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI - extra_js_urls: - - url: https://code.jquery.com/jquery-3.2.1.slim.min.js - sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_css_urls: - - url: https://simonwillison.net/static/css/all.bf8cd891642c.css - sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI - extra_js_urls: - - url: https://code.jquery.com/jquery-3.2.1.slim.min.js - sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_css_urls": [ - { - "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", - "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" - } - ], - "extra_js_urls": [ - { - "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", - "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" - } - ] - } -.. [[[end]]] - -This will produce: - -.. code-block:: html - - - - -Modern browsers will only execute the stylesheet or JavaScript if the SRI hash -matches the content served. You can generate hashes using `www.srihash.org `_ - -Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: - -.. [[[cog - config_example(cog, """ - extra_js_urls: - - url: https://example.datasette.io/module.js - module: true - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_js_urls: - - url: https://example.datasette.io/module.js - module: true - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_js_urls": [ - { - "url": "https://example.datasette.io/module.js", - "module": true - } - ] - } -.. [[[end]]] - -Will produce this HTML: - -.. code-block:: html - - - CSS classes on the ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -258,9 +105,10 @@ The following URLs will now serve the content from those CSS and JS files:: http://localhost:8001/assets/styles.css http://localhost:8001/assets/app.js -You can reference those files from ``datasette.yaml`` like so: +You can reference those files from ``datasette.yaml`` like this, see :ref:`custom CSS and JavaScript ` for more details: .. [[[cog + from metadata_doc import config_example config_example(cog, """ extra_css_urls: - /assets/styles.css diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index a8f13414..ad85bf52 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -24,17 +24,19 @@ def metadata_example(cog, data=None, yaml=None): cog.out("\n") -def config_example(cog, input): +def config_example( + cog, input, yaml_title="datasette.yaml", json_title="datasette.json" +): if type(input) is str: data = YAML().load(input) output_yaml = input else: data = input output_yaml = safe_dump(input, sort_keys=False) - cog.out("\n.. tab:: datasette.yaml\n\n") + cog.out("\n.. tab:: {}\n\n".format(yaml_title)) cog.out(" .. code-block:: yaml\n\n") cog.out(textwrap.indent(output_yaml, " ")) - cog.out("\n\n.. tab:: datasette.json\n\n") + cog.out("\n\n.. tab:: {}\n\n".format(json_title)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1a88cd31..d9d135e5 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -270,7 +270,7 @@ you have one: Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. -If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. +If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`configuration_reference_css_js` for more details. .. code-block:: python diff --git a/docs/plugins.rst b/docs/plugins.rst index 1a72af95..03ddf8f0 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -328,7 +328,7 @@ To write that to a ``requirements.txt`` file, run this:: Plugin configuration -------------------- -Plugins can have their own configuration, embedded in a :ref:`configuration` file. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. +Plugins can have their own configuration, embedded in a :ref:`configuration file `. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. Here is an example of some plugin configuration for a specific table: From 4da581d09bbed2377682630c147e05d78c48f7e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 14:40:49 -0800 Subject: [PATCH 1911/2303] Link to config reference --- docs/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/settings.rst b/docs/settings.rst index 1d4baf90..d1553703 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -15,6 +15,8 @@ You can set multiple settings at once like this:: --setting sql_time_limit_ms 3500 \ --setting max_returned_rows 2000 +Settings can also be specified :ref:`in the database.yaml configuration file `. + .. _config_dir: Configuration directory mode From d4bc2b2dfc728017c8f669c1714f20b89655557c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 14:44:16 -0800 Subject: [PATCH 1912/2303] Remove fail_if_plugins_in_metadata, part of #2248 --- datasette/app.py | 8 ++------ datasette/cli.py | 3 +-- datasette/utils/__init__.py | 15 --------------- tests/test_plugins.py | 8 -------- 4 files changed, 3 insertions(+), 31 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 530f79bc..0143223a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -77,7 +77,6 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, - fail_if_plugins_in_metadata, tilde_decode, to_css_class, urlsafe_components, @@ -336,16 +335,13 @@ class Datasette: ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: - metadata = fail_if_plugins_in_metadata( - parse_metadata(fp.read()), metadata_files[0].name - ) + metadata = parse_metadata(fp.read()) if config_dir and config_files and not config: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - self._metadata_local = fail_if_plugins_in_metadata(metadata or {}) - + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: # Resolve spatialite, if requested diff --git a/datasette/cli.py b/datasette/cli.py index 91f38f69..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -33,7 +33,6 @@ from .utils import ( initial_path_for_datasette, pairs_to_nested_config, temporary_docker_directory, - fail_if_plugins_in_metadata, value_as_boolean, SpatialiteNotFound, StaticMount, @@ -543,7 +542,7 @@ def serve( metadata_data = None if metadata: - metadata_data = fail_if_plugins_in_metadata(parse_metadata(metadata.read())) + metadata_data = parse_metadata(metadata.read()) config_data = None if config: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 196e1682..75f1c2f4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1272,21 +1272,6 @@ def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> return result -def fail_if_plugins_in_metadata(metadata: dict, filename=None): - """If plugin config is inside metadata, raise an Exception""" - if metadata is not None and metadata.get("plugins") is not None: - suggested_extension = ( - ".yaml" - if filename is not None - and (filename.endswith(".yaml") or filename.endswith(".yml")) - else ".json" - ) - raise Exception( - f'Datasette no longer accepts plugin configuration in --metadata. Move your "plugins" configuration blocks to a separate file - we suggest calling that datasette.{suggested_extension} - and start Datasette with datasette -c datasette.{suggested_extension}. See https://docs.datasette.io/en/latest/configuration.html for more details.' - ) - return metadata - - def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm diff --git a/tests/test_plugins.py b/tests/test_plugins.py index dad4f2ca..f26e3652 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -883,14 +883,6 @@ def test_hook_forbidden(restore_working_directory): ) -def test_plugin_config_in_metadata(): - with pytest.raises( - Exception, - match="Datasette no longer accepts plugin configuration in --metadata", - ): - Datasette(memory=True, metadata={"plugins": {}}) - - @pytest.mark.asyncio async def test_hook_handle_exception(ds_client): await ds_client.get("/trigger-error?x=123") From be4f02335fb35d40a763d07b2d4e880b90083e53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:33:33 -0800 Subject: [PATCH 1913/2303] Treat plugins in metadata as if they were in config, closes #2248 --- datasette/app.py | 6 +++++ datasette/utils/__init__.py | 40 +++++++++++++++++++++++++++++ tests/test_plugins.py | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 0143223a..634283ff 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,6 +74,7 @@ from .utils import ( find_spatialite, format_bytes, module_from_path, + move_plugins, parse_metadata, resolve_env_secrets, resolve_routes, @@ -341,6 +342,11 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) + # Move any "plugins" settings from metadata to config - updates them in place + metadata = metadata or {} + config = config or {} + move_plugins(metadata, config) + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 75f1c2f4..cc175b01 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1287,3 +1287,43 @@ def make_slot_function(name, datasette, request, **kwargs): return markupsafe.Markup("".join(html_bits)) return inner + + +def move_plugins(source, destination): + """ + Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. + After moving, recursively remove any keys in the source that are left empty. + """ + + def recursive_move(src, dest, path=None): + if path is None: + path = [] + for key, value in list(src.items()): + new_path = path + [key] + if key == "plugins": + # Navigate and create the hierarchy in destination if needed + d = dest + for step in path: + d = d.setdefault(step, {}) + # Move the plugins + d[key] = value + # Remove the plugins from source + src.pop(key, None) + elif isinstance(value, dict): + recursive_move(value, dest, new_path) + # After moving, check if the current dictionary is empty and remove it if so + if not value: + src.pop(key, None) + + def prune_empty_dicts(d): + """ + Recursively prune all empty dictionaries from a given dictionary. + """ + for key, value in list(d.items()): + if isinstance(value, dict): + prune_empty_dicts(value) + if value == {}: + d.pop(key, None) + + recursive_move(source, destination) + prune_empty_dicts(source) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f26e3652..a53fc118 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1458,3 +1458,54 @@ async def test_hook_register_events(): datasette = Datasette(memory=True) await datasette.invoke_startup() assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) + + +@pytest.mark.parametrize( + "metadata,config,expected_metadata,expected_config", + ( + ( + # Instance level + {"plugins": {"datasette-foo": "bar"}}, + {}, + {}, + {"plugins": {"datasette-foo": "bar"}}, + ), + ( + # Database level + {"databases": {"foo": {"plugins": {"datasette-foo": "bar"}}}}, + {}, + {}, + {"databases": {"foo": {"plugins": {"datasette-foo": "bar"}}}}, + ), + ( + # Table level + { + "databases": { + "foo": {"tables": {"bar": {"plugins": {"datasette-foo": "bar"}}}} + } + }, + {}, + {}, + { + "databases": { + "foo": {"tables": {"bar": {"plugins": {"datasette-foo": "bar"}}}} + } + }, + ), + ( + # Keep other keys + {"plugins": {"datasette-foo": "bar"}, "other": "key"}, + {"original_config": "original"}, + {"other": "key"}, + {"original_config": "original", "plugins": {"datasette-foo": "bar"}}, + ), + ), +) +def test_metadata_plugin_config_treated_as_config( + metadata, config, expected_metadata, expected_config +): + ds = Datasette(metadata=metadata, config=config) + actual_metadata = ds.metadata() + assert "plugins" not in actual_metadata + assert actual_metadata == expected_metadata + assert ds.config == expected_config From 6ccef35cc92cc2357c2b2a9aa003b7334b2459eb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:42:45 -0800 Subject: [PATCH 1914/2303] More links between events documentation --- docs/events.rst | 2 +- docs/plugin_hooks.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/events.rst b/docs/events.rst index f150ac02..b86c8025 100644 --- a/docs/events.rst +++ b/docs/events.rst @@ -7,7 +7,7 @@ Datasette includes a mechanism for tracking events that occur while the software The core Datasette application triggers events when certain things happen. This page describes those events. -Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes (or additional classes registered by other plugins): +Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes - or additional classes :ref:`registered by other plugins `. .. automodule:: datasette.events :members: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d9d135e5..16f5cebb 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1769,7 +1769,7 @@ Datasette includes an internal mechanism for tracking analytical events. This ca Plugins can register to receive events using the ``track_event`` plugin hook. -They can also define their own events for other plugins to receive using the ``register_events`` plugin hook, combined with calls to the ``datasette.track_event(...)`` internal method. +They can also define their own events for other plugins to receive using the :ref:`register_events() plugin hook `, combined with calls to the :ref:`datasette.track_event() internal method `. .. _plugin_hook_track_event: @@ -1826,7 +1826,7 @@ register_events(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the ``datasette.track_event()`` method. +This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the :ref:`datasette.track_event() ` method. This example registers event subclasses for ``ban-user`` and ``unban-user`` events: From 4ea109ac4dd17392c85ca7d5934009f9a9488a9d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:47:41 -0800 Subject: [PATCH 1915/2303] Two spaces is aesthetically more pleasing here --- docs/settings.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index d1553703..c4b4ba82 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -11,9 +11,9 @@ Datasette supports a number of settings. These can be set using the ``--setting You can set multiple settings at once like this:: datasette mydatabase.db \ - --setting default_page_size 50 \ - --setting sql_time_limit_ms 3500 \ - --setting max_returned_rows 2000 + --setting default_page_size 50 \ + --setting sql_time_limit_ms 3500 \ + --setting max_returned_rows 2000 Settings can also be specified :ref:`in the database.yaml configuration file `. @@ -25,10 +25,10 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: datasette one.db two.db \ - --metadata=metadata.json \ - --template-dir=templates/ \ - --plugins-dir=plugins \ - --static css:css + --metadata=metadata.json \ + --template-dir=templates/ \ + --plugins-dir=plugins \ + --static css:css As an alternative to this, you can run Datasette in *configuration directory* mode. Create a directory with the following structure:: From 5ea7098e4da5fe8576f6452dbfac86e6aedba397 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Feb 2024 10:15:21 -0800 Subject: [PATCH 1916/2303] Fixed an unnecessary f-string --- demos/plugins/example_js_manager_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/plugins/example_js_manager_plugins.py b/demos/plugins/example_js_manager_plugins.py index 7db45464..2705f2c5 100644 --- a/demos/plugins/example_js_manager_plugins.py +++ b/demos/plugins/example_js_manager_plugins.py @@ -16,6 +16,6 @@ def extra_js_urls(view_name): if view_name in PERMITTED_VIEWS: return [ { - "url": f"/static/table-example-plugins.js", + "url": "/static/table-example-plugins.js", } ] From 7219a56d1e8b5d076037aeeec2583ad4fc3cacb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 10:34:10 -0800 Subject: [PATCH 1917/2303] 3 space indent, not 2 --- docs/configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index a835ace9..79e2a1ca 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -145,8 +145,8 @@ Settings """ # inside datasette.yaml settings: - default_allow_sql: off - default_page_size: 50 + default_allow_sql: off + default_page_size: 50 """).strip() ) .. ]]] @@ -157,8 +157,8 @@ Settings # inside datasette.yaml settings: - default_allow_sql: off - default_page_size: 50 + default_allow_sql: off + default_page_size: 50 .. tab:: datasette.json From 503545b20363ac15d3664bec7e6c4522ff271668 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 11:47:17 -0800 Subject: [PATCH 1918/2303] JavaScript plugins documentation, closes #2250 --- docs/index.rst | 1 + docs/javascript_plugins.rst | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 docs/javascript_plugins.rst diff --git a/docs/index.rst b/docs/index.rst index ce1ed2eb..e3036618 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,6 +60,7 @@ Contents custom_templates plugins writing_plugins + javascript_plugins plugin_hooks testing_plugins internals diff --git a/docs/javascript_plugins.rst b/docs/javascript_plugins.rst new file mode 100644 index 00000000..e7ee6817 --- /dev/null +++ b/docs/javascript_plugins.rst @@ -0,0 +1,159 @@ +.. _javascript_plugins: + +JavaScript plugins +================== + +Datasette can run custom JavaScript in several different ways: + +- Datasette plugins written in Python can use the :ref:`extra_js_urls() ` or :ref:`extra_body_script() ` plugin hooks to inject JavaScript into a page +- Datasette instances with :ref:`custom templates ` can include additional JavaScript in those templates +- The ``extra_js_urls`` key in ``datasette.yaml`` :ref:`can be used to include extra JavaScript ` + +There are no limitations on what this JavaScript can do. It is executed directly by the browser, so it can manipulate the DOM, fetch additional data and do anything else that JavaScript is capable of. + +.. warning:: + Custom JavaScript has security implications, especially for authenticated Datasette instances where the JavaScript might run in the context of the authenticated user. It's important to carefully review any JavaScript you run in your Datasette instance. + +.. _javascript_datasette_init: + +The datasette_init event +------------------------ + +Datasette emits a custom event called ``datasette_init`` when the page is loaded. This event is dispatched on the ``document`` object, and includes a ``detail`` object with a reference to the :ref:`datasetteManager ` object. + +Your JavaScript code can listen out for this event using ``document.addEventListener()`` like this: + +.. code-block:: javascript + + document.addEventListener("datasette_init", function (evt) { + const manager = evt.detail; + console.log("Datasette version:", manager.VERSION); + }); + +.. _javascript_datasette_manager: + +datasetteManager +---------------- + +The ``datasetteManager`` object + +``VERSION`` - string + The version of Datasette + +``plugins`` - ``Map()`` + A Map of currently loaded plugin names to plugin implementations + +``registerPlugin(name, implementation)`` + Call this to register a plugin, passing its name and implementation + +``selectors`` - object + An object providing named aliases to useful CSS selectors, :ref:`listed below ` + +.. _javascript_plugin_objects: + +JavaScript plugin objects +------------------------- + +JavaScript plugins are blocks of code that can be registered with Datasette using the ``registerPlugin()`` method on the :ref:`datasetteManager ` object. + +The ``implementation`` object passed to this method should include a ``version`` key defining the plugin version, and one or more of the following named functions providing the implementation of the plugin: + +.. _javascript_plugins_makeAboveTablePanelConfigs: + +makeAboveTablePanelConfigs() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This method should return a JavaScript array of objects defining additional panels to be added to the top of the table page. Each object should have the following: + +``id`` - string + A unique string ID for the panel, for example ``map-panel`` +``label`` - string + A human-readable label for the panel +``render(node)`` - function + A function that will be called with a DOM node to render the panel into + +This example shows how a plugin might define a single panel: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('panel-plugin', { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + id: 'first-panel', + label: 'First panel', + render: node => { + node.innerHTML = '

    My custom panel

    This is a custom panel that I added using a JavaScript plugin

    '; + } + } + ] + } + }); + }); + +When a page with a table loads, all registered plugins that implement ``makeAboveTablePanelConfigs()`` will be called and panels they return will be added to the top of the table page. + +.. _javascript_plugins_makeColumnActions: + +makeColumnActions(columnDetails) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This method, if present, will be called when Datasette is rendering the cog action menu icons that appear at the top of the table view. By default these include options like "Sort ascending/descending" and "Facet by this", but plugins can return additional actions to be included in this menu. + +The method will be called with a ``columnDetails`` object with the following keys: + +``columnName`` - string + The name of the column +``columnNotNull`` - boolean + True if the column is defined as NOT NULL +``columnType`` - string + The SQLite data type of the column +``isPk`` - boolean + True if the column is part of the primary key + +It should return a JavaScript array of objects each with a ``label`` and ``onClick`` property: + +``label`` - string + The human-readable label for the action +``onClick(evt)`` - function + A function that will be called when the action is clicked + +The ``evt`` object passed to the ``onClick`` is the standard browser event object that triggered the click. + +This example plugin adds two menu items - one to copy the column name to the clipboard and another that displays the column metadata in an ``alert()`` window: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('column-name-plugin', { + version: 0.1, + makeColumnActions: (columnDetails) => { + return [ + { + label: 'Copy column to clipboard', + onClick: async (evt) => { + await navigator.clipboard.writeText(columnDetails.columnName) + } + }, + { + label: 'Alert column metadata', + onClick: () => alert(JSON.stringify(columnDetails, null, 2)) + } + ]; + } + }); + }); + +.. _javascript_datasette_manager_selectors: + +Selectors +--------- + +These are available on the ``selectors`` property of the :ref:`javascript_datasette_manager` object. + +.. literalinclude:: ../datasette/static/datasette-manager.js + :language: javascript + :start-at: const DOM_SELECTORS = { + :end-at: }; From efc73575548b0bbaca4bdc8de40fc6939bb88428 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 13:01:03 -0800 Subject: [PATCH 1919/2303] Remove Using YAML for metadata section No longer necessary now we show YAML and JSON examples everywhere. --- docs/changelog.rst | 2 +- docs/metadata.rst | 29 +---------------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 04ce9583..f4b928e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1190,7 +1190,7 @@ Also in this release: 0.40 (2020-04-21) ----------------- -* Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. See :ref:`metadata_yaml`. (:issue:`713`) +* Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. (:issue:`713`) * Removed support for ``datasette publish now``, which used the the now-retired Zeit Now v1 hosting platform. A new plugin, `datasette-publish-now `__, can be installed to publish data to Zeit (`now Vercel `__) Now v2. (:issue:`710`) * Fixed a bug where the ``extra_template_vars(request, view_name)`` plugin hook was not receiving the correct ``view_name``. (:issue:`716`) * Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`setting_template_debug`). (:issue:`693`) diff --git a/docs/metadata.rst b/docs/metadata.rst index b4dc90f9..f3ca68ac 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -53,7 +53,7 @@ Your ``metadata.yaml`` file can look something like this: .. [[[end]]] -Choosing YAML over JSON adds support for multi-line strings and comments, see :ref:`metadata_yaml`. +Choosing YAML over JSON adds support for multi-line strings and comments. The above metadata will be displayed on the index page of your Datasette-powered site. The source and license information will also be included in the footer of @@ -664,33 +664,6 @@ SpatiaLite tables are automatically hidden) using ``"hidden": true``: } .. [[[end]]] -.. _metadata_yaml: - -Using YAML for metadata ------------------------ - -Datasette accepts YAML as an alternative to JSON for your metadata configuration file. -YAML is particularly useful for including multiline HTML and SQL strings, plus inline comments. - -Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`canned_queries`. - -.. code-block:: yaml - - title: Demonstrating Metadata from YAML - description_html: |- -

    This description includes a long HTML string

    -
      -
    • YAML is better for embedding HTML strings than JSON!
      • -
    - license: ODbL - license_url: https://opendatacommons.org/licenses/odbl/ - databases: - fixtures: - tables: - no_primary_key: - hidden: true - - .. _metadata_reference: Metadata reference From 85a1dfe6e07fcdd7ec8f83cb5b3a8f023659d064 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 13:43:50 -0800 Subject: [PATCH 1920/2303] Configuration via the command-line section Closes #2252 Closes #2156 --- docs/configuration.rst | 78 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 79e2a1ca..425024da 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,7 +5,7 @@ Configuration Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more. -Most configuration can be handled using a ``datasette.yaml`` configuration file, passed to datasette using the ``--config``/ ``-c`` flag: +Most configuration can be handled using a ``datasette.yaml`` configuration file, passed to datasette using the ``-c/--config`` flag: .. code-block:: bash @@ -13,12 +13,86 @@ Most configuration can be handled using a ``datasette.yaml`` configuration file, This file can also use JSON, as ``datasette.json``. YAML is recommended over JSON due to its support for comments and multi-line strings. +.. _configuration_cli: + +Configuration via the command-line +---------------------------------- + +The recommended way to configure Datasette is using a ``datasette.yaml`` file passed to ``-c/--config``. You can also pass individual settings to Datasette using the ``-s/--setting`` option, which can be used multiple times: + +.. code-block:: bash + + datasette mydatabase.db \ + --setting settings.default_page_size 50 \ + --setting settings.sql_time_limit_ms 3500 + +This option takes dotted-notation for the first argument and a value for the second argument. This means you can use it to set any configuration value that would be valid in a ``datasette.yaml`` file. + +It also works for plugin configuration, for example for `datasette-cluster-map `_: + +.. code-block:: bash + + datasette mydatabase.db \ + --setting plugins.datasette-cluster-map.latitude_column xlat \ + --setting plugins.datasette-cluster-map.longitude_column xlon + +If the value you provide is a valid JSON object or list it will be treated as nested data, allowing you to configure plugins that accept lists such as `datasette-proxy-url `_: + +.. code-block:: bash + + datasette mydatabase.db \ + -s plugins.datasette-proxy-url.paths '[{"path": "/proxy", "backend": "http://example.com/"}]' + +This is equivalent to a ``datasette.yaml`` file containing the following: + +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + plugins: + datasette-proxy-url: + paths: + - path: /proxy + backend: http://example.com/ + """).strip() + ) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + plugins: + datasette-proxy-url: + paths: + - path: /proxy + backend: http://example.com/ + +.. tab:: datasette.json + + .. code-block:: json + + { + "plugins": { + "datasette-proxy-url": { + "paths": [ + { + "path": "/proxy", + "backend": "http://example.com/" + } + ] + } + } + } +.. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference ---------------------------- -This example shows many of the valid configuration options that can exist inside ``datasette.yaml``. +The following example shows some of the valid configuration options that can exist inside ``datasette.yaml``. .. [[[cog from metadata_doc import config_example From 1e901aa690211db36f02cc1b25246d0f56cd8720 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 12:33:46 -0800 Subject: [PATCH 1921/2303] /-/config page, closes #2254 --- datasette/app.py | 14 ++++++----- datasette/utils/__init__.py | 28 +++++++++++++++++++++ datasette/views/special.py | 4 +-- docs/introspection.rst | 11 +++++++- tests/test_api.py | 50 ++++++++++++++++++++++++++----------- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 634283ff..2e20d402 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -81,6 +81,7 @@ from .utils import ( tilde_decode, to_css_class, urlsafe_components, + redact_keys, row_sql_params_pks, ) from .utils.asgi import ( @@ -1374,6 +1375,11 @@ class Datasette: output.append(script) return output + def _config(self): + return redact_keys( + self.config, ("secret", "key", "password", "token", "hash", "dsn") + ) + def _routes(self): routes = [] @@ -1433,12 +1439,8 @@ class Datasette: r"/-/settings(\.(?Pjson))?$", ) add_route( - permanent_redirect("/-/settings.json"), - r"/-/config.json", - ) - add_route( - permanent_redirect("/-/settings"), - r"/-/config", + JsonDataView.as_view(self, "config.json", lambda: self._config()), + r"/-/config(\.(?Pjson))?$", ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index cc175b01..4c940645 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -17,6 +17,7 @@ import time import types import secrets import shutil +from typing import Iterable import urllib import yaml from .shutil_backport import copytree @@ -1327,3 +1328,30 @@ def move_plugins(source, destination): recursive_move(source, destination) prune_empty_dicts(source) + + +def redact_keys(original: dict, key_patterns: Iterable) -> dict: + """ + Recursively redact sensitive keys in a dictionary based on given patterns + + :param original: The original dictionary + :param key_patterns: A list of substring patterns to redact + :return: A copy of the original dictionary with sensitive values redacted + """ + + def redact(data): + if isinstance(data, dict): + return { + k: ( + redact(v) + if not any(pattern in k for pattern in key_patterns) + else "***" + ) + for k, v in data.items() + } + elif isinstance(data, list): + return [redact(item) for item in data] + else: + return data + + return redact(original) diff --git a/datasette/views/special.py b/datasette/views/special.py index 4088a1f9..296652d0 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -42,7 +42,7 @@ class JsonDataView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response( - json.dumps(data), + json.dumps(data, default=repr), content_type="application/json; charset=utf-8", headers=headers, ) @@ -53,7 +53,7 @@ class JsonDataView(BaseView): request=request, context={ "filename": self.filename, - "data_json": json.dumps(data, indent=4), + "data_json": json.dumps(data, indent=4, default=repr), }, ) diff --git a/docs/introspection.rst b/docs/introspection.rst index e08ca911..b62197ea 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -87,7 +87,7 @@ Shows a list of currently installed plugins and their versions. `Plugins example Add ``?all=1`` to include details of the default plugins baked into Datasette. -.. _JsonDataView_config: +.. _JsonDataView_settings: /-/settings ----------- @@ -105,6 +105,15 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. + +Any keys that include the one of the following substrings in their names will be returned as redacted ``***`` output, to help avoid accidentally leaking private configuration information: ``secret``, ``key``, ``password``, ``token``, ``hash``, ``dsn``. + .. _JsonDataView_databases: /-/databases diff --git a/tests/test_api.py b/tests/test_api.py index 177dc95c..0a1f3725 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -846,20 +846,6 @@ async def test_settings_json(ds_client): } -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path,expected_redirect", - ( - ("/-/config.json", "/-/settings.json"), - ("/-/config", "/-/settings"), - ), -) -async def test_config_redirects_to_settings(ds_client, path, expected_redirect): - response = await ds_client.get(path) - assert response.status_code == 301 - assert response.headers["Location"] == expected_redirect - - test_json_columns_default_expected = [ {"intval": 1, "strval": "s", "floatval": 0.5, "jsonval": '{"foo": "bar"}'} ] @@ -1039,3 +1025,39 @@ async def test_tilde_encoded_database_names(db_name): # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config,expected", + ( + ({}, {}), + ({"plugins": {"datasette-foo": "bar"}}, {"plugins": {"datasette-foo": "bar"}}), + # Test redaction + ( + { + "plugins": { + "datasette-auth": {"secret_key": "key"}, + "datasette-foo": "bar", + "datasette-auth2": {"password": "password"}, + "datasette-sentry": { + "dsn": "sentry:///foo", + }, + } + }, + { + "plugins": { + "datasette-auth": {"secret_key": "***"}, + "datasette-foo": "bar", + "datasette-auth2": {"password": "***"}, + "datasette-sentry": {"dsn": "***"}, + } + }, + ), + ), +) +async def test_config_json(config, expected): + "/-/config.json should return redacted configuration" + ds = Datasette(config=config) + response = await ds.client.get("/-/config.json") + assert response.json() == expected From 5a63ecc5577d070f28bf5daa34aaaf3dadfd2e4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 15:03:19 -0800 Subject: [PATCH 1922/2303] Rename metadata= to table_config= in facet code, refs #2247 --- datasette/facets.py | 41 ++++++++++++++++++++-------------------- datasette/views/table.py | 2 +- tests/test_facets.py | 4 ++-- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index b23615fe..f1cfc68f 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -11,8 +11,8 @@ from datasette.utils import ( ) -def load_facet_configs(request, table_metadata): - # Given a request and the metadata configuration for a table, return +def load_facet_configs(request, table_config): + # Given a request and the configuration for a table, return # a dictionary of selected facets, their lists of configs and for each # config whether it came from the request or the metadata. # @@ -20,21 +20,21 @@ def load_facet_configs(request, table_metadata): # {"source": "metadata", "config": config1}, # {"source": "request", "config": config2}]} facet_configs = {} - table_metadata = table_metadata or {} - metadata_facets = table_metadata.get("facets", []) - for metadata_config in metadata_facets: - if isinstance(metadata_config, str): + table_config = table_config or {} + table_facet_configs = table_config.get("facets", []) + for facet_config in table_facet_configs: + if isinstance(facet_config, str): type = "column" - metadata_config = {"simple": metadata_config} + facet_config = {"simple": facet_config} else: assert ( - len(metadata_config.values()) == 1 + len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, metadata_config = list(metadata_config.items())[0] - if isinstance(metadata_config, str): - metadata_config = {"simple": metadata_config} + type, facet_config = list(facet_config.items())[0] + if isinstance(facet_config, str): + facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( - {"source": "metadata", "config": metadata_config} + {"source": "metadata", "config": facet_config} ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): @@ -45,13 +45,12 @@ def load_facet_configs(request, table_metadata): elif key.startswith("_facet_"): type = key[len("_facet_") :] for value in values: - # The value is the config - either JSON or not - if value.startswith("{"): - config = json.loads(value) - else: - config = {"simple": value} + # The value is the facet_config - either JSON or not + facet_config = ( + json.loads(value) if value.startswith("{") else {"simple": value} + ) facet_configs.setdefault(type, []).append( - {"source": "request", "config": config} + {"source": "request", "config": facet_config} ) return facet_configs @@ -75,7 +74,7 @@ class Facet: sql=None, table=None, params=None, - metadata=None, + table_config=None, row_count=None, ): assert table or sql, "Must provide either table= or sql=" @@ -86,12 +85,12 @@ class Facet: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] - self.metadata = metadata + self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: self.row_count = row_count def get_configs(self): - configs = load_facet_configs(self.request, self.metadata) + configs = load_facet_configs(self.request, self.table_config) return configs.get(self.type) or [] def get_querystring_pairs(self): diff --git a/datasette/views/table.py b/datasette/views/table.py index 3b812c01..22722847 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1275,7 +1275,7 @@ async def table_view_data( sql=sql_no_order_no_limit, params=params, table=table_name, - metadata=table_metadata, + table_config=table_metadata, row_count=extra_count, ) ) diff --git a/tests/test_facets.py b/tests/test_facets.py index 85c8f85b..76344108 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -82,7 +82,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(ds_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["_city_id"]}, + table_config={"facets": ["_city_id"]}, ) suggestions = [s["name"] for s in await facet.suggest()] assert [ @@ -278,7 +278,7 @@ async def test_column_facet_from_metadata_cannot_be_hidden(ds_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["_city_id"]}, + table_config={"facets": ["_city_id"]}, ) buckets, timed_out = await facet.facet_results() assert [] == timed_out From 5d21057cf1e4171bc2741d3d8d9da83aee1165b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 15:22:03 -0800 Subject: [PATCH 1923/2303] /-/config example, refs #2254 --- docs/introspection.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/introspection.rst b/docs/introspection.rst index b62197ea..ff78ec78 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -110,7 +110,17 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. +Shows the :ref:`configuration ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. `Config example `_: + +.. code-block:: json + + { + "settings": { + "template_debug": true, + "trace_debug": true, + "force_https_urls": true + } + } Any keys that include the one of the following substrings in their names will be returned as redacted ``***`` output, to help avoid accidentally leaking private configuration information: ``secret``, ``key``, ``password``, ``token``, ``hash``, ``dsn``. From 69c6e953231078ab18ba0807e5fe2a4e20e84093 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 17:27:20 -0800 Subject: [PATCH 1924/2303] Fixed a bunch of unused imports spotted with ruff --- datasette/cli.py | 1 - datasette/forbidden.py | 1 - datasette/handle_exception.py | 4 +--- datasette/permissions.py | 2 +- datasette/url_builder.py | 2 +- datasette/views/base.py | 1 - datasette/views/index.py | 3 --- ruff.toml | 1 + tests/conftest.py | 2 +- tests/test_black.py | 2 -- tests/test_cli.py | 2 -- tests/test_cli_serve_get.py | 2 +- tests/test_config_dir.py | 2 -- tests/test_internals_datasette.py | 1 - tests/test_plugins.py | 7 ++----- tests/test_table_html.py | 2 +- 16 files changed, 9 insertions(+), 26 deletions(-) create mode 100644 ruff.toml diff --git a/datasette/cli.py b/datasette/cli.py index 1a5a8af3..0c8a8541 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -15,7 +15,6 @@ import sys import textwrap import webbrowser from .app import ( - OBSOLETE_SETTINGS, Datasette, DEFAULT_SETTINGS, SETTINGS, diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 156a44d4..41c48396 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,4 +1,3 @@ -from os import stat from datasette import hookimpl, Response diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index bef6b4ee..1a0ac979 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,14 +1,12 @@ from datasette import hookimpl, Response -from .utils import await_me_maybe, add_cors_headers +from .utils import add_cors_headers from .utils.asgi import ( Base400, - Forbidden, ) from .views.base import DatasetteError from markupsafe import Markup import pdb import traceback -from .plugins import pm try: import rich diff --git a/datasette/permissions.py b/datasette/permissions.py index 152f1721..bd42158e 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, fields +from dataclasses import dataclass from typing import Optional diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 574bf3c1..9c6bbde0 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,4 +1,4 @@ -from .utils import tilde_encode, path_with_format, HASH_LENGTH, PrefixedUrlString +from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib diff --git a/datasette/views/base.py b/datasette/views/base.py index bdc1e9cf..9d7a854c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -10,7 +10,6 @@ from markupsafe import escape import pint -from datasette import __version__ from datasette.database import QueryInterrupted from datasette.utils.asgi import Request from datasette.utils import ( diff --git a/datasette/views/index.py b/datasette/views/index.py index 595cf234..2cb18b1c 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,12 +1,9 @@ import json -from datasette.plugins import pm from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ -from markupsafe import Markup - from .base import BaseView diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..0deb884c --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 160 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 445de057..168194d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import re import subprocess import tempfile import time -from dataclasses import dataclass, field +from dataclasses import dataclass from datasette import Event, hookimpl diff --git a/tests/test_black.py b/tests/test_black.py index d09b2514..ccf51171 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1,8 +1,6 @@ import black from click.testing import CliRunner from pathlib import Path -import pytest -import sys code_root = Path(__file__).parent.parent diff --git a/tests/test_cli.py b/tests/test_cli.py index 9cc18c6e..bda17eed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,6 @@ from .fixtures import ( TestClient as _TestClient, EXPECTED_PLUGINS, ) -import asyncio from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve @@ -19,7 +18,6 @@ import pytest import sys import textwrap from unittest import mock -import urllib def test_inspect_cli(app_client): diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ff2429c6..1088d906 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,4 +1,4 @@ -from datasette.cli import cli, serve +from datasette.cli import cli from datasette.plugins import pm from click.testing import CliRunner import textwrap diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 748412c3..66114a27 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -3,11 +3,9 @@ import pathlib import pytest from datasette.app import Datasette -from datasette.cli import cli from datasette.utils.sqlite import sqlite3 from datasette.utils import StartupError from .fixtures import TestClient as _TestClient -from click.testing import CliRunner PLUGIN = """ from datasette import hookimpl diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index c30bb748..2614e02e 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -7,7 +7,6 @@ from datasette import Forbidden, Context from datasette.app import Datasette, Database from itsdangerous import BadSignature import pytest -from typing import Optional @pytest.fixture diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a53fc118..c02c94cc 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,6 +1,5 @@ from bs4 import BeautifulSoup as Soup from .fixtures import ( - app_client, app_client, make_app_client, TABLES, @@ -9,14 +8,12 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from click.testing import CliRunner -from dataclasses import dataclass from datasette.app import Datasette -from datasette import cli, hookimpl, Event, Permission +from datasette import cli, hookimpl, Permission from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 -from datasette.utils import CustomRow, StartupError -from jinja2.environment import Template +from datasette.utils import StartupError from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 0604d34c..2a658663 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1,4 +1,4 @@ -from datasette.app import Datasette, Database +from datasette.app import Datasette from bs4 import BeautifulSoup as Soup from .fixtures import ( # noqa app_client, From f0491038523e000b97a18d2e9a23faee62208083 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 17:33:18 -0800 Subject: [PATCH 1925/2303] datasette.table_metadata() is now await datasette.table_config(), refs #2247 --- datasette/app.py | 2 +- datasette/database.py | 2 +- datasette/filters.py | 2 +- datasette/views/row.py | 2 +- datasette/views/table.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 2e20d402..b0b8f041 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1202,7 +1202,7 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - def table_metadata(self, database, table): + async def table_config(self, database, table): """Fetch table-specific metadata.""" return ( (self.metadata("databases") or {}) diff --git a/datasette/database.py b/datasette/database.py index f2c980d7..1c1b3e1b 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -418,7 +418,7 @@ class Database: return await self.execute_fn(lambda conn: detect_fts(conn, table)) async def label_column_for_table(self, table): - explicit_label_column = self.ds.table_metadata(self.name, table).get( + explicit_label_column = (await self.ds.table_config(self.name, table)).get( "label_column" ) if explicit_label_column: diff --git a/datasette/filters.py b/datasette/filters.py index 73eea857..4d9580d8 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -50,7 +50,7 @@ def search_filters(request, database, table, datasette): extra_context = {} # Figure out which fts_table to use - table_metadata = datasette.table_metadata(database, table) + table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) fts_table = request.args.get("_fts_table") fts_table = fts_table or table_metadata.get("fts_table") diff --git a/datasette/views/row.py b/datasette/views/row.py index 7b646641..7b43b893 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -89,7 +89,7 @@ class RowView(DataView): "columns": columns, "primary_keys": resolved.pks, "primary_key_values": pk_values, - "units": self.ds.table_metadata(database, table).get("units", {}), + "units": (await self.ds.table_config(database, table)).get("units", {}), } if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): diff --git a/datasette/views/table.py b/datasette/views/table.py index 22722847..2b5b7c24 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -142,7 +142,7 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) column_descriptions = table_metadata.get("columns") or {} column_details = { col.name: col for col in await db.table_column_details(table_name) @@ -663,7 +663,7 @@ async def _columns_to_select(table_columns, pks, request): async def _sortable_columns_for_table(datasette, database_name, table_name, use_rowid): db = datasette.databases[database_name] - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: @@ -962,7 +962,7 @@ async def table_view_data( nocount = True nofacet = True - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) units = table_metadata.get("units", {}) # Arguments that start with _ and don't contain a __ are From 52a1dac5d2bac7e106c8d6ce8e0c6f1dc0141a7e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 21:00:55 -0800 Subject: [PATCH 1926/2303] Test proving $env works for datasette.yml, closes #2255 --- tests/test_plugins.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c02c94cc..40d01c71 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -231,10 +231,18 @@ async def test_plugin_config(ds_client): @pytest.mark.asyncio -async def test_plugin_config_env(ds_client): - os.environ["FOO_ENV"] = "FROM_ENVIRONMENT" - assert {"foo": "FROM_ENVIRONMENT"} == ds_client.ds.plugin_config("env-plugin") - del os.environ["FOO_ENV"] +async def test_plugin_config_env(ds_client, monkeypatch): + monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT") + assert ds_client.ds.plugin_config("env-plugin") == {"foo": "FROM_ENVIRONMENT"} + + +@pytest.mark.asyncio +async def test_plugin_config_env_from_config(monkeypatch): + monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT_2") + datasette = Datasette( + config={"plugins": {"env-plugin": {"setting": {"$env": "FOO_ENV"}}}} + ) + assert datasette.plugin_config("env-plugin") == {"setting": "FROM_ENVIRONMENT_2"} @pytest.mark.asyncio From 60c6692f6802dfae6a433f648f287be30ef52325 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 21:57:09 -0800 Subject: [PATCH 1927/2303] table_config instead of table_metadata (#2257) Table configuration that was incorrectly placed in metadata is now treated as if it was in config. New await datasette.table_config() method. Closes #2247 --- datasette/app.py | 12 +++-- datasette/database.py | 10 ++-- datasette/utils/__init__.py | 69 +++++++++++++++++++----- datasette/views/table.py | 11 ++-- tests/test_api.py | 101 +++++++++++++++++++++++++++++++++++- tests/test_html.py | 2 +- 6 files changed, 175 insertions(+), 30 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b0b8f041..373b3e95 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -75,6 +75,7 @@ from .utils import ( format_bytes, module_from_path, move_plugins, + move_table_config, parse_metadata, resolve_env_secrets, resolve_routes, @@ -346,7 +347,9 @@ class Datasette: # Move any "plugins" settings from metadata to config - updates them in place metadata = metadata or {} config = config or {} - move_plugins(metadata, config) + metadata, config = move_plugins(metadata, config) + # Now migrate any known table configuration settings over as well + metadata, config = move_table_config(metadata, config) self._metadata_local = metadata or {} self.sqlite_extensions = [] @@ -1202,10 +1205,11 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - async def table_config(self, database, table): - """Fetch table-specific metadata.""" + async def table_config(self, database: str, table: str) -> dict: + """Return dictionary of configuration for specified table""" return ( - (self.metadata("databases") or {}) + (self.config or {}) + .get("databases", {}) .get(database, {}) .get("tables", {}) .get(table, {}) diff --git a/datasette/database.py b/datasette/database.py index 1c1b3e1b..fba81496 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -487,13 +487,11 @@ class Database: ) ).rows ] - # Add any from metadata.json - db_metadata = self.ds.metadata(database=self.name) - if "tables" in db_metadata: + # Add any tables marked as hidden in config + db_config = self.ds.config.get("databases", {}).get(self.name, {}) + if "tables" in db_config: hidden_tables += [ - t - for t in db_metadata["tables"] - if db_metadata["tables"][t].get("hidden") + t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] # Also mark as hidden any tables which start with the name of a hidden table # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 4c940645..fcaebe3f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -2,6 +2,7 @@ import asyncio from contextlib import contextmanager import click from collections import OrderedDict, namedtuple, Counter +import copy import base64 import hashlib import inspect @@ -17,7 +18,7 @@ import time import types import secrets import shutil -from typing import Iterable +from typing import Iterable, Tuple import urllib import yaml from .shutil_backport import copytree @@ -1290,11 +1291,24 @@ def make_slot_function(name, datasette, request, **kwargs): return inner -def move_plugins(source, destination): +def prune_empty_dicts(d: dict): + """ + Recursively prune all empty dictionaries from a given dictionary. + """ + for key, value in list(d.items()): + if isinstance(value, dict): + prune_empty_dicts(value) + if value == {}: + d.pop(key, None) + + +def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: """ Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys in the source that are left empty. """ + source = copy.deepcopy(source) + destination = copy.deepcopy(destination) def recursive_move(src, dest, path=None): if path is None: @@ -1316,18 +1330,49 @@ def move_plugins(source, destination): if not value: src.pop(key, None) - def prune_empty_dicts(d): - """ - Recursively prune all empty dictionaries from a given dictionary. - """ - for key, value in list(d.items()): - if isinstance(value, dict): - prune_empty_dicts(value) - if value == {}: - d.pop(key, None) - recursive_move(source, destination) prune_empty_dicts(source) + return source, destination + + +_table_config_keys = ( + "hidden", + "sort", + "sort_desc", + "size", + "sortable_columns", + "label_column", + "facets", + "fts_table", + "fts_pk", + "searchmode", + "units", +) + + +def move_table_config(metadata: dict, config: dict): + """ + Move all known table configuration keys from metadata to config. + """ + if "databases" not in metadata: + return metadata, config + metadata = copy.deepcopy(metadata) + config = copy.deepcopy(config) + for database_name, database in metadata["databases"].items(): + if "tables" not in database: + continue + for table_name, table in database["tables"].items(): + for key in _table_config_keys: + if key in table: + config.setdefault("databases", {}).setdefault( + database_name, {} + ).setdefault("tables", {}).setdefault(table_name, {})[ + key + ] = table.pop( + key + ) + prune_empty_dicts(metadata) + return metadata, config def redact_keys(original: dict, key_patterns: Iterable) -> dict: diff --git a/datasette/views/table.py b/datasette/views/table.py index 2b5b7c24..50d2b3c2 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -142,11 +142,11 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - table_metadata = await datasette.table_config(database_name, table_name) - column_descriptions = table_metadata.get("columns") or {} + column_descriptions = datasette.metadata("columns", database_name, table_name) or {} column_details = { col.name: col for col in await db.table_column_details(table_name) } + table_config = await datasette.table_config(database_name, table_name) pks = await db.primary_keys(table_name) pks_for_display = pks if not pks_for_display: @@ -193,7 +193,6 @@ async def display_columns_and_rows( "raw": pk_path, "value": markupsafe.Markup( '{flat_pks}'.format( - base_url=base_url, table_path=datasette.urls.table(database_name, table_name), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), @@ -274,9 +273,9 @@ async def display_columns_and_rows( ), ) ) - elif column in table_metadata.get("units", {}) and value != "": + elif column in table_config.get("units", {}) and value != "": # Interpret units using pint - value = value * ureg(table_metadata["units"][column]) + value = value * ureg(table_config["units"][column]) # Pint uses floating point which sometimes introduces errors in the compact # representation, which we have to round off to avoid ugliness. In the vast # majority of cases this rounding will be inconsequential. I hope. @@ -591,7 +590,7 @@ class TableDropView(BaseView): try: data = json.loads(await request.post_body()) confirm = data.get("confirm") - except json.JSONDecodeError as e: + except json.JSONDecodeError: pass if not confirm: diff --git a/tests/test_api.py b/tests/test_api.py index 0a1f3725..8cb73dbb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -771,7 +771,7 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): @pytest.mark.asyncio async def test_metadata_json(ds_client): response = await ds_client.get("/-/metadata.json") - assert response.json() == METADATA + assert response.json() == ds_client.ds.metadata() @pytest.mark.asyncio @@ -1061,3 +1061,102 @@ async def test_config_json(config, expected): ds = Datasette(config=config) response = await ds.client.get("/-/config.json") assert response.json() == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "metadata,expected_config,expected_metadata", + ( + ({}, {}, {}), + ( + # Metadata input + { + "title": "Datasette Fixtures", + "databases": { + "fixtures": { + "tables": { + "sortable": { + "sortable_columns": [ + "sortable", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + ], + }, + "no_primary_key": {"sortable_columns": [], "hidden": True}, + "units": {"units": {"distance": "m", "frequency": "Hz"}}, + "primary_key_multiple_columns_explicit_label": { + "label_column": "content2" + }, + "simple_view": {"sortable_columns": ["content"]}, + "searchable_view_configured_by_metadata": { + "fts_table": "searchable_fts", + "fts_pk": "pk", + }, + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, + "attraction_characteristic": {"sort_desc": "pk"}, + "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, + }, + } + }, + }, + # Should produce a config with just the table configuration keys + { + "databases": { + "fixtures": { + "tables": { + "sortable": { + "sortable_columns": [ + "sortable", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + ] + }, + "units": {"units": {"distance": "m", "frequency": "Hz"}}, + # These one get redacted: + "no_primary_key": "***", + "primary_key_multiple_columns_explicit_label": "***", + "simple_view": {"sortable_columns": ["content"]}, + "searchable_view_configured_by_metadata": { + "fts_table": "searchable_fts", + "fts_pk": "pk", + }, + "attraction_characteristic": {"sort_desc": "pk"}, + "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, + } + } + } + }, + # And metadata with everything else + { + "title": "Datasette Fixtures", + "databases": { + "fixtures": { + "tables": { + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, + } + } + }, + }, + ), + ), +) +async def test_upgrade_metadata(metadata, expected_config, expected_metadata): + ds = Datasette(metadata=metadata) + response = await ds.client.get("/-/config.json") + assert response.json() == expected_config + response2 = await ds.client.get("/-/metadata.json") + assert response2.json() == expected_metadata diff --git a/tests/test_html.py b/tests/test_html.py index 86895844..8229b166 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -753,7 +753,7 @@ async def test_metadata_json_html(ds_client): response = await ds_client.get("/-/metadata") assert response.status_code == 200 pre = Soup(response.content, "html.parser").find("pre") - assert METADATA == json.loads(pre.text) + assert ds_client.ds.metadata() == json.loads(pre.text) @pytest.mark.asyncio From 9ac9f0152f1d3396d01ceddfcaf07fd1cf3f7168 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:18:38 -0800 Subject: [PATCH 1928/2303] Migrate allow from metadata to config if necessary, closes #2249 --- datasette/app.py | 6 +++--- datasette/utils/__init__.py | 9 +++++---- tests/test_permissions.py | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 373b3e95..af8cfeab 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,7 +74,7 @@ from .utils import ( find_spatialite, format_bytes, module_from_path, - move_plugins, + move_plugins_and_allow, move_table_config, parse_metadata, resolve_env_secrets, @@ -344,10 +344,10 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - # Move any "plugins" settings from metadata to config - updates them in place + # Move any "plugins" and "allow" settings from metadata to config - updates them in place metadata = metadata or {} config = config or {} - metadata, config = move_plugins(metadata, config) + metadata, config = move_plugins_and_allow(metadata, config) # Now migrate any known table configuration settings over as well metadata, config = move_table_config(metadata, config) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index fcaebe3f..f2cd7eb0 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1302,10 +1302,11 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: """ - Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. - After moving, recursively remove any keys in the source that are left empty. + Move 'plugins' and 'allow' keys from source to destination dictionary. Creates + hierarchy in destination if needed. After moving, recursively remove any keys + in the source that are left empty. """ source = copy.deepcopy(source) destination = copy.deepcopy(destination) @@ -1315,7 +1316,7 @@ def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: path = [] for key, value in list(src.items()): new_path = path + [key] - if key == "plugins": + if key in ("plugins", "allow"): # Navigate and create the hierarchy in destination if needed d = dest for step in path: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 9917b749..6713b850 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -89,10 +89,11 @@ def test_view_padlock(allow, expected_anon, expected_auth, path, padlock_client) ({"id": "root"}, 403, 200), ], ) -def test_view_database(allow, expected_anon, expected_auth): - with make_app_client( - config={"databases": {"fixtures": {"allow": allow}}} - ) as client: +@pytest.mark.parametrize("use_metadata", (True, False)) +def test_view_database(allow, expected_anon, expected_auth, use_metadata): + key = "metadata" if use_metadata else "config" + kwargs = {key: {"databases": {"fixtures": {"allow": allow}}}} + with make_app_client(**kwargs) as client: for path in ( "/fixtures", "/fixtures/compound_three_primary_keys", @@ -173,16 +174,19 @@ def test_database_list_respects_view_table(): ({"id": "root"}, 403, 200), ], ) -def test_view_table(allow, expected_anon, expected_auth): - with make_app_client( - config={ +@pytest.mark.parametrize("use_metadata", (True, False)) +def test_view_table(allow, expected_anon, expected_auth, use_metadata): + key = "metadata" if use_metadata else "config" + kwargs = { + key: { "databases": { "fixtures": { "tables": {"compound_three_primary_keys": {"allow": allow}} } } } - ) as client: + } + with make_app_client(**kwargs) as client: anon_response = client.get("/fixtures/compound_three_primary_keys") assert expected_anon == anon_response.status if allow and anon_response.status == 200: From ad01f9d3217f2447b0a321ee7731900dac7e3e6d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:24:24 -0800 Subject: [PATCH 1929/2303] 1.0a8 release notes Closes #2243 * Changelog for jinja2_environment_from_request and plugin_hook_slots * track_event() in changelog * Remove Using YAML for metadata section - no longer necessary now we show YAML and JSON examples everywhere. * Configuration via the command-line section - #2252 * JavaScript plugins in release notes, refs #2052 * /-/config in changelog, refs #2254 Refs #2052, #2156, #2243, #2247, #2249, #2252, #2254 --- docs/changelog.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4b928e3..1f47b429 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,83 @@ Changelog ========= +.. _v1_0_a8: + +1.0a8 (2024-02-01) +------------------ + +This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. + +Configuration +~~~~~~~~~~~~~ + +- Plugin configuration now lives in the :ref:`datasette.yaml configuration file `, passed to Datasette using the ``-c/--config`` option. Thanks, Alex Garcia. (:issue:`2093`) + + .. code-block:: bash + + datasette -c datasette.yaml + + Where ``datasette.yaml`` contains configuration that looks like this: + + .. code-block:: yaml + + plugins: + datasette-cluster-map: + latitude_column: xlat + longitude_column: xlon + + Previously plugins were configured in ``metadata.yaml``, which was confusing as plugin settings were unrelated to database and table metadata. +- The ``-s/--setting`` option can now be used to set plugin configuration as well. See :ref:`configuration_cli` for details. (:issue:`2252`) + + The above YAML configuration example using ``-s/--setting`` looks like this: + + .. code-block:: bash + + datasette mydatabase.db \ + -s plugins.datasette-cluster-map.latitude_column xlat \ + -s plugins.datasette-cluster-map.longitude_column xlon + +- The new ``/-/config`` page shows the current instance configuration, after redacting keys that could contain sensitive data such as API keys or passwords. (:issue:`2254`) + +- Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`) + +JavaScript plugins +~~~~~~~~~~~~~~~~~~ + +Datasette now includes a :ref:`JavaScript plugins mechanism `, allowing JavaScript to customize Datasette in a way that can collaborate with other plugins. + +This provides two initial hooks, with more to come in the future: + +- :ref:`makeAboveTablePanelConfigs() ` can add additional panels to the top of the table page. +- :ref:`makeColumnActions() ` can add additional actions to the column menu. + +Thanks `Cameron Yick `__ for contributing this feature. (`#2052 `__) + +Plugin hooks +~~~~~~~~~~~~ + +- New :ref:`plugin_hook_jinja2_environment_from_request` plugin hook, which can be used to customize the current Jinja environment based on the incoming request. This can be used to modify the template lookup path based on the incoming request hostname, among other things. (:issue:`2225`) +- New :ref:`family of template slot plugin hooks `: ``top_homepage``, ``top_database``, ``top_table``, ``top_row``, ``top_query``, ``top_canned_query``. Plugins can use these to provide additional HTML to be injected at the top of the corresponding pages. (:issue:`1191`) +- New :ref:`track_event() mechanism ` for plugins to emit and receive events when certain events occur within Datasette. (:issue:`2240`) + - Plugins can register additional event classes using :ref:`plugin_hook_register_events`. + - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method. + - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook. +- New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`) + +Documentation +~~~~~~~~~~~~~ + +- Documentation describing :ref:`how to write tests that use signed actor cookies ` using ``datasette.client.actor_cookie()``. (:issue:`1830`) +- Documentation on how to :ref:`register a plugin for the duration of a test `. (:issue:`2234`) +- The :ref:`configuration documentation ` now shows examples of both YAML and JSON for each setting. + +Minor fixes +~~~~~~~~~~~ + +- Datasette no longer attempts to run SQL queries in parallel when rendering a table page, as this was leading to some rare crashing bugs. (:issue:`2189`) +- Fixed warning: ``DeprecationWarning: pkg_resources is deprecated as an API`` (:issue:`2057`) +- Fixed bug where ``?_extra=columns`` parameter returned an incorrectly shaped response. (:issue:`2230`) + .. _v0_64_6: 0.64.6 (2023-12-22) From c64453a4a137ea815f4d94a99cdc4c7709734839 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:28:22 -0800 Subject: [PATCH 1930/2303] Fix the date on the 1.0a8 release (due to go tomorrow) Refs #2258 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f47b429..5e9e3ba2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a8: -1.0a8 (2024-02-01) +1.0a8 (2024-02-07) ------------------ This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. From d0089ba7769d58b97c5dc1e07969246502d97544 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:30:30 -0800 Subject: [PATCH 1931/2303] Note in changelog about datasette publish, refs #2195 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5e9e3ba2..e17dc2f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,8 @@ Configuration - Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`) +Note that the ``datasette publish`` command has not yet been updated to accept a ``datasette.yaml`` configuration file. This will be addressed in :issue:`2195` but for the moment you can include those settings in ``metadata.yaml`` instead. + JavaScript plugins ~~~~~~~~~~~~~~~~~~ From df8d1c055a48a36693d9caec3915eb93c1acefc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:59:58 -0800 Subject: [PATCH 1932/2303] Mention JS plugins in release intro --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e17dc2f8..a73635a8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog 1.0a8 (2024-02-07) ------------------ -This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. +This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks. Configuration ~~~~~~~~~~~~~ From 1e31821d9ff2bc81c62bcd442214e9098cf785a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:25:47 -0800 Subject: [PATCH 1933/2303] Link to events docs from changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a73635a8..dfcf492f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,6 +67,7 @@ Plugin hooks - Plugins can register additional event classes using :ref:`plugin_hook_register_events`. - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method. - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook. + - Datasette core now emits ``login``, ``logout``, ``create-token``, ``create-table``, ``drop-table``, ``insert-rows``, ``upsert-rows``, ``update-row``, ``delete-row`` events, :ref:`documented here `. - New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`) Documentation From e0794ddd52697812848c0b59f68e49a2e9361693 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:32:47 -0800 Subject: [PATCH 1934/2303] Link to annotated release notes blog post --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index dfcf492f..d164f71d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,8 @@ Changelog This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks. +See `Datasette 1.0a8: JavaScript plugins, new plugin hooks and plugin configuration in datasette.yaml `__ for an annotated version of these release notes. + Configuration ~~~~~~~~~~~~~ From 9989f257094daaf26e0cb0cebe31f17f19d4cad2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:34:05 -0800 Subject: [PATCH 1935/2303] Release 1.0a8 Refs Refs #2052, #2156, #2243, #2247, #2249, #2252, #2254, #2258 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 75d44727..e43b9918 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a8.dev1" +__version__ = "1.0a8" __version_info__ = tuple(__version__.split(".")) From 569aacd39bcc5529b2f463c89c616e3ada21c560 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 22:53:14 -0800 Subject: [PATCH 1936/2303] Link to /en/latest/ changelog --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57f17a5c..662f2a11 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) From 900d15bcb81c90d26cfebc3fe463c4b0465832c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:21:13 -0800 Subject: [PATCH 1937/2303] alter table support for /db/-/create API, refs #2101 --- datasette/default_permissions.py | 10 ++++- datasette/events.py | 25 +++++++++++ datasette/views/database.py | 61 +++++++++++++++++++++++--- docs/authentication.rst | 12 ++++++ tests/test_api_write.py | 74 ++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 8 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index d29dbe84..c13f2ed2 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -8,7 +8,6 @@ from typing import Union, Tuple @hookimpl def register_permissions(): return ( - # name, abbr, description, takes_database, takes_resource, default Permission( name="view-instance", abbr="vi", @@ -109,6 +108,14 @@ def register_permissions(): takes_resource=False, default=False, ), + Permission( + name="alter-table", + abbr="at", + description="Alter tables", + takes_database=True, + takes_resource=True, + default=False, + ), Permission( name="drop-table", abbr="dt", @@ -129,6 +136,7 @@ def permission_allowed_default(datasette, actor, action, resource): "debug-menu", "insert-row", "create-table", + "alter-table", "drop-table", "delete-row", "update-row", diff --git a/datasette/events.py b/datasette/events.py index 96244779..ae90972d 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -108,6 +108,30 @@ class DropTableEvent(Event): table: str +@dataclass +class AlterTableEvent(Event): + """ + Event name: ``alter-table`` + + A table has been altered. + + :ivar database: The name of the database where the table was altered + :type database: str + :ivar table: The name of the table that was altered + :type table: str + :ivar before_schema: The table's SQL schema before the alteration + :type before_schema: str + :ivar after_schema: The table's SQL schema after the alteration + :type after_schema: str + """ + + name = "alter-table" + database: str + table: str + before_schema: str + after_schema: str + + @dataclass class InsertRowsEvent(Event): """ @@ -203,6 +227,7 @@ def register_events(): LogoutEvent, CreateTableEvent, CreateTokenEvent, + AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/views/database.py b/datasette/views/database.py index 6d17b16c..bd55064f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,7 @@ import re import sqlite_utils import textwrap -from datasette.events import CreateTableEvent +from datasette.events import AlterTableEvent, CreateTableEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -792,7 +792,17 @@ class MagicParameters(dict): class TableCreateView(BaseView): name = "table-create" - _valid_keys = {"table", "rows", "row", "columns", "pk", "pks", "ignore", "replace"} + _valid_keys = { + "table", + "rows", + "row", + "columns", + "pk", + "pks", + "ignore", + "replace", + "alter", + } _supported_column_types = { "text", "integer", @@ -876,6 +886,20 @@ class TableCreateView(BaseView): ): return _error(["Permission denied - need insert-row"], 403) + alter = False + if rows or row: + if not table_exists: + # if table is being created for the first time, alter=True + alter = True + else: + # alter=True only if they request it AND they have permission + if data.get("alter"): + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=database_name + ): + return _error(["Permission denied - need alter-table"], 403) + alter = True + if columns: if rows or row: return _error(["Cannot specify columns with rows or row"]) @@ -939,10 +963,18 @@ class TableCreateView(BaseView): return _error(["pk cannot be changed for existing table"]) pks = actual_pks + initial_schema = None + if table_exists: + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + def create_table(conn): table = sqlite_utils.Database(conn)[table_name] if rows: - table.insert_all(rows, pk=pks or pk, ignore=ignore, replace=replace) + table.insert_all( + rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter + ) else: table.create( {c["name"]: c["type"] for c in columns}, @@ -954,6 +986,18 @@ class TableCreateView(BaseView): schema = await db.execute_write_fn(create_table) except Exception as e: return _error([str(e)]) + + if initial_schema is not None and initial_schema != schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=schema, + ) + ) + table_url = self.ds.absolute_url( request, self.ds.urls.table(db.name, table_name) ) @@ -970,11 +1014,14 @@ class TableCreateView(BaseView): } if rows: details["row_count"] = len(rows) - await self.ds.track_event( - CreateTableEvent( - request.actor, database=db.name, table=table_name, schema=schema + + if not table_exists: + # Only log creation if we created a table + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) ) - ) return Response.json(details, status=201) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8758765d..87ee6385 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1217,6 +1217,18 @@ Actor is allowed to create a database table. Default *deny*. +.. _permissions_alter_table: + +alter-table +----------- + +Actor is allowed to alter a database table. + +``resource`` - tuple: (string, string) + The name of the database, then the name of the table + +Default *deny*. + .. _permissions_drop_table: drop-table diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9caf9fdf..30cbfbab 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1349,3 +1349,77 @@ async def test_method_not_allowed(ds_write, path): "ok": False, "error": "Method not allowed", } + + +@pytest.mark.asyncio +async def test_create_uses_alter_by_default_for_new_table(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [ + { + "name": "Row 1", + } + ] + * 100 + + [ + {"name": "Row 2", "extra": "Extra"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + event = last_event(ds_write) + assert event.name == "create-table" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("has_alter_permission", (True,)) # False)) +async def test_create_using_alter_against_existing_table( + ds_write, has_alter_permission +): + token = write_token( + ds_write, permissions=["ir", "ct"] + (["at"] if has_alter_permission else []) + ) + # First create the table + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [ + { + "name": "Row 1", + } + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + # Now try to insert more rows using /-/create with alter=True + response2 = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [{"name": "Row 2", "extra": "extra"}], + "pk": "id", + "alter": True, + }, + headers=_headers(token), + ) + if not has_alter_permission: + assert response2.status_code == 403 + assert response2.json() == { + "ok": False, + "errors": ["Permission denied - need alter-table"], + } + else: + assert response2.status_code == 201 + # It should have altered the table + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema From 574687834f4bd8e73281731b8ff01bfe093fecb5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:33:41 -0800 Subject: [PATCH 1938/2303] Docs for /db/-/create alter: true option, refs #2101 --- docs/json_api.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index 16b997eb..68a0c984 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -834,19 +834,22 @@ To create a table, make a ``POST`` to ``//-/create``. This requires th The JSON here describes the table that will be created: -* ``table`` is the name of the table to create. This field is required. -* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. +* ``table`` is the name of the table to create. This field is required. +* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. - - ``name`` is the name of the column. This is required. - - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``name`` is the name of the column. This is required. + - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. -* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. +* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. - If the primary key is an integer column, it will be configured to automatically increment for each new record. + If the primary key is an integer column, it will be configured to automatically increment for each new record. - If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an integer ID column for you. + If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an auto-incrementing integer ID column for you. -* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. +* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. +* ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. +* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: @@ -925,6 +928,8 @@ You can avoid this error by passing the same ``"ignore": true`` or ``"replace": To use the ``"replace": true`` option you will also need the :ref:`permissions_update_row` permission. +Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`permissions_alter_table` permission. + .. _TableDropView: Dropping tables From b5ccc4d60844a24fdf91c3f317d8cda4a285a58d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:35:12 -0800 Subject: [PATCH 1939/2303] Test for Permission denied - need alter-table --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 30cbfbab..abf9a88a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1377,7 +1377,7 @@ async def test_create_uses_alter_by_default_for_new_table(ds_write): @pytest.mark.asyncio -@pytest.mark.parametrize("has_alter_permission", (True,)) # False)) +@pytest.mark.parametrize("has_alter_permission", (True, False)) async def test_create_using_alter_against_existing_table( ds_write, has_alter_permission ): From 528d89d1a3d6ff85047a7eef9a7623efdd2fb19f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:14:12 -0800 Subject: [PATCH 1940/2303] alter: true support for /-/insert and /-/upsert, refs #2101 --- datasette/views/table.py | 48 +++++++++++++++++++++++++++++++++++----- docs/json_api.rst | 6 ++++- tests/test_api_write.py | 48 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 50d2b3c2..fcbe253d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,7 +8,12 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted -from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent +from datasette.events import ( + AlterTableEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, +) from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -388,7 +393,7 @@ class TableInsertView(BaseView): extras = { key: value for key, value in data.items() if key not in ("row", "rows") } - valid_extras = {"return", "ignore", "replace"} + valid_extras = {"return", "ignore", "replace", "alter"} invalid_extras = extras.keys() - valid_extras if invalid_extras: return _errors( @@ -397,7 +402,6 @@ class TableInsertView(BaseView): if extras.get("ignore") and extras.get("replace"): return _errors(['Cannot use "ignore" and "replace" at the same time']) - # Validate columns of each row columns = set(await db.table_columns(table_name)) columns.update(pks_list) @@ -412,7 +416,7 @@ class TableInsertView(BaseView): ) ) invalid_columns = set(row.keys()) - columns - if invalid_columns: + if invalid_columns and not extras.get("alter"): errors.append( "Row {} has invalid columns: {}".format( i, ", ".join(sorted(invalid_columns)) @@ -476,10 +480,23 @@ class TableInsertView(BaseView): ignore = extras.get("ignore") replace = extras.get("replace") + alter = extras.get("alter") if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + initial_schema = None + if alter: + # Must have alter-table permission + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(database_name, table_name) + ): + return _error(["Permission denied for alter-table"], 403) + # Track initial schema to check if it changed later + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + should_return = bool(extras.get("return", False)) row_pk_values_for_later = [] if should_return and upsert: @@ -489,9 +506,13 @@ class TableInsertView(BaseView): table = sqlite_utils.Database(conn)[table_name] kwargs = {} if upsert: - kwargs["pk"] = pks[0] if len(pks) == 1 else pks + kwargs = { + "pk": pks[0] if len(pks) == 1 else pks, + "alter": alter, + } else: - kwargs = {"ignore": ignore, "replace": replace} + # Insert + kwargs = {"ignore": ignore, "replace": replace, "alter": alter} if should_return and not upsert: rowids = [] method = table.upsert if upsert else table.insert @@ -552,6 +573,21 @@ class TableInsertView(BaseView): ) ) + if initial_schema is not None: + after_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + if initial_schema != after_schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=after_schema, + ) + ) + return Response.json(result, status=200 if upsert else 201) diff --git a/docs/json_api.rst b/docs/json_api.rst index 68a0c984..000f532d 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -618,6 +618,8 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _TableUpsertView: Upserting rows @@ -728,6 +730,8 @@ When using upsert you must provide the primary key column (or columns if the tab If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowUpdateView: Updating a row @@ -849,7 +853,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. -* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index abf9a88a..9e1d73e0 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -60,6 +60,27 @@ async def test_insert_row(ds_write): assert not event.replace +@pytest.mark.asyncio +async def test_insert_row_alter(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/docs/-/insert", + json={ + "row": {"title": "Test", "score": 1.2, "age": 5, "extra": "extra"}, + "alter": True, + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert response.json()["ok"] is True + assert response.json()["rows"][0]["extra"] == "extra" + # Analytics event + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema + + @pytest.mark.asyncio @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): @@ -278,16 +299,27 @@ async def test_insert_rows(ds_write, return_rows): 403, ["Permission denied: need both insert-row and update-row"], ), + # Alter table forbidden without alter permission + ( + "/data/docs/-/upsert", + {"rows": [{"id": 1, "title": "One", "extra": "extra"}], "alter": True}, + "update-and-insert-but-no-alter", + 403, + ["Permission denied for alter-table"], + ), ), ) async def test_insert_or_upsert_row_errors( ds_write, path, input, special_case, expected_status, expected_errors ): - token = write_token(ds_write) + token_permissions = [] if special_case == "insert-but-not-update": - token = write_token(ds_write, permissions=["ir", "vi"]) + token_permissions = ["ir", "vi"] if special_case == "update-but-not-insert": - token = write_token(ds_write, permissions=["ur", "vi"]) + token_permissions = ["ur", "vi"] + if special_case == "update-and-insert-but-no-alter": + token_permissions = ["ur", "ir"] + token = write_token(ds_write, permissions=token_permissions) if special_case == "duplicate_id": await ds_write.get_database("data").execute_write( "insert into docs (id) values (1)" @@ -309,7 +341,9 @@ async def test_insert_or_upsert_row_errors( actor_response = ( await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) ).json() - print(actor_response) + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -434,6 +468,12 @@ async def test_insert_ignore_replace( {"id": 1, "title": "Two", "score": 1}, ], ), + ( + # Upsert with an alter + {"rows": [{"id": 1, "title": "One"}], "pk": "id"}, + {"rows": [{"id": 1, "title": "Two", "extra": "extra"}], "alter": True}, + [{"id": 1, "title": "Two", "extra": "extra"}], + ), ), ) @pytest.mark.parametrize("should_return", (False, True)) From 4e944c29e4a208f173f15ac2df6253ff90f6466f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:19:47 -0800 Subject: [PATCH 1941/2303] Corrected path used in test_update_row_check_permission --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9e1d73e0..b43ee5a6 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -633,7 +633,7 @@ async def test_update_row_check_permission(ds_write, scenario): pk = await _insert_row(ds_write) - path = "/data/{}/{}/-/delete".format( + path = "/data/{}/{}/-/update".format( "docs" if scenario != "bad_table" else "bad_table", pk ) From c954795f9af9007e7c04d9b472bfd2faef647a87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:30:48 -0800 Subject: [PATCH 1942/2303] alter: true for row/-/update, refs #2101 --- datasette/views/row.py | 12 +++++++++++- docs/json_api.rst | 2 ++ tests/test_api_write.py | 43 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 7b43b893..4d20e41a 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -237,11 +237,21 @@ class RowUpdateView(BaseView): if not "update" in data or not isinstance(data["update"], dict): return _error(["JSON must contain an update dictionary"]) + invalid_keys = set(data.keys()) - {"update", "return", "alter"} + if invalid_keys: + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + update = data["update"] + alter = data.get("alter") + if alter and not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(resolved.db.name, resolved.table) + ): + return _error(["Permission denied for alter-table"], 403) + def update_row(conn): sqlite_utils.Database(conn)[resolved.table].update( - resolved.pk_values, update + resolved.pk_values, update, alter=alter ) try: diff --git a/docs/json_api.rst b/docs/json_api.rst index 000f532d..c401d97e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -787,6 +787,8 @@ The returned JSON will look like this: Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowDeleteView: Deleting a row diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b43ee5a6..7cc38674 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -622,12 +622,17 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table")) +@pytest.mark.parametrize( + "scenario", ("no_token", "no_perm", "bad_table", "cannot_alter") +) async def test_update_row_check_permission(ds_write, scenario): if scenario == "no_token": token = "bad_token" elif scenario == "no_perm": token = write_token(ds_write, actor_id="not-root") + elif scenario == "cannot_alter": + # update-row but no alter-table: + token = write_token(ds_write, permissions=["ur"]) else: token = write_token(ds_write) @@ -637,9 +642,13 @@ async def test_update_row_check_permission(ds_write, scenario): "docs" if scenario != "bad_table" else "bad_table", pk ) + json_body = {"update": {"title": "New title"}} + if scenario == "cannot_alter": + json_body["alter"] = True + response = await ds_write.client.post( path, - json={"update": {"title": "New title"}}, + json=json_body, headers=_headers(token), ) assert response.status_code == 403 if scenario in ("no_token", "bad_token") else 404 @@ -651,6 +660,36 @@ async def test_update_row_check_permission(ds_write, scenario): ) +@pytest.mark.asyncio +async def test_update_row_invalid_key(ds_write): + token = write_token(ds_write) + + pk = await _insert_row(ds_write) + + path = "/data/docs/{}/-/update".format(pk) + response = await ds_write.client.post( + path, + json={"update": {"title": "New title"}, "bad_key": 1}, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == {"ok": False, "errors": ["Invalid keys: bad_key"]} + + +@pytest.mark.asyncio +async def test_update_row_alter(ds_write): + token = write_token(ds_write, permissions=["ur", "at"]) + pk = await _insert_row(ds_write) + path = "/data/docs/{}/-/update".format(pk) + response = await ds_write.client.post( + path, + json={"update": {"title": "New title", "extra": "extra"}, "alter": True}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == {"ok": True} + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", From c62cfa6de836667834b5b9a7fef2b861307ac998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:32:36 -0800 Subject: [PATCH 1943/2303] Fix upsert test to detect new alter-table event --- tests/test_api_write.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 7cc38674..2d127e1a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -499,10 +499,14 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): # Analytics event event = last_event(ds_write) - assert event.name == "upsert-rows" - assert event.num_rows == 1 assert event.database == "data" assert event.table == "upsert_test" + if input.get("alter"): + assert event.name == "alter-table" + assert "extra" in event.after_schema + else: + assert event.name == "upsert-rows" + assert event.num_rows == 1 if should_return: # We only expect it to return rows corresponding to those we sent From dcd9ea3622520c99a1f921766dc36ca4c0e3b796 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 14:14:58 -0800 Subject: [PATCH 1944/2303] datasette-events-db as an example of track_events() --- docs/plugin_hooks.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 16f5cebb..960dc9b6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1817,6 +1817,7 @@ This example plugin logs details of all events to standard error: ) print(msg, file=sys.stderr, flush=True) +Example: `datasette-events-db `_ .. _plugin_hook_register_events: From bd9ed62e5d8821f9dc9e035b195452980c900b3c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 18:58:12 -0800 Subject: [PATCH 1945/2303] Make ds.pemrission_allawed(..., default=) a keyword-only argument, refs #2262 --- datasette/app.py | 2 +- datasette/views/table.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index af8cfeab..d943b97b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -896,7 +896,7 @@ class Datasette: await await_me_maybe(hook) async def permission_allowed( - self, actor, action, resource=None, default=DEFAULT_NOT_SET + self, actor, action, resource=None, *, default=DEFAULT_NOT_SET ): """Check permissions using the permissions_allowed plugin hook""" result = None diff --git a/datasette/views/table.py b/datasette/views/table.py index fcbe253d..1c187692 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -444,10 +444,10 @@ class TableInsertView(BaseView): # Must have insert-row AND upsert-row permissions if not ( await self.ds.permission_allowed( - request.actor, "insert-row", database_name, table_name + request.actor, "insert-row", resource=(database_name, table_name) ) and await self.ds.permission_allowed( - request.actor, "update-row", database_name, table_name + request.actor, "update-row", resource=(database_name, table_name) ) ): return _error( From 398a92cf1e54f868ff80f01634d6a814d1c61998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 20:12:22 -0800 Subject: [PATCH 1946/2303] Include database in name of _execute_writes thread, closes #2265 --- datasette/database.py | 3 +++ tests/test_api.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index fba81496..becb552c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -196,6 +196,9 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) + self._write_thread.name = "_execute_writes for database {}".format( + self.name + ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() diff --git a/tests/test_api.py b/tests/test_api.py index 8cb73dbb..7a25b55e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -780,7 +780,11 @@ async def test_threads_json(ds_client): expected_keys = {"threads", "num_threads"} if sys.version_info >= (3, 7, 0): expected_keys.update({"tasks", "num_tasks"}) - assert set(response.json().keys()) == expected_keys + data = response.json() + assert set(data.keys()) == expected_keys + # Should be at least one _execute_writes thread for __INTERNAL__ + thread_names = [thread["name"] for thread in data["threads"]] + assert "_execute_writes for database __INTERNAL__" in thread_names @pytest.mark.asyncio From 5d7997418664bcdfdba714c16bd5a67c241e8740 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Feb 2024 07:19:47 -0800 Subject: [PATCH 1947/2303] Call them "notable events" --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 960dc9b6..5372ea5e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1765,7 +1765,7 @@ Returns HTML to be displayed at the top of the canned query page. Event tracking -------------- -Datasette includes an internal mechanism for tracking analytical events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. +Datasette includes an internal mechanism for tracking notable events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. Plugins can register to receive events using the ``track_event`` plugin hook. From b89cac3b6a63929325c067d0cf2d5748e4bf4d2e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Feb 2024 18:23:54 -0800 Subject: [PATCH 1948/2303] Use MD5 usedforsecurity=False on Python 3.9 and higher to pass FIPS Closes #2270 --- datasette/database.py | 4 ++-- datasette/utils/__init__.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index becb552c..707d8f85 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,7 +1,6 @@ import asyncio from collections import namedtuple from pathlib import Path -import hashlib import janus import queue import sys @@ -15,6 +14,7 @@ from .utils import ( detect_spatialite, get_all_foreign_keys, get_outbound_foreign_keys, + md5_not_usedforsecurity, sqlite_timelimit, sqlite3, table_columns, @@ -74,7 +74,7 @@ class Database: def color(self): if self.hash: return self.hash[:6] - return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] + return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): if self.path: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index f2cd7eb0..e3637f7a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -713,7 +713,7 @@ def to_css_class(s): """ if css_class_re.match(s): return s - md5_suffix = hashlib.md5(s.encode("utf8")).hexdigest()[:6] + md5_suffix = md5_not_usedforsecurity(s)[:6] # Strip leading _, - s = s.lstrip("_").lstrip("-") # Replace any whitespace with hyphens @@ -1401,3 +1401,11 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict: return data return redact(original) + + +def md5_not_usedforsecurity(s): + try: + return hashlib.md5(s.encode("utf8"), usedforsecurity=False).hexdigest() + except TypeError: + # For Python 3.8 which does not support usedforsecurity=False + return hashlib.md5(s.encode("utf8")).hexdigest() From 97de4d6362ce5a6c1e3520ecdc73b305ab269910 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 15 Feb 2024 21:35:49 -0800 Subject: [PATCH 1949/2303] Use transaction in delete_everything(), closes #2273 --- datasette/utils/internal_db.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 2e5ac53b..dd0d3a9d 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -69,18 +69,20 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) + with conn: + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) await internal_db.execute_write_fn(delete_everything) From 47e29e948b26e8c003a03b4fc46cb635134a3958 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 10:05:18 -0800 Subject: [PATCH 1950/2303] Better comments in permission_allowed_default() --- datasette/default_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index c13f2ed2..757b3a46 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -144,7 +144,7 @@ def permission_allowed_default(datasette, actor, action, resource): if actor and actor.get("id") == "root": return True - # Resolve metadata view permissions + # Resolve view permissions in allow blocks in configuration if action in ( "view-instance", "view-database", @@ -158,7 +158,7 @@ def permission_allowed_default(datasette, actor, action, resource): if result is not None: return result - # Check custom permissions: blocks + # Resolve custom permissions: blocks in configuration result = await _resolve_config_permissions_blocks( datasette, actor, action, resource ) From 232a30459babebece653795d136fb6516444ecf0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 12:56:39 -0800 Subject: [PATCH 1951/2303] DATASETTE_TRACE_PLUGINS setting, closes #2274 --- datasette/plugins.py | 24 ++++++++++++++++++++++++ docs/writing_plugins.rst | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/datasette/plugins.py b/datasette/plugins.py index f7a1905f..3769a209 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,6 +1,7 @@ import importlib import os import pluggy +from pprint import pprint import sys from . import hookspecs @@ -33,6 +34,29 @@ DEFAULT_PLUGINS = ( pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) +DATASETTE_TRACE_PLUGINS = os.environ.get("DATASETTE_TRACE_PLUGINS", None) + + +def before(hook_name, hook_impls, kwargs): + print(file=sys.stderr) + print(f"{hook_name}:", file=sys.stderr) + pprint(kwargs, width=40, indent=4, stream=sys.stderr) + print("Hook implementations:", file=sys.stderr) + pprint(hook_impls, width=40, indent=4, stream=sys.stderr) + + +def after(outcome, hook_name, hook_impls, kwargs): + results = outcome.get_result() + if not isinstance(results, list): + results = [results] + print(f"Results:", file=sys.stderr) + pprint(results, width=40, indent=4, stream=sys.stderr) + + +if DATASETTE_TRACE_PLUGINS: + pm.add_hookcall_monitoring(before, after) + + DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 5c8bc4c6..2bc6bd24 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -7,6 +7,30 @@ You can write one-off plugins that apply to just one Datasette instance, or you Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 90 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. +.. _writing_plugins_tracing: + +Tracing plugin hooks +-------------------- + +The ``DATASETTE_TRACE_PLUGINS`` environment variable turns on detailed tracing showing exactly which hooks are being run. This can be useful for understanding how Datasette is using your plugin. + +.. code-block:: bash + + DATASETTE_TRACE_PLUGINS=1 datasette mydb.db + +Example output:: + + actor_from_request: + { 'datasette': , + 'request': } + Hook implementations: + [ >, + >, + >] + Results: + [{'id': 'root'}] + + .. _writing_plugins_one_off: Writing one-off plugins From 8bfa3a51c222d653f45fb48ebcb6957a85f9ea6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:29:39 -0800 Subject: [PATCH 1952/2303] Consider every plugins opinion in datasette.permission_allowed() Closes #2275, refs #2262 --- datasette/app.py | 14 +++++++++++++- docs/authentication.rst | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index d943b97b..8591af6a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -903,6 +903,8 @@ class Datasette: # Use default from registered permission, if available if default is DEFAULT_NOT_SET and action in self.permissions: default = self.permissions[action].default + opinions = [] + # Every plugin is consulted for their opinion for check in pm.hook.permission_allowed( datasette=self, actor=actor, @@ -911,9 +913,19 @@ class Datasette: ): check = await await_me_maybe(check) if check is not None: - result = check + opinions.append(check) + + result = None + # If any plugin said False it's false - the veto rule + if any(not r for r in opinions): + result = False + elif any(r for r in opinions): + # Otherwise, if any plugin said True it's true + result = True + used_default = False if result is None: + # No plugin expressed an opinion, so use the default result = default used_default = True self._permission_checks.append( diff --git a/docs/authentication.rst b/docs/authentication.rst index 87ee6385..a8dc5637 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -71,6 +71,23 @@ Datasette's built-in view permissions (``view-database``, ``view-table`` etc) de Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file. +.. _authentication_permissions_explained: + +How permissions are resolved +---------------------------- + +The :ref:`datasette.permission_allowed(actor, action, resource=None, default=...)` method is called to check if an actor is allowed to perform a specific action. + +This method asks every plugin that implements the :ref:`plugin_hook_permission_allowed` hook if the actor is allowed to perform the action. + +Each plugin can return ``True`` to indicate that the actor is allowed to perform the action, ``False`` if they are not allowed and ``None`` if the plugin has no opinion on the matter. + +``False`` acts as a veto - if any plugin returns ``False`` then the permission check is denied. Otherwise, if any plugin returns ``True`` then the permission check is allowed. + +The ``resource`` argument can be used to specify a specific resource that the action is being performed against. Some permissions, such as ``view-instance``, do not involve a resource. Others such as ``view-database`` have a resource that is a string naming the database. Permissions that take both a database name and the name of a table, view or canned query within that database use a resource that is a tuple of two strings, ``(database_name, resource_name)``. + +Plugins that implement the ``permission_allowed()`` hook can decide if they are going to consider the provided resource or not. + .. _authentication_permissions_allow: Defining permissions with "allow" blocks From 244f3ff83aac19e96fab85a95ddde349079a9827 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:39:57 -0800 Subject: [PATCH 1953/2303] Test demonstrating fix for permisisons bug in #2262 --- tests/test_api_write.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2d127e1a..2aea699b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -365,6 +365,41 @@ async def test_insert_or_upsert_row_errors( assert before_count == after_count +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed", (True, False)) +async def test_upsert_permissions_per_table(ds_write, allowed): + # https://github.com/simonw/datasette/issues/2262 + token = "dstok_{}".format( + ds_write.sign( + { + "a": "root", + "token": "dstok", + "t": int(time.time()), + "_r": { + "r": { + "data": { + "docs" if allowed else "other": ["ir", "ur"], + } + } + }, + }, + namespace="token", + ) + ) + response = await ds_write.client.post( + "/data/docs/-/upsert", + json={"rows": [{"id": 1, "title": "One"}]}, + headers={ + "Authorization": "Bearer {}".format(token), + }, + ) + if allowed: + assert response.status_code == 200 + assert response.json()["ok"] is True + else: + assert response.status_code == 403 + + @pytest.mark.asyncio @pytest.mark.parametrize( "ignore,replace,expected_rows", From 3a999a85fb431594ccee1adf38721de03de19500 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:58:33 -0800 Subject: [PATCH 1954/2303] Fire insert-rows on /db/-/create if rows were inserted, refs #2260 --- datasette/views/database.py | 13 ++++++- tests/test_api_write.py | 71 +++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index bd55064f..2a8b40cc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,7 @@ import re import sqlite_utils import textwrap -from datasette.events import AlterTableEvent, CreateTableEvent +from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -1022,6 +1022,17 @@ class TableCreateView(BaseView): request.actor, database=db.name, table=table_name, schema=schema ) ) + if rows: + await self.ds.track_event( + InsertRowsEvent( + request.actor, + database=db.name, + table=table_name, + num_rows=len(rows), + ignore=ignore, + replace=replace, + ) + ) return Response.json(details, status=201) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2aea699b..0eb915ba 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -857,13 +857,14 @@ async def test_drop_table(ds_write, scenario): @pytest.mark.asyncio @pytest.mark.parametrize( - "input,expected_status,expected_response", + "input,expected_status,expected_response,expected_events", ( # Permission error with a bad token ( {"table": "bad", "row": {"id": 1}}, 403, {"ok": False, "errors": ["Permission denied"]}, + [], ), # Successful creation with columns: ( @@ -910,6 +911,7 @@ async def test_drop_table(ds_write, scenario): ")" ), }, + ["create-table"], ), # Successful creation with rows: ( @@ -945,6 +947,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 2, }, + ["create-table", "insert-rows"], ), # Successful creation with row: ( @@ -973,6 +976,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Create with row and no primary key ( @@ -992,6 +996,7 @@ async def test_drop_table(ds_write, scenario): "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Create table with compound primary key ( @@ -1013,6 +1018,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Error: Table is required ( @@ -1024,6 +1030,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Table is required"], }, + [], ), # Error: Invalid table name ( @@ -1036,6 +1043,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Invalid table name"], }, + [], ), # Error: JSON must be an object ( @@ -1045,6 +1053,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["JSON must be an object"], }, + [], ), # Error: Cannot specify columns with rows or row ( @@ -1058,6 +1067,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Cannot specify columns with rows or row"], }, + [], ), # Error: columns, rows or row is required ( @@ -1069,6 +1079,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns, rows or row is required"], }, + [], ), # Error: columns must be a list ( @@ -1081,6 +1092,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns must be a list"], }, + [], ), # Error: columns must be a list of objects ( @@ -1093,6 +1105,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns must be a list of objects"], }, + [], ), # Error: Column name is required ( @@ -1105,6 +1118,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Column name is required"], }, + [], ), # Error: Unsupported column type ( @@ -1117,6 +1131,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Unsupported column type: bad"], }, + [], ), # Error: Duplicate column name ( @@ -1132,6 +1147,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Duplicate column name: id"], }, + [], ), # Error: rows must be a list ( @@ -1144,6 +1160,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["rows must be a list"], }, + [], ), # Error: rows must be a list of objects ( @@ -1156,6 +1173,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["rows must be a list of objects"], }, + [], ), # Error: pk must be a string ( @@ -1169,6 +1187,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["pk must be a string"], }, + [], ), # Error: Cannot specify both pk and pks ( @@ -1183,6 +1202,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Cannot specify both pk and pks"], }, + [], ), # Error: pks must be a list ( @@ -1196,12 +1216,14 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["pks must be a list"], }, + [], ), # Error: pks must be a list of strings ( {"table": "bad", "row": {"id": 1, "name": "Row 1"}, "pks": [1, 2]}, 400, {"ok": False, "errors": ["pks must be a list of strings"]}, + [], ), # Error: ignore and replace are mutually exclusive ( @@ -1217,6 +1239,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace are mutually exclusive"], }, + [], ), # ignore and replace require row or rows ( @@ -1230,6 +1253,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require row or rows"], }, + [], ), # ignore and replace require pk or pks ( @@ -1243,6 +1267,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require pk or pks"], }, + [], ), ( { @@ -1255,10 +1280,14 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require pk or pks"], }, + [], ), ), ) -async def test_create_table(ds_write, input, expected_status, expected_response): +async def test_create_table( + ds_write, input, expected_status, expected_response, expected_events +): + ds_write._tracked_events = [] # Special case for expected status of 403 if expected_status == 403: token = "bad_token" @@ -1272,12 +1301,9 @@ async def test_create_table(ds_write, input, expected_status, expected_response) assert response.status_code == expected_status data = response.json() assert data == expected_response - # create-table event - if expected_status == 201: - event = last_event(ds_write) - assert event.name == "create-table" - assert event.actor == {"id": "root", "token": "dstok"} - assert event.schema.startswith("CREATE TABLE ") + # Should have tracked the expected events + events = ds_write._tracked_events + assert [e.name for e in events] == expected_events @pytest.mark.asyncio @@ -1376,6 +1402,8 @@ async def test_create_table_ignore_replace(ds_write, input, expected_rows_after) ) assert first_response.status_code == 201 + ds_write._tracked_events = [] + # Try a second time second_response = await ds_write.client.post( "/data/-/create", @@ -1387,6 +1415,10 @@ async def test_create_table_ignore_replace(ds_write, input, expected_rows_after) rows = await ds_write.client.get("/data/test_insert_replace.json?_shape=array") assert rows.json() == expected_rows_after + # Check it fired the right events + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["insert-rows"] + @pytest.mark.asyncio async def test_create_table_error_if_pk_changed(ds_write): @@ -1471,6 +1503,7 @@ async def test_method_not_allowed(ds_write, path): @pytest.mark.asyncio async def test_create_uses_alter_by_default_for_new_table(ds_write): + ds_write._tracked_events = [] token = write_token(ds_write) response = await ds_write.client.post( "/data/-/create", @@ -1490,8 +1523,8 @@ async def test_create_uses_alter_by_default_for_new_table(ds_write): headers=_headers(token), ) assert response.status_code == 201 - event = last_event(ds_write) - assert event.name == "create-table" + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["create-table", "insert-rows"] @pytest.mark.asyncio @@ -1517,6 +1550,8 @@ async def test_create_using_alter_against_existing_table( headers=_headers(token), ) assert response.status_code == 201 + + ds_write._tracked_events = [] # Now try to insert more rows using /-/create with alter=True response2 = await ds_write.client.post( "/data/-/create", @@ -1536,8 +1571,16 @@ async def test_create_using_alter_against_existing_table( } else: assert response2.status_code == 201 + + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["alter-table", "insert-rows"] + # It should have altered the table - event = last_event(ds_write) - assert event.name == "alter-table" - assert "extra" not in event.before_schema - assert "extra" in event.after_schema + alter_event = ds_write._tracked_events[0] + assert alter_event.name == "alter-table" + assert "extra" not in alter_event.before_schema + assert "extra" in alter_event.after_schema + + insert_rows_event = ds_write._tracked_events[1] + assert insert_rows_event.name == "insert-rows" + assert insert_rows_event.num_rows == 1 From 9906f937d92c79dcc457cb057d7222ed70aef0e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 14:32:47 -0800 Subject: [PATCH 1955/2303] Release 1.0a9 Refs #2101, #2260, #2262, #2265, #2270, #2273, #2274, #2275 Closes #2276 --- datasette/version.py | 2 +- docs/changelog.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e43b9918..f5e07ac8 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a8" +__version__ = "1.0a9" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index d164f71d..e567f422 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,51 @@ Changelog ========= +.. _v1_0_a9: + +1.0a9 (2024-02-16) +------------------ + +This alpha release adds basic alter table support to the Datasette Write API and fixes a permissions bug relating to the ``/upsert`` API endpoint. + +Alter table support for create, insert, upsert and update +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`JSON write API ` can now be used to apply simple alter table schema changes, provided the acting actor has the new :ref:`permissions_alter_table` permission. (:issue:`2101`) + +The only alter operation supported so far is adding new columns to an existing table. + +* The :ref:`/db/-/create ` API now adds new columns during large operations to create a table based on incoming example ``"rows"``, in the case where one of the later rows includes columns that were not present in the earlier batches. This requires the ``create-table`` but not the ``alter-table`` permission. +* When ``/db/-/create`` is called with rows in a situation where the table may have been already created, an ``"alter": true`` key can be included to indicate that any missing columns from the new rows should be added to the table. This requires the ``alter-table`` permission. +* :ref:`/db/table/-/insert ` and :ref:`/db/table/-/upsert ` and :ref:`/db/table/row-pks/-/update ` all now also accept ``"alter": true``, depending on the ``alter-table`` permission. + +Operations that alter a table now fire the new :ref:`alter-table event `. + +Permissions fix for the upsert API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`/database/table/-/upsert API ` had a minor permissions bug, only affecting Datasette instances that had configured the ``insert-row`` and ``update-row`` permissions to apply to a specific table rather than the database or instance as a whole. Full details in issue :issue:`2262`. + +To avoid similar mistakes in the future the :ref:`datasette.permission_allowed() ` method now specifies ``default=`` as a keyword-only argument. + +Permission checks now consider opinions from every plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`datasette.permission_allowed() ` method previously consulted every plugin that implemented the :ref:`permission_allowed() ` plugin hook and obeyed the opinion of the last plugin to return a value. (:issue:`2275`) + +Datasette now consults every plugin and checks to see if any of them returned ``False`` (the veto rule), and if none of them did, it then checks to see if any of them returned ``True``. + +This is explained at length in the new documentation covering :ref:`authentication_permissions_explained`. + +Other changes +~~~~~~~~~~~~~ + +- The new :ref:`DATASETTE_TRACE_PLUGINS=1 environment variable ` turns on detailed trace output for every executed plugin hook, useful for debugging and understanding how the plugin system works at a low level. (:issue:`2274`) +- Datasette on Python 3.9 or above marks its non-cryptographic uses of the MD5 hash function as ``usedforsecurity=False``, for compatibility with FIPS systems. (:issue:`2270`) +- SQL relating to :ref:`internals_internal` now executes inside a transaction, avoiding a potential database locked error. (:issue:`2273`) +- The ``/-/threads`` debug page now identifies the database in the name associated with each dedicated write thread. (:issue:`2265`) +- The ``/db/-/create`` API now fires a ``insert-rows`` event if rows were inserted after the table was created. (:issue:`2260`) + .. _v1_0_a8: 1.0a8 (2024-02-07) From e1c80efff8f4b0a53619546bb03e6dfd6cb42a32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 14:43:36 -0800 Subject: [PATCH 1956/2303] Note about activating alpha documentation versions on ReadTheDocs --- docs/contributing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ef022a4d..b678e637 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -254,6 +254,7 @@ Datasette releases are performed using tags. When a new release is published on * Re-point the "latest" tag on Docker Hub to the new image * Build a wheel bundle of the underlying Python source code * Push that new wheel up to PyPI: https://pypi.org/project/datasette/ +* If the release is an alpha, navigate to https://readthedocs.org/projects/datasette/versions/ and search for the tag name in the "Activate a version" filter, then mark that version as "active" to ensure it will appear on the public ReadTheDocs documentation site. To deploy new releases you will need to have push access to the main Datasette GitHub repository. From 5e0e440f2c8a0771b761b02801456e55e95e2a04 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:28:15 -0800 Subject: [PATCH 1957/2303] database.execute_write_fn(transaction=True) parameter, closes #2277 --- datasette/database.py | 31 +++++++++++++++++++++++-------- docs/internals.rst | 15 ++++++++++----- tests/test_internals_database.py | 27 +++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 707d8f85..d34aac73 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -179,17 +179,25 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) - async def execute_write_fn(self, fn, block=True): + async def execute_write_fn(self, fn, block=True, transaction=True): if self.ds.executor is None: # non-threaded mode if self._write_connection is None: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) - return fn(self._write_connection) + if transaction: + with self._write_connection: + return fn(self._write_connection) + else: + return fn(self._write_connection) else: - return await self._send_to_write_thread(fn, block) + return await self._send_to_write_thread( + fn, block=block, transaction=transaction + ) - async def _send_to_write_thread(self, fn, block=True, isolated_connection=False): + async def _send_to_write_thread( + self, fn, block=True, isolated_connection=False, transaction=True + ): if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: @@ -202,7 +210,9 @@ class Database: self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() - self._write_queue.put(WriteTask(fn, task_id, reply_queue, isolated_connection)) + self._write_queue.put( + WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + ) if block: result = await reply_queue.async_q.get() if isinstance(result, Exception): @@ -244,7 +254,11 @@ class Database: pass else: try: - result = task.fn(conn) + if task.transaction: + with conn: + result = task.fn(conn) + else: + result = task.fn(conn) except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() @@ -554,13 +568,14 @@ class Database: class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection") + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") - def __init__(self, fn, task_id, reply_queue, isolated_connection): + def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): self.fn = fn self.task_id = task_id self.reply_queue = reply_queue self.isolated_connection = isolated_connection + self.transaction = transaction class QueryInterrupted(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index bd7a70b5..6ca62423 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1010,7 +1010,9 @@ You can pass additional SQL parameters as a tuple or dictionary. The method will block until the operation is completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. -If you pass ``block=False`` this behaviour changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. +If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. + +Each call to ``execute_write()`` will be executed inside a transaction. .. _database_execute_write_script: @@ -1019,6 +1021,8 @@ await db.execute_write_script(sql, block=True) Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() `__ method. +Each call to ``execute_write_script()`` will be executed inside a transaction. + .. _database_execute_write_many: await db.execute_write_many(sql, params_seq, block=True) @@ -1033,10 +1037,12 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() 5") - conn.commit() return conn.execute( "select count(*) from some_table" ).fetchone()[0] @@ -1069,7 +1074,7 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. -If you see ``OperationalError: database table is locked`` errors you should check that you remembered to explicitly call ``conn.commit()`` in your write function. +By default your function will be executed inside a transaction. You can pass ``transaction=False`` to disable this behavior, though if you do that you should be careful to manually apply transactions - ideally using the ``with conn:`` pattern, or you may see ``OperationalError: database table is locked`` errors. If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index dd68a6cb..57e75046 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -66,6 +66,33 @@ async def test_execute_fn(db): assert 2 == await db.execute_fn(get_1_plus_1) +@pytest.mark.asyncio +async def test_execute_fn_transaction_false(): + datasette = Datasette(memory=True) + db = datasette.add_memory_database("test_execute_fn_transaction_false") + + def run(conn): + try: + with conn: + conn.execute("create table foo (id integer primary key)") + conn.execute("insert into foo (id) values (44)") + # Table should exist + assert ( + conn.execute( + 'select count(*) from sqlite_master where name = "foo"' + ).fetchone()[0] + == 1 + ) + assert conn.execute("select id from foo").fetchall()[0][0] == 44 + raise ValueError("Cancel commit") + except ValueError: + pass + # Row should NOT exist + assert conn.execute("select count(*) from foo").fetchone()[0] == 0 + + await db.execute_write_fn(run, transaction=False) + + @pytest.mark.parametrize( "tables,exists", ( From 10f9ba1a0050724ba47a089861606bef58a4087f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:51:19 -0800 Subject: [PATCH 1958/2303] Take advantage of execute_write_fn(transaction=True) A bunch of places no longer need to do manual transaction handling thanks to this change. Refs #2277 --- datasette/database.py | 9 +++------ datasette/utils/internal_db.py | 27 +++++++++++++-------------- tests/test_internals_database.py | 10 ++++------ 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d34aac73..4e590d3a 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -123,8 +123,7 @@ class Database: async def execute_write(self, sql, params=None, block=True): def _inner(conn): - with conn: - return conn.execute(sql, params or []) + return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn(_inner, block=block) @@ -132,8 +131,7 @@ class Database: async def execute_write_script(self, sql, block=True): def _inner(conn): - with conn: - return conn.executescript(sql) + return conn.executescript(sql) with trace("sql", database=self.name, sql=sql.strip(), executescript=True): results = await self.execute_write_fn(_inner, block=block) @@ -149,8 +147,7 @@ class Database: count += 1 yield param - with conn: - return conn.executemany(sql, count_params(params_seq)), count + return conn.executemany(sql, count_params(params_seq)), count with trace( "sql", database=self.name, sql=sql.strip(), executemany=True diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index dd0d3a9d..dbfcceb4 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -69,20 +69,19 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - with conn: - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) await internal_db.execute_write_fn(delete_everything) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 57e75046..1c155cf3 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -501,9 +501,8 @@ async def test_execute_write_has_correctly_prepared_connection(db): @pytest.mark.asyncio async def test_execute_write_fn_block_false(db): def write_fn(conn): - with conn: - conn.execute("delete from roadside_attractions where pk = 1;") - row = conn.execute("select count(*) from roadside_attractions").fetchone() + conn.execute("delete from roadside_attractions where pk = 1;") + row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] task_id = await db.execute_write_fn(write_fn, block=False) @@ -513,9 +512,8 @@ async def test_execute_write_fn_block_false(db): @pytest.mark.asyncio async def test_execute_write_fn_block_true(db): def write_fn(conn): - with conn: - conn.execute("delete from roadside_attractions where pk = 1;") - row = conn.execute("select count(*) from roadside_attractions").fetchone() + conn.execute("delete from roadside_attractions where pk = 1;") + row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] new_count = await db.execute_write_fn(write_fn) From a4fa1ef3bd6a6117118b5cdd64aca2308c21604b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:56:15 -0800 Subject: [PATCH 1959/2303] Release 1.0a10 Refs #2277 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index f5e07ac8..809c434f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a9" +__version__ = "1.0a10" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index e567f422..92f198af 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a10: + +1.0a10 (2024-02-17) +------------------- + +The only changes in this alpha correspond to the way Datasette handles database transactions. (:issue:`2277`) + +- The :ref:`database.execute_write_fn() ` method has a new ``transaction=True`` parameter. This defaults to ``True`` which means all functions executed using this method are now automatically wrapped in a transaction - previously the functions needed to roll transaction handling on their own, and many did not. +- Pass ``transaction=False`` to ``execute_write_fn()`` if you want to manually handle transactions in your function. +- Several internal Datasette features, including parts of the :ref:`JSON write API `, had been failing to wrap their operations in a transaction. This has been fixed by the new ``transaction=True`` default. + .. _v1_0_a9: 1.0a9 (2024-02-16) From 81629dbeffb5cee9086bc956ce3a9ab7d051f4d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 21:03:41 -0800 Subject: [PATCH 1960/2303] Upgrade GitHub Actions, including PyPI publishing --- .github/workflows/publish.yml | 60 ++++++++++++++--------------------- .github/workflows/test.yml | 13 +++----- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 64a03a77..55fc0eb9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,20 +12,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | pip install -e '.[test]' @@ -36,47 +31,38 @@ jobs: deploy: runs-on: ubuntu-latest needs: [test] + environment: release + permissions: + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + python-version: '3.12' + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | - pip install setuptools wheel twine - - name: Publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + pip install setuptools wheel build + - name: Build run: | - python setup.py sdist bdist_wheel - twine upload dist/* + python -m build + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.9' - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | python -m pip install -e .[docs] @@ -105,7 +91,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 656b0b1c..3ac8756d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,19 +12,14 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: setup.py - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) From 3856a8cb244f1338d2c4bceb76a510022d88ade5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 12:51:14 -0800 Subject: [PATCH 1961/2303] Consistent Permission denied:, refs #2279 --- datasette/views/database.py | 6 +++--- tests/test_api_write.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 2a8b40cc..56fc6f8c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -860,7 +860,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "update-row", resource=database_name ): - return _error(["Permission denied - need update-row"], 403) + return _error(["Permission denied: need update-row"], 403) table_name = data.get("table") if not table_name: @@ -884,7 +884,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "insert-row", resource=database_name ): - return _error(["Permission denied - need insert-row"], 403) + return _error(["Permission denied: need insert-row"], 403) alter = False if rows or row: @@ -897,7 +897,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "alter-table", resource=database_name ): - return _error(["Permission denied - need alter-table"], 403) + return _error(["Permission denied: need alter-table"], 403) alter = True if columns: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 0eb915ba..634f5ee9 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1316,7 +1316,7 @@ async def test_create_table( ["create-table"], {"table": "t", "rows": [{"name": "c"}]}, 403, - ["Permission denied - need insert-row"], + ["Permission denied: need insert-row"], ), # This should work: ( @@ -1330,7 +1330,7 @@ async def test_create_table( ["create-table", "insert-row"], {"table": "t", "rows": [{"id": 1}], "pk": "id", "replace": True}, 403, - ["Permission denied - need update-row"], + ["Permission denied: need update-row"], ), ), ) @@ -1567,7 +1567,7 @@ async def test_create_using_alter_against_existing_table( assert response2.status_code == 403 assert response2.json() == { "ok": False, - "errors": ["Permission denied - need alter-table"], + "errors": ["Permission denied: need alter-table"], } else: assert response2.status_code == 201 From b36a2d8f4b566a3a4902cdaa7a549241e0e8c881 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 12:55:51 -0800 Subject: [PATCH 1962/2303] Require update-row to use insert replace, closes #2279 --- datasette/views/table.py | 5 +++++ docs/json_api.rst | 4 ++-- tests/test_api_write.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1c187692..6d0d9885 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -485,6 +485,11 @@ class TableInsertView(BaseView): if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + if replace and not await self.ds.permission_allowed( + request.actor, "update-row", resource=(database_name, table_name) + ): + return _error(['Permission denied: need update-row to use "replace"'], 403) + initial_schema = None if alter: # Must have alter-table permission diff --git a/docs/json_api.rst b/docs/json_api.rst index c401d97e..366f74b2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -616,7 +616,7 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: "ignore": true } -Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. This requires the :ref:`permissions_update_row` permission. Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. @@ -854,7 +854,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. -* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. +* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`permissions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 634f5ee9..6a7ddeb6 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -221,6 +221,14 @@ async def test_insert_rows(ds_write, return_rows): 400, ['Cannot use "ignore" and "replace" at the same time'], ), + ( + # Replace is not allowed if you don't have update-row + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "replace": True}, + "insert-but-not-update", + 403, + ['Permission denied: need update-row to use "replace"'], + ), ( "/data/docs/-/insert", {"rows": [{"title": "Test"}], "invalid_param": True}, From 392ca2e24cc93a3918d07718f40524857d626d14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 13:40:48 -0800 Subject: [PATCH 1963/2303] Improvements to table column cog menu display, closes #2263 - Repositions if menu would cause a horizontal scrollbar - Arrow tip on menu now attempts to align with cog icon on column --- datasette/static/table.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/datasette/static/table.js b/datasette/static/table.js index 778457c5..0c54a472 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -217,6 +217,17 @@ const initDatasetteTable = function (manager) { menuList.appendChild(menuItem); }); + // Measure width of menu and adjust position if too far right + const menuWidth = menu.offsetWidth; + const windowWidth = window.innerWidth; + if (menuLeft + menuWidth > windowWidth) { + menu.style.left = windowWidth - menuWidth - 20 + "px"; + } + // Align menu .hook arrow with the column cog icon + const hook = menu.querySelector('.hook'); + const icon = th.querySelector('.dropdown-menu-icon'); + const iconRect = icon.getBoundingClientRect(); + hook.style.left = (iconRect.left - menuLeft + 1) + 'px'; } var svg = document.createElement("div"); From 27409a78929b4baa017cce2cc0ca636603ed6d37 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 14:01:55 -0800 Subject: [PATCH 1964/2303] Fix for hook position in wide column names, refs #2263 --- datasette/static/table.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/datasette/static/table.js b/datasette/static/table.js index 0c54a472..4f81b2e5 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -227,7 +227,15 @@ const initDatasetteTable = function (manager) { const hook = menu.querySelector('.hook'); const icon = th.querySelector('.dropdown-menu-icon'); const iconRect = icon.getBoundingClientRect(); - hook.style.left = (iconRect.left - menuLeft + 1) + 'px'; + const hookLeft = (iconRect.left - menuLeft + 1) + 'px'; + hook.style.left = hookLeft; + // Move the whole menu right if the hook is too far right + const menuRect = menu.getBoundingClientRect(); + if (iconRect.right > menuRect.right) { + menu.style.left = (iconRect.right - menuWidth) + 'px'; + // And move hook tip as well + hook.style.left = (menuWidth - 13) + 'px'; + } } var svg = document.createElement("div"); From 26300738e3c6e7ad515bd513063f57249a05000a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 14:17:37 -0800 Subject: [PATCH 1965/2303] Fixes for permissions debug page, closes #2278 --- datasette/templates/permissions_debug.html | 10 +++++----- datasette/views/special.py | 17 +++++++++-------- tests/test_permissions.py | 8 ++++++++ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 36a12acc..5a5c1aa6 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -57,7 +57,7 @@ textarea {

    @@ -71,19 +71,19 @@ textarea { +{% endif %} + {% endblock %} diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 9c6bbde0..16b3d42b 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -31,6 +31,12 @@ class Urls: db = self.ds.get_database(database) return self.path(tilde_encode(db.route), format=format) + def database_query(self, database, sql, format=None): + path = f"{self.database(database)}/-/query?" + urllib.parse.urlencode( + {"sql": sql} + ) + return self.path(path, format=format) + def table(self, database, table, format=None): path = f"{self.database(database)}/{tilde_encode(table)}" if format is not None: diff --git a/datasette/views/table.py b/datasette/views/table.py index d71efeb0..ea044b36 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -929,6 +929,7 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), + count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1280,6 +1281,9 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) + async def extra_count_sql(): + return count_sql + async def extra_count(): "Total count of rows matching these filters" # Calculate the total count for this query @@ -1299,8 +1303,11 @@ async def table_view_data( # Otherwise run a select count(*) ... if count_sql and count is None and not nocount: + count_sql_limited = ( + f"select count(*) from (select * {from_sql} limit 10001)" + ) try: - count_rows = list(await db.execute(count_sql, from_sql_params)) + count_rows = list(await db.execute(count_sql_limited, from_sql_params)) count = count_rows[0][0] except QueryInterrupted: pass @@ -1615,6 +1622,7 @@ async def table_view_data( "facet_results", "facets_timed_out", "count", + "count_sql", "human_description_en", "next_url", "metadata", @@ -1647,6 +1655,7 @@ async def table_view_data( registry = Registry( extra_count, + extra_count_sql, extra_facet_results, extra_facets_timed_out, extra_suggested_facets, From dc288056b81a3635bdb02a6d0121887db2720e5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 21 Aug 2024 19:56:02 -0700 Subject: [PATCH 2064/2303] Better handling of errors for count all button, refs #2408 --- datasette/templates/table.html | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 187f0143..7246ff5d 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -42,7 +42,7 @@ {% if count or human_description_en %}

    {% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows - {% if allow_execute_sql and query.sql %} count all rows{% endif %} + {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %}

    @@ -180,7 +180,7 @@ document.addEventListener('DOMContentLoaded', function() { const countLink = document.querySelector('a.count-sql'); if (countLink) { - countLink.addEventListener('click', function(ev) { + countLink.addEventListener('click', async function(ev) { ev.preventDefault(); // Replace countLink with span with same style attribute const span = document.createElement('span'); @@ -189,14 +189,23 @@ document.addEventListener('DOMContentLoaded', function() { countLink.replaceWith(span); countLink.setAttribute('disabled', 'disabled'); let url = countLink.href.replace(/(\?|$)/, '.json$1'); - fetch(url) - .then(response => response.json()) - .then(data => { - const count = data['rows'][0]['count(*)']; - const formattedCount = count.toLocaleString(); - span.closest('h3').textContent = formattedCount + ' rows'; - }) - .catch(error => countLink.textContent = 'error'); + try { + const response = await fetch(url); + console.log({response}); + const data = await response.json(); + console.log({data}); + if (!response.ok) { + console.log('throw error'); + throw new Error(data.title || data.error); + } + const count = data['rows'][0]['count(*)']; + const formattedCount = count.toLocaleString(); + span.closest('h3').textContent = formattedCount + ' rows'; + } catch (error) { + console.log('Update', span, 'with error message', error); + span.textContent = error.message; + span.style.color = 'red'; + } }); } }); From 92c4d41ca605e0837a2711ee52fde9cf1eea74d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Sep 2024 17:20:41 -0700 Subject: [PATCH 2065/2303] results.dicts() method, closes #2414 --- datasette/database.py | 3 +++ datasette/views/row.py | 3 +-- datasette/views/table.py | 2 +- docs/internals.rst | 3 +++ tests/test_api_write.py | 23 +++++++++-------------- tests/test_internals_database.py | 11 +++++++++++ 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index da0ab1de..a2e899bc 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -677,6 +677,9 @@ class Results: else: raise MultipleValues + def dicts(self): + return [dict(row) for row in self.rows] + def __iter__(self): return iter(self.rows) diff --git a/datasette/views/row.py b/datasette/views/row.py index d802994e..f374fd94 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -277,8 +277,7 @@ class RowUpdateView(BaseView): results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) - rows = list(results.rows) - result["row"] = dict(rows[0]) + result["row"] = results.dicts()[0] await self.ds.track_event( UpdateRowEvent( diff --git a/datasette/views/table.py b/datasette/views/table.py index ea044b36..82dab613 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -558,7 +558,7 @@ class TableInsertView(BaseView): ), args, ) - result["rows"] = [dict(r) for r in fetched_rows.rows] + result["rows"] = fetched_rows.dicts() else: result["rows"] = rows # We track the number of rows requested, but do not attempt to show which were actually diff --git a/docs/internals.rst b/docs/internals.rst index 4289c815..facbc224 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1093,6 +1093,9 @@ The ``Results`` object also has the following properties and methods: ``.rows`` - list of ``sqlite3.Row`` This property provides direct access to the list of rows returned by the database. You can access specific rows by index using ``results.rows[0]``. +``.dicts()`` - list of ``dict`` + This method returns a list of Python dictionaries, one for each row. + ``.first()`` - row or None Returns the first row in the results, or ``None`` if no rows were returned. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9c2b9b45..04e61261 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -58,8 +58,8 @@ async def test_insert_row(ds_write, content_type): assert response.status_code == 201 assert response.json()["ok"] is True assert response.json()["rows"] == [expected_row] - rows = (await ds_write.get_database("data").execute("select * from docs")).rows - assert dict(rows[0]) == expected_row + rows = (await ds_write.get_database("data").execute("select * from docs")).dicts() + assert rows[0] == expected_row # Analytics event event = last_event(ds_write) assert event.name == "insert-rows" @@ -118,12 +118,9 @@ async def test_insert_rows(ds_write, return_rows): assert not event.ignore assert not event.replace - actual_rows = [ - dict(r) - for r in ( - await ds_write.get_database("data").execute("select * from docs") - ).rows - ] + actual_rows = ( + await ds_write.get_database("data").execute("select * from docs") + ).dicts() assert len(actual_rows) == 20 assert actual_rows == [ {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} @@ -469,12 +466,10 @@ async def test_insert_ignore_replace( assert event.ignore == ignore assert event.replace == replace - actual_rows = [ - dict(r) - for r in ( - await ds_write.get_database("data").execute("select * from docs") - ).rows - ] + actual_rows = ( + await ds_write.get_database("data").execute("select * from docs") + ).dicts() + assert actual_rows == expected_rows assert response.json()["ok"] is True if should_return: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 0020668a..edfc6bc7 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -40,6 +40,17 @@ async def test_results_bool(db, expected): assert bool(results) is expected +@pytest.mark.asyncio +async def test_results_dicts(db): + results = await db.execute("select pk, name from roadside_attractions") + assert results.dicts() == [ + {"pk": 1, "name": "The Mystery Spot"}, + {"pk": 2, "name": "Winchester Mystery House"}, + {"pk": 3, "name": "Burlingame Museum of PEZ Memorabilia"}, + {"pk": 4, "name": "Bigfoot Discovery Museum"}, + ] + + @pytest.mark.parametrize( "query,expected", [ From 2170269258d1de38f4e518aa3e55e6b3ed202841 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Sep 2024 08:37:26 -0700 Subject: [PATCH 2066/2303] New .core CSS class for inputs and buttons * Initial .core input/button classes, refs #2415 * Docs for the new .core CSS class, refs #2415 * Applied .core class everywhere that needs it, closes #2415 --- datasette/static/app.css | 33 +++++++++++++++------- datasette/templates/allow_debug.html | 2 +- datasette/templates/api_explorer.html | 4 +-- datasette/templates/create_token.html | 2 +- datasette/templates/database.html | 2 +- datasette/templates/logout.html | 2 +- datasette/templates/messages_debug.html | 2 +- datasette/templates/permissions_debug.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/table.html | 4 +-- docs/custom_templates.rst | 9 ++++++ docs/writing_plugins.rst | 3 +- tests/test_permissions.py | 2 +- 13 files changed, 46 insertions(+), 23 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 562d6adb..f975f0ad 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -528,8 +528,11 @@ label.sort_by_desc { pre#sql-query { margin-bottom: 1em; } -form input[type=text], -form input[type=search] { + +.core input[type=text], +input.core[type=text], +.core input[type=search], +input.core[type=search] { border: 1px solid #ccc; border-radius: 3px; width: 60%; @@ -540,17 +543,25 @@ form input[type=search] { } /* Stop Webkit from styling search boxes in an inconsistent way */ /* https://css-tricks.com/webkit-html5-search-inputs/ comments */ -input[type=search] { +.core input[type=search], +input.core[type=search] { -webkit-appearance: textfield; } -input[type="search"]::-webkit-search-decoration, -input[type="search"]::-webkit-search-cancel-button, -input[type="search"]::-webkit-search-results-button, -input[type="search"]::-webkit-search-results-decoration { +.core input[type="search"]::-webkit-search-decoration, +input.core[type="search"]::-webkit-search-decoration, +.core input[type="search"]::-webkit-search-cancel-button, +input.core[type="search"]::-webkit-search-cancel-button, +.core input[type="search"]::-webkit-search-results-button, +input.core[type="search"]::-webkit-search-results-button, +.core input[type="search"]::-webkit-search-results-decoration, +input.core[type="search"]::-webkit-search-results-decoration { display: none; } -form input[type=submit], form button[type=button] { +.core input[type=submit], +.core button[type=button], +input.core[type=submit], +button.core[type=button] { font-weight: 400; cursor: pointer; text-align: center; @@ -563,14 +574,16 @@ form input[type=submit], form button[type=button] { border-radius: .25rem; } -form input[type=submit] { +.core input[type=submit], +input.core[type=submit] { color: #fff; background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); border-color: #007bff; -webkit-appearance: button; } -form button[type=button] { +.core button[type=button], +button.core[type=button] { color: #007bff; background-color: #fff; border-color: #007bff; diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 04181531..610417d2 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -35,7 +35,7 @@ p.message-warning {

    Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

    - +

    diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 109fb1e9..dc393c20 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -19,7 +19,7 @@

    GET - +
    @@ -29,7 +29,7 @@
    POST - +
    diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html index 2be98d38..409fb8a9 100644 --- a/datasette/templates/create_token.html +++ b/datasette/templates/create_token.html @@ -39,7 +39,7 @@ {% endfor %} {% endif %} - +

    diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html index 4c4a7d11..c8fc642a 100644 --- a/datasette/templates/logout.html +++ b/datasette/templates/logout.html @@ -8,7 +8,7 @@

    You are logged in as {{ display_actor(actor) }}

    - +
    diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html index e0ab9a40..2940cd69 100644 --- a/datasette/templates/messages_debug.html +++ b/datasette/templates/messages_debug.html @@ -8,7 +8,7 @@

    Set a message:

    - +
    diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 5a5c1aa6..83891181 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -47,7 +47,7 @@ textarea {

    This tool lets you simulate an actor and a permission check for that actor.

    - +

    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f7c8d0a3..a6e9a3aa 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -36,7 +36,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 7246ff5d..c9e0e87b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -48,7 +48,7 @@

    {% endif %} - + {% if supports_search %}
    {% endif %} @@ -152,7 +152,7 @@ object {% endif %}

    - +

    CSV options: diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 534d8b33..8cc40f0f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -83,6 +83,15 @@ database column they are representing, for example: +.. _customization_css: + +Writing custom CSS +~~~~~~~~~~~~~~~~~~ + +Custom templates need to take Datasette's default CSS into account. The pattern portfolio at ``/-/patterns`` (`example here `__) is a useful reference for understanding the available CSS classes. + +The ``core`` class is particularly useful - you can apply this directly to a ```` or ``