Fixed typo in docs

Refs #2189

.github/workflows/publish.yml

@@ -31,7 +31,10 @@ jobs:
         pip install -e '.[test]'
     - name: Run tests
       run: |
-        pytest
+        pytest -n auto -m "not serial"
+        pytest -m "serial"
+        # And the test that exceeds a localhost HTTPS server
+        tests/test_datasette_https_server.sh
 
   deploy:
     runs-on: ubuntu-latest
@@ -69,7 +72,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10'
+        python-version: '3.9'
     - uses: actions/cache@v2
       name: Configure pip caching
       with:
@@ -90,7 +93,7 @@ jobs:
     - name: Set up Cloud Run
       uses: google-github-actions/setup-gcloud@v0
       with:
-        version: '275.0.0'
+        version: '318.0.0'
         service_account_email: ${{ secrets.GCP_SA_EMAIL }}
         service_account_key: ${{ secrets.GCP_SA_KEY }}
     - name: Deploy stable-docs.datasette.io to Cloud Run

.github/workflows/test.yml

@@ -35,6 +35,8 @@ jobs:
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
+        # And the test that exceeds a localhost HTTPS server
+        tests/test_datasette_https_server.sh
     - name: Check if cog needs to be run
       run: |
         cog --check docs/*.rst

datasette/app.py

@@ -64,16 +64,14 @@ from .utils import (
 )
 from .utils.asgi import (
     AsgiLifespan,
-    Base400,
     Forbidden,
     NotFound,
     Request,
     Response,
+    AsgiRunOnFirstRequest,
     asgi_static,
     asgi_send,
     asgi_send_file,
-    asgi_send_html,
-    asgi_send_json,
     asgi_send_redirect,
 )
 from .utils.internal_db import init_internal_db, populate_schema_tables
@@ -118,6 +116,11 @@ SETTINGS = (
         True,
         "Allow users to specify columns to facet using ?_facet= parameter",
     ),
+    Setting(
+        "default_allow_sql",
+        True,
+        "Allow anyone to run arbitrary SQL queries",
+    ),
     Setting(
         "allow_download",
         True,
@@ -215,6 +218,8 @@ class Datasette:
         self.config_dir = config_dir
         self.pdb = pdb
         self._secret = secret or secrets.token_hex(32)
+        if files is not None and isinstance(files, str):
+            raise ValueError("files= must be a list of paths, not a string")
         self.files = tuple(files or []) + tuple(immutables or [])
         if config_dir:
             db_files = []
@@ -1260,7 +1265,7 @@ class Datasette:
 
         async def setup_db():
             # First time server starts up, calculate table counts for immutable databases
-            for dbname, database in self.databases.items():
+            for database in self.databases.values():
                 if not database.is_mutable:
                     await database.table_counts(limit=60 * 60 * 1000)
 
@@ -1274,10 +1279,8 @@ class Datasette:
         )
         if self.setting("trace_debug"):
             asgi = AsgiTracer(asgi)
-        asgi = AsgiLifespan(
-            asgi,
-            on_startup=setup_db,
-        )
+        asgi = AsgiLifespan(asgi)
+        asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup])
         for wrapper in pm.hook.asgi_wrapper(datasette=self):
             asgi = wrapper(asgi)
         return asgi
@@ -1566,42 +1569,34 @@ class DatasetteClient:
         return path
 
     async def get(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.get(self._fix(path), **kwargs)
 
     async def options(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.options(self._fix(path), **kwargs)
 
     async def head(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.head(self._fix(path), **kwargs)
 
     async def post(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.post(self._fix(path), **kwargs)
 
     async def put(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.put(self._fix(path), **kwargs)
 
     async def patch(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.patch(self._fix(path), **kwargs)
 
     async def delete(self, path, **kwargs):
-        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.delete(self._fix(path), **kwargs)
 
     async def request(self, method, path, **kwargs):
-        await self.ds.invoke_startup()
         avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None)
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.request(

datasette/cli.py

@@ -4,6 +4,7 @@ import click
 from click import formatting
 from click.types import CompositeParamType
 from click_default_group import DefaultGroup
+import functools
 import json
 import os
 import pathlib
@@ -11,6 +12,7 @@ import shutil
 from subprocess import call
 import sys
 from runpy import run_module
+import textwrap
 import webbrowser
 from .app import (
     OBSOLETE_SETTINGS,
@@ -126,7 +128,7 @@ class Setting(CompositeParamType):
 
 
 def sqlite_extensions(fn):
-    return click.option(
+    fn = click.option(
         "sqlite_extensions",
         "--load-extension",
         type=LoadExtension(),
@@ -135,6 +137,26 @@ def sqlite_extensions(fn):
         help="Path to a SQLite extension to load, and optional entrypoint",
     )(fn)
 
+    # Wrap it in a custom error handler
+    @functools.wraps(fn)
+    def wrapped(*args, **kwargs):
+        try:
+            return fn(*args, **kwargs)
+        except AttributeError as e:
+            if "enable_load_extension" in str(e):
+                raise click.ClickException(
+                    textwrap.dedent(
+                        """
+                    Your Python installation does not have the ability to load SQLite extensions.
+
+                    More information: https://datasette.io/help/extensions
+                    """
+                    ).strip()
+                )
+            raise
+
+    return wrapped
+
 
 @click.group(cls=DefaultGroup, default="serve", default_if_no_args=True)
 @click.version_option(version=__version__)
@@ -607,7 +629,7 @@ def serve(
         url = "http://{}:{}{}?token={}".format(
             host, port, ds.urls.path("-/auth-token"), ds._root_token
         )
-        print(url)
+        click.echo(url)
     if open_browser:
         if url is None:
             # Figure out most convenient URL - to table, database or homepage

datasette/default_permissions.py

@@ -36,12 +36,16 @@ def permission_allowed(datasette, actor, action, resource):
                 return None
             return actor_matches_allow(actor, allow)
         elif action == "execute-sql":
+            # Only use default_allow_sql setting if it is set to False:
+            default_allow_sql = (
+                None if datasette.setting("default_allow_sql") else False
+            )
             # Use allow_sql block from database block, or from top-level
             database_allow_sql = datasette.metadata("allow_sql", database=resource)
             if database_allow_sql is None:
                 database_allow_sql = datasette.metadata("allow_sql")
             if database_allow_sql is None:
-                return None
+                return default_allow_sql
             return actor_matches_allow(actor, database_allow_sql)
 
     return inner

datasette/publish/cloudrun.py

@@ -173,7 +173,7 @@ def publish_subcommand(publish):
                     print(fp.read())
                 print("\n====================\n")
 
-            image_id = f"gcr.io/{project}/{name}"
+            image_id = f"gcr.io/{project}/datasette-{service}"
             check_call(
                 "gcloud builds submit --tag {}{}".format(
                     image_id, " --timeout {}".format(timeout) if timeout else ""

datasette/publish/heroku.py

@@ -3,7 +3,9 @@ from datasette import hookimpl
 import click
 import json
 import os
+import pathlib
 import shlex
+import shutil
 from subprocess import call, check_output
 import tempfile
 
@@ -28,6 +30,11 @@ def publish_subcommand(publish):
         "--tar",
         help="--tar option to pass to Heroku, e.g. --tar=/usr/local/bin/gtar",
     )
+    @click.option(
+        "--generate-dir",
+        type=click.Path(dir_okay=True, file_okay=False),
+        help="Output generated application files and stop without deploying",
+    )
     def heroku(
         files,
         metadata,
@@ -49,6 +56,7 @@ def publish_subcommand(publish):
         about_url,
         name,
         tar,
+        generate_dir,
     ):
         "Publish databases to Datasette running on Heroku"
         fail_if_publish_binary_not_installed(
@@ -105,6 +113,16 @@ def publish_subcommand(publish):
             secret,
             extra_metadata,
         ):
+            if generate_dir:
+                # Recursively copy files from current working directory to it
+                if pathlib.Path(generate_dir).exists():
+                    raise click.ClickException("Directory already exists")
+                shutil.copytree(".", generate_dir)
+                click.echo(
+                    f"Generated files written to {generate_dir}, stopping without deploying",
+                    err=True,
+                )
+                return
             app_name = None
             if name:
                 # Check to see if this app already exists
@@ -176,7 +194,7 @@ def temporary_heroku_directory(
                 fp.write(json.dumps(metadata_content, indent=2))
 
         with open("runtime.txt", "w") as fp:
-            fp.write("python-3.8.10")
+            fp.write("python-3.11.0")
 
         if branch:
             install = [

datasette/static/app.css

@@ -573,6 +573,9 @@ form button[type=button] {
     display: inline-block;
     margin-right: 0.3em;
 }
+.select-wrapper:focus-within {
+    border: 1px solid black;
+}
 .select-wrapper.filter-op {
     width: 80px;
 }

datasette/utils/asgi.py

@@ -428,3 +428,18 @@ class AsgiFileDownload:
             content_type=self.content_type,
             headers=self.headers,
         )
+
+
+class AsgiRunOnFirstRequest:
+    def __init__(self, asgi, on_startup):
+        assert isinstance(on_startup, list)
+        self.asgi = asgi
+        self.on_startup = on_startup
+        self._started = False
+
+    async def __call__(self, scope, receive, send):
+        if not self._started:
+            self._started = True
+            for hook in self.on_startup:
+                await hook()
+        return await self.asgi(scope, receive, send)

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "0.63.1"
+__version__ = "0.64.4"
 __version_info__ = tuple(__version__.split("."))

datasette/views/table.py

@@ -172,20 +172,14 @@ class TableView(DataView):
             raise NotFound("Database not found: {}".format(database_route))
         database_name = db.name
 
-        # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather
-        async def _gather_parallel(*args):
-            return await asyncio.gather(*args)
-
-        async def _gather_sequential(*args):
+        # We always now run queries sequentially, rather than with asyncio.gather() -
+        # see https://github.com/simonw/datasette/issues/2189
+        async def gather(*args):
             results = []
             for fn in args:
                 results.append(await fn)
             return results
 
-        gather = (
-            _gather_sequential if request.args.get("_noparallel") else _gather_parallel
-        )
-
         # If this is a canned query, not a table, then dispatch to QueryView instead
         canned_query = await self.ds.get_canned_query(
             database_name, table_name, request.actor

docs/authentication.rst

@@ -307,7 +307,21 @@ To limit access to the ``add_name`` canned query in your ``dogs.db`` database to
 Controlling the ability to execute arbitrary SQL
 ------------------------------------------------
 
-The ``"allow_sql"`` block can be used to control who is allowed to execute arbitrary SQL queries, both using the form on the database page e.g. https://latest.datasette.io/fixtures or by appending a ``?_where=`` parameter to the table page as seen on https://latest.datasette.io/fixtures/facetable?_where=city_id=1.
+Datasette defaults to allowing any site visitor to execute their own custom SQL queries, for example using the form on `the database page <https://latest.datasette.io/fixtures>`__ or by appending a ``?_where=`` parameter to the table page `like this <https://latest.datasette.io/fixtures/facetable?_where=_city_id=1>`__.
+
+Access to this ability is controlled by the :ref:`permissions_execute_sql` permission.
+
+The easiest way to disable arbitrary SQL queries is using the :ref:`default_allow_sql setting <setting_default_allow_sql>` when you first start Datasette running.
+
+You can alternatively use an ``"allow_sql"`` block to control who is allowed to execute arbitrary SQL queries.
+
+To prevent any user from executing arbitrary SQL queries, use this:
+
+.. code-block:: json
+
+    {
+        "allow_sql": false
+    }
 
 To enable just the :ref:`root user<authentication_root>` to execute SQL for all databases in your instance, use the following:
 
@@ -515,7 +529,7 @@ Actor is allowed to run arbitrary SQL queries against a specific database, e.g.
 ``resource`` - string
     The name of the database
 
-Default *allow*.
+Default *allow*. See also :ref:`the default_allow_sql setting <setting_default_allow_sql>`.
 
 .. _permissions_permissions_debug:
 

docs/changelog.rst

@@ -4,6 +4,62 @@
 Changelog
 =========
 
+.. _v0_64_4:
+
+0.64.4 (2023-09-21)
+-------------------
+
+- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`)
+
+.. _v0_64_3:
+
+0.64.3 (2023-04-27)
+-------------------
+
+- Added ``pip`` and ``setuptools`` as explicit dependencies. This fixes a bug where Datasette could not be installed using `Rye <https://github.com/mitsuhiko/rye>`__. (:issue:`2065`)
+
+.. _v0_64_2:
+
+0.64.2 (2023-03-08)
+-------------------
+
+- Fixed a bug with ``datasette publish cloudrun`` where deploys all used the same Docker image tag. This was mostly inconsequential as the service is deployed as soon as the image has been pushed to the registry, but could result in the incorrect image being deployed if two different deploys for two separate services ran at exactly the same time. (:issue:`2036`)
+
+.. _v0_64_1:
+
+0.64.1 (2023-01-11)
+-------------------
+
+- Documentation now links to a current source of information for installing Python 3. (:issue:`1987`)
+- Incorrectly calling the Datasette constructor using ``Datasette("path/to/data.db")`` instead of ``Datasette(["path/to/data.db"])`` now returns a useful error message. (:issue:`1985`)
+
+.. _v0_64:
+
+0.64 (2023-01-09)
+-----------------
+
+- Datasette now **strongly recommends against allowing arbitrary SQL queries if you are using SpatiaLite**. SpatiaLite includes SQL functions that could cause the Datasette server to crash. See :ref:`spatialite` for more details.
+- New :ref:`setting_default_allow_sql` setting, providing an easier way to disable all arbitrary SQL execution by end users: ``datasette --setting default_allow_sql off``. See also :ref:`authentication_permissions_execute_sql`. (:issue:`1409`)
+- `Building a location to time zone API with SpatiaLite <https://datasette.io/tutorials/spatialite>`__ is a new Datasette tutorial showing how to safely use SpatiaLite to create a location to time zone API.
+- New documentation about :ref:`how to debug problems loading SQLite extensions <installation_extensions>`. The error message shown when an extension cannot be loaded has also been improved. (:issue:`1979`)
+- Fixed an accessibility issue: the ``<select>`` elements in the table filter form now show an outline when they are currently focused. (:issue:`1771`)
+
+.. _v0_63_3:
+
+0.63.3 (2022-12-17)
+-------------------
+
+- Fixed a bug where ``datasette --root``, when running in Docker, would only output the URL to sign in as root when the server shut down, not when it started up. (:issue:`1958`)
+- You no longer need to ensure ``await datasette.invoke_startup()`` has been called in order for Datasette to start correctly serving requests - this is now handled automatically the first time the server receives a request. This fixes a bug experienced when Datasette is served directly by an ASGI application server such as Uvicorn or Gunicorn. It also fixes a bug with the `datasette-gunicorn <https://datasette.io/plugins/datasette-gunicorn>`__ plugin. (:issue:`1955`)
+
+.. _v0_63_2:
+
+0.63.2 (2022-11-18)
+-------------------
+
+- Fixed a bug in ``datasette publish heroku`` where deployments failed due to an older version of Python being requested. (:issue:`1905`)
+- New ``datasette publish heroku --generate-dir <dir>`` option for generating a Heroku deployment directory without deploying it.
+
 .. _v0_63_1:
 
 0.63.1 (2022-11-10)

docs/cli-reference.rst

@@ -224,6 +224,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
                                    (default=50)
       allow_facet                  Allow users to specify columns to facet using
                                    ?_facet= parameter (default=True)
+      default_allow_sql            Allow anyone to run arbitrary SQL queries
+                                   (default=True)
       allow_download               Allow users to download the original SQLite
                                    database files (default=True)
       suggest_facets               Calculate and display suggested facets
@@ -501,6 +503,8 @@ See :ref:`publish_heroku`.
       -n, --name TEXT                 Application name to use when deploying
       --tar TEXT                      --tar option to pass to Heroku, e.g.
                                       --tar=/usr/local/bin/gtar
+      --generate-dir DIRECTORY        Output generated application files and stop
+                                      without deploying
       --help                          Show this message and exit.
 
 

docs/conf.py

@@ -34,7 +34,7 @@
 extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"]
 
 extlinks = {
-    "issue": ("https://github.com/simonw/datasette/issues/%s", "#"),
+    "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"),
 }
 
 # Add any paths that contain templates here, relative to this directory.

docs/installation.rst

@@ -57,7 +57,7 @@ If the latest packaged release of Datasette has not yet been made available thro
 Using pip
 ---------
 
-Datasette requires Python 3.7 or higher. Visit `InstallPython3.com <https://installpython3.com/>`__ for step-by-step installation guides for your operating system.
+Datasette requires Python 3.7 or higher. The `Python.org Python For Beginners <https://www.python.org/about/gettingstarted/>`__ page has instructions for getting started.
 
 You can install Datasette and its dependencies using ``pip``::
 
@@ -230,3 +230,60 @@ Some plugins such as `datasette-ripgrep <https://datasette.io/plugins/datasette-
         pip install datasette-ripgrep'
 
     docker commit $(docker ps -lq) datasette-with-ripgrep
+
+.. _installation_extensions:
+
+A note about extensions
+=======================
+
+SQLite supports extensions, such as :ref:`spatialite` for geospatial operations.
+
+These can be loaded using the ``--load-extension`` argument, like so::
+
+    datasette --load-extension=/usr/local/lib/mod_spatialite.dylib
+
+Some Python installations do not include support for SQLite extensions. If this is the case you will see the following error when you attempt to load an extension:
+
+    Your Python installation does not have the ability to load SQLite extensions.
+
+In some cases you may see the following error message instead::
+
+    AttributeError: 'sqlite3.Connection' object has no attribute 'enable_load_extension'
+
+On macOS the easiest fix for this is to install Datasette using Homebrew::
+
+    brew install datasette
+
+Use ``which datasette`` to confirm that ``datasette`` will run that version. The output should look something like this::
+
+    /usr/local/opt/datasette/bin/datasette
+
+If you get a different location here such as ``/Library/Frameworks/Python.framework/Versions/3.10/bin/datasette`` you can run the following command to cause ``datasette`` to execute the Homebrew version instead::
+
+    alias datasette=$(echo $(brew --prefix datasette)/bin/datasette)
+
+You can undo this operation using::
+
+    unalias datasette
+
+If you need to run SQLite with extension support for other Python code, you can do so by install Python itself using Homebrew::
+
+    brew install python
+
+Then executing Python using::
+
+    /usr/local/opt/python@3/libexec/bin/python
+
+A more convenient way to work with this version of Python may be to use it to create a virtual environment::
+
+    /usr/local/opt/python@3/libexec/bin/python -m venv datasette-venv
+
+Then activate it like this::
+
+    source datasette-venv/bin/activate
+
+Now running ``python`` and ``pip`` will work against a version of Python 3 that includes support for SQLite extensions::
+
+    pip install datasette
+    which datasette
+    datasette --version

docs/json_api.rst

@@ -357,8 +357,8 @@ Special table arguments
 
     Some examples:
 
-    * `facetable?_where=neighborhood like "%c%"&_where=city_id=3 <https://latest.datasette.io/fixtures/facetable?_where=neighborhood%20like%20%22%c%%22&_where=city_id=3>`__
-    * `facetable?_where=city_id in (select id from facet_cities where name != "Detroit") <https://latest.datasette.io/fixtures/facetable?_where=city_id%20in%20(select%20id%20from%20facet_cities%20where%20name%20!=%20%22Detroit%22)>`__
+    * `facetable?_where=_neighborhood like "%c%"&_where=_city_id=3 <https://latest.datasette.io/fixtures/facetable?_where=_neighborhood%20like%20%22%c%%22&_where=_city_id=3>`__
+    * `facetable?_where=_city_id in (select id from facet_cities where name != "Detroit") <https://latest.datasette.io/fixtures/facetable?_where=_city_id%20in%20(select%20id%20from%20facet_cities%20where%20name%20!=%20%22Detroit%22)>`__
 
 ``?_through={json}``
     This can be used to filter rows via a join against another table.

docs/metadata.rst

@@ -189,7 +189,7 @@ Or use ``"sort_desc"`` to sort in descending order:
 Setting a custom page size
 --------------------------
 
-Datasette defaults to displaing 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``:
+Datasette defaults to displaying 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``:
 
 .. code-block:: json
 

docs/plugin_hooks.rst

@@ -855,13 +855,14 @@ Potential use-cases:
 
 .. note::
 
-   If you are writing :ref:`unit tests <testing_plugins>` for a plugin that uses this hook you will need to explicitly call ``await ds.invoke_startup()`` in your tests. An example:
+   If you are writing :ref:`unit tests <testing_plugins>` for a plugin that uses this hook and doesn't exercise Datasette by sending
+   any simulated requests through it you will need to explicitly call ``await ds.invoke_startup()`` in your tests. An example:
 
    .. code-block:: python
 
         @pytest.mark.asyncio
         async def test_my_plugin():
-            ds = Datasette([], metadata={})
+            ds = Datasette()
             await ds.invoke_startup()
             # Rest of test goes here
 
@@ -1345,7 +1346,7 @@ This example adds a new table action if the signed in user is ``"root"``:
 
 
     @hookimpl
-    def table_actions(datasette, actor):
+    def table_actions(datasette, actor, database, table):
         if actor and actor.get("id") == "root":
             return [
                 {

docs/publish.rst

@@ -73,6 +73,10 @@ This will output some details about the new deployment, including a URL like thi
 
 You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app.
 
+Rather than deploying directly you can use the ``--generate-dir`` option to output the files that would be deployed to a directory::
+
+    datasette publish heroku mydatabase.db --generate-dir=/tmp/deploy-this-to-heroku
+
 See :ref:`cli_help_publish_heroku___help` for the full list of options for this command.
 
 .. _publish_vercel:

docs/settings.rst

@@ -59,6 +59,21 @@ Settings
 
 The following options can be set using ``--setting name value``, or by storing them in the ``settings.json`` file for use with :ref:`config_dir`.
 
+.. _setting_default_allow_sql:
+
+default_allow_sql
+~~~~~~~~~~~~~~~~~
+
+Should users be able to execute arbitrary SQL queries by default?
+
+Setting this to ``off`` causes permission checks for :ref:`permissions_execute_sql` to fail by default.
+
+::
+
+    datasette mydatabase.db --setting default_allow_sql off
+
+There are two ways to achieve this: the other is to add ``"allow_sql": false`` to your ``metadata.json`` file, as described in :ref:`authentication_permissions_execute_sql`. This setting offers a more convenient way to do this.
+
 .. _setting_default_page_size:
 
 default_page_size

docs/spatialite.rst

@@ -4,17 +4,37 @@
  SpatiaLite
 ============
 
-The `SpatiaLite module <https://www.gaia-gis.it/fossil/libspatialite/index>`_ for SQLite adds features for handling geographic and spatial data. For an example of what you can do with it, see the tutorial `Building a location to time zone API with SpatiaLite, OpenStreetMap and Datasette <https://simonwillison.net/2017/Dec/12/location-time-zone-api/>`_.
+The `SpatiaLite module <https://www.gaia-gis.it/fossil/libspatialite/index>`_ for SQLite adds features for handling geographic and spatial data. For an example of what you can do with it, see the tutorial `Building a location to time zone API with SpatiaLite <https://datasette.io/tutorials/spatialite>`__.
 
 To use it with Datasette, you need to install the ``mod_spatialite`` dynamic library. This can then be loaded into Datasette using the ``--load-extension`` command-line option.
 
 Datasette can look for SpatiaLite in common installation locations if you run it like this::
 
-    datasette --load-extension=spatialite
+    datasette --load-extension=spatialite --setting default_allow_sql off
 
 If SpatiaLite is in another location, use the full path to the extension instead::
 
-    datasette --load-extension=/usr/local/lib/mod_spatialite.dylib
+    datasette --setting default_allow_sql off \
+      --load-extension=/usr/local/lib/mod_spatialite.dylib
+
+.. _spatialite_warning:
+
+Warning
+=======
+
+.. warning::
+    The SpatiaLite extension adds `a large number of additional SQL functions <https://www.gaia-gis.it/gaia-sins/spatialite-sql-5.0.1.html>`__, some of which are not be safe for untrusted users to execute: they may cause the Datasette server to crash.
+
+    You should not expose a SpatiaLite-enabled Datasette instance to the public internet without taking extra measures to secure it against potentially harmful SQL queries.
+
+    The following steps are recommended:
+
+    - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option.
+    - Define :ref:`canned_queries` with the SQL queries that use SpatiaLite functions that you want people to be able to execute.
+
+    The `Datasette SpatiaLite tutorial <https://datasette.io/tutorials/spatialite>`__ includes detailed instructions for running SpatiaLite safely using these techniques
+
+.. _spatialite_installation:
 
 Installation
 ============

docs/testing_plugins.rst

@@ -80,7 +80,7 @@ Creating a ``Datasette()`` instance like this as useful shortcut in tests, but t
 
 This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls.
 
-If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you.
+If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request.
 
 .. _testing_plugins_pdb:
 

setup.py

@@ -57,6 +57,8 @@ setup(
         "PyYAML>=5.3",
         "mergedeep>=1.1.1",
         "itsdangerous>=1.1",
+        "setuptools",
+        "pip",
     ],
     entry_points="""
         [console_scripts]

test-in-pyodide-with-shot-scraper.sh

@@ -25,6 +25,7 @@ async () => {
   let output = await pyodide.runPythonAsync(\`
     import micropip
     await micropip.install('h11==0.12.0')
+    await micropip.install('httpx==0.23')
     await micropip.install('http://localhost:8529/$wheel')
     import ssl
     import setuptools

tests/conftest.py

@@ -23,6 +23,17 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
+def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs):
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            client.get(url, **kwargs)
+            return
+        except httpx.ConnectError:
+            time.sleep(0.1)
+    raise AssertionError("Timed out waiting for {} to respond".format(url))
+
+
 def pytest_report_header(config):
     return "SQLite: {}".format(
         sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0]
@@ -111,13 +122,7 @@ def ds_localhost_http_server():
         # Avoid FileNotFoundError: [Errno 2] No such file or directory:
         cwd=tempfile.gettempdir(),
     )
-    # Loop until port 8041 serves traffic
-    while True:
-        try:
-            httpx.get("http://localhost:8041/")
-            break
-        except httpx.ConnectError:
-            time.sleep(0.1)
+    wait_until_responds("http://localhost:8041/")
     # Check it started successfully
     assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
     yield ds_proc
@@ -125,46 +130,6 @@ def ds_localhost_http_server():
     ds_proc.terminate()
 
 
-@pytest.fixture(scope="session")
-def ds_localhost_https_server(tmp_path_factory):
-    cert_directory = tmp_path_factory.mktemp("certs")
-    ca = trustme.CA()
-    server_cert = ca.issue_cert("localhost")
-    keyfile = str(cert_directory / "server.key")
-    certfile = str(cert_directory / "server.pem")
-    client_cert = str(cert_directory / "client.pem")
-    server_cert.private_key_pem.write_to_path(path=keyfile)
-    for blob in server_cert.cert_chain_pems:
-        blob.write_to_path(path=certfile, append=True)
-    ca.cert_pem.write_to_path(path=client_cert)
-    ds_proc = subprocess.Popen(
-        [
-            "datasette",
-            "--memory",
-            "-p",
-            "8042",
-            "--ssl-keyfile",
-            keyfile,
-            "--ssl-certfile",
-            certfile,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        cwd=tempfile.gettempdir(),
-    )
-    while True:
-        try:
-            httpx.get("https://localhost:8042/", verify=client_cert)
-            break
-        except httpx.ConnectError:
-            time.sleep(0.1)
-    # Check it started successfully
-    assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
-    yield ds_proc, client_cert
-    # Shut it down at the end of the pytest session
-    ds_proc.terminate()
-
-
 @pytest.fixture(scope="session")
 def ds_unix_domain_socket_server(tmp_path_factory):
     # This used to use tmp_path_factory.mktemp("uds") but that turned out to
@@ -181,12 +146,7 @@ def ds_unix_domain_socket_server(tmp_path_factory):
     # Poll until available
     transport = httpx.HTTPTransport(uds=uds)
     client = httpx.Client(transport=transport)
-    while True:
-        try:
-            client.get("http://localhost/_memory.json")
-            break
-        except httpx.ConnectError:
-            time.sleep(0.1)
+    wait_until_responds("http://localhost/_memory.json", client=client)
     # Check it started successfully
     assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
     yield ds_proc, uds

tests/test_api.py

@@ -805,6 +805,7 @@ def test_settings_json(app_client):
     assert {
         "default_page_size": 50,
         "default_facet_size": 30,
+        "default_allow_sql": True,
         "facet_suggest_time_limit_ms": 50,
         "facet_time_limit_ms": 200,
         "max_returned_rows": 100,
@@ -966,7 +967,10 @@ async def test_hidden_sqlite_stat1_table():
     await db.execute_write("analyze")
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
-    assert tables == [("normal", False), ("sqlite_stat1", True)]
+    assert tables in (
+        [("normal", False), ("sqlite_stat1", True)],
+        [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)],
+    )
 
 
 @pytest.mark.asyncio

tests/test_cli.py

@@ -215,6 +215,28 @@ def test_setting_type_validation():
     assert '"default_page_size" should be an integer' in result.stderr
 
 
+@pytest.mark.parametrize("default_allow_sql", (True, False))
+def test_setting_default_allow_sql(default_allow_sql):
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        [
+            "--setting",
+            "default_allow_sql",
+            "on" if default_allow_sql else "off",
+            "--get",
+            "/_memory.json?sql=select+21&_shape=objects",
+        ],
+    )
+    if default_allow_sql:
+        assert result.exit_code == 0, result.output
+        assert json.loads(result.output)["rows"][0] == {"21": 21}
+    else:
+        assert result.exit_code == 1, result.output
+        # This isn't JSON at the moment, maybe it should be though
+        assert "Forbidden" in result.output
+
+
 def test_config_deprecated():
     # The --config option should show a deprecation message
     runner = CliRunner(mix_stderr=False)

tests/test_cli_serve_server.py

@@ -13,17 +13,6 @@ def test_serve_localhost_http(ds_localhost_http_server):
     }.items() <= response.json().items()
 
 
-@pytest.mark.serial
-def test_serve_localhost_https(ds_localhost_https_server):
-    _, client_cert = ds_localhost_https_server
-    response = httpx.get("https://localhost:8042/_memory.json", verify=client_cert)
-    assert {
-        "database": "_memory",
-        "path": "/_memory",
-        "tables": [],
-    }.items() <= response.json().items()
-
-
 @pytest.mark.serial
 @pytest.mark.skipif(
     not hasattr(socket, "AF_UNIX"), reason="Requires socket.AF_UNIX support"

tests/test_csv.py

@@ -5,6 +5,7 @@ from .fixtures import (  # noqa
     app_client_with_cors,
     app_client_with_trace,
 )
+import urllib.parse
 
 EXPECTED_TABLE_CSV = """id,content
 1,hello
@@ -142,11 +143,24 @@ def test_csv_with_non_ascii_characters(app_client):
 
 
 def test_max_csv_mb(app_client_csv_max_mb_one):
+    # This query deliberately generates a really long string
+    # should be 100*100*100*2 = roughly 2MB
     response = app_client_csv_max_mb_one.get(
-        (
-            "/fixtures.csv?sql=select+'{}'+"
-            "from+compound_three_primary_keys&_stream=1&_size=max"
-        ).format("abcdefg" * 10000)
+        "/fixtures.csv?"
+        + urllib.parse.urlencode(
+            {
+                "sql": """
+            select group_concat('ab', '')
+            from json_each(json_array({lots})),
+                json_each(json_array({lots})),
+                json_each(json_array({lots}))
+            """.format(
+                    lots=", ".join(str(i) for i in range(100))
+                ),
+                "_stream": 1,
+                "_size": "max",
+            }
+        ),
     )
     # It's a 200 because we started streaming before we knew the error
     assert response.status == 200

tests/test_datasette_https_server.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Generate certificates
+python -m trustme
+# This creates server.pem, server.key, client.pem
+
+# Start the server in the background
+datasette --memory \
+    --ssl-keyfile=server.key \
+    --ssl-certfile=server.pem \
+    -p 8152 &
+
+# Store the background process ID in a variable
+server_pid=$!
+
+# Wait for the server to start
+sleep 2
+
+# Make a test request using curl
+curl -f --cacert client.pem 'https://localhost:8152/_memory.json'
+
+# Save curl's exit code (-f option causes it to return one on HTTP errors)
+curl_exit_code=$?
+
+# Shut down the server
+kill $server_pid
+sleep 1
+
+# Clean up the certificates
+rm server.pem server.key client.pem
+
+echo $curl_exit_code
+exit $curl_exit_code

tests/test_filters.py

@@ -83,13 +83,11 @@ async def test_through_filters_from_request(app_client):
     request = Request.fake(
         '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
     )
-    filter_args = await (
-        through_filters(
-            request=request,
-            datasette=app_client.ds,
-            table="roadside_attractions",
-            database="fixtures",
-        )
+    filter_args = await through_filters(
+        request=request,
+        datasette=app_client.ds,
+        table="roadside_attractions",
+        database="fixtures",
     )()
     assert filter_args.where_clauses == [
         "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)"
@@ -106,13 +104,11 @@ async def test_through_filters_from_request(app_client):
     request = Request.fake(
         '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
     )
-    filter_args = await (
-        through_filters(
-            request=request,
-            datasette=app_client.ds,
-            table="roadside_attractions",
-            database="fixtures",
-        )
+    filter_args = await through_filters(
+        request=request,
+        datasette=app_client.ds,
+        table="roadside_attractions",
+        database="fixtures",
     )()
     assert filter_args.where_clauses == [
         "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)"
@@ -127,12 +123,10 @@ async def test_through_filters_from_request(app_client):
 @pytest.mark.asyncio
 async def test_where_filters_from_request(app_client):
     request = Request.fake("/?_where=pk+>+3")
-    filter_args = await (
-        where_filters(
-            request=request,
-            datasette=app_client.ds,
-            database="fixtures",
-        )
+    filter_args = await where_filters(
+        request=request,
+        datasette=app_client.ds,
+        database="fixtures",
     )()
     assert filter_args.where_clauses == ["pk > 3"]
     assert filter_args.params == {}
@@ -145,13 +139,11 @@ async def test_where_filters_from_request(app_client):
 @pytest.mark.asyncio
 async def test_search_filters_from_request(app_client):
     request = Request.fake("/?_search=bobcat")
-    filter_args = await (
-        search_filters(
-            request=request,
-            datasette=app_client.ds,
-            database="fixtures",
-            table="searchable",
-        )
+    filter_args = await search_filters(
+        request=request,
+        datasette=app_client.ds,
+        database="fixtures",
+        table="searchable",
     )()
     assert filter_args.where_clauses == [
         "rowid in (select rowid from searchable_fts where searchable_fts match escape_fts(:search))"

tests/test_internals_datasette.py

@@ -130,7 +130,14 @@ async def test_datasette_ensure_permissions_check_visibility(
 @pytest.mark.asyncio
 async def test_datasette_render_template_no_request():
     # https://github.com/simonw/datasette/issues/1849
-    ds = Datasette([], memory=True)
+    ds = Datasette(memory=True)
     await ds.invoke_startup()
     rendered = await ds.render_template("error.html")
     assert "Error " in rendered
+
+
+def test_datasette_error_if_string_not_list(tmpdir):
+    # https://github.com/simonw/datasette/issues/1985
+    db_path = str(tmpdir / "data.db")
+    with pytest.raises(ValueError):
+        ds = Datasette(db_path)

tests/test_load_extensions.py

@@ -8,6 +8,7 @@ from pathlib import Path
 # this resolves to "./ext", which is enough for SQLite to calculate the rest
 COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext")
 
+
 # See if ext.c has been compiled, based off the different possible suffixes.
 def has_compiled_ext():
     for ext in ["dylib", "so", "dll"]:
@@ -20,7 +21,6 @@ def has_compiled_ext():
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
 async def test_load_extension_default_entrypoint():
-
     # The default entrypoint only loads a() and NOT b() or c(), so those
     # should fail.
     ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
@@ -41,7 +41,6 @@ async def test_load_extension_default_entrypoint():
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
 async def test_load_extension_multiple_entrypoints():
-
     # Load in the default entrypoint and the other 2 custom entrypoints, now
     # all a(), b(), and c() should run successfully.
     ds = Datasette(

tests/test_publish_cloudrun.py

@@ -57,7 +57,7 @@ def test_publish_cloudrun_prompts_for_service(
         "Service name: input-service"
     ) == result.output.strip()
     assert 0 == result.exit_code
-    tag = "gcr.io/myproject/datasette"
+    tag = "gcr.io/myproject/datasette-input-service"
     mock_call.assert_has_calls(
         [
             mock.call(f"gcloud builds submit --tag {tag}", shell=True),
@@ -86,7 +86,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory):
         cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"]
     )
     assert 0 == result.exit_code
-    tag = f"gcr.io/{mock_output.return_value}/datasette"
+    tag = f"gcr.io/{mock_output.return_value}/datasette-test"
     mock_call.assert_has_calls(
         [
             mock.call(f"gcloud builds submit --tag {tag}", shell=True),
@@ -167,7 +167,7 @@ def test_publish_cloudrun_memory_cpu(
         assert 2 == result.exit_code
         return
     assert 0 == result.exit_code
-    tag = f"gcr.io/{mock_output.return_value}/datasette"
+    tag = f"gcr.io/{mock_output.return_value}/datasette-test"
     expected_call = (
         "gcloud run deploy --allow-unauthenticated --platform=managed"
         " --image {} test".format(tag)

tests/test_publish_heroku.py

@@ -2,6 +2,7 @@ from click.testing import CliRunner
 from datasette import cli
 from unittest import mock
 import os
+import pathlib
 import pytest
 
 
@@ -128,3 +129,55 @@ def test_publish_heroku_plugin_secrets(
             mock.call(["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"]),
         ]
     )
+
+
+@pytest.mark.serial
+@mock.patch("shutil.which")
+@mock.patch("datasette.publish.heroku.check_output")
+@mock.patch("datasette.publish.heroku.call")
+def test_publish_heroku_generate_dir(
+    mock_call, mock_check_output, mock_which, tmp_path_factory
+):
+    mock_which.return_value = True
+    mock_check_output.side_effect = lambda s: {
+        "['heroku', 'plugins']": b"heroku-builds",
+    }[repr(s)]
+    runner = CliRunner()
+    os.chdir(tmp_path_factory.mktemp("runner"))
+    with open("test.db", "w") as fp:
+        fp.write("data")
+    output = str(tmp_path_factory.mktemp("generate_dir") / "output")
+    result = runner.invoke(
+        cli.cli,
+        [
+            "publish",
+            "heroku",
+            "test.db",
+            "--generate-dir",
+            output,
+        ],
+    )
+    assert result.exit_code == 0
+    path = pathlib.Path(output)
+    assert path.exists()
+    file_names = {str(r.relative_to(path)) for r in path.glob("*")}
+    assert file_names == {
+        "requirements.txt",
+        "bin",
+        "runtime.txt",
+        "Procfile",
+        "test.db",
+    }
+    for name, expected in (
+        ("requirements.txt", "datasette"),
+        ("runtime.txt", "python-3.11.0"),
+        (
+            "Procfile",
+            (
+                "web: datasette serve --host 0.0.0.0 -i test.db "
+                "--cors --port $PORT --inspect-file inspect-data.json"
+            ),
+        ),
+    ):
+        with open(path / name) as fp:
+            assert fp.read().strip() == expected