diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fa608055..b7d99c1c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,7 +31,10 @@ jobs: pip install -e '.[test]' - name: Run tests run: | - pytest + pytest -n auto -m "not serial" + pytest -m "serial" + # And the test that exceeds a localhost HTTPS server + tests/test_datasette_https_server.sh deploy: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 886f649a..39aa8b13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,8 @@ jobs: run: | pytest -n auto -m "not serial" pytest -m "serial" + # And the test that exceeds a localhost HTTPS server + tests/test_datasette_https_server.sh - name: Check if cog needs to be run run: | cog --check docs/*.rst diff --git a/datasette/app.py b/datasette/app.py index 246269f3..ea9bb6d2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -64,16 +64,14 @@ from .utils import ( ) from .utils.asgi import ( AsgiLifespan, - Base400, Forbidden, NotFound, Request, Response, + AsgiRunOnFirstRequest, asgi_static, asgi_send, asgi_send_file, - asgi_send_html, - asgi_send_json, asgi_send_redirect, ) from .utils.internal_db import init_internal_db, populate_schema_tables @@ -1260,7 +1258,7 @@ class Datasette: async def setup_db(): # First time server starts up, calculate table counts for immutable databases - for dbname, database in self.databases.items(): + for database in self.databases.values(): if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) @@ -1274,10 +1272,8 @@ class Datasette: ) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan( - asgi, - on_startup=setup_db, - ) + asgi = AsgiLifespan(asgi) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) return asgi @@ -1566,42 +1562,34 @@ class DatasetteClient: return path async def get(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.get(self._fix(path), **kwargs) async def options(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.options(self._fix(path), **kwargs) async def head(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.head(self._fix(path), **kwargs) async def post(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.post(self._fix(path), **kwargs) async def put(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.put(self._fix(path), **kwargs) async def patch(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.patch(self._fix(path), **kwargs) async def delete(self, path, **kwargs): - await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): - await self.ds.invoke_startup() avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) async with httpx.AsyncClient(app=self.app) as client: return await client.request( diff --git a/datasette/cli.py b/datasette/cli.py index 6eb42712..2b61292b 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -607,7 +607,7 @@ def serve( url = "http://{}:{}{}?token={}".format( host, port, ds.urls.path("-/auth-token"), ds._root_token ) - print(url) + click.echo(url) if open_browser: if url is None: # Figure out most convenient URL - to table, database or homepage diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 171252ce..f576a346 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -3,7 +3,9 @@ from datasette import hookimpl import click import json import os +import pathlib import shlex +import shutil from subprocess import call, check_output import tempfile @@ -28,6 +30,11 @@ def publish_subcommand(publish): "--tar", help="--tar option to pass to Heroku, e.g. --tar=/usr/local/bin/gtar", ) + @click.option( + "--generate-dir", + type=click.Path(dir_okay=True, file_okay=False), + help="Output generated application files and stop without deploying", + ) def heroku( files, metadata, @@ -49,6 +56,7 @@ def publish_subcommand(publish): about_url, name, tar, + generate_dir, ): "Publish databases to Datasette running on Heroku" fail_if_publish_binary_not_installed( @@ -105,6 +113,16 @@ def publish_subcommand(publish): secret, extra_metadata, ): + if generate_dir: + # Recursively copy files from current working directory to it + if pathlib.Path(generate_dir).exists(): + raise click.ClickException("Directory already exists") + shutil.copytree(".", generate_dir) + click.echo( + f"Generated files written to {generate_dir}, stopping without deploying", + err=True, + ) + return app_name = None if name: # Check to see if this app already exists @@ -176,7 +194,7 @@ def temporary_heroku_directory( fp.write(json.dumps(metadata_content, indent=2)) with open("runtime.txt", "w") as fp: - fp.write("python-3.8.10") + fp.write("python-3.11.0") if branch: install = [ diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 8a2fa060..16f90077 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -428,3 +428,18 @@ class AsgiFileDownload: content_type=self.content_type, headers=self.headers, ) + + +class AsgiRunOnFirstRequest: + def __init__(self, asgi, on_startup): + assert isinstance(on_startup, list) + self.asgi = asgi + self.on_startup = on_startup + self._started = False + + async def __call__(self, scope, receive, send): + if not self._started: + self._started = True + for hook in self.on_startup: + await hook() + return await self.asgi(scope, receive, send) diff --git a/datasette/version.py b/datasette/version.py index 3a4f06dc..8aaf793d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.63.1" +__version__ = "0.63.3" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0e0393ef..270765c8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,22 @@ Changelog ========= +.. _v0_63_3: + +0.63.3 (2022-12-17) +------------------- + +- Fixed a bug where ``datasette --root``, when running in Docker, would only output the URL to sign in as root when the server shut down, not when it started up. (:issue:`1958`) +- You no longer need to ensure ``await datasette.invoke_startup()`` has been called in order for Datasette to start correctly serving requests - this is now handled automatically the first time the server receives a request. This fixes a bug experienced when Datasette is served directly by an ASGI application server such as Uvicorn or Gunicorn. It also fixes a bug with the `datasette-gunicorn `__ plugin. (:issue:`1955`) + +.. _v0_63_2: + +0.63.2 (2022-11-18) +------------------- + +- Fixed a bug in ``datasette publish heroku`` where deployments failed due to an older version of Python being requested. (:issue:`1905`) +- New ``datasette publish heroku --generate-dir `` option for generating a Heroku deployment directory without deploying it. + .. _v0_63_1: 0.63.1 (2022-11-10) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 4a8465cb..a6885fc8 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -501,6 +501,8 @@ See :ref:`publish_heroku`. -n, --name TEXT Application name to use when deploying --tar TEXT --tar option to pass to Heroku, e.g. --tar=/usr/local/bin/gtar + --generate-dir DIRECTORY Output generated application files and stop + without deploying --help Show this message and exit. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b61f953a..101911cc 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -855,13 +855,14 @@ Potential use-cases: .. note:: - If you are writing :ref:`unit tests ` for a plugin that uses this hook you will need to explicitly call ``await ds.invoke_startup()`` in your tests. An example: + If you are writing :ref:`unit tests ` for a plugin that uses this hook and doesn't exercise Datasette by sending + any simulated requests through it you will need to explicitly call ``await ds.invoke_startup()`` in your tests. An example: .. code-block:: python @pytest.mark.asyncio async def test_my_plugin(): - ds = Datasette([], metadata={}) + ds = Datasette() await ds.invoke_startup() # Rest of test goes here diff --git a/docs/publish.rst b/docs/publish.rst index 4ba94792..7ae0399e 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -73,6 +73,10 @@ This will output some details about the new deployment, including a URL like thi You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app. +Rather than deploying directly you can use the ``--generate-dir`` option to output the files that would be deployed to a directory:: + + datasette publish heroku mydatabase.db --generate-dir=/tmp/deploy-this-to-heroku + See :ref:`cli_help_publish_heroku___help` for the full list of options for this command. .. _publish_vercel: diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 41f50e56..6d2097ad 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -80,7 +80,7 @@ Creating a ``Datasette()`` instance like this as useful shortcut in tests, but t This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. -If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you. +If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. .. _testing_plugins_pdb: diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index e5df7398..0c140818 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -25,6 +25,7 @@ async () => { let output = await pyodide.runPythonAsync(\` import micropip await micropip.install('h11==0.12.0') + await micropip.install('httpx==0.23') await micropip.install('http://localhost:8529/$wheel') import ssl import setuptools diff --git a/tests/conftest.py b/tests/conftest.py index f4638a14..ee13df11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,17 @@ UNDOCUMENTED_PERMISSIONS = { } +def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): + start = time.time() + while time.time() - start < timeout: + try: + client.get(url, **kwargs) + return + except httpx.ConnectError: + time.sleep(0.1) + raise AssertionError("Timed out waiting for {} to respond".format(url)) + + def pytest_report_header(config): return "SQLite: {}".format( sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] @@ -111,13 +122,7 @@ def ds_localhost_http_server(): # Avoid FileNotFoundError: [Errno 2] No such file or directory: cwd=tempfile.gettempdir(), ) - # Loop until port 8041 serves traffic - while True: - try: - httpx.get("http://localhost:8041/") - break - except httpx.ConnectError: - time.sleep(0.1) + wait_until_responds("http://localhost:8041/") # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc @@ -125,46 +130,6 @@ def ds_localhost_http_server(): ds_proc.terminate() -@pytest.fixture(scope="session") -def ds_localhost_https_server(tmp_path_factory): - cert_directory = tmp_path_factory.mktemp("certs") - ca = trustme.CA() - server_cert = ca.issue_cert("localhost") - keyfile = str(cert_directory / "server.key") - certfile = str(cert_directory / "server.pem") - client_cert = str(cert_directory / "client.pem") - server_cert.private_key_pem.write_to_path(path=keyfile) - for blob in server_cert.cert_chain_pems: - blob.write_to_path(path=certfile, append=True) - ca.cert_pem.write_to_path(path=client_cert) - ds_proc = subprocess.Popen( - [ - "datasette", - "--memory", - "-p", - "8042", - "--ssl-keyfile", - keyfile, - "--ssl-certfile", - certfile, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=tempfile.gettempdir(), - ) - while True: - try: - httpx.get("https://localhost:8042/", verify=client_cert) - break - except httpx.ConnectError: - time.sleep(0.1) - # Check it started successfully - assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") - yield ds_proc, client_cert - # Shut it down at the end of the pytest session - ds_proc.terminate() - - @pytest.fixture(scope="session") def ds_unix_domain_socket_server(tmp_path_factory): # This used to use tmp_path_factory.mktemp("uds") but that turned out to @@ -181,12 +146,7 @@ def ds_unix_domain_socket_server(tmp_path_factory): # Poll until available transport = httpx.HTTPTransport(uds=uds) client = httpx.Client(transport=transport) - while True: - try: - client.get("http://localhost/_memory.json") - break - except httpx.ConnectError: - time.sleep(0.1) + wait_until_responds("http://localhost/_memory.json", client=client) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") yield ds_proc, uds diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 1c31e2a3..47f23c08 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -13,17 +13,6 @@ def test_serve_localhost_http(ds_localhost_http_server): }.items() <= response.json().items() -@pytest.mark.serial -def test_serve_localhost_https(ds_localhost_https_server): - _, client_cert = ds_localhost_https_server - response = httpx.get("https://localhost:8042/_memory.json", verify=client_cert) - assert { - "database": "_memory", - "path": "/_memory", - "tables": [], - }.items() <= response.json().items() - - @pytest.mark.serial @pytest.mark.skipif( not hasattr(socket, "AF_UNIX"), reason="Requires socket.AF_UNIX support" diff --git a/tests/test_datasette_https_server.sh b/tests/test_datasette_https_server.sh new file mode 100755 index 00000000..a701ad4c --- /dev/null +++ b/tests/test_datasette_https_server.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Generate certificates +python -m trustme +# This creates server.pem, server.key, client.pem + +# Start the server in the background +datasette --memory \ + --ssl-keyfile=server.key \ + --ssl-certfile=server.pem \ + -p 8152 & + +# Store the background process ID in a variable +server_pid=$! + +# Wait for the server to start +sleep 2 + +# Make a test request using curl +curl -f --cacert client.pem 'https://localhost:8152/_memory.json' + +# Save curl's exit code (-f option causes it to return one on HTTP errors) +curl_exit_code=$? + +# Shut down the server +kill $server_pid +sleep 1 + +# Clean up the certificates +rm server.pem server.key client.pem + +echo $curl_exit_code +exit $curl_exit_code diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index b5a8af73..cab83654 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -2,6 +2,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock import os +import pathlib import pytest @@ -128,3 +129,55 @@ def test_publish_heroku_plugin_secrets( mock.call(["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"]), ] ) + + +@pytest.mark.serial +@mock.patch("shutil.which") +@mock.patch("datasette.publish.heroku.check_output") +@mock.patch("datasette.publish.heroku.call") +def test_publish_heroku_generate_dir( + mock_call, mock_check_output, mock_which, tmp_path_factory +): + mock_which.return_value = True + mock_check_output.side_effect = lambda s: { + "['heroku', 'plugins']": b"heroku-builds", + }[repr(s)] + runner = CliRunner() + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + output = str(tmp_path_factory.mktemp("generate_dir") / "output") + result = runner.invoke( + cli.cli, + [ + "publish", + "heroku", + "test.db", + "--generate-dir", + output, + ], + ) + assert result.exit_code == 0 + path = pathlib.Path(output) + assert path.exists() + file_names = {str(r.relative_to(path)) for r in path.glob("*")} + assert file_names == { + "requirements.txt", + "bin", + "runtime.txt", + "Procfile", + "test.db", + } + for name, expected in ( + ("requirements.txt", "datasette"), + ("runtime.txt", "python-3.11.0"), + ( + "Procfile", + ( + "web: datasette serve --host 0.0.0.0 -i test.db " + "--cors --port $PORT --inspect-file inspect-data.json" + ), + ), + ): + with open(path / name) as fp: + assert fp.read().strip() == expected