From e139a7619f63d45ca2ff1ee108b933e17b5675b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2020 17:24:40 -0700 Subject: [PATCH] 'datasette --get' option, closes #926 Also made a start on the datasette.utils.testing module, refs #898 --- datasette/cli.py | 11 +++ datasette/utils/testing.py | 151 +++++++++++++++++++++++++++++++++ docs/datasette-serve-help.txt | 3 + docs/getting_started.rst | 64 ++++++++++++-- docs/index.rst | 2 +- docs/installation.rst | 2 +- setup.py | 2 +- tests/fixtures.py | 152 +--------------------------------- tests/test_cli.py | 15 ++++ 9 files changed, 242 insertions(+), 160 deletions(-) create mode 100644 datasette/utils/testing.py diff --git a/datasette/cli.py b/datasette/cli.py index 4480dec9..5c81489e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .utils import ( StaticMount, ValueAsBooleanError, ) +from .utils.testing import TestClient class Config(click.ParamType): @@ -335,6 +336,9 @@ def uninstall(packages, yes): help="Output URL that sets a cookie authenticating the root user", is_flag=True, ) +@click.option( + "--get", help="Run an HTTP GET request against this path, print results and exit", +) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-config", is_flag=True, help="Show available config options") def serve( @@ -355,6 +359,7 @@ def serve( config, secret, root, + get, version_note, help_config, return_instance=False, @@ -411,6 +416,12 @@ def serve( ds = Datasette(files, **kwargs) + if get: + client = TestClient(ds.app()) + response = client.get(get) + click.echo(response.text) + return + if return_instance: # Private utility mechanism for writing unit tests return ds diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py new file mode 100644 index 00000000..dc261dc8 --- /dev/null +++ b/datasette/utils/testing.py @@ -0,0 +1,151 @@ +from datasette.utils import MultiParams +from asgiref.testing import ApplicationCommunicator +from asgiref.sync import async_to_sync +from urllib.parse import unquote, quote, urlencode +from http.cookies import SimpleCookie +import json + + +class TestResponse: + def __init__(self, status, headers, body): + self.status = status + self.headers = headers + self.body = body + + @property + def cookies(self): + cookie = SimpleCookie() + for header in self.headers.getlist("set-cookie"): + cookie.load(header) + return {key: value.value for key, value in cookie.items()} + + @property + def json(self): + return json.loads(self.text) + + @property + def text(self): + return self.body.decode("utf8") + + +class TestClient: + max_redirects = 5 + + def __init__(self, asgi_app): + self.asgi_app = asgi_app + + def actor_cookie(self, actor): + return self.ds.sign({"a": actor}, "actor") + + @async_to_sync + async def get( + self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None + ): + return await self._request( + path, allow_redirects, redirect_count, method, cookies + ) + + @async_to_sync + async def post( + self, + path, + post_data=None, + allow_redirects=True, + redirect_count=0, + content_type="application/x-www-form-urlencoded", + cookies=None, + csrftoken_from=None, + ): + cookies = cookies or {} + post_data = post_data or {} + # Maybe fetch a csrftoken first + if csrftoken_from is not None: + if csrftoken_from is True: + csrftoken_from = path + token_response = await self._request(csrftoken_from, cookies=cookies) + csrftoken = token_response.cookies["ds_csrftoken"] + cookies["ds_csrftoken"] = csrftoken + post_data["csrftoken"] = csrftoken + return await self._request( + path, + allow_redirects, + redirect_count, + "POST", + cookies, + post_data, + content_type, + ) + + async def _request( + self, + path, + allow_redirects=True, + redirect_count=0, + method="GET", + cookies=None, + post_data=None, + content_type=None, + ): + query_string = b"" + if "?" in path: + path, _, query_string = path.partition("?") + query_string = query_string.encode("utf8") + if "%" in path: + raw_path = path.encode("latin-1") + else: + raw_path = quote(path, safe="/:,").encode("latin-1") + headers = [[b"host", b"localhost"]] + if content_type: + headers.append((b"content-type", content_type.encode("utf-8"))) + if cookies: + sc = SimpleCookie() + for key, value in cookies.items(): + sc[key] = value + headers.append([b"cookie", sc.output(header="").encode("utf-8")]) + scope = { + "type": "http", + "http_version": "1.0", + "method": method, + "path": unquote(path), + "raw_path": raw_path, + "query_string": query_string, + "headers": headers, + } + instance = ApplicationCommunicator(self.asgi_app, scope) + + if post_data: + body = urlencode(post_data, doseq=True).encode("utf-8") + await instance.send_input({"type": "http.request", "body": body}) + else: + await instance.send_input({"type": "http.request"}) + + # First message back should be response.start with headers and status + messages = [] + start = await instance.receive_output(2) + messages.append(start) + assert start["type"] == "http.response.start" + response_headers = MultiParams( + [(k.decode("utf8"), v.decode("utf8")) for k, v in start["headers"]] + ) + status = start["status"] + # Now loop until we run out of response.body + body = b"" + while True: + message = await instance.receive_output(2) + messages.append(message) + assert message["type"] == "http.response.body" + body += message["body"] + if not message.get("more_body"): + break + response = TestResponse(status, response_headers, body) + if allow_redirects and response.status in (301, 302): + assert ( + redirect_count < self.max_redirects + ), "Redirected {} times, max_redirects={}".format( + redirect_count, self.max_redirects + ) + location = response.headers["Location"] + return await self._request( + location, allow_redirects=True, redirect_count=redirect_count + 1 + ) + return response diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 183ecc14..0f84fd42 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -33,6 +33,9 @@ Options: cookies --root Output URL that sets a cookie authenticating the root user + --get TEXT Run an HTTP GET request against this path, print results and + exit + --version-note TEXT Additional note to show on /-/versions --help-config Show available config options --help Show this message and exit. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index fdf7d23c..22cf6bb2 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -9,7 +9,7 @@ The best way to experience Datasette for the first time is with a demo: * `fivethirtyeight.datasettes.com `__ shows Datasette running against over 400 datasets imported from the `FiveThirtyEight GitHub repository `__. * `sf-trees.datasettes.com `__ demonstrates the `datasette-cluster-map `__ plugin running against 190,000 trees imported from `data.sfgov.org `__. -.. _glitch: +.. _getting_started_glitch: Try Datasette without installing anything using Glitch ------------------------------------------------------ @@ -33,6 +33,8 @@ Need some data? Try this `Public Art Data `__. +.. _getting_started_your_computer: + Using Datasette on your own computer ------------------------------------ @@ -40,13 +42,11 @@ First, follow the :ref:`installation` instructions. Now you can run Datasette ag :: - datasette serve path/to/database.db + datasette path/to/database.db This will start a web server on port 8001 - visit http://localhost:8001/ to access the web interface. -``serve`` is the default subcommand, you can omit it if you like. - Use Chrome on OS X? You can run datasette against your browser history like so: @@ -90,7 +90,7 @@ JSON: } http://localhost:8001/History/downloads.json?_shape=objects will return that data as -JSON in a more convenient but less efficient format: +JSON in a more convenient format: :: @@ -109,7 +109,57 @@ JSON in a more convenient but less efficient format: ] } -datasette serve options ------------------------ +.. _getting_started_datasette_get: + +datasette --get +--------------- + +The ``--get`` option can specify the path to a page within Datasette and cause Datasette to output the content from that path without starting the web server. This means that all of Datasette's functionality can be accessed directly from the command-line. For example:: + + $ datasette --get '/-/versions.json' | jq . + { + "python": { + "version": "3.8.5", + "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" + }, + "datasette": { + "version": "0.46+15.g222a84a.dirty" + }, + "asgi": "3.0", + "uvicorn": "0.11.8", + "sqlite": { + "version": "3.32.3", + "fts_versions": [ + "FTS5", + "FTS4", + "FTS3" + ], + "extensions": { + "json1": null + }, + "compile_options": [ + "COMPILER=clang-11.0.3", + "ENABLE_COLUMN_METADATA", + "ENABLE_FTS3", + "ENABLE_FTS3_PARENTHESIS", + "ENABLE_FTS4", + "ENABLE_FTS5", + "ENABLE_GEOPOLY", + "ENABLE_JSON1", + "ENABLE_PREUPDATE_HOOK", + "ENABLE_RTREE", + "ENABLE_SESSION", + "MAX_VARIABLE_NUMBER=250000", + "THREADSAFE=1" + ] + } + } + +.. _getting_started_serve_help: + +datasette serve --help +---------------------- + +Running ``datasette downloads.db`` executes the default ``serve`` sub-command, and is equivalent to running ``datasette serve downloads.db``. The full list of options to that command is shown below. .. literalinclude:: datasette-serve-help.txt diff --git a/docs/index.rst b/docs/index.rst index 26c74043..834dad81 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,7 @@ Datasette is a tool for exploring and publishing data. It helps people take data Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world. It is part of a :ref:`wider ecosystem of tools and plugins ` dedicated to making working with structured data as productive as possible. -`Explore a demo `__, watch `a presentation about the project `__ or :ref:`glitch`. +`Explore a demo `__, watch `a presentation about the project `__ or :ref:`getting_started_glitch`. More examples: https://github.com/simonw/datasette/wiki/Datasettes diff --git a/docs/installation.rst b/docs/installation.rst index 2ca92e5e..7fc5b3e1 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -5,7 +5,7 @@ ============== .. note:: - If you just want to try Datasette out you don't need to install anything: see :ref:`glitch` + If you just want to try Datasette out you don't need to install anything: see :ref:`getting_started_glitch` There are two main options for installing Datasette. You can install it directly on to your machine, or you can install it using Docker. diff --git a/setup.py b/setup.py index 5ebb76db..7c352d87 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ setup( package_data={"datasette": ["templates/*.html"]}, include_package_data=True, install_requires=[ + "asgiref~=3.2.10", "click~=7.1.1", "click-default-group~=1.2.2", "Jinja2>=2.10.3,<2.12.0", @@ -70,7 +71,6 @@ setup( "pytest>=5.2.2,<6.1.0", "pytest-asyncio>=0.10,<0.15", "beautifulsoup4>=4.8.1,<4.10.0", - "asgiref~=3.2.3", "black~=19.10b0", ], }, diff --git a/tests/fixtures.py b/tests/fixtures.py index e29ea45d..139eff83 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,10 +1,8 @@ from datasette.app import Datasette -from datasette.utils import sqlite3, MultiParams -from asgiref.testing import ApplicationCommunicator -from asgiref.sync import async_to_sync +from datasette.utils import sqlite3 +from datasette.utils.testing import TestClient import click import contextlib -from http.cookies import SimpleCookie import itertools import json import os @@ -16,7 +14,6 @@ import string import tempfile import textwrap import time -from urllib.parse import unquote, quote, urlencode # This temp file is used by one of the plugin config tests @@ -89,151 +86,6 @@ EXPECTED_PLUGINS = [ ] -class TestResponse: - def __init__(self, status, headers, body): - self.status = status - self.headers = headers - self.body = body - - @property - def cookies(self): - cookie = SimpleCookie() - for header in self.headers.getlist("set-cookie"): - cookie.load(header) - return {key: value.value for key, value in cookie.items()} - - @property - def json(self): - return json.loads(self.text) - - @property - def text(self): - return self.body.decode("utf8") - - -class TestClient: - max_redirects = 5 - - def __init__(self, asgi_app): - self.asgi_app = asgi_app - - def actor_cookie(self, actor): - return self.ds.sign({"a": actor}, "actor") - - @async_to_sync - async def get( - self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None - ): - return await self._request( - path, allow_redirects, redirect_count, method, cookies - ) - - @async_to_sync - async def post( - self, - path, - post_data=None, - allow_redirects=True, - redirect_count=0, - content_type="application/x-www-form-urlencoded", - cookies=None, - csrftoken_from=None, - ): - cookies = cookies or {} - post_data = post_data or {} - # Maybe fetch a csrftoken first - if csrftoken_from is not None: - if csrftoken_from is True: - csrftoken_from = path - token_response = await self._request(csrftoken_from, cookies=cookies) - csrftoken = token_response.cookies["ds_csrftoken"] - cookies["ds_csrftoken"] = csrftoken - post_data["csrftoken"] = csrftoken - return await self._request( - path, - allow_redirects, - redirect_count, - "POST", - cookies, - post_data, - content_type, - ) - - async def _request( - self, - path, - allow_redirects=True, - redirect_count=0, - method="GET", - cookies=None, - post_data=None, - content_type=None, - ): - query_string = b"" - if "?" in path: - path, _, query_string = path.partition("?") - query_string = query_string.encode("utf8") - if "%" in path: - raw_path = path.encode("latin-1") - else: - raw_path = quote(path, safe="/:,").encode("latin-1") - headers = [[b"host", b"localhost"]] - if content_type: - headers.append((b"content-type", content_type.encode("utf-8"))) - if cookies: - sc = SimpleCookie() - for key, value in cookies.items(): - sc[key] = value - headers.append([b"cookie", sc.output(header="").encode("utf-8")]) - scope = { - "type": "http", - "http_version": "1.0", - "method": method, - "path": unquote(path), - "raw_path": raw_path, - "query_string": query_string, - "headers": headers, - } - instance = ApplicationCommunicator(self.asgi_app, scope) - - if post_data: - body = urlencode(post_data, doseq=True).encode("utf-8") - await instance.send_input({"type": "http.request", "body": body}) - else: - await instance.send_input({"type": "http.request"}) - - # First message back should be response.start with headers and status - messages = [] - start = await instance.receive_output(2) - messages.append(start) - assert start["type"] == "http.response.start" - response_headers = MultiParams( - [(k.decode("utf8"), v.decode("utf8")) for k, v in start["headers"]] - ) - status = start["status"] - # Now loop until we run out of response.body - body = b"" - while True: - message = await instance.receive_output(2) - messages.append(message) - assert message["type"] == "http.response.body" - body += message["body"] - if not message.get("more_body"): - break - response = TestResponse(status, response_headers, body) - if allow_redirects and response.status in (301, 302): - assert ( - redirect_count < self.max_redirects - ), "Redirected {} times, max_redirects={}".format( - redirect_count, self.max_redirects - ) - location = response.headers["Location"] - return await self._request( - location, allow_redirects=True, redirect_count=redirect_count + 1 - ) - return response - - @contextlib.contextmanager def make_app_client( sql_time_limit_ms=None, diff --git a/tests/test_cli.py b/tests/test_cli.py index c3d27f09..4dda4a71 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -50,6 +50,20 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache(): assert {"hithere": 44} == db.cached_table_counts +def test_serve_with_get(): + runner = CliRunner() + result = runner.invoke( + cli, + ["serve", "--memory", "--get", "/:memory:.json?sql=select+sqlite_version()"], + ) + assert 0 == result.exit_code, result.output + assert { + "database": ":memory:", + "truncated": False, + "columns": ["sqlite_version()"], + }.items() <= json.loads(result.output).items() + + def test_spatialite_error_if_attempt_to_open_spatialite(): runner = CliRunner() result = runner.invoke( @@ -102,6 +116,7 @@ def test_metadata_yaml(): secret=None, root=False, version_note=None, + get=None, help_config=False, return_instance=True, )