Mechanism for creating custom pages using templates

Closes #648

datasette/app.py

@@ -17,6 +17,7 @@ from markupsafe import Markup
 import jinja2
 from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape
 from jinja2.environment import Template
+from jinja2.exceptions import TemplateNotFound
 import uvicorn
 
 from .views.base import DatasetteError, ureg, AsgiRouter
@@ -745,7 +746,7 @@ class DatasetteRouter(AsgiRouter):
             path = "/" + path[len(base_url) :]
         return await super().route_path(scope, receive, send, path)
 
-    async def handle_404(self, scope, receive, send):
+    async def handle_404(self, scope, receive, send, exception=None):
         # If URL has a trailing slash, redirect to URL without it
         path = scope.get("raw_path", scope["path"].encode("utf8"))
         if path.endswith(b"/"):
@@ -754,7 +755,54 @@ class DatasetteRouter(AsgiRouter):
                 path += b"?" + scope["query_string"]
             await asgi_send_redirect(send, path.decode("latin1"))
         else:
-            await super().handle_404(scope, receive, send)
+            # Is there a pages/* template matching this path?
+            template_path = os.path.join("pages", *scope["path"].split("/")) + ".html"
+            try:
+                template = self.ds.jinja_env.select_template([template_path])
+            except TemplateNotFound:
+                template = None
+            if template:
+                headers = {}
+                status = [200]
+
+                def custom_header(name, value):
+                    headers[name] = value
+                    return ""
+
+                def custom_status(code):
+                    status[0] = code
+                    return ""
+
+                def custom_redirect(location, code=302):
+                    status[0] = code
+                    headers["Location"] = location
+                    return ""
+
+                body = await self.ds.render_template(
+                    template,
+                    {
+                        "custom_header": custom_header,
+                        "custom_status": custom_status,
+                        "custom_redirect": custom_redirect,
+                    },
+                    view_name="page",
+                )
+                # Pull content-type out into separate parameter
+                content_type = "text/html"
+                matches = [k for k in headers if k.lower() == "content-type"]
+                if matches:
+                    content_type = headers[matches[0]]
+                await asgi_send(
+                    send,
+                    body,
+                    status=status[0],
+                    headers=headers,
+                    content_type=content_type,
+                )
+            else:
+                await self.handle_500(
+                    scope, receive, send, exception or NotFound("404")
+                )
 
     async def handle_500(self, scope, receive, send, exception):
         title = None

datasette/utils/asgi.py

@@ -119,11 +119,13 @@ class AsgiRouter:
                 new_scope = dict(scope, url_route={"kwargs": match.groupdict()})
                 try:
                     return await view(new_scope, receive, send)
+                except NotFound as exception:
+                    return await self.handle_404(scope, receive, send, exception)
                 except Exception as exception:
                     return await self.handle_500(scope, receive, send, exception)
         return await self.handle_404(scope, receive, send)
 
-    async def handle_404(self, scope, receive, send):
+    async def handle_404(self, scope, receive, send, exception=None):
         await send(
             {
                 "type": "http.response.start",

docs/custom_templates.rst

@@ -265,3 +265,64 @@ Here is an example of a custom ``_table.html`` template::
             <p>Category: {{ row.display("category_id") }}</p>
         </div>
     {% endfor %}
+
+.. _custom_pages:
+
+Custom pages
+------------
+
+You can add templated pages to your Datasette instance by creating HTML files in a ``pages`` directory within your ``templates`` directory.
+
+For example, to add a custom page that is served at ``http://localhost/about`` you would create a file in ``templates/pages/about.html``, then start Datasette like this::
+
+    $ datasette mydb.db --template-dir=templates/
+
+You can nest directories within pages to create a nested structure. To create a ``http://localhost:8001/about/map`` page you would create ``templates/pages/about/map.html``.
+
+Custom headers and status codes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Custom pages default to being served with a content-type of ``text/html`` and a ``200`` status code. You can change these by calling a custom function from within your template.
+
+For example, to serve a custom page with a ``418 I'm a teapot`` HTTP status code, create a file in ``pages/teapot.html`` containing the following::
+
+    {{ custom_status(418) }}
+    <html>
+    <head><title>Teapot</title></head>
+    <body>
+    I'm a teapot
+    </body>
+    </html>
+
+To serve a custom HTTP header, add a ``custom_header(name, value)`` function call. For example::
+
+    {{ custom_status(418) }}
+    {{ custom_header("x-teapot", "I am") }}
+    <html>
+    <head><title>Teapot</title></head>
+    <body>
+    I'm a teapot
+    </body>
+    </html>
+
+You can verify this is working using ``curl`` like this::
+
+    $ curl -I 'http://127.0.0.1:8001/teapot'
+    HTTP/1.1 418
+    date: Sun, 26 Apr 2020 18:38:30 GMT
+    server: uvicorn
+    x-teapot: I am
+    content-type: text/html
+
+Custom redirects
+~~~~~~~~~~~~~~~~
+
+You can use the ``custom_redirect(location)`` function to redirect users to another page, for example in a file called ``pages/datasette.html``::
+
+    {{ custom_redirect("https://github.com/simonw/datasette") }}
+
+Now requests to ``http://localhost:8001/datasette`` will result in a redirect.
+
+These redirects are served with a ``301 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function::
+
+    {{ custom_redirect("https://github.com/simonw/datasette", 301) }}

tests/fixtures.py

@@ -109,6 +109,7 @@ def make_app_client(
     inspect_data=None,
     static_mounts=None,
     template_dir=None,
+    extra_plugins=None,
 ):
     with tempfile.TemporaryDirectory() as tmpdir:
         filepath = os.path.join(tmpdir, filename)
@@ -133,6 +134,8 @@ def make_app_client(
         os.mkdir(plugins_dir)
         open(os.path.join(plugins_dir, "my_plugin.py"), "w").write(PLUGIN1)
         open(os.path.join(plugins_dir, "my_plugin_2.py"), "w").write(PLUGIN2)
+        for filename, content in (extra_plugins or {}).items():
+            open(os.path.join(plugins_dir, filename), "w").write(content)
         config = config or {}
         config.update(
             {

tests/test_custom_pages.py

@@ -0,0 +1,90 @@
+import pytest
+from .fixtures import make_app_client
+
+VIEW_NAME_PLUGIN = """
+from datasette import hookimpl
+
+@hookimpl
+def extra_template_vars(view_name):
+    return {
+        "view_name": view_name,
+    }
+"""
+
+
+@pytest.fixture(scope="session")
+def custom_pages_client(tmp_path_factory):
+    template_dir = tmp_path_factory.mktemp("page-templates")
+    extra_plugins = {"view_name.py": VIEW_NAME_PLUGIN}
+    pages_dir = template_dir / "pages"
+    pages_dir.mkdir()
+    (pages_dir / "about.html").write_text("ABOUT! view_name:{{ view_name }}", "utf-8")
+    (pages_dir / "202.html").write_text("{{ custom_status(202) }}202!", "utf-8")
+    (pages_dir / "headers.html").write_text(
+        '{{ custom_header("x-this-is-foo", "foo") }}FOO'
+        '{{ custom_header("x-this-is-bar", "bar") }}BAR',
+        "utf-8",
+    )
+    (pages_dir / "atom.html").write_text(
+        '{{ custom_header("content-type", "application/xml") }}<?xml ...>', "utf-8",
+    )
+    (pages_dir / "redirect.html").write_text(
+        '{{ custom_redirect("/example") }}', "utf-8"
+    )
+    (pages_dir / "redirect2.html").write_text(
+        '{{ custom_redirect("/example", 301) }}', "utf-8"
+    )
+    nested_dir = pages_dir / "nested"
+    nested_dir.mkdir()
+    (nested_dir / "nest.html").write_text("Nest!", "utf-8")
+    for client in make_app_client(
+        template_dir=str(template_dir), extra_plugins=extra_plugins
+    ):
+        yield client
+
+
+def test_custom_pages_view_name(custom_pages_client):
+    response = custom_pages_client.get("/about")
+    assert 200 == response.status
+    assert "ABOUT! view_name:page" == response.text
+
+
+def test_custom_pages_nested(custom_pages_client):
+    response = custom_pages_client.get("/nested/nest")
+    assert 200 == response.status
+    assert "Nest!" == response.text
+    response = custom_pages_client.get("/nested/nest2")
+    assert 404 == response.status
+
+
+def test_custom_status(custom_pages_client):
+    response = custom_pages_client.get("/202")
+    assert 202 == response.status
+    assert "202!" == response.text
+
+
+def test_custom_headers(custom_pages_client):
+    response = custom_pages_client.get("/headers")
+    assert 200 == response.status
+    assert "foo" == response.headers["x-this-is-foo"]
+    assert "bar" == response.headers["x-this-is-bar"]
+    assert "FOOBAR" == response.text
+
+
+def test_custom_content_type(custom_pages_client):
+    response = custom_pages_client.get("/atom")
+    assert 200 == response.status
+    assert response.headers["content-type"] == "application/xml"
+    assert "<?xml ...>" == response.text
+
+
+def test_redirect(custom_pages_client):
+    response = custom_pages_client.get("/redirect", allow_redirects=False)
+    assert 302 == response.status
+    assert "/example" == response.headers["Location"]
+
+
+def test_redirect2(custom_pages_client):
+    response = custom_pages_client.get("/redirect2", allow_redirects=False)
+    assert 301 == response.status
+    assert "/example" == response.headers["Location"]

tests/test_html.py

@@ -11,6 +11,7 @@ import json
 import pathlib
 import pytest
 import re
+import textwrap
 import urllib.parse