First partially working version of ASGI-powered Datasette #272

Lots still to do:

* Static files are not being served
* Streaming CSV files don't work
* Tests all fail
* Some URLs (e.g. the 'next' link on tables) are incorrect

But... the server does start up and you can browse databases/tables

datasette/app.py

@@ -17,7 +17,7 @@ from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
 from sanic import Sanic, response
 from sanic.exceptions import InvalidUsage, NotFound
 
-from .views.base import DatasetteError, ureg
+from .views.base import DatasetteError, ureg, AsgiRouter
 from .views.database import DatabaseDownload, DatabaseView
 from .views.index import IndexView
 from .views.special import JsonDataView
@@ -126,8 +126,15 @@ CONFIG_OPTIONS = (
 DEFAULT_CONFIG = {option.name: option.default for option in CONFIG_OPTIONS}
 
 
-async def favicon(request):
+async def favicon(scope, recieve, send):
-    return response.text("")
+    await send(
+        {
+            "type": "http.response.start",
+            "status": 200,
+            "headers": [[b"content-type", b"text/plain"]],
+        }
+    )
+    await send({"type": "http.response.body", "body": b""})
 
 
 class Datasette:
@@ -543,21 +550,8 @@ class Datasette:
             self.renderers[renderer["extension"]] = renderer["callback"]
 
     def app(self):
-        class TracingSanic(Sanic):
+        "Returns an ASGI app function that serves the whole of Datasette"
-            async def handle_request(self, request, write_callback, stream_callback):
+        # TODO: re-implement ?_trace= mechanism, see class TracingSanic
-                if request.args.get("_trace"):
-                    request["traces"] = []
-                    request["trace_start"] = time.time()
-                    with capture_traces(request["traces"]):
-                        await super().handle_request(
-                            request, write_callback, stream_callback
-                        )
-                else:
-                    await super().handle_request(
-                        request, write_callback, stream_callback
-                    )
-
-        app = TracingSanic(__name__)
         default_templates = str(app_root / "datasette" / "templates")
         template_paths = []
         if self.template_dir:
@@ -588,134 +582,86 @@ class Datasette:
         pm.hook.prepare_jinja2_environment(env=self.jinja_env)
 
         self.register_renderers()
+
+        routes = []
+
+        def add_route(view, regex):
+            routes.append((regex, view))
+
         # Generate a regex snippet to match all registered renderer file extensions
         renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
 
-        app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>")
+        add_route(IndexView.as_asgi(self), r"/(?P<as_format>(\.jsono?)?$)")
         # TODO: /favicon.ico and /-/static/ deserve far-future cache expires
-        app.add_route(favicon, "/favicon.ico")
+        add_route(favicon, "/favicon.ico")
-        app.static("/-/static/", str(app_root / "datasette" / "static"))
+        # # TODO: re-enable the static bits
-        for path, dirname in self.static_mounts:
+        # app.static("/-/static/", str(app_root / "datasette" / "static"))
-            app.static(path, dirname)
+        # for path, dirname in self.static_mounts:
-        # Mount any plugin static/ directories
+        #     app.static(path, dirname)
-        for plugin in get_plugins(pm):
+        # # Mount any plugin static/ directories
-            if plugin["static_path"]:
+        # for plugin in get_plugins(pm):
-                modpath = "/-/static-plugins/{}/".format(plugin["name"])
+        #     if plugin["static_path"]:
-                app.static(modpath, plugin["static_path"])
+        #         modpath = "/-/static-plugins/{}/".format(plugin["name"])
-        app.add_route(
+        #         app.static(modpath, plugin["static_path"])
-            JsonDataView.as_view(self, "metadata.json", lambda: self._metadata),
+        add_route(
-            r"/-/metadata<as_format:(\.json)?$>",
+            JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata),
+            r"/-/metadata(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
+        add_route(
-            JsonDataView.as_view(self, "versions.json", self.versions),
+            JsonDataView.as_asgi(self, "versions.json", self.versions),
-            r"/-/versions<as_format:(\.json)?$>",
+            r"/-/versions(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
+        add_route(
-            JsonDataView.as_view(self, "plugins.json", self.plugins),
+            JsonDataView.as_asgi(self, "plugins.json", self.plugins),
-            r"/-/plugins<as_format:(\.json)?$>",
+            r"/-/plugins(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
+        add_route(
-            JsonDataView.as_view(self, "config.json", lambda: self._config),
+            JsonDataView.as_asgi(self, "config.json", lambda: self._config),
-            r"/-/config<as_format:(\.json)?$>",
+            r"/-/config(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
+        add_route(
-            JsonDataView.as_view(self, "databases.json", self.connected_databases),
+            JsonDataView.as_asgi(self, "databases.json", self.connected_databases),
-            r"/-/databases<as_format:(\.json)?$>",
+            r"/-/databases(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
+        add_route(
-            DatabaseDownload.as_view(self), r"/<db_name:[^/]+?><as_db:(\.db)$>"
+            DatabaseDownload.as_asgi(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$"
         )
-        app.add_route(
+        add_route(
-            DatabaseView.as_view(self),
+            DatabaseView.as_asgi(self),
-            r"/<db_name:[^/]+?><as_format:(" + renderer_regex + r"|.jsono|\.csv)?$>",
+            r"/(?P<db_name>[^/]+?)(?P<as_format>"
+            + renderer_regex
+            + r"|.jsono|\.csv)?$",
         )
-        app.add_route(
+        add_route(
-            TableView.as_view(self), r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>"
+            TableView.as_asgi(self),
+            r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)",
         )
-        app.add_route(
+        add_route(
-            RowView.as_view(self),
+            RowView.as_asgi(self),
             r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:("
             + renderer_regex
             + r")?$>",
         )
         self.register_custom_units()
 
+        app = AsgiRouter(routes)
         # On 404 with a trailing slash redirect to path without that slash:
         # pylint: disable=unused-variable
-        @app.middleware("response")
+        # TODO: re-enable this
-        def redirect_on_404_with_trailing_slash(request, original_response):
+        # @app.middleware("response")
-            if original_response.status == 404 and request.path.endswith("/"):
+        # def redirect_on_404_with_trailing_slash(request, original_response):
-                path = request.path.rstrip("/")
+        #     if original_response.status == 404 and request.path.endswith("/"):
-                if request.query_string:
+        #         path = request.path.rstrip("/")
-                    path = "{}?{}".format(path, request.query_string)
+        #         if request.query_string:
-                return response.redirect(path)
+        #             path = "{}?{}".format(path, request.query_string)
-
+        #         return response.redirect(path)
-        @app.middleware("response")
-        async def add_traces_to_response(request, response):
-            if request.get("traces") is None:
-                return
-            traces = request["traces"]
-            trace_info = {
-                "request_duration_ms": 1000 * (time.time() - request["trace_start"]),
-                "sum_trace_duration_ms": sum(t["duration_ms"] for t in traces),
-                "num_traces": len(traces),
-                "traces": traces,
-            }
-            if "text/html" in response.content_type and b"</body>" in response.body:
-                extra = json.dumps(trace_info, indent=2)
-                extra_html = "<pre>{}</pre></body>".format(extra).encode("utf8")
-                response.body = response.body.replace(b"</body>", extra_html)
-            elif "json" in response.content_type and response.body.startswith(b"{"):
-                data = json.loads(response.body.decode("utf8"))
-                if "_trace" not in data:
-                    data["_trace"] = trace_info
-                    response.body = json.dumps(data).encode("utf8")
-
-        @app.exception(Exception)
-        def on_exception(request, exception):
-            title = None
-            help = None
-            if isinstance(exception, NotFound):
-                status = 404
-                info = {}
-                message = exception.args[0]
-            elif isinstance(exception, InvalidUsage):
-                status = 405
-                info = {}
-                message = exception.args[0]
-            elif isinstance(exception, DatasetteError):
-                status = exception.status
-                info = exception.error_dict
-                message = exception.message
-                if exception.messagge_is_html:
-                    message = Markup(message)
-                title = exception.title
-            else:
-                status = 500
-                info = {}
-                message = str(exception)
-                traceback.print_exc()
-            templates = ["500.html"]
-            if status != 500:
-                templates = ["{}.html".format(status)] + templates
-            info.update(
-                {"ok": False, "error": message, "status": status, "title": title}
-            )
-            if request is not None and request.path.split("?")[0].endswith(".json"):
-                r = response.json(info, status=status)
-
-            else:
-                template = self.jinja_env.select_template(templates)
-                r = response.html(template.render(info), status=status)
-            if self.cors:
-                r.headers["Access-Control-Allow-Origin"] = "*"
-            return r
 
         # First time server starts up, calculate table counts for immutable databases
-        @app.listener("before_server_start")
+        # TODO: re-enable this mechanism
-        async def setup_db(app, loop):
+        # @app.listener("before_server_start")
-            for dbname, database in self.databases.items():
+        # async def setup_db(app, loop):
-                if not database.is_mutable:
+        #     for dbname, database in self.databases.items():
-                    await database.table_counts(limit=60 * 60 * 1000)
+        #         if not database.is_mutable:
+        #             await database.table_counts(limit=60 * 60 * 1000)
 
         return app

datasette/cli.py

@@ -1,4 +1,5 @@
 import asyncio
+import uvicorn
 import click
 from click import formatting
 from click_default_group import DefaultGroup
@@ -354,4 +355,4 @@ def serve(
     asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks())
 
     # Start the server
-    ds.app().run(host=host, port=port, debug=debug)
+    uvicorn.run(ds.app(), host=host, port=port, log_level="info")

datasette/views/base.py

@@ -10,6 +10,7 @@ import pint
 from sanic import response
 from sanic.exceptions import NotFound
 from sanic.views import HTTPMethodView
+from sanic.request import Request as SanicRequest
 
 from datasette import __version__
 from datasette.plugins import pm
@@ -54,7 +55,7 @@ class AsgiRouter:
         routes = routes or []
         self.routes = [
             # Compile any strings to regular expressions
-            (re.compile(pattern) if isinstance(pattern, str) else pattern, view)
+            ((re.compile(pattern) if isinstance(pattern, str) else pattern), view)
             for pattern, view in routes
         ]
 
@@ -77,29 +78,48 @@ class AsgiRouter:
         await send({"type": "http.response.body", "body": b"<h1>404</h1>"})
 
 
-async def hello_world(scope, receive, send):
-    assert scope["type"] == "http"
-    await send(
-        {
-            "type": "http.response.start",
-            "status": 200,
-            "headers": [[b"content-type", b"text/html"]],
-        }
-    )
-    await send({"type": "http.response.body", "body": b"<h1>Hello world!</h1>"})
-
-
-app = AsgiRouter([("/hello/", hello_world)])
-
-
 class AsgiView(HTTPMethodView):
-    async def asgi(self, scope, receive, send):
+    @classmethod
-        # Uses scope to create a Sanic-compatible request object,
+    def as_asgi(cls, *class_args, **class_kwargs):
-        # then dispatches that to self.get(...) or self.options(...)
+        async def view(scope, receive, send):
-        # along with keyword arguments that were already tucked
+            # Uses scope to create a Sanic-compatible request object,
-        # into scope["url_route"]["kwargs"] by the router
+            # then dispatches that to self.get(...) or self.options(...)
-        # https://channels.readthedocs.io/en/latest/topics/routing.html#urlrouter
+            # along with keyword arguments that were already tucked
-        pass
+            # into scope["url_route"]["kwargs"] by the router
+            # https://channels.readthedocs.io/en/latest/topics/routing.html#urlrouter
+            path = scope.get("raw_path", scope["path"].encode("utf8"))
+            if scope["query_string"]:
+                path = path + b"?" + scope["query_string"]
+            request = SanicRequest(path, {}, "1.1", scope["method"], None)
+
+            class Woo:
+                def get_extra_info(self, key):
+                    return False
+
+            request.app = Woo()
+            request.app.websocket_enabled = False
+            request.transport = Woo()
+            self = view.view_class(*class_args, **class_kwargs)
+            response = await self.dispatch_request(
+                request, **scope["url_route"]["kwargs"]
+            )
+            await send(
+                {
+                    "type": "http.response.start",
+                    "status": response.status,
+                    "headers": [
+                        [key.encode("utf-8"), value.encode("utf-8")]
+                        for key, value in response.headers.items()
+                    ],
+                }
+            )
+            await send({"type": "http.response.body", "body": response.body})
+
+        view.view_class = cls
+        view.__doc__ = cls.__doc__
+        view.__module__ = cls.__module__
+        view.__name__ = cls.__name__
+        return view
 
 
 class BaseView(AsgiView):
@@ -250,17 +270,17 @@ class DataView(BaseView):
                 kwargs["table"] = table
                 if _format:
                     kwargs["as_format"] = ".{}".format(_format)
-            elif "table" in kwargs:
+            elif kwargs.get("table"):
                 kwargs["table"] = urllib.parse.unquote_plus(kwargs["table"])
 
             should_redirect = "/{}-{}".format(name, expected)
-            if "table" in kwargs:
+            if kwargs.get("table"):
                 should_redirect += "/" + urllib.parse.quote_plus(kwargs["table"])
-            if "pk_path" in kwargs:
+            if kwargs.get("pk_path"):
                 should_redirect += "/" + kwargs["pk_path"]
-            if "as_format" in kwargs:
+            if kwargs.get("as_format"):
                 should_redirect += kwargs["as_format"]
-            if "as_db" in kwargs:
+            if kwargs.get("as_db"):
                 should_redirect += kwargs["as_db"]
 
             if (

setup.py

@@ -37,7 +37,7 @@ setup(
     author="Simon Willison",
     license="Apache License, Version 2.0",
     url="https://github.com/simonw/datasette",
-    packages=find_packages(exclude='tests'),
+    packages=find_packages(exclude="tests"),
     package_data={"datasette": ["templates/*.html"]},
     include_package_data=True,
     install_requires=[
@@ -48,6 +48,7 @@ setup(
         "hupper==1.0",
         "pint==0.8.1",
         "pluggy>=0.12.0",
+        "uvicorn>=0.8.1",
     ],
     entry_points="""
         [console_scripts]