Consistently return charset utf-8, closes #752

datasette/app.py

@@ -828,7 +828,7 @@ class DatasetteRouter(AsgiRouter):
                     view_name="page",
                 )
                 # Pull content-type out into separate parameter
-                content_type = "text/html"
+                content_type = "text/html; charset=utf-8"
                 matches = [k for k in headers if k.lower() == "content-type"]
                 if matches:
                     content_type = headers[matches[0]]

datasette/utils/asgi.py

@@ -130,7 +130,7 @@ class AsgiRouter:
             {
                 "type": "http.response.start",
                 "status": 404,
-                "headers": [[b"content-type", b"text/html"]],
+                "headers": [[b"content-type", b"text/html; charset=utf-8"]],
             }
         )
         await send({"type": "http.response.body", "body": b"<h1>404</h1>"})
@@ -140,11 +140,11 @@ class AsgiRouter:
             {
                 "type": "http.response.start",
                 "status": 404,
-                "headers": [[b"content-type", b"text/html"]],
+                "headers": [[b"content-type", b"text/html; charset=utf-8"]],
             }
         )
         html = "<h1>500</h1><pre{}></pre>".format(escape(repr(exception)))
-        await send({"type": "http.response.body", "body": html.encode("latin-1")})
+        await send({"type": "http.response.body", "body": html.encode("utf-8")})
 
 
 class AsgiLifespan:
@@ -259,7 +259,11 @@ async def asgi_send_json(send, info, status=200, headers=None):
 async def asgi_send_html(send, html, status=200, headers=None):
     headers = headers or {}
     await asgi_send(
-        send, html, status=status, headers=headers, content_type="text/html"
+        send,
+        html,
+        status=status,
+        headers=headers,
+        content_type="text/html; charset=utf-8",
     )
 
 
@@ -269,13 +273,13 @@ async def asgi_send_redirect(send, location, status=302):
         "",
         status=status,
         headers={"Location": location},
-        content_type="text/html",
+        content_type="text/html; charset=utf-8",
     )
 
 
 async def asgi_send(send, content, status, headers=None, content_type="text/plain"):
     await asgi_start(send, status, headers, content_type)
-    await send({"type": "http.response.body", "body": content.encode("latin-1")})
+    await send({"type": "http.response.body", "body": content.encode("utf-8")})
 
 
 async def asgi_start(send, status, headers=None, content_type="text/plain"):

docs/custom_templates.rst

@@ -284,7 +284,7 @@ You can nest directories within pages to create a nested structure. To create a
 Custom headers and status codes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Custom pages default to being served with a content-type of ``text/html`` and a ``200`` status code. You can change these by calling a custom function from within your template.
+Custom pages default to being served with a content-type of ``text/html; charset=utf-8`` and a ``200`` status code. You can change these by calling a custom function from within your template.
 
 For example, to serve a custom page with a ``418 I'm a teapot`` HTTP status code, create a file in ``pages/teapot.html`` containing the following::
 
@@ -314,7 +314,7 @@ You can verify this is working using ``curl`` like this::
     date: Sun, 26 Apr 2020 18:38:30 GMT
     server: uvicorn
     x-teapot: I am
-    content-type: text/html
+    content-type: text/html; charset=utf-8
 
 Custom redirects
 ~~~~~~~~~~~~~~~~

tests/test_html.py

@@ -963,6 +963,12 @@ def test_404_trailing_slash_redirect(app_client, path, expected_redirect):
     assert expected_redirect == response.headers["Location"]
 
 
+def test_404_content_type(app_client):
+    response = app_client.get("/404")
+    assert 404 == response.status
+    assert "text/html; charset=utf-8" == response.headers["content-type"]
+
+
 def test_canned_query_with_custom_metadata(app_client):
     response = app_client.get("/fixtures/neighborhood_search?text=town")
     assert response.status == 200