Magic parameters for canned queries

Closes #842

Includes a new plugin hook, register_magic_parameters()

datasette/database.py

@@ -132,7 +132,7 @@ class Database:
             with sqlite_timelimit(conn, time_limit_ms):
                 try:
                     cursor = conn.cursor()
-                    cursor.execute(sql, params or {})
+                    cursor.execute(sql, params if params is not None else {})
                     max_returned_rows = self.ds.max_returned_rows
                     if max_returned_rows == page_size:
                         max_returned_rows += 1

datasette/default_magic_parameters.py

@@ -0,0 +1,55 @@
+from datasette import hookimpl
+from datasette.utils import escape_fts
+import datetime
+import os
+import time
+
+
+def header(key, request):
+    key = key.replace("_", "-").encode("utf-8")
+    headers_dict = dict(request.scope["headers"])
+    return headers_dict[key].decode("utf-8")
+
+
+def actor(key, request):
+    if request.actor is None:
+        raise KeyError
+    return request.actor[key]
+
+
+def cookie(key, request):
+    return request.cookies[key]
+
+
+def timestamp(key, request):
+    if key == "epoch":
+        return int(time.time())
+    elif key == "date_utc":
+        return datetime.datetime.utcnow().date().isoformat()
+    elif key == "datetime_utc":
+        return datetime.datetime.utcnow().strftime(r"%Y-%m-%dT%H:%M:%S") + "Z"
+    else:
+        raise KeyError
+
+
+def random(key, request):
+    if key.startswith("chars_") and key.split("chars_")[-1].isdigit():
+        num_chars = int(key.split("chars_")[-1])
+        if num_chars % 2 == 1:
+            urandom_len = (num_chars + 1) / 2
+        else:
+            urandom_len = num_chars / 2
+        return os.urandom(int(urandom_len)).hex()[:num_chars]
+    else:
+        raise KeyError
+
+
+@hookimpl
+def register_magic_parameters():
+    return [
+        ("header", header),
+        ("actor", actor),
+        ("cookie", cookie),
+        ("timestamp", timestamp),
+        ("random", random),
+    ]

datasette/hookspecs.py

@@ -83,3 +83,8 @@ def permission_allowed(datasette, actor, action, resource):
 @hookspec
 def canned_queries(datasette, database, actor):
     "Return a dictonary of canned query definitions or an awaitable function that returns them"
+
+
+@hookspec
+def register_magic_parameters(datasette):
+    "Return a list of (name, function) magic parameter functions"

datasette/plugins.py

@@ -11,6 +11,7 @@ DEFAULT_PLUGINS = (
     "datasette.sql_functions",
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
+    "datasette.default_magic_parameters",
 )
 
 pm = pluggy.PluginManager("datasette")

datasette/utils/__init__.py

@@ -4,6 +4,7 @@ import base64
 import click
 import hashlib
 import inspect
+import itertools
 import json
 import mergedeep
 import os
@@ -17,6 +18,7 @@ import urllib
 import numbers
 import yaml
 from .shutil_backport import copytree
+from ..plugins import pm
 
 try:
     import pysqlite3 as sqlite3

datasette/views/database.py

@@ -1,4 +1,5 @@
 import os
+import itertools
 import jinja2
 
 from datasette.utils import (
@@ -165,11 +166,12 @@ class QueryView(DataView):
         named_parameter_values = {
             named_parameter: params.get(named_parameter) or ""
             for named_parameter in named_parameters
+            if not named_parameter.startswith("_")
         }
 
         # Set to blank string if missing from params
         for named_parameter in named_parameters:
-            if named_parameter not in params:
+            if named_parameter not in params and not named_parameter.startswith("_"):
                 params[named_parameter] = ""
 
         extra_args = {}
@@ -184,9 +186,13 @@ class QueryView(DataView):
         if write:
             if request.method == "POST":
                 params = await request.post_vars()
+                if canned_query:
+                    params_for_query = MagicParameters(params, request, self.ds)
+                else:
+                    params_for_query = params
                 try:
                     cursor = await self.ds.databases[database].execute_write(
-                        sql, params, block=True
+                        sql, params_for_query, block=True
                     )
                     message = metadata.get(
                         "on_success_message"
@@ -227,8 +233,12 @@ class QueryView(DataView):
                     templates,
                 )
         else:  # Not a write
+            if canned_query:
+                params_for_query = MagicParameters(params, request, self.ds)
+            else:
+                params_for_query = params
             results = await self.ds.execute(
-                database, sql, params, truncate=True, **extra_args
+                database, sql, params_for_query, truncate=True, **extra_args
             )
             columns = [r[0] for r in results.description]
 
@@ -298,3 +308,25 @@ class QueryView(DataView):
             extra_template,
             templates,
         )
+
+
+class MagicParameters(dict):
+    def __init__(self, data, request, datasette):
+        super().__init__(data)
+        self._request = request
+        self._magics = dict(
+            itertools.chain.from_iterable(
+                pm.hook.register_magic_parameters(datasette=datasette)
+            )
+        )
+
+    def __getitem__(self, key):
+        if key.startswith("_") and key.count("_") >= 2:
+            prefix, suffix = key[1:].split("_", 1)
+            if prefix in self._magics:
+                try:
+                    return self._magics[prefix](suffix, self._request)
+                except KeyError:
+                    return super().__getitem__(key)
+        else:
+            return super().__getitem__(key)

docs/plugin_hooks.rst

@@ -895,3 +895,42 @@ Here's an example that allows users to view the ``admin_log`` table only if thei
 See :ref:`built-in permissions <permissions>` for a full list of permissions that are included in Datasette core.
 
 Example: `datasette-permissions-sql <https://github.com/simonw/datasette-permissions-sql>`_
+
+.. _plugin_hook_register_magic_parameters:
+
+register_magic_parameters(datasette)
+------------------------------------
+
+``datasette`` - :ref:`internals_datasette`
+    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``.
+
+:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries <canned_queries>`. This plugin hook allows additional magic parameters to be defined by plugins.
+
+Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function.
+
+To register a new function, return it as a tuple of ``(string prefix, function)`` from this hook. The function you register should take two arguments: ``key`` and ``request``, where ``key`` is the ``rest_of_parameter`` portion of the parameter and ``request`` is the current :ref:`internals_request`.
+
+This example registers two new magic parameters: ``:_request_http_version`` returning the HTTP version of the current request, and ``:_uuid_new`` which returns a new UUID:
+
+.. code-block:: python
+
+    from uuid import uuid4
+
+    def uuid(key, request):
+        if key == "new":
+            return str(uuid4())
+        else:
+            raise KeyError
+
+    def request(key, request):
+        if key == "http_version":
+            return request.scope["http_version"]
+        else:
+            raise KeyError
+
+    @hookimpl
+    def register_magic_parameters(datasette):
+        return [
+            ("request", request),
+            ("uuid", uuid),
+        ]

docs/sql_queries.rst

@@ -114,8 +114,8 @@ rendered as HTML (rather than having HTML special characters escaped).
 
 .. _canned_queries_named_parameters:
 
-Named parameters
+Canned query parameters
-~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~
 
 Canned queries support named parameters, so if you include those in the SQL you
 will then be able to enter them using the form fields on the canned query page
@@ -274,6 +274,58 @@ You can use ``"params"`` to explicitly list the named parameters that should be
 
 You can pre-populate form fields when the page first loads using a querystring, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query.
 
+.. _canned_queries_magic_parameters:
+
+Magic parameters
+~~~~~~~~~~~~~~~~
+
+Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or querystring.
+
+Available magic parameters are:
+
+``_actor_*`` - e.g. ``_actor_id``, ``_actor_name``
+    Fields from the currently authenticated :ref:`authentication_actor`.
+
+``_header_*`` - e.g. ``_header_user_agent``
+    Header from the incoming HTTP request. The key should be in lower case and with hyphens converted to underscores e.g. ``_header_user_agent`` or ``_header_accept_language``.
+
+``_cookie_*`` - e.g. ``_cookie_lang``
+    The value of the incoming cookie of that name.
+
+``_timestamp_epoch``
+    The number of seconds since the Unix epoch.
+
+``_timestamp_date_utc``
+    The date in UTC, e.g. ``2020-06-01``
+
+``_timestamp_datetime_utc``
+    The ISO 8601 datetime in UTC, e.g. ``2020-06-24T18:01:07Z``
+
+``_random_chars_*`` - e.g. ``_random_chars_128``
+    A random string of characters of the specified length.
+
+Here's an example configuration (this time using ``metadata.yaml`` since that provides better support for multi-line SQL queries) that adds a message from the authenticated user, storing various pieces of additional metadata using magic parameters:
+
+.. code-block:: yaml
+
+    databases:
+      mydatabase:
+        queries:
+          add_message:
+            allow:
+              id: "*"
+            sql: |-
+              INSERT INTO messages (
+                user_id, ip, message, datetime
+              ) VALUES (
+                :_actor_id, :_request_ip, :message, :_timestamp_datetime_utc
+              )
+            write: true
+
+The form presented at ``/mydatabase/add_message`` will have just a field for ``message`` - the other parameters will be populated by the magic parameter mechanism.
+
+Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook.
+
 .. _pagination:
 
 Pagination

tests/fixtures.py

@@ -49,6 +49,7 @@ EXPECTED_PLUGINS = [
             "prepare_connection",
             "prepare_jinja2_environment",
             "register_facet_classes",
+            "register_magic_parameters",
             "register_routes",
             "render_cell",
             "startup",

tests/plugins/my_plugin.py

@@ -212,3 +212,25 @@ def canned_queries(datasette, database, actor):
             actor["id"] if actor else "null"
         )
     }
+
+
+@hookimpl
+def register_magic_parameters():
+    from uuid import uuid4
+
+    def uuid(key, request):
+        if key == "new":
+            return str(uuid4())
+        else:
+            raise KeyError
+
+    def request(key, request):
+        if key == "http_version":
+            return request.scope["http_version"]
+        else:
+            raise KeyError
+
+    return [
+        ("request", request),
+        ("uuid", uuid),
+    ]

tests/test_api.py

@@ -601,18 +601,6 @@ def test_custom_sql(app_client):
     assert not data["truncated"]
 
 
-def test_canned_query_with_named_parameter(app_client):
-    response = app_client.get("/fixtures/neighborhood_search.json?text=town")
-    assert [
-        ["Corktown", "Detroit", "MI"],
-        ["Downtown", "Los Angeles", "CA"],
-        ["Downtown", "Detroit", "MI"],
-        ["Greektown", "Detroit", "MI"],
-        ["Koreatown", "Los Angeles", "CA"],
-        ["Mexicantown", "Detroit", "MI"],
-    ] == response.json["rows"]
-
-
 def test_sql_time_limit(app_client_shorter_time_limit):
     response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)")
     assert 400 == response.status

tests/test_canned_queries.py

@@ -1,5 +1,7 @@
+from bs4 import BeautifulSoup as Soup
 import pytest
-from .fixtures import make_app_client
+import re
+from .fixtures import make_app_client, app_client
 
 
 @pytest.fixture
@@ -39,6 +41,18 @@ def canned_write_client():
         yield client
 
 
+def test_canned_query_with_named_parameter(app_client):
+    response = app_client.get("/fixtures/neighborhood_search.json?text=town")
+    assert [
+        ["Corktown", "Detroit", "MI"],
+        ["Downtown", "Los Angeles", "CA"],
+        ["Downtown", "Detroit", "MI"],
+        ["Greektown", "Detroit", "MI"],
+        ["Koreatown", "Los Angeles", "CA"],
+        ["Mexicantown", "Detroit", "MI"],
+    ] == response.json["rows"]
+
+
 def test_insert(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name", {"name": "Hello"}, allow_redirects=False, csrftoken_from=True,
@@ -147,3 +161,74 @@ def test_canned_query_permissions(canned_write_client):
     cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})}
     assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status
     assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status
+
+
+@pytest.fixture(scope="session")
+def magic_parameters_client():
+    with make_app_client(
+        extra_databases={"data.db": "create table logs (line text)"},
+        metadata={
+            "databases": {
+                "data": {
+                    "queries": {
+                        "runme_post": {"sql": "", "write": True},
+                        "runme_get": {"sql": ""},
+                    }
+                }
+            }
+        },
+    ) as client:
+        yield client
+
+
+@pytest.mark.parametrize(
+    "magic_parameter,expected_re",
+    [
+        ("_actor_id", "root"),
+        ("_header_host", "localhost"),
+        ("_cookie_foo", "bar"),
+        ("_timestamp_epoch", r"^\d+$"),
+        ("_timestamp_date_utc", r"^\d{4}-\d{2}-\d{2}$"),
+        ("_timestamp_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"),
+        ("_random_chars_1", r"^\w$"),
+        ("_random_chars_10", r"^\w{10}$"),
+    ],
+)
+def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re):
+    magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_post"][
+        "sql"
+    ] = "insert into logs (line) values (:{})".format(magic_parameter)
+    magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_get"][
+        "sql"
+    ] = "select :{} as result".format(magic_parameter)
+    cookies = {
+        "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}),
+        "foo": "bar",
+    }
+    # Test the GET version
+    get_response = magic_parameters_client.get(
+        "/data/runme_get.json?_shape=array", cookies=cookies
+    )
+    get_actual = get_response.json[0]["result"]
+    assert re.match(expected_re, str(get_actual))
+    # Test the form
+    form_response = magic_parameters_client.get("/data/runme_post")
+    soup = Soup(form_response.body, "html.parser")
+    # The magic parameter should not be represented as a form field
+    assert None is soup.find("input", {"name": magic_parameter})
+    # Submit the form to create a log line
+    response = magic_parameters_client.post(
+        "/data/runme_post", {}, csrftoken_from=True, cookies=cookies
+    )
+    post_actual = magic_parameters_client.get(
+        "/data/logs.json?_sort_desc=rowid&_shape=array"
+    ).json[0]["line"]
+    assert re.match(expected_re, post_actual)
+
+
+def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client):
+    response = magic_parameters_client.get(
+        "/data.json?sql=select+:_header_host&_shape=array"
+    )
+    assert 500 == response.status
+    assert "You did not supply a value for binding 1." == response.json["error"]

tests/test_plugins.py

@@ -638,3 +638,31 @@ def test_canned_queries_actor(app_client):
     assert [{"1": 1, "actor_id": "bot"}] == app_client.get(
         "/fixtures/from_hook.json?_bot=1&_shape=array"
     ).json
+
+
+def test_register_magic_parameters(restore_working_directory):
+    with make_app_client(
+        extra_databases={"data.db": "create table logs (line text)"},
+        metadata={
+            "databases": {
+                "data": {
+                    "queries": {
+                        "runme": {
+                            "sql": "insert into logs (line) values (:_request_http_version)",
+                            "write": True,
+                        },
+                        "get_uuid": {"sql": "select :_uuid_new",},
+                    }
+                }
+            }
+        },
+    ) as client:
+        response = client.post("/data/runme", {}, csrftoken_from=True)
+        assert 200 == response.status
+        actual = client.get("/data/logs.json?_sort_desc=rowid&_shape=array").json
+        assert [{"rowid": 1, "line": "1.0"}] == actual
+        # Now try the GET request against get_uuid
+        response_get = client.get("/data/get_uuid.json?_shape=array")
+        assert 200 == response_get.status
+        new_uuid = response_get.json[0][":_uuid_new"]
+        assert 4 == new_uuid.count("-")