github/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2025-12-10 16:51:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

.execute_write() and .execute_write_fn() methods on Database (#683)

Browse source 
Closes #682.
This commit is contained in:
Simon Willison 2020-02-24 20:45:08 -08:00 • committed by GitHub
commit a093c5f79f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 282 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

64
datasette/database.py
View file

@ -1,7 +1,10 @@
import asyncio
import contextlib
from pathlib import Path
import janus
import queue
import threading
import uuid
from .tracer import trace
from .utils import (
@ -30,6 +33,8 @@ class Database:
self.hash = None
self.cached_size = None
self.cached_table_counts = None
self._write_thread = None
self._write_queue = None
if not self.is_mutable:
p = Path(path)
self.hash = inspect_hash(p)
@ -41,18 +46,60 @@ class Database:
for key, value in self.ds.inspect_data[self.name]["tables"].items()
}
def connect(self):
def connect(self, write=False):
if self.is_memory:
return sqlite3.connect(":memory:")
# mode=ro or immutable=1?
if self.is_mutable:
qs = "mode=ro"
qs = "?mode=ro"
else:
qs = "immutable=1"
qs = "?immutable=1"
assert not (write and not self.is_mutable)
if write:
qs = ""
return sqlite3.connect(
"file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False
"file:{}{}".format(self.path, qs), uri=True, check_same_thread=False
)
async def execute_write(self, sql, params=None, block=False):
def _inner(conn):
with conn:
return conn.execute(sql, params or [])
return await self.execute_write_fn(_inner, block=block)
async def execute_write_fn(self, fn, block=False):
task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
if self._write_queue is None:
self._write_queue = queue.Queue()
if self._write_thread is None:
self._write_thread = threading.Thread(
target=self._execute_writes, daemon=True
)
self._write_thread.start()
reply_queue = janus.Queue()
self._write_queue.put(WriteTask(fn, task_id, reply_queue))
if block:
result = await reply_queue.async_q.get()
if isinstance(result, Exception):
raise result
else:
return result
else:
return task_id
def _execute_writes(self):
# Infinite looping thread that protects the single write connection
# to this database
conn = self.connect(write=True)
while True:
task = self._write_queue.get()
try:
result = task.fn(conn)
except Exception as e:
result = e
task.reply_queue.sync_q.put(result)
async def execute_against_connection_in_thread(self, fn):
def in_thread():
conn = getattr(connections, self.name, None)
@ -326,3 +373,12 @@ class Database:
if tags:
tags_str = " ({})".format(", ".join(tags))
return "<Database: {}{}>".format(self.name, tags_str)
class WriteTask:
__slots__ = ("fn", "task_id", "reply_queue")
def __init__(self, fn, task_id, reply_queue):
self.fn = fn
self.task_id = task_id
self.reply_queue = reply_queue

2
docs/changelog.rst
View file

@ -9,7 +9,7 @@ Changelog
0.36 (2020-02-21)
-----------------
* The ``datasette`` object passed to plugins now has API documentation: :ref:`datasette`. (`#576 <https://github.com/simonw/datasette/issues/576>`__)
* The ``datasette`` object passed to plugins now has API documentation: :ref:`internals_datasette`. (`#576 <https://github.com/simonw/datasette/issues/576>`__)
* New methods on ``datasette``: ``.add_database()`` and ``.remove_database()`` - :ref:`documentation <datasette_add_database>`. (`#671 <https://github.com/simonw/datasette/issues/671>`__)
* ``prepare_connection()`` plugin hook now takes optional ``datasette`` and ``database`` arguments - :ref:`plugin_hook_prepare_connection`. (`#678 <https://github.com/simonw/datasette/issues/678>`__)
* Added three new plugins and one new conversion tool to the :ref:`ecosystem`.

83
docs/datasette.rst
View file

@ -1,83 +0,0 @@
.. _datasette:
Datasette class
===============
Many of Datasette's :ref:`plugin_hooks` pass a ``datasette`` object to the plugin as an argument.
This object is an instance of the ``Datasette`` class. That class currently has a large number of methods on it, but it should not be considered stable (at least until Datasette 1.0) with the exception of the methods that are documented on this page.
.. _datasette_add_database:
.add_database(name, db)
-----------------------
``name`` - string
The unique name to use for this database. Also used in the URL.
``db`` - datasette.database.Database instance
The database to be attached.
The ``datasette.add_database(name, db)`` method lets you add a new database to the current Datasette instance. This database will then be served at URL path that matches the ``name`` parameter, e.g. ``/mynewdb/``.
The ``db`` parameter should be an instance of the ``datasette.database.Database`` class. For example:
.. code-block:: python
from datasette.database import Database
datasette.add_database("my-new-database", Database(
datasette,
path="path/to/my-new-database.db",
is_mutable=True
))
This will add a mutable database from the provided file path.
The ``Database()`` constructor takes four arguments: the first is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments.
Use ``is_mutable`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior.
Use ``is_memory`` if the connection is to an in-memory SQLite database.
.. _datasette_remove_database:
.remove_database(name)
----------------------
``name`` - string
The name of the database to be removed.
This removes a database that has been previously added. ``name=`` is the unique name of that database, also used in the URL for it.
.. _datasette_plugin_config:
.plugin_config(plugin_name, database=None, table=None)
------------------------------------------------------
``plugin_name`` - string
The name of the plugin to look up configuration for. Usually this is something similar to ``datasette-cluster-map``.
``database`` - None or string
The database the user is interacting with.
``table`` - None or string
The table the user is interacting with.
This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`plugins_plugin_config` for full details of how this method should be used.
.. _datasette_render_template:
.render_template(template, context=None, request=None)
------------------------------------------------------
``template`` - string
The template file to be rendered, e.g. ``my_plugin.html``. Datasette will search for this file first in the ``--template-dir=`` location, if it was specified - then in the plugin's bundled templates and finally in Datasette's set of default templates.
``conttext`` - None or a Python dictionary
The context variables to pass to the template.
``request`` - request object or None
If you pass a Datasette request object here it will be made available to the template.
Renders a `Jinja template <https://jinja.palletsprojects.com/en/2.11.x/>`__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins.

2
docs/index.rst
View file

@ -34,7 +34,7 @@ Contents
introspection
custom_templates
plugins
datasette
internals
contributing
changelog

148
docs/internals.rst Normal file
View file

@ -0,0 +1,148 @@
.. _internals:
Internals for plugins
=====================
Many :ref:`plugin_hooks` are passed objects that provide access to internal Datasette functionality. The interface to these objects should not be considered stable (at least until Datasette 1.0) with the exception of methods that are documented on this page.
.. _internals_datasette:
Datasette class
~~~~~~~~~~~~~~~
This object is an instance of the ``Datasette`` class, passed to many plugin hooks as an argument called ``datasette``.
.. _datasette_plugin_config:
.plugin_config(plugin_name, database=None, table=None)
------------------------------------------------------
``plugin_name`` - string
The name of the plugin to look up configuration for. Usually this is something similar to ``datasette-cluster-map``.
``database`` - None or string
The database the user is interacting with.
``table`` - None or string
The table the user is interacting with.
This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`plugins_plugin_config` for full details of how this method should be used.
.. _datasette_render_template:
.render_template(template, context=None, request=None)
------------------------------------------------------
``template`` - string
The template file to be rendered, e.g. ``my_plugin.html``. Datasette will search for this file first in the ``--template-dir=`` location, if it was specified - then in the plugin's bundled templates and finally in Datasette's set of default templates.
``conttext`` - None or a Python dictionary
The context variables to pass to the template.
``request`` - request object or None
If you pass a Datasette request object here it will be made available to the template.
Renders a `Jinja template <https://jinja.palletsprojects.com/en/2.11.x/>`__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins.
.. _datasette_add_database:
.add_database(name, db)
-----------------------
``name`` - string
The unique name to use for this database. Also used in the URL.
``db`` - datasette.database.Database instance
The database to be attached.
The ``datasette.add_database(name, db)`` method lets you add a new database to the current Datasette instance. This database will then be served at URL path that matches the ``name`` parameter, e.g. ``/mynewdb/``.
The ``db`` parameter should be an instance of the ``datasette.database.Database`` class. For example:
.. code-block:: python
from datasette.database import Database
datasette.add_database("my-new-database", Database(
datasette,
path="path/to/my-new-database.db",
is_mutable=True
))
This will add a mutable database from the provided file path.
The ``Database()`` constructor takes four arguments: the first is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments.
Use ``is_mutable`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior.
Use ``is_memory`` if the connection is to an in-memory SQLite database.
.. _datasette_remove_database:
.remove_database(name)
----------------------
``name`` - string
The name of the database to be removed.
This removes a database that has been previously added. ``name=`` is the unique name of that database, also used in the URL for it.
.. _internals_database:
Database class
~~~~~~~~~~~~~~
Instances of the ``Database`` class can be used to execute queries against attached SQLite databases, and to run introspection against their schemas.
SQLite only allows one databasae connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
.. _database_execute_write:
await db.execute_write(sql, params=None, block=False)
-----------------------------------------------------
This method can be used to queue up a non-SELECT SQL query to be executed against a single write connection to the database.
You can pass additional SQL parametercs as a tuple or list.
By default queries are considered to be "fire and forget" - they will be added to the queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task.
If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library.
.. _database_execute_write:
await db.execute_write_fn(fn, block=False)
------------------------------------------
This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. This function will be queued up and then called when the write connection is available, passing that connection as the argument to the function.
The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection as long as it is executing.
For example:
.. code-block:: python
def my_action(conn):
conn.execute("delete from some_table")
conn.execute("delete from other_table")
await database.execute_write_fn(my_action)
This method is fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned.
If you pass ``block=True`` your calling code will block until the function has been executed. The return value to the ``await`` will be the return value of your function.
If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=False`` any exceptions will be silently ignored.
Here's an example of ``block=True`` in action:
.. code-block:: python
def my_action(conn):
conn.execute("delete from some_table where id > 5")
return conn.execute("select count(*) from some_table").fetchone()[0]
try:
num_rows_left = await database.execute_write_fn(my_action, block=True)
except Exception as e:
print("An error occurred:", e)

12
docs/plugins.rst
View file

@ -349,7 +349,7 @@ prepare_connection(conn, database, datasette)
``database`` - string
The name of the database
``datasette`` - :ref:`datasette`
``datasette`` - :ref:`internals_datasette`
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
This hook is called when a new SQLite database connection is created. You can
@ -409,7 +409,7 @@ extra_css_urls(template, database, table, datasette)
``table`` - string or None
The name of the table
``datasette`` - :ref:`datasette`
``datasette`` - :ref:`internals_datasette`
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
Return a list of extra CSS URLs that should be included on the page. These can
@ -545,7 +545,7 @@ Lets you customize the display of values within table cells in the HTML table vi
``database`` - string
The name of the database
``datasette`` - :ref:`datasette`
``datasette`` - :ref:`internals_datasette`
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value.
@ -615,7 +615,7 @@ Extra JavaScript to be added to a ``<script>`` block at the end of the ``<body>`
``view_name`` - string
The name of the view being displayed. (`index`, `database`, `table`, and `row` are the most important ones.)
``datasette`` - :ref:`datasette`
``datasette`` - :ref:`internals_datasette`
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
The ``template``, ``database`` and ``table`` options can be used to return different code depending on which template is being rendered and which database or table are being processed.
@ -647,7 +647,7 @@ Extra template variables that should be made available in the rendered template
``request`` - object
The current HTTP request object. ``request.scope`` provides access to the ASGI scope.
``datasette`` - :ref:`datasette`
``datasette`` - :ref:`internals_datasette`
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
This hook can return one of three different types:
@ -692,7 +692,7 @@ You can then use the new function in a template like so::
register_output_renderer(datasette)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``datasette`` - :ref:`datasette`
``datasette`` - :ref:`internals_datasette`
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
Allows the plugin to register a new output renderer, to output data in a custom format. The hook function should return a dictionary, or a list of dictionaries, which contain the file extension you want to handle and a callback function:

1
setup.py
View file

@ -44,6 +44,7 @@ setup(
"pluggy~=0.13.0",
"uvicorn~=0.11",
"aiofiles~=0.4.0",
"janus~=0.4.0",
],
entry_points="""
[console_scripts]

65
tests/test_database.py
View file

@ -1,5 +1,7 @@
from .fixtures import app_client
import pytest
import time
import uuid
@pytest.mark.parametrize(
@ -83,3 +85,66 @@ async def test_table_names(app_client):
"attraction_characteristic",
"roadside_attraction_characteristics",
] == table_names
@pytest.mark.asyncio
async def test_execute_write_block_true(app_client):
db = app_client.ds.databases["fixtures"]
await db.execute_write(
"update roadside_attractions set name = ? where pk = ?",
["Mystery!", 1],
block=True,
)
rows = await db.execute("select name from roadside_attractions where pk = 1")
assert "Mystery!" == rows.rows[0][0]
@pytest.mark.asyncio
async def test_execute_write_block_false(app_client):
db = app_client.ds.databases["fixtures"]
await db.execute_write(
"update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1],
)
time.sleep(0.1)
rows = await db.execute("select name from roadside_attractions where pk = 1")
assert "Mystery!" == rows.rows[0][0]
@pytest.mark.asyncio
async def test_execute_write_fn_block_false(app_client):
db = app_client.ds.databases["fixtures"]
def write_fn(conn):
with conn:
conn.execute("delete from roadside_attractions where id = 1;")
row = conn.execute("select count(*) from roadside_attractions").fetchone()
print("row = ", row)
return row[0]
task_id = await db.execute_write_fn(write_fn)
assert isinstance(task_id, uuid.UUID)
@pytest.mark.asyncio
async def test_execute_write_fn_block_true(app_client):
db = app_client.ds.databases["fixtures"]
def write_fn(conn):
with conn:
conn.execute("delete from roadside_attractions where pk = 1;")
row = conn.execute("select count(*) from roadside_attractions").fetchone()
return row[0]
new_count = await db.execute_write_fn(write_fn, block=True)
assert 3 == new_count
@pytest.mark.asyncio
async def test_execute_write_fn_exception(app_client):
db = app_client.ds.databases["fixtures"]
def write_fn(conn):
assert False
with pytest.raises(AssertionError):
await db.execute_write_fn(write_fn, block=True)