From 6119bd797366a899119f1bba51c1c8cba2efc8fc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 16 Dec 2020 13:44:39 -0800 Subject: [PATCH 0001/1364] Update pytest requirement from <6.2.0,>=5.2.2 to >=5.2.2,<6.3.0 (#1145) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...6.2.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e9eb1597..be94c1c6 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ - "pytest>=5.2.2,<6.2.0", + "pytest>=5.2.2,<6.3.0", "pytest-asyncio>=0.10,<0.15", "beautifulsoup4>=4.8.1,<4.10.0", "black==20.8b1", From 5e9895c67f08e9f42acedd3d6d29512ac446e15f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Dec 2020 17:01:18 -0800 Subject: [PATCH 0002/1364] Database(memory_name=) for shared in-memory databases, closes #1151 --- datasette/database.py | 24 +++++++++++++++++++-- docs/internals.rst | 37 +++++++++++++++++++++++++++++--- tests/test_internals_database.py | 30 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 412e0c59..a977b362 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -24,11 +24,18 @@ connections = threading.local() class Database: - def __init__(self, ds, path=None, is_mutable=False, is_memory=False): + def __init__( + self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None + ): self.ds = ds self.path = path self.is_mutable = is_mutable self.is_memory = is_memory + self.memory_name = memory_name + if memory_name is not None: + self.path = memory_name + self.is_memory = True + self.is_mutable = True self.hash = None self.cached_size = None self.cached_table_counts = None @@ -46,6 +53,16 @@ class Database: } def connect(self, write=False): + if self.memory_name: + uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) + conn = sqlite3.connect( + uri, + uri=True, + check_same_thread=False, + ) + if not write: + conn.execute("PRAGMA query_only=1") + return conn if self.is_memory: return sqlite3.connect(":memory:") # mode=ro or immutable=1? @@ -215,7 +232,10 @@ class Database: @property def name(self): if self.is_memory: - return ":memory:" + if self.memory_name: + return ":memory:{}".format(self.memory_name) + else: + return ":memory:" else: return Path(self.path).stem diff --git a/docs/internals.rst b/docs/internals.rst index ff566f69..b68a1d8a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -270,11 +270,16 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` This will add a mutable database from the provided file path. -The ``Database()`` constructor takes four arguments: the first is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. +To create a shared in-memory database named ``statistics``, use the following: -Use ``is_mutable`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. +.. code-block:: python -Use ``is_memory`` if the connection is to an in-memory SQLite database. + from datasette.database import Database + + datasette.add_database("statistics", Database( + datasette, + memory_name="statistics" + )) .. _datasette_remove_database: @@ -480,6 +485,32 @@ Database class Instances of the ``Database`` class can be used to execute queries against attached SQLite databases, and to run introspection against their schemas. +.. _database_constructor: + +Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None) +---------------------------------------------------------------------------- + +The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. + +The arguments are as follows: + +``ds`` - :ref:`internals_datasette` (required) + The Datasette instance you are attaching this database to. + +``path`` - string + Path to a SQLite database file on disk. + +``is_mutable`` - boolean + Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. + +``is_memory`` - boolean + Use this to create non-shared memory connections. + +``memory_name`` - string or ``None`` + Use this to create a named in-memory database. Unlike regular memory databases these can be accessed by multiple threads and will persist an changes made to them for the lifetime of the Datasette server process. + +The first argument is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. + .. _database_execute: await db.execute(sql, ...) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 49b8a1b3..dc1af48c 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -464,3 +464,33 @@ def test_mtime_ns_is_none_for_memory(app_client): def test_is_mutable(app_client): assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False + + +@pytest.mark.asyncio +async def test_database_memory_name(app_client): + ds = app_client.ds + foo1 = Database(ds, memory_name="foo") + foo2 = Database(ds, memory_name="foo") + bar1 = Database(ds, memory_name="bar") + bar2 = Database(ds, memory_name="bar") + for db in (foo1, foo2, bar1, bar2): + table_names = await db.table_names() + assert table_names == [] + # Now create a table in foo + await foo1.execute_write("create table foo (t text)", block=True) + assert await foo1.table_names() == ["foo"] + assert await foo2.table_names() == ["foo"] + assert await bar1.table_names() == [] + assert await bar2.table_names() == [] + + +@pytest.mark.asyncio +async def test_in_memory_databases_forbid_writes(app_client): + ds = app_client.ds + db = Database(ds, memory_name="test") + with pytest.raises(sqlite3.OperationalError): + await db.execute("create table foo (t text)") + assert await db.table_names() == [] + # Using db.execute_write() should work: + await db.execute_write("create table foo (t text)", block=True) + assert await db.table_names() == ["foo"] From ebc7aa287c99fe6114b79aeab8efb8d4489a6182 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Dec 2020 14:34:05 -0800 Subject: [PATCH 0003/1364] In-memory _schemas database tracking schemas of attached tables, closes #1150 --- datasette/app.py | 39 +++++++- datasette/cli.py | 3 + datasette/default_permissions.py | 2 + datasette/utils/__init__.py | 7 +- datasette/utils/schemas.py | 162 +++++++++++++++++++++++++++++++ datasette/views/base.py | 2 + tests/test_plugins.py | 2 +- tests/test_schemas.py | 68 +++++++++++++ 8 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 datasette/utils/schemas.py create mode 100644 tests/test_schemas.py diff --git a/datasette/app.py b/datasette/app.py index 9bc84df0..cc8506e2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,6 +74,7 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) +from .utils.schemas import init_schemas, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, @@ -222,6 +223,11 @@ class Datasette: elif memory: self.files = (MEMORY,) + self.files self.databases = collections.OrderedDict() + # memory_name is a random string so that each Datasette instance gets its own + # unique in-memory named database - otherwise unit tests can fail with weird + # errors when different instances accidentally share an in-memory database + self.add_database("_schemas", Database(self, memory_name=secrets.token_hex())) + self._schemas_created = False for file in self.files: path = file is_memory = False @@ -326,6 +332,33 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + async def refresh_schemas(self): + schema_db = self.databases["_schemas"] + if not self._schemas_created: + await init_schemas(schema_db) + self._schemas_created = True + + current_schema_versions = { + row["database_name"]: row["schema_version"] + for row in await schema_db.execute( + "select database_name, schema_version from databases" + ) + } + for database_name, db in self.databases.items(): + schema_version = (await db.execute("PRAGMA schema_version")).first()[0] + # Compare schema versions to see if we should skip it + if schema_version == current_schema_versions.get(database_name): + continue + await schema_db.execute_write( + """ + INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) + VALUES (?, ?, ?, ?) + """, + [database_name, db.path, db.is_memory, schema_version], + block=True, + ) + await populate_schema_tables(schema_db, db) + @property def urls(self): return Urls(self) @@ -342,7 +375,8 @@ class Datasette: def get_database(self, name=None): if name is None: - return next(iter(self.databases.values())) + # Return first no-_schemas database + name = [key for key in self.databases.keys() if key != "_schemas"][0] return self.databases[name] def add_database(self, name, db): @@ -590,7 +624,8 @@ class Datasette: "is_memory": d.is_memory, "hash": d.hash, } - for d in sorted(self.databases.values(), key=lambda d: d.name) + for name, d in sorted(self.databases.items(), key=lambda p: p[1].name) + if name != "_schemas" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 32408d23..50367fb3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -134,6 +134,9 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): + if name == "_schemas": + # Don't include the in-memory _schemas database + continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 9f1d9c62..62cab83a 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -13,6 +13,8 @@ def permission_allowed(datasette, actor, action, resource): if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": + if resource == "_schemas" and (actor is None or actor.get("id") != "root"): + return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: return None diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2576090a..ac1d82f7 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1024,11 +1024,12 @@ def find_spatialite(): async def initial_path_for_datasette(datasette): "Return suggested path for opening this Datasette, based on number of DBs and tables" - if len(datasette.databases) == 1: - db_name = next(iter(datasette.databases.keys())) + databases = dict([p for p in datasette.databases.items() if p[0] != "_schemas"]) + if len(databases) == 1: + db_name = next(iter(databases.keys())) path = datasette.urls.database(db_name) # Does this DB only have one table? - db = next(iter(datasette.databases.values())) + db = next(iter(databases.values())) tables = await db.table_names() if len(tables) == 1: path = datasette.urls.table(db_name, tables[0]) diff --git a/datasette/utils/schemas.py b/datasette/utils/schemas.py new file mode 100644 index 00000000..4612e236 --- /dev/null +++ b/datasette/utils/schemas.py @@ -0,0 +1,162 @@ +async def init_schemas(db): + await db.execute_write( + """ + CREATE TABLE databases ( + "database_name" TEXT PRIMARY KEY, + "path" TEXT, + "is_memory" INTEGER, + "schema_version" INTEGER + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE tables ( + "database_name" TEXT, + "table_name" TEXT, + "rootpage" INTEGER, + "sql" TEXT, + PRIMARY KEY (database_name, table_name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE columns ( + "database_name" TEXT, + "table_name" TEXT, + "cid" INTEGER, + "name" TEXT, + "type" TEXT, + "notnull" INTEGER, + "default_value" TEXT, -- renamed from dflt_value + "is_pk" INTEGER, -- renamed from pk + "hidden" INTEGER, + PRIMARY KEY (database_name, table_name, name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE indexes ( + "database_name" TEXT, + "table_name" TEXT, + "seq" INTEGER, + "name" TEXT, + "unique" INTEGER, + "origin" TEXT, + "partial" INTEGER, + PRIMARY KEY (database_name, table_name, name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE foreign_keys ( + "database_name" TEXT, + "table_name" TEXT, + "id" INTEGER, + "seq" INTEGER, + "table" TEXT, + "from" TEXT, + "to" TEXT, + "on_update" TEXT, + "on_delete" TEXT, + "match" TEXT + ) + """, + block=True, + ) + + +async def populate_schema_tables(schema_db, db): + database_name = db.name + await schema_db.execute_write( + "delete from tables where database_name = ?", [database_name], block=True + ) + tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows + for table in tables: + table_name = table["name"] + await schema_db.execute_write( + """ + insert into tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + [database_name, table_name, table["rootpage"], table["sql"]], + block=True, + ) + # And the columns + await schema_db.execute_write( + "delete from columns where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + columns = await db.table_column_details(table_name) + for column in columns: + params = { + **{"database_name": database_name, "table_name": table_name}, + **column._asdict(), + } + await schema_db.execute_write( + """ + insert into columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + params, + block=True, + ) + # And the foreign_keys + await schema_db.execute_write( + "delete from foreign_keys where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + foreign_keys = ( + await db.execute(f"PRAGMA foreign_key_list([{table_name}])") + ).rows + for foreign_key in foreign_keys: + params = { + **{"database_name": database_name, "table_name": table_name}, + **dict(foreign_key), + } + await schema_db.execute_write( + """ + insert into foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + params, + block=True, + ) + # And the indexes + await schema_db.execute_write( + "delete from indexes where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows + for index in indexes: + params = { + **{"database_name": database_name, "table_name": table_name}, + **dict(index), + } + await schema_db.execute_write( + """ + insert into indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + params, + block=True, + ) diff --git a/datasette/views/base.py b/datasette/views/base.py index 76e03206..73bf9459 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -115,6 +115,8 @@ class BaseView: return Response.text("Method not allowed", status=405) async def dispatch_request(self, request, *args, **kwargs): + if self.ds: + await self.ds.refresh_schemas() handler = getattr(self, request.method.lower(), None) return await handler(request, *args, **kwargs) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 93b444ab..61e7d4b5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -293,7 +293,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): def test_hook_asgi_wrapper(app_client): response = app_client.get("/fixtures") - assert "fixtures" == response.headers["x-databases"] + assert "_schemas, fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): diff --git a/tests/test_schemas.py b/tests/test_schemas.py new file mode 100644 index 00000000..87656784 --- /dev/null +++ b/tests/test_schemas.py @@ -0,0 +1,68 @@ +from .fixtures import app_client +import pytest + + +def test_schemas_only_available_to_root(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + assert app_client.get("/_schemas").status == 403 + assert app_client.get("/_schemas", cookies={"ds_actor": cookie}).status == 200 + + +def test_schemas_databases(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + databases = app_client.get( + "/_schemas/databases.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(databases) == 2 + assert databases[0]["database_name"] == "_schemas" + assert databases[1]["database_name"] == "fixtures" + + +def test_schemas_tables(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + tables = app_client.get( + "/_schemas/tables.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(tables) > 5 + table = tables[0] + assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} + + +def test_schemas_indexes(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + indexes = app_client.get( + "/_schemas/indexes.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(indexes) > 5 + index = indexes[0] + assert set(index.keys()) == { + "partial", + "name", + "table_name", + "unique", + "seq", + "database_name", + "origin", + } + + +def test_schemas_foreign_keys(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + foreign_keys = app_client.get( + "/_schemas/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(foreign_keys) > 5 + foreign_key = foreign_keys[0] + assert set(foreign_key.keys()) == { + "table", + "seq", + "on_update", + "on_delete", + "to", + "rowid", + "id", + "match", + "database_name", + "table_name", + "from", + } From dcdfb2c301341d45b66683e3e3be72f9c7585b2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Dec 2020 11:48:06 -0800 Subject: [PATCH 0004/1364] Rename _schemas to _internal, closes #1156 --- datasette/app.py | 39 +++++++------------ datasette/cli.py | 4 +- datasette/default_permissions.py | 2 +- datasette/utils/__init__.py | 2 +- .../utils/{schemas.py => internal_db.py} | 20 +++++----- .../{test_schemas.py => test_internal_db.py} | 24 ++++++------ tests/test_plugins.py | 2 +- 7 files changed, 42 insertions(+), 51 deletions(-) rename datasette/utils/{schemas.py => internal_db.py} (91%) rename tests/{test_schemas.py => test_internal_db.py} (63%) diff --git a/datasette/app.py b/datasette/app.py index cc8506e2..f995e79d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,7 +74,7 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) -from .utils.schemas import init_schemas, populate_schema_tables +from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, @@ -85,8 +85,6 @@ from .version import __version__ app_root = Path(__file__).parent.parent -MEMORY = object() - Setting = collections.namedtuple("Setting", ("name", "default", "help")) SETTINGS = ( Setting("default_page_size", 100, "Default page size for the table view"), @@ -218,24 +216,17 @@ class Datasette: ] self.inspect_data = inspect_data self.immutables = set(immutables or []) - if not self.files: - self.files = [MEMORY] - elif memory: - self.files = (MEMORY,) + self.files self.databases = collections.OrderedDict() + if memory or not self.files: + self.add_database(":memory:", Database(self, ":memory:", is_memory=True)) # memory_name is a random string so that each Datasette instance gets its own # unique in-memory named database - otherwise unit tests can fail with weird # errors when different instances accidentally share an in-memory database - self.add_database("_schemas", Database(self, memory_name=secrets.token_hex())) - self._schemas_created = False + self.add_database("_internal", Database(self, memory_name=secrets.token_hex())) + self._interna_db_created = False for file in self.files: path = file - is_memory = False - if file is MEMORY: - path = None - is_memory = True - is_mutable = path not in self.immutables - db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) + db = Database(self, path, is_mutable=path not in self.immutables) if db.name in self.databases: raise Exception(f"Multiple files with same stem: {db.name}") self.add_database(db.name, db) @@ -333,14 +324,14 @@ class Datasette: self.client = DatasetteClient(self) async def refresh_schemas(self): - schema_db = self.databases["_schemas"] - if not self._schemas_created: - await init_schemas(schema_db) - self._schemas_created = True + internal_db = self.databases["_internal"] + if not self._interna_db_created: + await init_internal_db(internal_db) + self._interna_db_created = True current_schema_versions = { row["database_name"]: row["schema_version"] - for row in await schema_db.execute( + for row in await internal_db.execute( "select database_name, schema_version from databases" ) } @@ -349,7 +340,7 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - await schema_db.execute_write( + await internal_db.execute_write( """ INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) VALUES (?, ?, ?, ?) @@ -357,7 +348,7 @@ class Datasette: [database_name, db.path, db.is_memory, schema_version], block=True, ) - await populate_schema_tables(schema_db, db) + await populate_schema_tables(internal_db, db) @property def urls(self): @@ -376,7 +367,7 @@ class Datasette: def get_database(self, name=None): if name is None: # Return first no-_schemas database - name = [key for key in self.databases.keys() if key != "_schemas"][0] + name = [key for key in self.databases.keys() if key != "_internal"][0] return self.databases[name] def add_database(self, name, db): @@ -625,7 +616,7 @@ class Datasette: "hash": d.hash, } for name, d in sorted(self.databases.items(), key=lambda p: p[1].name) - if name != "_schemas" + if name != "_internal" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 50367fb3..c342a35a 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -134,8 +134,8 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - if name == "_schemas": - # Don't include the in-memory _schemas database + if name == "_internal": + # Don't include the in-memory _internal database continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 62cab83a..b58d8d1b 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -13,7 +13,7 @@ def permission_allowed(datasette, actor, action, resource): if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - if resource == "_schemas" and (actor is None or actor.get("id") != "root"): + if resource == "_internal" and (actor is None or actor.get("id") != "root"): return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index ac1d82f7..34ee4630 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1024,7 +1024,7 @@ def find_spatialite(): async def initial_path_for_datasette(datasette): "Return suggested path for opening this Datasette, based on number of DBs and tables" - databases = dict([p for p in datasette.databases.items() if p[0] != "_schemas"]) + databases = dict([p for p in datasette.databases.items() if p[0] != "_internal"]) if len(databases) == 1: db_name = next(iter(databases.keys())) path = datasette.urls.database(db_name) diff --git a/datasette/utils/schemas.py b/datasette/utils/internal_db.py similarity index 91% rename from datasette/utils/schemas.py rename to datasette/utils/internal_db.py index 4612e236..a60fe1fe 100644 --- a/datasette/utils/schemas.py +++ b/datasette/utils/internal_db.py @@ -1,4 +1,4 @@ -async def init_schemas(db): +async def init_internal_db(db): await db.execute_write( """ CREATE TABLE databases ( @@ -73,15 +73,15 @@ async def init_schemas(db): ) -async def populate_schema_tables(schema_db, db): +async def populate_schema_tables(internal_db, db): database_name = db.name - await schema_db.execute_write( + await internal_db.execute_write( "delete from tables where database_name = ?", [database_name], block=True ) tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows for table in tables: table_name = table["name"] - await schema_db.execute_write( + await internal_db.execute_write( """ insert into tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) @@ -90,7 +90,7 @@ async def populate_schema_tables(schema_db, db): block=True, ) # And the columns - await schema_db.execute_write( + await internal_db.execute_write( "delete from columns where database_name = ? and table_name = ?", [database_name, table_name], block=True, @@ -101,7 +101,7 @@ async def populate_schema_tables(schema_db, db): **{"database_name": database_name, "table_name": table_name}, **column._asdict(), } - await schema_db.execute_write( + await internal_db.execute_write( """ insert into columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden @@ -113,7 +113,7 @@ async def populate_schema_tables(schema_db, db): block=True, ) # And the foreign_keys - await schema_db.execute_write( + await internal_db.execute_write( "delete from foreign_keys where database_name = ? and table_name = ?", [database_name, table_name], block=True, @@ -126,7 +126,7 @@ async def populate_schema_tables(schema_db, db): **{"database_name": database_name, "table_name": table_name}, **dict(foreign_key), } - await schema_db.execute_write( + await internal_db.execute_write( """ insert into foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match @@ -138,7 +138,7 @@ async def populate_schema_tables(schema_db, db): block=True, ) # And the indexes - await schema_db.execute_write( + await internal_db.execute_write( "delete from indexes where database_name = ? and table_name = ?", [database_name, table_name], block=True, @@ -149,7 +149,7 @@ async def populate_schema_tables(schema_db, db): **{"database_name": database_name, "table_name": table_name}, **dict(index), } - await schema_db.execute_write( + await internal_db.execute_write( """ insert into indexes ( database_name, table_name, seq, name, "unique", origin, partial diff --git a/tests/test_schemas.py b/tests/test_internal_db.py similarity index 63% rename from tests/test_schemas.py rename to tests/test_internal_db.py index 87656784..9349fa3c 100644 --- a/tests/test_schemas.py +++ b/tests/test_internal_db.py @@ -2,36 +2,36 @@ from .fixtures import app_client import pytest -def test_schemas_only_available_to_root(app_client): +def test_internal_only_available_to_root(app_client): cookie = app_client.actor_cookie({"id": "root"}) - assert app_client.get("/_schemas").status == 403 - assert app_client.get("/_schemas", cookies={"ds_actor": cookie}).status == 200 + assert app_client.get("/_internal").status == 403 + assert app_client.get("/_internal", cookies={"ds_actor": cookie}).status == 200 -def test_schemas_databases(app_client): +def test_internal_databases(app_client): cookie = app_client.actor_cookie({"id": "root"}) databases = app_client.get( - "/_schemas/databases.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/databases.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(databases) == 2 - assert databases[0]["database_name"] == "_schemas" + assert databases[0]["database_name"] == "_internal" assert databases[1]["database_name"] == "fixtures" -def test_schemas_tables(app_client): +def test_internal_tables(app_client): cookie = app_client.actor_cookie({"id": "root"}) tables = app_client.get( - "/_schemas/tables.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/tables.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(tables) > 5 table = tables[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} -def test_schemas_indexes(app_client): +def test_internal_indexes(app_client): cookie = app_client.actor_cookie({"id": "root"}) indexes = app_client.get( - "/_schemas/indexes.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/indexes.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(indexes) > 5 index = indexes[0] @@ -46,10 +46,10 @@ def test_schemas_indexes(app_client): } -def test_schemas_foreign_keys(app_client): +def test_internal_foreign_keys(app_client): cookie = app_client.actor_cookie({"id": "root"}) foreign_keys = app_client.get( - "/_schemas/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(foreign_keys) > 5 foreign_key = foreign_keys[0] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 61e7d4b5..8063460b 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -293,7 +293,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): def test_hook_asgi_wrapper(app_client): response = app_client.get("/fixtures") - assert "_schemas, fixtures" == response.headers["x-databases"] + assert "_internal, fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): From 810853c5f2fa560c6d303331c037f6443c145930 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Dec 2020 13:49:14 -0800 Subject: [PATCH 0005/1364] Use time.perf_counter() instead of time.time(), closes #1157 --- datasette/tracer.py | 8 ++++---- datasette/utils/__init__.py | 4 ++-- datasette/views/base.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 8f666767..772f0405 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -37,9 +37,9 @@ def trace(type, **kwargs): if tracer is None: yield return - start = time.time() + start = time.perf_counter() yield - end = time.time() + end = time.perf_counter() trace_info = { "type": type, "start": start, @@ -74,7 +74,7 @@ class AsgiTracer: if b"_trace=1" not in scope.get("query_string", b"").split(b"&"): await self.app(scope, receive, send) return - trace_start = time.time() + trace_start = time.perf_counter() traces = [] accumulated_body = b"" @@ -109,7 +109,7 @@ class AsgiTracer: # We have all the body - modify it and send the result # TODO: What to do about Content-Type or other cases? trace_info = { - "request_duration_ms": 1000 * (time.time() - trace_start), + "request_duration_ms": 1000 * (time.perf_counter() - trace_start), "sum_trace_duration_ms": sum(t["duration_ms"] for t in traces), "num_traces": len(traces), "traces": traces, diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 34ee4630..0d45e11a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -138,7 +138,7 @@ class CustomJSONEncoder(json.JSONEncoder): @contextmanager def sqlite_timelimit(conn, ms): - deadline = time.time() + (ms / 1000) + deadline = time.perf_counter() + (ms / 1000) # n is the number of SQLite virtual machine instructions that will be # executed between each check. It's hard to know what to pick here. # After some experimentation, I've decided to go with 1000 by default and @@ -148,7 +148,7 @@ def sqlite_timelimit(conn, ms): n = 1 def handler(): - if time.time() >= deadline: + if time.perf_counter() >= deadline: return 1 conn.set_progress_handler(handler, n) diff --git a/datasette/views/base.py b/datasette/views/base.py index 73bf9459..8a64f88e 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -425,7 +425,7 @@ class DataView(BaseView): kwargs["default_labels"] = True extra_template_data = {} - start = time.time() + start = time.perf_counter() status_code = 200 templates = [] try: @@ -457,7 +457,7 @@ class DataView(BaseView): except DatasetteError: raise - end = time.time() + end = time.perf_counter() data["query_ms"] = (end - start) * 1000 for key in ("source", "source_url", "license", "license_url"): value = self.ds.metadata(key) From bc1f1e1ce8562872b7532a167873193e787cef20 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 11:04:29 -0800 Subject: [PATCH 0006/1364] Compound primary key for foreign_keys table in _internal --- datasette/utils/internal_db.py | 3 ++- tests/test_internal_db.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index a60fe1fe..959f422e 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -66,7 +66,8 @@ async def init_internal_db(db): "to" TEXT, "on_update" TEXT, "on_delete" TEXT, - "match" TEXT + "match" TEXT, + PRIMARY KEY (database_name, table_name, id, seq) ) """, block=True, diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 9349fa3c..755ddae5 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -59,7 +59,6 @@ def test_internal_foreign_keys(app_client): "on_update", "on_delete", "to", - "rowid", "id", "match", "database_name", From 270de6527bc2afb8c5996c400099321c320ded31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 11:48:54 -0800 Subject: [PATCH 0007/1364] Foreign keys for _internal database Refs #1099 - Datasette now uses compound foreign keys internally, so it would be great to link them correctly. --- datasette/utils/internal_db.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 959f422e..5cd32381 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -17,7 +17,8 @@ async def init_internal_db(db): "table_name" TEXT, "rootpage" INTEGER, "sql" TEXT, - PRIMARY KEY (database_name, table_name) + PRIMARY KEY (database_name, table_name), + FOREIGN KEY (database_name) REFERENCES databases(database_name) ) """, block=True, @@ -34,7 +35,9 @@ async def init_internal_db(db): "default_value" TEXT, -- renamed from dflt_value "is_pk" INTEGER, -- renamed from pk "hidden" INTEGER, - PRIMARY KEY (database_name, table_name, name) + PRIMARY KEY (database_name, table_name, name), + FOREIGN KEY (database_name) REFERENCES databases(database_name), + FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) """, block=True, @@ -49,7 +52,9 @@ async def init_internal_db(db): "unique" INTEGER, "origin" TEXT, "partial" INTEGER, - PRIMARY KEY (database_name, table_name, name) + PRIMARY KEY (database_name, table_name, name), + FOREIGN KEY (database_name) REFERENCES databases(database_name), + FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) """, block=True, @@ -67,7 +72,9 @@ async def init_internal_db(db): "on_update" TEXT, "on_delete" TEXT, "match" TEXT, - PRIMARY KEY (database_name, table_name, id, seq) + PRIMARY KEY (database_name, table_name, id, seq), + FOREIGN KEY (database_name) REFERENCES databases(database_name), + FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) """, block=True, From 8919f99c2f7f245aca7f94bd53d5ac9d04aa42b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 12:04:18 -0800 Subject: [PATCH 0008/1364] Improved .add_database() method design Closes #1155 - _internal now has a sensible name Closes #509 - Support opening multiple databases with the same stem --- datasette/app.py | 34 +++++++++++++++++--------- datasette/database.py | 42 +++++++++++++++++--------------- docs/internals.rst | 29 ++++++++++++++-------- tests/test_cli.py | 15 ++++++++++++ tests/test_internals_database.py | 12 ++++----- 5 files changed, 86 insertions(+), 46 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f995e79d..ad3ba07e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -218,18 +218,18 @@ class Datasette: self.immutables = set(immutables or []) self.databases = collections.OrderedDict() if memory or not self.files: - self.add_database(":memory:", Database(self, ":memory:", is_memory=True)) + self.add_database(Database(self, is_memory=True), name=":memory:") # memory_name is a random string so that each Datasette instance gets its own # unique in-memory named database - otherwise unit tests can fail with weird # errors when different instances accidentally share an in-memory database - self.add_database("_internal", Database(self, memory_name=secrets.token_hex())) - self._interna_db_created = False + self.add_database( + Database(self, memory_name=secrets.token_hex()), name="_internal" + ) + self.internal_db_created = False for file in self.files: - path = file - db = Database(self, path, is_mutable=path not in self.immutables) - if db.name in self.databases: - raise Exception(f"Multiple files with same stem: {db.name}") - self.add_database(db.name, db) + self.add_database( + Database(self, file, is_mutable=file not in self.immutables) + ) self.cache_headers = cache_headers self.cors = cors metadata_files = [] @@ -325,9 +325,9 @@ class Datasette: async def refresh_schemas(self): internal_db = self.databases["_internal"] - if not self._interna_db_created: + if not self.internal_db_created: await init_internal_db(internal_db) - self._interna_db_created = True + self.internal_db_created = True current_schema_versions = { row["database_name"]: row["schema_version"] @@ -370,8 +370,20 @@ class Datasette: name = [key for key in self.databases.keys() if key != "_internal"][0] return self.databases[name] - def add_database(self, name, db): + def add_database(self, db, name=None): + if name is None: + # Pick a unique name for this database + suggestion = db.suggest_name() + name = suggestion + else: + suggestion = name + i = 2 + while name in self.databases: + name = "{}_{}".format(suggestion, i) + i += 1 + db.name = name self.databases[name] = db + return db def remove_database(self, name): self.databases.pop(name) diff --git a/datasette/database.py b/datasette/database.py index a977b362..cda36e6e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -27,30 +27,44 @@ class Database: def __init__( self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None ): + self.name = None self.ds = ds self.path = path self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name if memory_name is not None: - self.path = memory_name self.is_memory = True self.is_mutable = True self.hash = None self.cached_size = None - self.cached_table_counts = None + self._cached_table_counts = None self._write_thread = None self._write_queue = None if not self.is_mutable and not self.is_memory: p = Path(path) self.hash = inspect_hash(p) self.cached_size = p.stat().st_size - # Maybe use self.ds.inspect_data to populate cached_table_counts - if self.ds.inspect_data and self.ds.inspect_data.get(self.name): - self.cached_table_counts = { - key: value["count"] - for key, value in self.ds.inspect_data[self.name]["tables"].items() - } + + @property + def cached_table_counts(self): + if self._cached_table_counts is not None: + return self._cached_table_counts + # Maybe use self.ds.inspect_data to populate cached_table_counts + if self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self._cached_table_counts = { + key: value["count"] + for key, value in self.ds.inspect_data[self.name]["tables"].items() + } + return self._cached_table_counts + + def suggest_name(self): + if self.path: + return Path(self.path).stem + elif self.memory_name: + return self.memory_name + else: + return "db" def connect(self, write=False): if self.memory_name: @@ -220,7 +234,7 @@ class Database: except (QueryInterrupted, sqlite3.OperationalError, sqlite3.DatabaseError): counts[table] = None if not self.is_mutable: - self.cached_table_counts = counts + self._cached_table_counts = counts return counts @property @@ -229,16 +243,6 @@ class Database: return None return Path(self.path).stat().st_mtime_ns - @property - def name(self): - if self.is_memory: - if self.memory_name: - return ":memory:{}".format(self.memory_name) - else: - return ":memory:" - else: - return Path(self.path).stem - async def table_exists(self, table): results = await self.execute( "select 1 from sqlite_master where type='table' and name=?", params=(table,) diff --git a/docs/internals.rst b/docs/internals.rst index b68a1d8a..05cb8bd7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -245,16 +245,16 @@ Returns the specified database object. Raises a ``KeyError`` if the database doe .. _datasette_add_database: -.add_database(name, db) ------------------------ - -``name`` - string - The unique name to use for this database. Also used in the URL. +.add_database(db, name=None) +---------------------------- ``db`` - datasette.database.Database instance The database to be attached. -The ``datasette.add_database(name, db)`` method lets you add a new database to the current Datasette instance. This database will then be served at URL path that matches the ``name`` parameter, e.g. ``/mynewdb/``. +``name`` - string, optional + The name to be used for this database - this will be used in the URL path, e.g. ``/dbname``. If not specified Datasette will pick one based on the filename or memory name. + +The ``datasette.add_database(db)`` method lets you add a new database to the current Datasette instance. The ``db`` parameter should be an instance of the ``datasette.database.Database`` class. For example: @@ -262,13 +262,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` from datasette.database import Database - datasette.add_database("my-new-database", Database( + datasette.add_database(Database( datasette, path="path/to/my-new-database.db", is_mutable=True )) -This will add a mutable database from the provided file path. +This will add a mutable database and serve it at ``/my-new-database``. To create a shared in-memory database named ``statistics``, use the following: @@ -276,11 +276,20 @@ To create a shared in-memory database named ``statistics``, use the following: from datasette.database import Database - datasette.add_database("statistics", Database( + datasette.add_database(Database( datasette, memory_name="statistics" )) +This database will be served at ``/statistics``. + +``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: + +.. code-block:: python + + db = datasette.add_database(Database(datasette, memory_name="statistics")) + await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) + .. _datasette_remove_database: .remove_database(name) @@ -289,7 +298,7 @@ To create a shared in-memory database named ``statistics``, use the following: ``name`` - string The name of the database to be removed. -This removes a database that has been previously added. ``name=`` is the unique name of that database, also used in the URL for it. +This removes a database that has been previously added. ``name=`` is the unique name of that database, used in its URL path. .. _datasette_sign: diff --git a/tests/test_cli.py b/tests/test_cli.py index 3f6b1840..ff46d76f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,6 +8,7 @@ import asyncio from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ +from datasette.utils.sqlite import sqlite3 from click.testing import CliRunner import io import json @@ -240,3 +241,17 @@ def test_serve_create(ensure_eventloop, tmpdir): "hash": None, }.items() <= databases[0].items() assert db_path.exists() + + +def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): + runner = CliRunner() + db_1_path = str(tmpdir / "db.db") + nested = tmpdir / "nested" + nested.mkdir() + db_2_path = str(tmpdir / "nested" / "db.db") + for path in (db_1_path, db_2_path): + sqlite3.connect(path).execute("vacuum") + result = runner.invoke(cli, [db_1_path, db_2_path, "--get", "/-/databases.json"]) + assert result.exit_code == 0, result.output + databases = json.loads(result.output) + assert {db["name"] for db in databases} == {"db", "db_2"} diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index dc1af48c..7eff9f7e 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -439,7 +439,7 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") sqlite3.connect(path).execute("vacuum") db = Database(app_client.ds, path=path, is_mutable=False) - app_client.ds.add_database("immutable-db", db) + app_client.ds.add_database(db, name="immutable-db") def write_fn(conn): assert False @@ -469,10 +469,10 @@ def test_is_mutable(app_client): @pytest.mark.asyncio async def test_database_memory_name(app_client): ds = app_client.ds - foo1 = Database(ds, memory_name="foo") - foo2 = Database(ds, memory_name="foo") - bar1 = Database(ds, memory_name="bar") - bar2 = Database(ds, memory_name="bar") + foo1 = ds.add_database(Database(ds, memory_name="foo")) + foo2 = ds.add_database(Database(ds, memory_name="foo")) + bar1 = ds.add_database(Database(ds, memory_name="bar")) + bar2 = ds.add_database(Database(ds, memory_name="bar")) for db in (foo1, foo2, bar1, bar2): table_names = await db.table_names() assert table_names == [] @@ -487,7 +487,7 @@ async def test_database_memory_name(app_client): @pytest.mark.asyncio async def test_in_memory_databases_forbid_writes(app_client): ds = app_client.ds - db = Database(ds, memory_name="test") + db = ds.add_database(Database(ds, memory_name="test")) with pytest.raises(sqlite3.OperationalError): await db.execute("create table foo (t text)") assert await db.table_names() == [] From 90eba4c3ca569c57e96bce314e7ac8caf67d884e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 15:55:43 -0800 Subject: [PATCH 0009/1364] Prettier CREATE TABLE SQL for _internal --- datasette/utils/internal_db.py | 109 ++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 5cd32381..e92625d5 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,82 +1,95 @@ +import textwrap + + async def init_internal_db(db): await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE databases ( - "database_name" TEXT PRIMARY KEY, - "path" TEXT, - "is_memory" INTEGER, - "schema_version" INTEGER + database_name TEXT PRIMARY KEY, + path TEXT, + is_memory INTEGER, + schema_version INTEGER ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE tables ( - "database_name" TEXT, - "table_name" TEXT, - "rootpage" INTEGER, - "sql" TEXT, + database_name TEXT, + table_name TEXT, + rootpage INTEGER, + sql TEXT, PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE columns ( - "database_name" TEXT, - "table_name" TEXT, - "cid" INTEGER, - "name" TEXT, - "type" TEXT, + database_name TEXT, + table_name TEXT, + cid INTEGER, + name TEXT, + type TEXT, "notnull" INTEGER, - "default_value" TEXT, -- renamed from dflt_value - "is_pk" INTEGER, -- renamed from pk - "hidden" INTEGER, + default_value TEXT, -- renamed from dflt_value + is_pk INTEGER, -- renamed from pk + hidden INTEGER, PRIMARY KEY (database_name, table_name, name), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE indexes ( - "database_name" TEXT, - "table_name" TEXT, - "seq" INTEGER, - "name" TEXT, + database_name TEXT, + table_name TEXT, + seq INTEGER, + name TEXT, "unique" INTEGER, - "origin" TEXT, - "partial" INTEGER, + origin TEXT, + partial INTEGER, PRIMARY KEY (database_name, table_name, name), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE foreign_keys ( - "database_name" TEXT, - "table_name" TEXT, - "id" INTEGER, - "seq" INTEGER, + database_name TEXT, + table_name TEXT, + id INTEGER, + seq INTEGER, "table" TEXT, "from" TEXT, "to" TEXT, - "on_update" TEXT, - "on_delete" TEXT, - "match" TEXT, + on_update TEXT, + on_delete TEXT, + match TEXT, PRIMARY KEY (database_name, table_name, id, seq), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) - """, + """ + ), block=True, ) @@ -84,14 +97,14 @@ async def init_internal_db(db): async def populate_schema_tables(internal_db, db): database_name = db.name await internal_db.execute_write( - "delete from tables where database_name = ?", [database_name], block=True + "DELETE FROM tables WHERE database_name = ?", [database_name], block=True ) - tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows + tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows for table in tables: table_name = table["name"] await internal_db.execute_write( """ - insert into tables (database_name, table_name, rootpage, sql) + INSERT INTO tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, [database_name, table_name, table["rootpage"], table["sql"]], @@ -99,7 +112,7 @@ async def populate_schema_tables(internal_db, db): ) # And the columns await internal_db.execute_write( - "delete from columns where database_name = ? and table_name = ?", + "DELETE FROM columns WHERE database_name = ? and table_name = ?", [database_name, table_name], block=True, ) @@ -111,7 +124,7 @@ async def populate_schema_tables(internal_db, db): } await internal_db.execute_write( """ - insert into columns ( + INSERT INTO columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -122,7 +135,7 @@ async def populate_schema_tables(internal_db, db): ) # And the foreign_keys await internal_db.execute_write( - "delete from foreign_keys where database_name = ? and table_name = ?", + "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?", [database_name, table_name], block=True, ) @@ -136,7 +149,7 @@ async def populate_schema_tables(internal_db, db): } await internal_db.execute_write( """ - insert into foreign_keys ( + INSERT INTO foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -147,7 +160,7 @@ async def populate_schema_tables(internal_db, db): ) # And the indexes await internal_db.execute_write( - "delete from indexes where database_name = ? and table_name = ?", + "DELETE FROM indexes WHERE database_name = ? and table_name = ?", [database_name, table_name], block=True, ) @@ -159,7 +172,7 @@ async def populate_schema_tables(internal_db, db): } await internal_db.execute_write( """ - insert into indexes ( + INSERT INTO indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial From a882d679626438ba0d809944f06f239bcba8ee96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= <6774676+eumiro@users.noreply.github.com> Date: Wed, 23 Dec 2020 18:04:32 +0100 Subject: [PATCH 0010/1364] Modernize code to Python 3.6+ (#1158) * Compact dict and set building * Remove redundant parentheses * Simplify chained conditions * Change method name to lowercase * Use triple double quotes for docstrings Thanks, @eumiro! --- datasette/app.py | 16 +++++++------- datasette/cli.py | 10 ++++----- datasette/facets.py | 4 +--- datasette/filters.py | 6 +++--- datasette/hookspecs.py | 42 ++++++++++++++++++------------------- datasette/inspect.py | 6 +++--- datasette/renderer.py | 2 +- datasette/utils/__init__.py | 20 +++++++++--------- datasette/utils/asgi.py | 18 +++++++--------- datasette/views/base.py | 6 +++--- datasette/views/table.py | 4 ++-- tests/fixtures.py | 2 +- tests/plugins/my_plugin.py | 2 +- tests/test_api.py | 4 ++-- tests/test_auth.py | 4 ++-- tests/test_cli.py | 2 +- tests/test_docs.py | 6 +++--- tests/test_permissions.py | 2 +- tests/test_plugins.py | 2 +- 19 files changed, 76 insertions(+), 82 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ad3ba07e..bd62fd3b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -429,7 +429,7 @@ class Datasette: return m def plugin_config(self, plugin_name, database=None, table=None, fallback=True): - "Return config for plugin, falling back from specified database/table" + """Return config for plugin, falling back from specified database/table""" plugins = self.metadata( "plugins", database=database, table=table, fallback=fallback ) @@ -523,7 +523,7 @@ class Datasette: return [] async def permission_allowed(self, actor, action, resource=None, default=False): - "Check permissions using the permissions_allowed plugin hook" + """Check permissions using the permissions_allowed plugin hook""" result = None for check in pm.hook.permission_allowed( datasette=self, @@ -570,7 +570,7 @@ class Datasette: ) async def expand_foreign_keys(self, database, table, column, values): - "Returns dict mapping (column, value) -> label" + """Returns dict mapping (column, value) -> label""" labeled_fks = {} db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) @@ -613,7 +613,7 @@ class Datasette: return url def _register_custom_units(self): - "Register any custom units defined in the metadata.json with Pint" + """Register any custom units defined in the metadata.json with Pint""" for unit in self.metadata("custom_units") or []: ureg.define(unit) @@ -730,7 +730,7 @@ class Datasette: return {"actor": request.actor} def table_metadata(self, database, table): - "Fetch table-specific metadata." + """Fetch table-specific metadata.""" return ( (self.metadata("databases") or {}) .get(database, {}) @@ -739,7 +739,7 @@ class Datasette: ) def _register_renderers(self): - """ Register output renderers which output data in custom formats. """ + """Register output renderers which output data in custom formats.""" # Built-in renderers self.renderers["json"] = (json_renderer, lambda: True) @@ -880,7 +880,7 @@ class Datasette: return output def app(self): - "Returns an ASGI app function that serves the whole of Datasette" + """Returns an ASGI app function that serves the whole of Datasette""" routes = [] for routes_to_add in pm.hook.register_routes(): @@ -1287,7 +1287,7 @@ def permanent_redirect(path): ) -_curly_re = re.compile(r"(\{.*?\})") +_curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): diff --git a/datasette/cli.py b/datasette/cli.py index c342a35a..2a84bf30 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -152,7 +152,7 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): - "Publish specified SQLite database files to the internet along with a Datasette-powered interface and API" + """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" pass @@ -168,7 +168,7 @@ pm.hook.publish_subcommand(publish=publish) help="Path to directory containing custom plugins", ) def plugins(all, plugins_dir): - "List currently available plugins" + """List currently available plugins""" app = Datasette([], plugins_dir=plugins_dir) click.echo(json.dumps(app._plugins(all=all), indent=4)) @@ -244,7 +244,7 @@ def package( port, **extra_metadata, ): - "Package specified SQLite files into a new datasette Docker container" + """Package specified SQLite files into a new datasette Docker container""" if not shutil.which("docker"): click.secho( ' The package command requires "docker" to be installed and configured ', @@ -284,7 +284,7 @@ def package( "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" ) def install(packages, upgrade): - "Install Python packages - e.g. Datasette plugins - into the same environment as Datasette" + """Install Python packages - e.g. Datasette plugins - into the same environment as Datasette""" args = ["pip", "install"] if upgrade: args += ["--upgrade"] @@ -297,7 +297,7 @@ def install(packages, upgrade): @click.argument("packages", nargs=-1, required=True) @click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") def uninstall(packages, yes): - "Uninstall Python packages (e.g. plugins) from the Datasette environment" + """Uninstall Python packages (e.g. plugins) from the Datasette environment""" sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) run_module("pip", run_name="__main__") diff --git a/datasette/facets.py b/datasette/facets.py index 8ad5a423..207d819d 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -162,10 +162,8 @@ class ColumnFacet(Facet): ) num_distinct_values = len(distinct_values) if ( - num_distinct_values - and num_distinct_values > 1 + 1 < num_distinct_values < row_count and num_distinct_values <= facet_size - and num_distinct_values < row_count # And at least one has n > 1 and any(r["n"] > 1 for r in distinct_values) ): diff --git a/datasette/filters.py b/datasette/filters.py index edf2de99..152a26b4 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -208,7 +208,7 @@ class Filters: self.ureg = ureg def lookups(self): - "Yields (lookup, display, no_argument) pairs" + """Yields (lookup, display, no_argument) pairs""" for filter in self._filters: yield filter.key, filter.display, filter.no_argument @@ -233,7 +233,7 @@ class Filters: return f"where {s}" def selections(self): - "Yields (column, lookup, value) tuples" + """Yields (column, lookup, value) tuples""" for key, value in self.pairs: if "__" in key: column, lookup = key.rsplit("__", 1) @@ -246,7 +246,7 @@ class Filters: return bool(self.pairs) def convert_unit(self, column, value): - "If the user has provided a unit in the query, convert it into the column unit, if present." + """If the user has provided a unit in the query, convert it into the column unit, if present.""" if column not in self.units: return value diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a305ca6a..13a10680 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -7,108 +7,108 @@ hookimpl = HookimplMarker("datasette") @hookspec def startup(datasette): - "Fires directly after Datasette first starts running" + """Fires directly after Datasette first starts running""" @hookspec def asgi_wrapper(datasette): - "Returns an ASGI middleware callable to wrap our ASGI application with" + """Returns an ASGI middleware callable to wrap our ASGI application with""" @hookspec def prepare_connection(conn, database, datasette): - "Modify SQLite connection in some way e.g. register custom SQL functions" + """Modify SQLite connection in some way e.g. register custom SQL functions""" @hookspec def prepare_jinja2_environment(env): - "Modify Jinja2 template environment e.g. register custom template tags" + """Modify Jinja2 template environment e.g. register custom template tags""" @hookspec def extra_css_urls(template, database, table, columns, view_name, request, datasette): - "Extra CSS URLs added by this plugin" + """Extra CSS URLs added by this plugin""" @hookspec def extra_js_urls(template, database, table, columns, view_name, request, datasette): - "Extra JavaScript URLs added by this plugin" + """Extra JavaScript URLs added by this plugin""" @hookspec def extra_body_script( template, database, table, columns, view_name, request, datasette ): - "Extra JavaScript code to be included in + {% endfor %} {% block extra_head %}{% endblock %} diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index d37bb729..a7236873 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -5,6 +5,8 @@ Custom pages and templates Datasette provides a number of ways of customizing the way data is displayed. +.. _customization_css_and_javascript: + Custom CSS and JavaScript ------------------------- @@ -25,7 +27,12 @@ Your ``metadata.json`` file can include links that look like this: ] } -The extra CSS and JavaScript files will be linked in the ```` of every page. +The extra CSS and JavaScript files will be linked in the ```` of every page: + +.. code-block:: html + + + You can also specify a SRI (subresource integrity hash) for these assets: @@ -46,9 +53,39 @@ You can also specify a SRI (subresource integrity hash) for these assets: ] } +This will produce: + +.. code-block:: html + + + + Modern browsers will only execute the stylesheet or JavaScript if the SRI hash matches the content served. You can generate hashes using `www.srihash.org `_ +Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: + +.. code-block:: json + + { + "extra_js_urls": [ + { + "url": "https://example.datasette.io/module.js", + "module": true + } + ] + } + +Will produce this HTML: + +.. code-block:: html + + + CSS classes on the ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 72b09367..d465307b 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -182,7 +182,7 @@ This can be a list of URLs: @hookimpl def extra_css_urls(): return [ - 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css' + "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css" ] Or a list of dictionaries defining both a URL and an @@ -190,21 +190,17 @@ Or a list of dictionaries defining both a URL and an .. code-block:: python - from datasette import hookimpl - @hookimpl def extra_css_urls(): return [{ - 'url': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css', - 'sri': 'sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4', + "url": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css", + "sri": "sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4", }] This function can also return an awaitable function, useful if it needs to run any async code: .. code-block:: python - from datasette import hookimpl - @hookimpl def extra_css_urls(datasette): async def inner(): @@ -233,8 +229,8 @@ return a list of URLs, a list of dictionaries or an awaitable function that retu @hookimpl def extra_js_urls(): return [{ - 'url': 'https://code.jquery.com/jquery-3.3.1.slim.min.js', - 'sri': 'sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo', + "url": "https://code.jquery.com/jquery-3.3.1.slim.min.js", + "sri": "sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo", }] You can also return URLs to files from your plugin's ``static/`` directory, if @@ -242,12 +238,21 @@ you have one: .. code-block:: python - from datasette import hookimpl - @hookimpl def extra_js_urls(): return [ - '/-/static-plugins/your-plugin/app.js' + "/-/static-plugins/your-plugin/app.js" + ] + +If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. + +.. code-block:: python + + @hookimpl + def extra_js_urls(): + return [{ + "url": "/-/static-plugins/your-plugin/app.js", + "module": True ] Examples: `datasette-cluster-map `_, `datasette-vega `_ diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 2e653e2b..1c86b4bc 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -61,6 +61,7 @@ def extra_js_urls(): "sri": "SRIHASH", }, "https://plugin-example.datasette.io/plugin1.js", + {"url": "https://plugin-example.datasette.io/plugin.module.js", "module": True}, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 212de2b5..648e7abd 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -118,16 +118,19 @@ def test_hook_extra_css_urls(app_client, path, expected_decoded_object): def test_hook_extra_js_urls(app_client): response = app_client.get("/") scripts = Soup(response.body, "html.parser").findAll("script") - assert [ - s - for s in scripts - if s.attrs - == { + script_attrs = [s.attrs for s in scripts] + for attrs in [ + { "integrity": "SRIHASH", "crossorigin": "anonymous", "src": "https://plugin-example.datasette.io/jquery.js", - } - ] + }, + { + "src": "https://plugin-example.datasette.io/plugin.module.js", + "type": "module", + }, + ]: + assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs) def test_plugins_with_duplicate_js_urls(app_client): From c38c42948cbfddd587729413fd6082ba352eaece Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Jan 2021 18:14:33 -0800 Subject: [PATCH 0032/1364] extra_body_script module support, closes #1187 --- datasette/app.py | 8 +++++++- datasette/templates/base.html | 2 +- docs/plugin_hooks.rst | 25 ++++++++++++++++++++----- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 2 +- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f8549fac..cfce8e0b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -781,7 +781,13 @@ class Datasette: datasette=self, ): extra_script = await await_me_maybe(extra_script) - body_scripts.append(Markup(extra_script)) + if isinstance(extra_script, dict): + script = extra_script["script"] + module = bool(extra_script.get("module")) + else: + script = extra_script + module = False + body_scripts.append({"script": Markup(script), "module": module}) extra_template_vars = {} # pylint: disable=no-member diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 3f3d4507..e61edc4f 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -62,7 +62,7 @@ {% include "_close_open_menus.html" %} {% for body_script in body_scripts %} - + {{ body_script.script }} {% endfor %} {% if select_templates %}{% endif %} diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d465307b..0206daaa 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -168,7 +168,7 @@ Examples: `datasette-search-all extra_css_urls(template, database, table, columns, view_name, request, datasette) --------------------------------------------------------------------------------- -Same arguments as :ref:`extra_template_vars(...) ` +This takes the same arguments as :ref:`extra_template_vars(...) ` Return a list of extra CSS URLs that should be included on the page. These can take advantage of the CSS class hooks described in :ref:`customization`. @@ -217,7 +217,7 @@ Examples: `datasette-cluster-map ` +This takes the same arguments as :ref:`extra_template_vars(...) ` This works in the same way as ``extra_css_urls()`` but for JavaScript. You can return a list of URLs, a list of dictionaries or an awaitable function that returns those things: @@ -264,15 +264,30 @@ extra_body_script(template, database, table, columns, view_name, request, datase Extra JavaScript to be added to a ```` element: + +.. code-block:: python + + @hookimpl + def extra_body_script(): + return { + "module": True, + "script": "console.log('Your JavaScript goes here...')" + } + +This will add the following to the end of your page: + +.. code-block:: html + + Example: `datasette-cluster-map `_ diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 1c86b4bc..8d192d28 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -70,7 +70,7 @@ def extra_body_script( template, database, table, view_name, columns, request, datasette ): async def inner(): - return "var extra_body_script = {};".format( + script = "var extra_body_script = {};".format( json.dumps( { "template": template, @@ -90,6 +90,7 @@ def extra_body_script( } ) ) + return {"script": script, "module": True} return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 648e7abd..715c7c17 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -288,7 +288,7 @@ def test_plugin_config_file(app_client): ], ) def test_hook_extra_body_script(app_client, path, expected_extra_body_script): - r = re.compile(r"") + r = re.compile(r"") json_data = r.search(app_client.get(path).text).group(1) actual_data = json.loads(json_data) assert expected_extra_body_script == actual_data From 7e3cfd9cf7aeddf153d907bc3ee08ae0cd489370 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Jan 2021 12:27:45 -0800 Subject: [PATCH 0033/1364] Clarify the name of plugin used in /-/static-plugins/ --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 0206daaa..23e57278 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -244,6 +244,8 @@ you have one: "/-/static-plugins/your-plugin/app.js" ] +Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. + If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. .. code-block:: python From 57f4d7b82f9c74298c67c5640207241925b70c02 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Jan 2021 12:47:30 -0800 Subject: [PATCH 0034/1364] Release 0.54a0 Refs #1091, #1145, #1151, #1156, #1157, #1158, #1166, #1170, #1178, #1182, #1184, #1185, #1186, #1187 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index a5edecfa..b19423a9 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.53" +__version__ = "0.54a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 40b9c5a3..ac2ac8c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v0_54_a0: + +0.54a0 (2020-12-19) +------------------- + +**Alpha release**. Release notes in progress. + +- Improved support for named in-memory databases. (`#1151 `__) +- New ``_internal`` in-memory database tracking attached databases, tables and columns. (`#1150 `__) +- Support for JavaScript modules. (`#1186 `__, `#1187 `__) + .. _v0_53: 0.53 (2020-12-10) From 5378f023529107ff7edbd6ee4ecab6ac170a83db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Jan 2021 12:50:12 -0800 Subject: [PATCH 0035/1364] Better tool for extracting issue numbers --- docs/contributing.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 24d5c8f0..3a4b2caa 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -169,17 +169,7 @@ To release a new version, first create a commit that updates the version number Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. -You can generate the list of issue references for a specific release by pasting the following into the browser devtools while looking at the :ref:`changelog` page (replace ``v0-44`` with the most recent version): - -.. code-block:: javascript - - [ - ...new Set( - Array.from( - document.getElementById("v0-44").querySelectorAll("a[href*=issues]") - ).map((a) => "#" + a.href.split("/issues/")[1]) - ), - ].sort().join(", "); +You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text `__ tool. To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. From 25c2933667680db045851b2cedcf4666d737d352 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Jan 2021 16:46:16 -0800 Subject: [PATCH 0036/1364] publish heroku now uses python-3.8.7 --- datasette/publish/heroku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index c772b476..c0c70e12 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -173,7 +173,7 @@ def temporary_heroku_directory( if metadata_content: open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) - open("runtime.txt", "w").write("python-3.8.6") + open("runtime.txt", "w").write("python-3.8.7") if branch: install = [ From f78e956eca1f363e3a3f93c69fd9fc31bed14629 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 12:38:29 -0800 Subject: [PATCH 0037/1364] Plugin testing documentation on using pytest-httpx Closes #1198 --- docs/testing_plugins.rst | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index bacfd57b..4261f639 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -118,3 +118,74 @@ If you want to create that test database repeatedly for every individual test fu @pytest.fixture def datasette(tmp_path_factory): # This fixture will be executed repeatedly for every test + +.. _testing_plugins_pytest_httpx: + +Testing outbound HTTP calls with pytest-httpx +--------------------------------------------- + +If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests. + +The `pytest-httpx `__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally. + +To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture. + +As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content: + +.. code-block:: python + + from datasette import hookimpl + from datasette.utils.asgi import Response + import httpx + + + @hookimpl + def register_routes(): + return [ + (r"^/-/fetch-url$", fetch_url), + ] + + + async def fetch_url(datasette, request): + if request.method == "GET": + return Response.html( + """ +
+ + +
""".format( + request.scope["csrftoken"]() + ) + ) + vars = await request.post_vars() + url = vars["url"] + return Response.text(httpx.get(url).text) + +Here's a test for that plugin that mocks the HTTPX outbound request: + +.. code-block:: python + + from datasette.app import Datasette + import pytest + + + @pytest.fixture + def non_mocked_hosts(): + # This ensures httpx-mock will not affect Datasette's own + # httpx calls made in the tests by datasette.client: + return ["localhost"] + + + async def test_outbound_http_call(httpx_mock): + httpx_mock.add_response( + url='https://www.example.com/', + data='Hello world', + ) + datasette = Datasette([], memory=True) + response = await datasette.client.post("/-/fetch-url", data={ + "url": "https://www.example.com/" + }) + asert response.text == "Hello world" + + outbound_request = httpx_mock.get_request() + assert outbound_request.url == "https://www.example.com/" From b6a7b58fa01af0cd5a5e94bd17d686d283a46819 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 16:08:29 -0800 Subject: [PATCH 0038/1364] Initial docs for _internal database, closes #1154 --- docs/internals.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index f7b0cc0b..4a2c0a8e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -747,3 +747,19 @@ If your plugin implements a ``
`` anywhere you will need to i .. code-block:: html + +.. _internals_internal: + +The _internal database +====================== + +.. warning:: + This API should be considered unstable - the structure of these tables may change prior to the release of Datasette 1.0. + +Datasette maintains an in-memory SQLite database with details of the the databases, tables and columns for all of the attached databases. + +By default all actors are denied access to the ``view-database`` permission for the ``_internal`` database, so the database is not visible to anyone unless they :ref:`sign in as root `. + +Plugins can access this database by calling ``db = datasette.get_database("_internal")`` and then executing queries using the :ref:`Database API `. + +You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. \ No newline at end of file From ffff3a4c5398a9f40b61d59736f386444da19289 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 17:41:46 -0800 Subject: [PATCH 0039/1364] Easier way to run Prettier locally (#1203) Thanks, Ben Pickles - refs #1167 --- .github/workflows/prettier.yml | 2 +- package.json | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index d846cca7..9dfe7ee0 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -19,4 +19,4 @@ jobs: run: npm ci - name: Run prettier run: |- - npx --no-install prettier --check 'datasette/static/*[!.min].js' + npm run prettier -- --check diff --git a/package.json b/package.json index 67452d2f..5c6dfe61 100644 --- a/package.json +++ b/package.json @@ -3,5 +3,9 @@ "private": true, "devDependencies": { "prettier": "^2.2.1" + }, + "scripts": { + "fix": "npm run prettier -- --write", + "prettier": "prettier 'datasette/static/*[!.min].js'" } } From f3a155531807c586e62b8ff0e97b96a76e949c8d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 17:58:15 -0800 Subject: [PATCH 0040/1364] Contributing docs for Black and Prettier, closes #1167 Refs #1203 --- docs/contributing.rst | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3a4b2caa..2cf641fd 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -97,6 +97,58 @@ You can tell Datasette to open an interactive ``pdb`` debugger session if an err datasette --pdb fixtures.db +.. _contributing_formatting: + +Code formatting +--------------- + +Datasette uses opinionated code formatters: `Black `__ for Python and `Prettier `__ for JavaScript. + +These formatters are enforced by Datasette's continuous integration: if a commit includes Python or JavaScript code that does not match the style enforced by those tools, the tests will fail. + +When developing locally, you can verify and correct the formatting of your code using these tools. + +.. _contributing_formatting_black: + +Running Black +~~~~~~~~~~~~~ + +Black will be installed when you run ``pip install -e '.[test]'``. To test that your code complies with Black, run the following in your root ``datasette`` repository checkout:: + + $ black . --check + All done! ✨ 🍰 ✨ + 95 files would be left unchanged. + +If any of your code does not conform to Black you can run this to automatically fix those problems:: + + $ black . + reformatted ../datasette/setup.py + All done! ✨ 🍰 ✨ + 1 file reformatted, 94 files left unchanged. + +.. _contributing_formatting_prettier: + +Prettier +~~~~~~~~ + +To install Prettier, `install Node.js `__ and then run the following in the root of your ``datasette`` repository checkout:: + + $ npm install + +This will install Prettier in a ``node_modules`` directory. You can then check that your code matches the coding style like so:: + + $ npm run prettier -- --check + > prettier + > prettier 'datasette/static/*[!.min].js' "--check" + + Checking formatting... + [warn] datasette/static/plugins.js + [warn] Code style issues found in the above file(s). Forgot to run Prettier? + +You can fix any problems by running:: + + $ npm run fix + .. _contributing_documentation: Editing and building the documentation From 07e163561592c743e4117f72102fcd350a600909 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 19:10:10 -0800 Subject: [PATCH 0041/1364] All ?_ parameters now copied to hidden form fields, closes #1194 --- datasette/views/table.py | 17 +++++------------ tests/test_html.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index cc8ef9f1..0a3504b3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -812,19 +812,12 @@ class TableView(RowTableShared): .get(table, {}) ) self.ds.update_with_inherited_metadata(metadata) + form_hidden_args = [] - # Add currently selected facets - for arg in special_args: - if arg == "_facet" or arg.startswith("_facet_"): - form_hidden_args.extend( - (arg, item) for item in request.args.getlist(arg) - ) - for arg in ("_fts_table", "_fts_pk"): - if arg in special_args: - form_hidden_args.append((arg, special_args[arg])) - if request.args.get("_where"): - for where_text in request.args.getlist("_where"): - form_hidden_args.append(("_where", where_text)) + for key in request.args: + if key.startswith("_"): + for value in request.args.getlist(key): + form_hidden_args.append((key, value)) # if no sort specified AND table has a single primary key, # set sort to that so arrow is displayed diff --git a/tests/test_html.py b/tests/test_html.py index c7dd9d97..08d17ca7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1250,6 +1250,28 @@ def test_extra_where_clauses(app_client): ] +@pytest.mark.parametrize( + "path,expected_hidden", + [ + ("/fixtures/facetable?_size=10", [("_size", "10")]), + ( + "/fixtures/facetable?_size=10&_ignore=1&_ignore=2", + [ + ("_size", "10"), + ("_ignore", "1"), + ("_ignore", "2"), + ], + ), + ], +) +def test_other_hidden_form_fields(app_client, path, expected_hidden): + response = app_client.get(path) + soup = Soup(response.body, "html.parser") + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == expected_hidden + + def test_binary_data_display_in_table(app_client): response = app_client.get("/fixtures/binary_data") assert response.status == 200 From a5ede3cdd455e2bb1a1fb2f4e1b5a9855caf5179 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 21:13:05 -0800 Subject: [PATCH 0042/1364] Fixed bug loading database called 'test-database (1).sqlite' Closes #1181. Also now ensures that database URLs have special characters URL-quoted. --- datasette/url_builder.py | 6 ++++-- datasette/views/base.py | 3 ++- docs/changelog.rst | 10 ++++++---- tests/test_api.py | 14 +++++++------- tests/test_cli.py | 23 +++++++++++++++++++++++ tests/test_html.py | 6 +++--- tests/test_internals_urls.py | 20 ++++++++++---------- 7 files changed, 55 insertions(+), 27 deletions(-) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 3034b664..2bcda869 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -30,9 +30,11 @@ class Urls: def database(self, database, format=None): db = self.ds.databases[database] if self.ds.setting("hash_urls") and db.hash: - path = self.path(f"{database}-{db.hash[:HASH_LENGTH]}", format=format) + path = self.path( + f"{urllib.parse.quote(database)}-{db.hash[:HASH_LENGTH]}", format=format + ) else: - path = self.path(database, format=format) + path = self.path(urllib.parse.quote(database), format=format) return path def table(self, database, table, format=None): diff --git a/datasette/views/base.py b/datasette/views/base.py index a21b9298..ba0f7d4c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -181,6 +181,7 @@ class DataView(BaseView): async def resolve_db_name(self, request, db_name, **kwargs): hash = None name = None + db_name = urllib.parse.unquote_plus(db_name) if db_name not in self.ds.databases and "-" in db_name: # No matching DB found, maybe it's a name-hash? name_bit, hash_bit = db_name.rsplit("-", 1) @@ -191,7 +192,7 @@ class DataView(BaseView): hash = hash_bit else: name = db_name - name = urllib.parse.unquote_plus(name) + try: db = self.ds.databases[name] except KeyError: diff --git a/docs/changelog.rst b/docs/changelog.rst index ac2ac8c9..abc2f4f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,12 +4,14 @@ Changelog ========= -.. _v0_54_a0: +.. _v0_54: + +0.54 (2021-01-24) +----------------- + + -0.54a0 (2020-12-19) -------------------- -**Alpha release**. Release notes in progress. - Improved support for named in-memory databases. (`#1151 `__) - New ``_internal`` in-memory database tracking attached databases, tables and columns. (`#1150 `__) diff --git a/tests/test_api.py b/tests/test_api.py index 3b4f3437..0d1bddd3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -609,17 +609,17 @@ def test_no_files_uses_memory_database(app_client_no_files): assert response.status == 200 assert { ":memory:": { + "name": ":memory:", "hash": None, "color": "f7935d", + "path": "/%3Amemory%3A", + "tables_and_views_truncated": [], + "tables_and_views_more": False, + "tables_count": 0, + "table_rows_sum": 0, + "show_table_row_counts": False, "hidden_table_rows_sum": 0, "hidden_tables_count": 0, - "name": ":memory:", - "show_table_row_counts": False, - "path": "/:memory:", - "table_rows_sum": 0, - "tables_count": 0, - "tables_and_views_more": False, - "tables_and_views_truncated": [], "views_count": 0, "private": False, } diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d806bff..c42c22ea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,6 +17,7 @@ import pytest import sys import textwrap from unittest import mock +import urllib @pytest.fixture @@ -255,3 +256,25 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): assert result.exit_code == 0, result.output databases = json.loads(result.output) assert {db["name"] for db in databases} == {"db", "db_2"} + + +@pytest.mark.parametrize( + "filename", ["test-database (1).sqlite", "database (1).sqlite"] +) +def test_weird_database_names(ensure_eventloop, tmpdir, filename): + # https://github.com/simonw/datasette/issues/1181 + runner = CliRunner() + db_path = str(tmpdir / filename) + sqlite3.connect(db_path).execute("vacuum") + result1 = runner.invoke(cli, [db_path, "--get", "/"]) + assert result1.exit_code == 0, result1.output + filename_no_stem = filename.rsplit(".", 1)[0] + expected_link = '{}'.format( + urllib.parse.quote(filename_no_stem), filename_no_stem + ) + assert expected_link in result1.output + # Now try hitting that database page + result2 = runner.invoke( + cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))] + ) + assert result2.exit_code == 0, result2.output diff --git a/tests/test_html.py b/tests/test_html.py index 08d17ca7..6c33fba7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -30,7 +30,7 @@ def test_homepage(app_client_two_attached_databases): # Should be two attached databases assert [ {"href": "/fixtures", "text": "fixtures"}, - {"href": "/extra database", "text": "extra database"}, + {"href": r"/extra%20database", "text": "extra database"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables h2 = soup.select("h2")[1] @@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": "/extra database/searchable", "text": "searchable"}, - {"href": "/extra database/searchable_view", "text": "searchable_view"}, + {"href": r"/extra%20database/searchable", "text": "searchable"}, + {"href": r"/extra%20database/searchable_view", "text": "searchable_view"}, ] == table_links diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index fd05c1b6..e6f405b3 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -103,9 +103,9 @@ def test_logout(ds, base_url, expected): @pytest.mark.parametrize( "base_url,format,expected", [ - ("/", None, "/:memory:"), - ("/prefix/", None, "/prefix/:memory:"), - ("/", "json", "/:memory:.json"), + ("/", None, "/%3Amemory%3A"), + ("/prefix/", None, "/prefix/%3Amemory%3A"), + ("/", "json", "/%3Amemory%3A.json"), ], ) def test_database(ds, base_url, format, expected): @@ -118,10 +118,10 @@ def test_database(ds, base_url, format, expected): @pytest.mark.parametrize( "base_url,name,format,expected", [ - ("/", "name", None, "/:memory:/name"), - ("/prefix/", "name", None, "/prefix/:memory:/name"), - ("/", "name", "json", "/:memory:/name.json"), - ("/", "name.json", "json", "/:memory:/name.json?_format=json"), + ("/", "name", None, "/%3Amemory%3A/name"), + ("/prefix/", "name", None, "/prefix/%3Amemory%3A/name"), + ("/", "name", "json", "/%3Amemory%3A/name.json"), + ("/", "name.json", "json", "/%3Amemory%3A/name.json?_format=json"), ], ) def test_table_and_query(ds, base_url, name, format, expected): @@ -137,9 +137,9 @@ def test_table_and_query(ds, base_url, name, format, expected): @pytest.mark.parametrize( "base_url,format,expected", [ - ("/", None, "/:memory:/facetable/1"), - ("/prefix/", None, "/prefix/:memory:/facetable/1"), - ("/", "json", "/:memory:/facetable/1.json"), + ("/", None, "/%3Amemory%3A/facetable/1"), + ("/prefix/", None, "/prefix/%3Amemory%3A/facetable/1"), + ("/", "json", "/%3Amemory%3A/facetable/1.json"), ], ) def test_row(ds, base_url, format, expected): From 0b9ac1b2e9c855f1b823a06a898891da87c720ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 Jan 2021 09:33:29 -0800 Subject: [PATCH 0043/1364] Release 0.54 Refs #509, #1091, #1150, #1151, #1166, #1167, #1178, #1181, #1182, #1184, #1185, #1186, #1187, #1194, #1198 --- datasette/version.py | 2 +- docs/changelog.rst | 54 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index b19423a9..8fb7217d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.54a0" +__version__ = "0.54" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index abc2f4f9..8fca312d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,16 +6,61 @@ Changelog .. _v0_54: -0.54 (2021-01-24) +0.54 (2021-01-25) ----------------- +The two big new features in this release are the ``_internal`` SQLite in-memory database storing details of all connected databases and tables, and support for JavaScript modules in plugins and additional scripts. +For additional commentary on this release, see `Datasette 0.54, the annotated release notes `__. +The _internal database +~~~~~~~~~~~~~~~~~~~~~~ +As part of ongoing work to help Datasette handle much larger numbers of connected databases and tables (see `Datasette Library `__) Datasette now maintains an in-memory SQLite database with details of all of the attached databases, tables, columns, indexes and foreign keys. (`#1150 `__) + +This will support future improvements such as a searchable, paginated homepage of all available tables. + +You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. + +Plugins can use these tables to introspect attached data in an efficient way. Plugin authors should note that this is not yet considered a stable interface, so any plugins that use this may need to make changes prior to Datasette 1.0 if the ``_internal`` table schemas change. + +Named in-memory database support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As part of the work building the ``_internal`` database, Datasette now supports named in-memory databases that can be shared across multiple connections. This allows plugins to create in-memory databases which will persist data for the lifetime of the Datasette server process. (`#1151 `__) + +The new ``memory_name=`` parameter to the :ref:`internals_database` can be used to create named, shared in-memory databases. + +JavaScript modules +~~~~~~~~~~~~~~~~~~ + +`JavaScript modules `__ were introduced in ECMAScript 2015 and provide native browser support for the ``import`` and ``export`` keywords. + +To use modules, JavaScript needs to be included in `` + diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 4019d448..ee09cff1 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -23,6 +23,7 @@ window.onload = () => { editor.setValue(sqlFormatter.format(editor.getValue())); }) } + cmResize(editor, {resizableWidth: false}); } if (sqlFormat && readOnly) { const formatted = sqlFormatter.format(readOnly.innerHTML); From 42caabf7e9e6e4d69ef6dd7de16f2cd96bc79d5b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Feb 2021 09:35:41 -0800 Subject: [PATCH 0062/1364] Fixed typo --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 8ea5e79b..1291a875 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -79,7 +79,7 @@ Using pytest fixtures A common pattern for Datasette plugins is to create a fixture which sets up a temporary test database and wraps it in a Datasette instance. -Here's an example that uses the `sqlite-utils library `__ to populate a temporary test database. It also sets the title of that table using a simulated ``metadata.json`` congiguration: +Here's an example that uses the `sqlite-utils library `__ to populate a temporary test database. It also sets the title of that table using a simulated ``metadata.json`` configuration: .. code-block:: python From 726f781c50e88f557437f6490b8479c3d6fabfc2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Feb 2021 16:22:47 -0800 Subject: [PATCH 0063/1364] Fix for arraycontains bug, closes #1239 --- datasette/filters.py | 4 ++-- tests/test_filters.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index 152a26b4..2b859d99 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -150,7 +150,7 @@ class Filters: "arraycontains", "array contains", """rowid in ( - select {t}.rowid from {t}, json_each({t}.{c}) j + select {t}.rowid from {t}, json_each([{t}].[{c}]) j where j.value = :{p} )""", '{c} contains "{v}"', @@ -159,7 +159,7 @@ class Filters: "arraynotcontains", "array does not contain", """rowid not in ( - select {t}.rowid from {t}, json_each({t}.{c}) j + select {t}.rowid from {t}, json_each([{t}].[{c}]) j where j.value = :{p} )""", '{c} does not contain "{v}"', diff --git a/tests/test_filters.py b/tests/test_filters.py index 75a779b9..f22b7b5c 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -56,6 +56,14 @@ import pytest # Not in, and JSON array not in ((("foo__notin", "1,2,3"),), ["foo not in (:p0, :p1, :p2)"], ["1", "2", "3"]), ((("foo__notin", "[1,2,3]"),), ["foo not in (:p0, :p1, :p2)"], [1, 2, 3]), + # JSON arraycontains + ( + (("Availability+Info__arraycontains", "yes"),), + [ + "rowid in (\n select table.rowid from table, json_each([table].[Availability+Info]) j\n where j.value = :p0\n )" + ], + ["yes"], + ), ], ) def test_build_where(args, expected_where, expected_params): From afed51b1e36cf275c39e71c7cb262d6c5bdbaa31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Feb 2021 09:27:09 -0800 Subject: [PATCH 0064/1364] Note about where to find plugin examples, closes #1244 --- docs/writing_plugins.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index b43ecb27..6afee1c3 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -5,6 +5,8 @@ Writing plugins You can write one-off plugins that apply to just one Datasette instance, or you can write plugins which can be installed using ``pip`` and can be shipped to the Python Package Index (`PyPI `__) for other people to install. +Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 50 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. + .. _writing_plugins_one_off: Writing one-off plugins From cc6774cbaaba2359e0a92cfcc41ad988680075d6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Feb 2021 14:34:44 -0800 Subject: [PATCH 0065/1364] Upgrade httpx and remove xfail from tests, refs #1005 --- setup.py | 2 +- tests/test_api.py | 2 -- tests/test_html.py | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 34b6b396..15ee63fe 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ setup( "click-default-group~=1.2.2", "Jinja2>=2.10.3,<2.12.0", "hupper~=1.9", - "httpx>=0.15", + "httpx>=0.17", "pint~=0.9", "pluggy~=0.13.0", "uvicorn~=0.11", diff --git a/tests/test_api.py b/tests/test_api.py index 0b5401d6..caf23329 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -810,7 +810,6 @@ def test_table_shape_object_compound_primary_key(app_client): assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json -@pytest.mark.xfail def test_table_with_slashes_in_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json" @@ -1286,7 +1285,6 @@ def test_row_format_in_querystring(app_client): assert [{"id": "1", "content": "hello"}] == response.json["rows"] -@pytest.mark.xfail def test_row_strange_table_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects" diff --git a/tests/test_html.py b/tests/test_html.py index e21bd64d..3482ec35 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -158,7 +158,6 @@ def test_row_redirects_with_url_hash(app_client_with_hash): assert response.status == 200 -@pytest.mark.xfail def test_row_strange_table_name_with_url_hash(app_client_with_hash): response = app_client_with_hash.get( "/fixtures/table%2Fwith%2Fslashes.csv/3", allow_redirects=False @@ -552,7 +551,6 @@ def test_facets_persist_through_filter_form(app_client): ] -@pytest.mark.xfail @pytest.mark.parametrize( "path,expected_classes", [ @@ -584,7 +582,6 @@ def test_css_classes_on_body(app_client, path, expected_classes): assert classes == expected_classes -@pytest.mark.xfail @pytest.mark.parametrize( "path,expected_considered", [ From 47eb885cc2c3aafa03645c330c6f597bee9b3b25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Feb 2021 19:44:04 -0800 Subject: [PATCH 0066/1364] JSON faceting now suggested even if column has blank strings, closes #1246 --- datasette/facets.py | 11 ++++++++--- tests/test_facets.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 207d819d..01628760 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -279,6 +279,7 @@ class ArrayFacet(Facet): suggested_facet_sql = """ select distinct json_type({column}) from ({sql}) + where {column} is not null and {column} != '' """.format( column=escape_sqlite(column), sql=self.sql ) @@ -298,9 +299,13 @@ class ArrayFacet(Facet): v[0] for v in await self.ds.execute( self.database, - "select {column} from ({sql}) where {column} is not null and json_array_length({column}) > 0 limit 100".format( - column=escape_sqlite(column), sql=self.sql - ), + ( + "select {column} from ({sql}) " + "where {column} is not null " + "and {column} != '' " + "and json_array_length({column}) > 0 " + "limit 100" + ).format(column=escape_sqlite(column), sql=self.sql), self.params, truncate=False, custom_time_limit=self.ds.setting( diff --git a/tests/test_facets.py b/tests/test_facets.py index 1e19dc3a..31518682 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,3 +1,5 @@ +from datasette.app import Datasette +from datasette.database import Database from datasette.facets import ColumnFacet, ArrayFacet, DateFacet from datasette.utils.asgi import Request from datasette.utils import detect_json1 @@ -325,3 +327,23 @@ async def test_date_facet_results(app_client): "truncated": False, } } == buckets + + +@pytest.mark.asyncio +async def test_json_array_with_blanks_and_nulls(): + ds = Datasette([], memory=True) + db = ds.add_database(Database(ds, memory_name="test_json_array")) + await db.execute_write("create table foo(json_column text)", block=True) + for value in ('["a", "b", "c"]', '["a", "b"]', "", None): + await db.execute_write( + "insert into foo (json_column) values (?)", [value], block=True + ) + response = await ds.client.get("/test_json_array/foo.json") + data = response.json() + assert data["suggested_facets"] == [ + { + "name": "json_column", + "type": "array", + "toggle_url": "http://localhost/test_json_array/foo.json?_facet_array=json_column", + } + ] From 7c87532acc4e9d92caa1c4ee29a3446200928018 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Feb 2021 20:02:18 -0800 Subject: [PATCH 0067/1364] New .add_memory_database() method, closes #1247 --- datasette/app.py | 3 +++ docs/internals.rst | 29 ++++++++++++++++++++--------- tests/test_internals_database.py | 4 ++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index e3272c6e..02d432df 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -390,6 +390,9 @@ class Datasette: self.databases[name] = db return db + def add_memory_database(self, memory_name): + return self.add_database(Database(self, memory_name=memory_name)) + def remove_database(self, name): self.databases.pop(name) diff --git a/docs/internals.rst b/docs/internals.rst index 713f5d7d..e3bb83fd 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -273,7 +273,25 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` This will add a mutable database and serve it at ``/my-new-database``. -To create a shared in-memory database named ``statistics``, use the following: +``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: + +.. code-block:: python + + db = datasette.add_database(Database(datasette, memory_name="statistics")) + await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) + +.. _datasette_add_memory_database: + +.add_memory_database(name) +-------------------------- + +Adds a shared in-memory database with the specified name: + +.. code-block:: python + + datasette.add_memory_database("statistics") + +This is a shortcut for the following: .. code-block:: python @@ -284,14 +302,7 @@ To create a shared in-memory database named ``statistics``, use the following: memory_name="statistics" )) -This database will be served at ``/statistics``. - -``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: - -.. code-block:: python - - db = datasette.add_database(Database(datasette, memory_name="statistics")) - await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) +Using either of these pattern will result in the in-memory database being served at ``/statistics``. .. _datasette_remove_database: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 086f1a48..b60aaa8e 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -479,9 +479,9 @@ async def test_attached_databases(app_client_two_attached_databases_crossdb_enab async def test_database_memory_name(app_client): ds = app_client.ds foo1 = ds.add_database(Database(ds, memory_name="foo")) - foo2 = ds.add_database(Database(ds, memory_name="foo")) + foo2 = ds.add_memory_database("foo") bar1 = ds.add_database(Database(ds, memory_name="bar")) - bar2 = ds.add_database(Database(ds, memory_name="bar")) + bar2 = ds.add_memory_database("bar") for db in (foo1, foo2, bar1, bar2): table_names = await db.table_names() assert table_names == [] From 4f9a2f1f47dcf7e8561d68a8a07f5009a13cfdb3 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 3 Mar 2021 22:46:10 -0700 Subject: [PATCH 0068/1364] Fix small typo (#1243) Thanks, @UtahDave --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 4e04ea1d..0f892f83 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -58,7 +58,7 @@ Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Data $ python3 -c 'import secrets; print(secrets.token_hex(32))' -This configuration will run Datasette against all database files contained in the ``/home/ubunt/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. +This configuration will run Datasette against all database files contained in the ``/home/ubuntu/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. You can start the Datasette process running using the following:: From d0fd833b8cdd97e1b91d0f97a69b494895d82bee Mon Sep 17 00:00:00 2001 From: Bob Whitelock Date: Sun, 7 Mar 2021 07:41:17 +0000 Subject: [PATCH 0069/1364] Add compile option to Dockerfile to fix failing test (fixes #696) (#1223) This test was failing when run inside the Docker container: `test_searchable[/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw-expected_rows3]`, with this error: ``` def test_searchable(app_client, path, expected_rows): response = app_client.get(path) > assert expected_rows == response.json["rows"] E AssertionError: assert [[1, 'barry c...sel', 'puma']] == [] E Left contains 2 more items, first extra item: [1, 'barry cat', 'terry dog', 'panther'] E Full diff: E + [] E - [[1, 'barry cat', 'terry dog', 'panther'], E - [2, 'terry dog', 'sara weasel', 'puma']] ``` The issue was that the version of sqlite3 built inside the Docker container was built with FTS3 and FTS4 enabled, but without the `SQLITE_ENABLE_FTS3_PARENTHESIS` compile option passed, which adds support for using `AND` and `NOT` within `match` expressions (see https://sqlite.org/fts3.html#compiling_and_enabling_fts3_and_fts4 and https://www.sqlite.org/compile.html). Without this, the `AND` used in the search in this test was being interpreted as a literal string, and so no matches were found. Adding this compile option fixes this. Thanks, @bobwhitelock --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aba701ab..f4b14146 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt update \ RUN wget "https://www.sqlite.org/2020/sqlite-autoconf-3310100.tar.gz" && tar xzf sqlite-autoconf-3310100.tar.gz \ - && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ + && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ && make && make install RUN wget "http://www.gaia-gis.it/gaia-sins/freexl-sources/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \ From a1bcd2fbe5e47bb431045f65eeceb5eb3a6718d5 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Pressac Date: Wed, 10 Mar 2021 19:26:39 +0100 Subject: [PATCH 0070/1364] Minor typo in IP adress (#1256) 127.0.01 replaced by 127.0.0.1 --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 0f892f83..48261b59 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -67,7 +67,7 @@ You can start the Datasette process running using the following:: You can confirm that Datasette is running on port 8000 like so:: - curl 127.0.01:8000/-/versions.json + curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``. From 8e18c7943181f228ce5ebcea48deb59ce50bee1f Mon Sep 17 00:00:00 2001 From: Konstantin Baikov <4488943+kbaikov@users.noreply.github.com> Date: Thu, 11 Mar 2021 17:15:49 +0100 Subject: [PATCH 0071/1364] Use context manager instead of plain open (#1211) Context manager with open closes the files after usage. When the object is already a pathlib.Path i used read_text write_text functions In some cases pathlib.Path.open were used in context manager, it is basically the same as builtin open. Thanks, Konstantin Baikov! --- datasette/app.py | 13 ++++++------- datasette/cli.py | 13 +++++++------ datasette/publish/cloudrun.py | 6 ++++-- datasette/publish/heroku.py | 17 ++++++++++------- datasette/utils/__init__.py | 6 ++++-- setup.py | 3 ++- tests/conftest.py | 6 ++---- tests/fixtures.py | 5 +++-- tests/test_cli.py | 3 ++- tests/test_cli_serve_get.py | 3 ++- tests/test_docs.py | 8 ++++---- tests/test_package.py | 6 ++++-- tests/test_plugins.py | 3 ++- tests/test_publish_cloudrun.py | 32 ++++++++++++++++++++------------ tests/test_publish_heroku.py | 12 ++++++++---- tests/test_utils.py | 18 ++++++++++++------ update-docs-help.py | 2 +- 17 files changed, 93 insertions(+), 63 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 02d432df..f43ec205 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -212,7 +212,7 @@ class Datasette: and (config_dir / "inspect-data.json").exists() and not inspect_data ): - inspect_data = json.load((config_dir / "inspect-data.json").open()) + inspect_data = json.loads((config_dir / "inspect-data.json").read_text()) if immutables is None: immutable_filenames = [i["file"] for i in inspect_data.values()] immutables = [ @@ -269,7 +269,7 @@ class Datasette: if config_dir and (config_dir / "config.json").exists(): raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: - config = json.load((config_dir / "settings.json").open()) + config = json.loads((config_dir / "settings.json").read_text()) self._settings = dict(DEFAULT_SETTINGS, **(config or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note @@ -450,11 +450,10 @@ class Datasette: def app_css_hash(self): if not hasattr(self, "_app_css_hash"): - self._app_css_hash = hashlib.sha1( - open(os.path.join(str(app_root), "datasette/static/app.css")) - .read() - .encode("utf8") - ).hexdigest()[:6] + with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: + self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ + :6 + ] return self._app_css_hash async def get_canned_queries(self, database_name, actor): diff --git a/datasette/cli.py b/datasette/cli.py index 96a41740..2fa039a0 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -125,13 +125,13 @@ def cli(): @sqlite_extensions def inspect(files, inspect_file, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) - if inspect_file == "-": - out = sys.stdout - else: - out = open(inspect_file, "w") loop = asyncio.get_event_loop() inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions)) - out.write(json.dumps(inspect_data, indent=2)) + if inspect_file == "-": + sys.stdout.write(json.dumps(inspect_data, indent=2)) + else: + with open(inspect_file, "w") as fp: + fp.write(json.dumps(inspect_data, indent=2)) async def inspect_(files, sqlite_extensions): @@ -475,7 +475,8 @@ def serve( inspect_data = None if inspect_file: - inspect_data = json.load(open(inspect_file)) + with open(inspect_file) as fp: + inspect_data = json.load(fp) metadata_data = None if metadata: diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 7f9e89e2..bad223a1 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -141,9 +141,11 @@ def publish_subcommand(publish): if show_files: if os.path.exists("metadata.json"): print("=== metadata.json ===\n") - print(open("metadata.json").read()) + with open("metadata.json") as fp: + print(fp.read()) print("\n==== Dockerfile ====\n") - print(open("Dockerfile").read()) + with open("Dockerfile") as fp: + print(fp.read()) print("\n====================\n") image_id = f"gcr.io/{project}/{name}" diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index c0c70e12..19fe3fbe 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -171,9 +171,11 @@ def temporary_heroku_directory( os.chdir(tmp.name) if metadata_content: - open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) + with open("metadata.json", "w") as fp: + fp.write(json.dumps(metadata_content, indent=2)) - open("runtime.txt", "w").write("python-3.8.7") + with open("runtime.txt", "w") as fp: + fp.write("python-3.8.7") if branch: install = [ @@ -182,11 +184,11 @@ def temporary_heroku_directory( else: install = ["datasette"] + list(install) - open("requirements.txt", "w").write("\n".join(install)) + with open("requirements.txt", "w") as fp: + fp.write("\n".join(install)) os.mkdir("bin") - open("bin/post_compile", "w").write( - "datasette inspect --inspect-file inspect-data.json" - ) + with open("bin/post_compile", "w") as fp: + fp.write("datasette inspect --inspect-file inspect-data.json") extras = [] if template_dir: @@ -218,7 +220,8 @@ def temporary_heroku_directory( procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) ) - open("Procfile", "w").write(procfile_cmd) + with open("Procfile", "w") as fp: + fp.write(procfile_cmd) for path, filename in zip(file_paths, file_names): link_or_copy(path, os.path.join(tmp.name, filename)) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 47ca0551..1fedb69c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -428,8 +428,10 @@ def temporary_docker_directory( ) os.chdir(datasette_dir) if metadata_content: - open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) - open("Dockerfile", "w").write(dockerfile) + with open("metadata.json", "w") as fp: + fp.write(json.dumps(metadata_content, indent=2)) + with open("Dockerfile", "w") as fp: + fp.write(dockerfile) for path, filename in zip(file_paths, file_names): link_or_copy(path, os.path.join(datasette_dir, filename)) if template_dir: diff --git a/setup.py b/setup.py index 15ee63fe..3540e30a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ def get_version(): os.path.dirname(os.path.abspath(__file__)), "datasette", "version.py" ) g = {} - exec(open(path).read(), g) + with open(path) as fp: + exec(fp.read(), g) return g["__version__"] diff --git a/tests/conftest.py b/tests/conftest.py index b00ea006..ad3eb9f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,10 +75,8 @@ def check_permission_actions_are_documented(): from datasette.plugins import pm content = ( - (pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst") - .open() - .read() - ) + pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" + ).read_text() permissions_re = re.compile(r"\.\. _permissions_([^\s:]+):") documented_permission_actions = set(permissions_re.findall(content)).union( UNDOCUMENTED_PERMISSIONS diff --git a/tests/fixtures.py b/tests/fixtures.py index 30113ff2..2fd8e9cb 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -789,7 +789,8 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): conn.executescript(GENERATED_COLUMNS_SQL) print(f"Test tables written to {db_filename}") if metadata: - open(metadata, "w").write(json.dumps(METADATA, indent=4)) + with open(metadata, "w") as fp: + fp.write(json.dumps(METADATA, indent=4)) print(f"- metadata written to {metadata}") if plugins_path: path = pathlib.Path(plugins_path) @@ -798,7 +799,7 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): test_plugins = pathlib.Path(__file__).parent / "plugins" for filepath in test_plugins.glob("*.py"): newpath = path / filepath.name - newpath.write_text(filepath.open().read()) + newpath.write_text(filepath.read_text()) print(f" Wrote plugin: {newpath}") if extra_db_filename: if pathlib.Path(extra_db_filename).exists(): diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ddd32f6..e094ccb6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -49,7 +49,8 @@ def test_inspect_cli_writes_to_file(app_client): cli, ["inspect", "fixtures.db", "--inspect-file", "foo.json"] ) assert 0 == result.exit_code, result.output - data = json.load(open("foo.json")) + with open("foo.json") as fp: + data = json.load(fp) assert ["fixtures"] == list(data.keys()) diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index aaa692e5..90fbfe3b 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -14,7 +14,8 @@ def test_serve_with_get(tmp_path_factory): @hookimpl def startup(datasette): - open("{}", "w").write("hello") + with open("{}", "w") as fp: + fp.write("hello") """.format( str(plugins_dir / "hello.txt") ), diff --git a/tests/test_docs.py b/tests/test_docs.py index 44b0810a..efd267b9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -19,13 +19,13 @@ def get_headings(content, underline="-"): def get_labels(filename): - content = (docs_path / filename).open().read() + content = (docs_path / filename).read_text() return set(label_re.findall(content)) @pytest.fixture(scope="session") def settings_headings(): - return get_headings((docs_path / "settings.rst").open().read(), "~") + return get_headings((docs_path / "settings.rst").read_text(), "~") @pytest.mark.parametrize("setting", app.SETTINGS) @@ -43,7 +43,7 @@ def test_settings_are_documented(settings_headings, setting): ), ) def test_help_includes(name, filename): - expected = open(str(docs_path / filename)).read() + expected = (docs_path / filename).read_text() runner = CliRunner() result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) actual = f"$ datasette {name} --help\n\n{result.output}" @@ -55,7 +55,7 @@ def test_help_includes(name, filename): @pytest.fixture(scope="session") def plugin_hooks_content(): - return (docs_path / "plugin_hooks.rst").open().read() + return (docs_path / "plugin_hooks.rst").read_text() @pytest.mark.parametrize( diff --git a/tests/test_package.py b/tests/test_package.py index 3248b3a4..bb939643 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -32,7 +32,8 @@ def test_package(mock_call, mock_which): capture = CaptureDockerfile() mock_call.side_effect = capture with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["package", "test.db", "--secret", "sekrit"]) assert 0 == result.exit_code mock_call.assert_has_calls([mock.call(["docker", "build", "."])]) @@ -47,7 +48,8 @@ def test_package_with_port(mock_call, mock_which): mock_call.side_effect = capture runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["package", "test.db", "-p", "8080", "--secret", "sekrit"] ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 715c7c17..ee6f1efa 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -225,7 +225,8 @@ def test_plugin_config_env_from_list(app_client): def test_plugin_config_file(app_client): - open(TEMP_PLUGIN_SECRET_FILE, "w").write("FROM_FILE") + with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: + fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == app_client.ds.plugin_config("file-plugin") # Ensure secrets aren't visible in /-/metadata.json metadata = app_client.get("/-/metadata.json") diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 2ef90705..7881ebae 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -11,7 +11,8 @@ def test_publish_cloudrun_requires_gcloud(mock_which): mock_which.return_value = False runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) assert result.exit_code == 1 assert "Publishing to Google Cloud requires gcloud" in result.output @@ -40,7 +41,8 @@ def test_publish_cloudrun_prompts_for_service( mock_which.return_value = True runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "cloudrun", "test.db"], input="input-service" ) @@ -81,7 +83,8 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): mock_which.return_value = True runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] ) @@ -120,7 +123,8 @@ def test_publish_cloudrun_memory( mock_which.return_value = True runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "cloudrun", "test.db", "--service", "test", "--memory", memory], @@ -152,17 +156,19 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") - open("metadata.yml", "w").write( - textwrap.dedent( - """ + with open("test.db", "w") as fp: + fp.write("data") + with open("metadata.yml", "w") as fp: + fp.write( + textwrap.dedent( + """ title: Hello from metadata YAML plugins: datasette-auth-github: foo: bar """ - ).strip() - ) + ).strip() + ) result = runner.invoke( cli.cli, [ @@ -228,7 +234,8 @@ def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, [ @@ -295,7 +302,8 @@ def test_publish_cloudrun_extra_options( runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, [ diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index c7a38031..c011ab43 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -8,7 +8,8 @@ def test_publish_heroku_requires_heroku(mock_which): mock_which.return_value = False runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) assert result.exit_code == 1 assert "Publishing to Heroku requires heroku" in result.output @@ -22,7 +23,8 @@ def test_publish_heroku_installs_plugin(mock_call, mock_check_output, mock_which mock_check_output.side_effect = lambda s: {"['heroku', 'plugins']": b""}[repr(s)] runner = CliRunner() with runner.isolated_filesystem(): - open("t.db", "w").write("data") + with open("t.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "t.db"], input="y\n") assert 0 != result.exit_code mock_check_output.assert_has_calls( @@ -54,7 +56,8 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): }[repr(s)] runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "heroku", "test.db", "--tar", "gtar"] ) @@ -88,7 +91,8 @@ def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which) }[repr(s)] runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, [ diff --git a/tests/test_utils.py b/tests/test_utils.py index 56306339..ecef6f7a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -232,7 +232,8 @@ def test_to_css_class(s, expected): def test_temporary_docker_directory_uses_hard_link(): with tempfile.TemporaryDirectory() as td: os.chdir(td) - open("hello", "w").write("world") + with open("hello", "w") as fp: + fp.write("world") # Default usage of this should use symlink with utils.temporary_docker_directory( files=["hello"], @@ -249,7 +250,8 @@ def test_temporary_docker_directory_uses_hard_link(): secret="secret", ) as temp_docker: hello = os.path.join(temp_docker, "hello") - assert "world" == open(hello).read() + with open(hello) as fp: + assert "world" == fp.read() # It should be a hard link assert 2 == os.stat(hello).st_nlink @@ -260,7 +262,8 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): mock_link.side_effect = OSError with tempfile.TemporaryDirectory() as td: os.chdir(td) - open("hello", "w").write("world") + with open("hello", "w") as fp: + fp.write("world") # Default usage of this should use symlink with utils.temporary_docker_directory( files=["hello"], @@ -277,7 +280,8 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): secret=None, ) as temp_docker: hello = os.path.join(temp_docker, "hello") - assert "world" == open(hello).read() + with open(hello) as fp: + assert "world" == fp.read() # It should be a copy, not a hard link assert 1 == os.stat(hello).st_nlink @@ -285,7 +289,8 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): def test_temporary_docker_directory_quotes_args(): with tempfile.TemporaryDirectory() as td: os.chdir(td) - open("hello", "w").write("world") + with open("hello", "w") as fp: + fp.write("world") with utils.temporary_docker_directory( files=["hello"], name="t", @@ -301,7 +306,8 @@ def test_temporary_docker_directory_quotes_args(): secret="secret", ) as temp_docker: df = os.path.join(temp_docker, "Dockerfile") - df_contents = open(df).read() + with open(df) as fp: + df_contents = fp.read() assert "'$PWD'" in df_contents assert "'--$HOME'" in df_contents assert "ENV DATASETTE_SECRET 'secret'" in df_contents diff --git a/update-docs-help.py b/update-docs-help.py index 3a192575..292d1dcd 100644 --- a/update-docs-help.py +++ b/update-docs-help.py @@ -18,7 +18,7 @@ def update_help_includes(): result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) actual = f"$ datasette {name} --help\n\n{result.output}" actual = actual.replace("Usage: cli ", "Usage: datasette ") - open(docs_path / filename, "w").write(actual) + (docs_path / filename).write_text(actual) if __name__ == "__main__": From c4f1ec7f33fd7d5b93f0f895dafb5351cc3bfc5b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Mar 2021 14:32:23 -0700 Subject: [PATCH 0072/1364] Documentation for Response.asgi_send(), closes #1266 --- docs/internals.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index e3bb83fd..18032406 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -138,6 +138,28 @@ Each of these responses will use the correct corresponding content-type - ``text Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above. +.. _internals_response_asgi_send: + +Returning a response with .asgi_send(send) +------------------------------------------ + + +In most cases you will return ``Response`` objects from your own view functions. You can also use a ``Response`` instance to respond at a lower level via ASGI, for example if you are writing code that uses the :ref:`plugin_asgi_wrapper` hook. + +Create a ``Response`` object and then use ``await response.asgi_send(send)``, passing the ASGI ``send`` function. For example: + +.. code-block:: python + + async def require_authorization(scope, recieve, send): + response = Response.text( + "401 Authorization Required", + headers={ + "www-authenticate": 'Basic realm="Datasette", charset="UTF-8"' + }, + status=401, + ) + await response.asgi_send(send) + .. _internals_response_set_cookie: Setting cookies with response.set_cookie() From 6ad544df5e6bd027a8e27317041e6168aee07459 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Mar 2021 09:19:41 -0700 Subject: [PATCH 0073/1364] Fixed master -> main in a bunch of places, mainly docs --- datasette/cli.py | 2 +- datasette/publish/common.py | 2 +- datasette/templates/patterns.html | 16 ++++++++-------- docs/contributing.rst | 2 +- docs/custom_templates.rst | 2 +- docs/datasette-package-help.txt | 2 +- docs/datasette-publish-cloudrun-help.txt | 2 +- docs/datasette-publish-heroku-help.txt | 2 +- docs/plugin_hooks.rst | 4 ++-- docs/publish.rst | 4 ++-- docs/spatialite.rst | 2 +- tests/fixtures.py | 4 ++-- tests/test_html.py | 9 ++++----- 13 files changed, 26 insertions(+), 27 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 2fa039a0..42b5c115 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -191,7 +191,7 @@ def plugins(all, plugins_dir): help="Path to JSON/YAML file containing metadata to publish", ) @click.option("--extra-options", help="Extra options to pass to datasette serve") -@click.option("--branch", help="Install datasette from a GitHub branch e.g. master") +@click.option("--branch", help="Install datasette from a GitHub branch e.g. main") @click.option( "--template-dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), diff --git a/datasette/publish/common.py b/datasette/publish/common.py index b6570290..29665eb3 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -19,7 +19,7 @@ def add_common_publish_arguments_and_options(subcommand): "--extra-options", help="Extra options to pass to datasette serve" ), click.option( - "--branch", help="Install datasette from a GitHub branch e.g. master" + "--branch", help="Install datasette from a GitHub branch e.g. main" ), click.option( "--template-dir", diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 984c1bf6..3f9b5a16 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -70,10 +70,10 @@

Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: @@ -118,10 +118,10 @@

Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: @@ -177,10 +177,10 @@

Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: @@ -478,10 +478,10 @@