From 1000d50220faef339ee688aae9150e83a205be3a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 May 2026 23:05:37 -0700 Subject: [PATCH] datasette.fixtures module, closes #2733 https://gist.github.com/simonw/613be79094d491dd08f45e05f4f70691 --- datasette/fixtures.py | 415 ++++++++++++++++++++++++++++++++++++ datasette/utils/__init__.py | 12 +- docs/changelog.rst | 1 + docs/testing_plugins.rst | 42 ++++ tests/conftest.py | 9 +- tests/fixtures.py | 401 ++-------------------------------- tests/test_docs.py | 18 +- tests/test_fixtures.py | 49 +++++ tests/test_plugins.py | 2 +- tests/test_table_api.py | 3 +- 10 files changed, 545 insertions(+), 407 deletions(-) create mode 100644 datasette/fixtures.py create mode 100644 tests/test_fixtures.py diff --git a/datasette/fixtures.py b/datasette/fixtures.py new file mode 100644 index 00000000..7c85e16a --- /dev/null +++ b/datasette/fixtures.py @@ -0,0 +1,415 @@ +from datasette.utils.sqlite import sqlite3 +from datasette.utils import documented +import itertools +import random +import string + +__all__ = [ + "EXTRA_DATABASE_SQL", + "TABLES", + "TABLE_PARAMETERIZED_SQL", + "generate_compound_rows", + "generate_sortable_rows", + "populate_extra_database", + "populate_fixture_database", + "write_extra_database", + "write_fixture_database", +] + + +def generate_compound_rows(num): + """Generate rows for the compound_three_primary_keys fixture table.""" + for a, b, c in itertools.islice( + itertools.product(string.ascii_lowercase, repeat=3), num + ): + yield a, b, c, f"{a}-{b}-{c}" + + +def generate_sortable_rows(num): + """Generate rows for the sortable fixture table.""" + rand = random.Random(42) + for a, b in itertools.islice( + itertools.product(string.ascii_lowercase, repeat=2), num + ): + yield { + "pk1": a, + "pk2": b, + "content": f"{a}-{b}", + "sortable": rand.randint(-100, 100), + "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), + "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), + "text": rand.choice(["$null", "$blah"]), + } + + +TABLES = ( + """ +CREATE TABLE simple_primary_key ( + id integer primary key, + content text +); + +CREATE TABLE primary_key_multiple_columns ( + id varchar(30) primary key, + content text, + content2 text +); + +CREATE TABLE primary_key_multiple_columns_explicit_label ( + id varchar(30) primary key, + content text, + content2 text +); + +CREATE TABLE compound_primary_key ( + pk1 varchar(30), + pk2 varchar(30), + content text, + PRIMARY KEY (pk1, pk2) +); + +INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); +INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); +INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); + +CREATE TABLE compound_three_primary_keys ( + pk1 varchar(30), + pk2 varchar(30), + pk3 varchar(30), + content text, + PRIMARY KEY (pk1, pk2, pk3) +); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); + +CREATE TABLE foreign_key_references ( + pk varchar(30) primary key, + foreign_key_with_label integer, + foreign_key_with_blank_label integer, + foreign_key_with_no_label varchar(30), + foreign_key_compound_pk1 varchar(30), + foreign_key_compound_pk2 varchar(30), + FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) + FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) +); + +CREATE TABLE sortable ( + pk1 varchar(30), + pk2 varchar(30), + content text, + sortable integer, + sortable_with_nulls real, + sortable_with_nulls_2 real, + text text, + PRIMARY KEY (pk1, pk2) +); + +CREATE TABLE no_primary_key ( + content text, + a text, + b text, + c text +); + +CREATE TABLE [123_starts_with_digits] ( + content text +); + +CREATE VIEW paginated_view AS + SELECT + content, + '- ' || content || ' -' AS content_extra + FROM no_primary_key; + +CREATE TABLE "Table With Space In Name" ( + pk varchar(30) primary key, + content text +); + +CREATE TABLE "table/with/slashes.csv" ( + pk varchar(30) primary key, + content text +); + +CREATE TABLE "complex_foreign_keys" ( + pk varchar(30) primary key, + f1 integer, + f2 integer, + f3 integer, + FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), + FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), + FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) +); + +CREATE TABLE "custom_foreign_key_label" ( + pk varchar(30) primary key, + foreign_key_with_custom_label text, + FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) +); + +CREATE TABLE tags ( + tag TEXT PRIMARY KEY +); + +CREATE TABLE searchable ( + pk integer primary key, + text1 text, + text2 text, + [name with . and spaces] text +); + +CREATE TABLE searchable_tags ( + searchable_id integer, + tag text, + PRIMARY KEY (searchable_id, tag), + FOREIGN KEY (searchable_id) REFERENCES searchable(pk), + FOREIGN KEY (tag) REFERENCES tags(tag) +); + +INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); +INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); + +INSERT INTO tags VALUES ("canine"); +INSERT INTO tags VALUES ("feline"); + +INSERT INTO searchable_tags (searchable_id, tag) VALUES + (1, "feline"), + (2, "canine") +; + +CREATE VIRTUAL TABLE "searchable_fts" + USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); +INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); + +CREATE TABLE [select] ( + [group] text, + [having] text, + [and] text, + [json] text +); +INSERT INTO [select] VALUES ('group', 'having', 'and', + '{"href": "http://example.com/", "label":"Example"}' +); + +CREATE TABLE infinity ( + value REAL +); +INSERT INTO infinity VALUES + (1e999), + (-1e999), + (1.5) +; + +CREATE TABLE facet_cities ( + id integer primary key, + name text +); +INSERT INTO facet_cities (id, name) VALUES + (1, 'San Francisco'), + (2, 'Los Angeles'), + (3, 'Detroit'), + (4, 'Memnonia') +; + +CREATE TABLE facetable ( + pk integer primary key, + created text, + planet_int integer, + on_earth integer, + state text, + _city_id integer, + _neighborhood text, + tags text, + complex_array text, + distinct_some_null, + n text, + FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) +); +INSERT INTO facetable + (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) +VALUES + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) +; + +CREATE TABLE binary_data ( + data BLOB +); + +-- Many 2 Many demo: roadside attractions! + +CREATE TABLE roadside_attractions ( + pk integer primary key, + name text, + address text, + url text, + latitude real, + longitude real +); +INSERT INTO roadside_attractions VALUES ( + 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", + 37.0167, -122.0024 +); +INSERT INTO roadside_attractions VALUES ( + 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", + 37.3184, -121.9511 +); +INSERT INTO roadside_attractions VALUES ( + 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, + 37.5793, -122.3442 +); +INSERT INTO roadside_attractions VALUES ( + 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", + 37.0414, -122.0725 +); + +CREATE TABLE attraction_characteristic ( + pk integer primary key, + name text +); +INSERT INTO attraction_characteristic VALUES ( + 1, "Museum" +); +INSERT INTO attraction_characteristic VALUES ( + 2, "Paranormal" +); + +CREATE TABLE roadside_attraction_characteristics ( + attraction_id INTEGER REFERENCES roadside_attractions(pk), + characteristic_id INTEGER REFERENCES attraction_characteristic(pk) +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 1, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 2, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 4, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 3, 1 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 4, 1 +); + +INSERT INTO simple_primary_key VALUES (1, 'hello'); +INSERT INTO simple_primary_key VALUES (2, 'world'); +INSERT INTO simple_primary_key VALUES (3, ''); +INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); +INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); + +INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); +INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); + +INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); +INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); + +INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); +INSERT INTO custom_foreign_key_label VALUES (1, 1); + +INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); + +CREATE VIEW simple_view AS + SELECT content, upper(content) AS upper_content FROM simple_primary_key; + +CREATE VIEW searchable_view AS + SELECT * from searchable; + +CREATE VIEW searchable_view_configured_by_metadata AS + SELECT * from searchable; + +""" + + "\n".join( + [ + 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( + i=i + 1 + ) + for i in range(201) + ] + ) + + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + + "\n".join( + [ + 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + a=a, b=b, c=c, content=content + ) + for a, b, c, content in generate_compound_rows(1001) + ] + ) + + "\n".join(["""INSERT INTO sortable VALUES ( + "{pk1}", "{pk2}", "{content}", {sortable}, + {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); + """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) +) + +TABLE_PARAMETERIZED_SQL = [ + ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (null);", []), +] + +EXTRA_DATABASE_SQL = """ +CREATE TABLE searchable ( + pk integer primary key, + text1 text, + text2 text +); + +CREATE VIEW searchable_view AS SELECT * FROM searchable; + +INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); +INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); + +CREATE VIRTUAL TABLE "searchable_fts" + USING FTS3 (text1, text2, content="searchable"); +INSERT INTO "searchable_fts" (rowid, text1, text2) + SELECT rowid, text1, text2 FROM searchable; +""" + + +@documented(label="datasette_fixtures_populate_fixture_database") +def populate_fixture_database(conn): + """Populate a SQLite connection with Datasette's test fixture tables.""" + conn.executescript(TABLES) + for sql, params in TABLE_PARAMETERIZED_SQL: + with conn: + conn.execute(sql, params) + + +def populate_extra_database(conn): + """Populate a SQLite connection with the extra database used in tests.""" + conn.executescript(EXTRA_DATABASE_SQL) + + +def write_fixture_database(db_filename): + """Write Datasette's test fixture tables to a SQLite database file.""" + conn = sqlite3.connect(db_filename) + try: + populate_fixture_database(conn) + finally: + conn.close() + + +def write_extra_database(db_filename): + """Write the extra test database tables to a SQLite database file.""" + conn = sqlite3.connect(db_filename) + try: + populate_extra_database(conn) + finally: + conn.close() diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1fea992e..9d189459 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -155,9 +155,15 @@ Column = namedtuple( functions_marked_as_documented = [] -def documented(fn): - functions_marked_as_documented.append(fn) - return fn +def documented(fn=None, *, label=None): + def decorate(fn): + fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) + functions_marked_as_documented.append(fn) + return fn + + if fn is None: + return decorate + return decorate(fn) @documented diff --git a/docs/changelog.rst b/docs/changelog.rst index f2ba23ec..51188461 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Unreleased - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) +- New documented :ref:`datasette.fixtures.populate_fixture_database(conn) ` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites. .. _v1_0_a29: diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index b82a6e0c..15891963 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,48 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_plugins_datasette_fixtures_database: + +Using Datasette's fixtures database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's own test suite uses a SQLite database with tables that exercise features such as compound primary keys, foreign keys, sortable columns, facets, full-text search and binary data. + +You can use those same tables in your plugin tests using the ``populate_fixture_database(conn)`` helper in ``datasette.fixtures``: + +Be aware that future Datasette releases may change details of these tables, so try not to rely on their exact structure in your own tests. + +.. _datasette_fixtures_populate_fixture_database: + +``populate_fixture_database(conn)`` + Populates an existing SQLite connection with the fixture tables. + +For an in-memory test database: + +.. code-block:: python + + from datasette.app import Datasette + from datasette.fixtures import populate_fixture_database + import pytest + import pytest_asyncio + + + @pytest_asyncio.fixture + async def datasette(): + datasette = Datasette() + db = datasette.add_memory_database("fixtures") + await db.execute_write_fn(populate_fixture_database) + await datasette.invoke_startup() + return datasette + + + @pytest.mark.asyncio + async def test_facetable(datasette): + response = await datasette.client.get( + "/fixtures/facetable.json?_shape=array" + ) + assert response.status_code == 200 + .. _testing_plugins_autoclose: Automatic cleanup of Datasette instances diff --git a/tests/conftest.py b/tests/conftest.py index 4ea89458..b9b3c35e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,7 +72,7 @@ async def ds_client(): "num_sql_threads": 1, }, ) - from .fixtures import TABLES, TABLE_PARAMETERIZED_SQL + from datasette.fixtures import populate_fixture_database # Use a unique memory_name to avoid collisions between different # Datasette instances in the same process, but use "fixtures" for routing @@ -82,10 +82,7 @@ async def ds_client(): def prepare(conn): if not conn.execute("select count(*) from sqlite_master").fetchone()[0]: - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) + populate_fixture_database(conn) await db.execute_write_fn(prepare) await ds.invoke_startup() @@ -264,8 +261,6 @@ from .fixtures import ( # noqa: E402, F401 app_client_with_cors, app_client_with_dot, app_client_with_trace, - generate_compound_rows, - generate_sortable_rows, make_app_client, TEMP_PLUGIN_SECRET_FILE, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index f61ec0c7..71884294 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,15 +1,16 @@ from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3 +from datasette.fixtures import ( + EXTRA_DATABASE_SQL, + write_extra_database, + write_fixture_database, +) from datasette.utils.testing import TestClient import click import contextlib -import itertools import json import os import pathlib import pytest -import random -import string import tempfile import textwrap @@ -128,19 +129,18 @@ def make_app_client( else: files = [filepath] immutables = [] - conn = sqlite3.connect(filepath) - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - # Close the connection to avoid "too many open files" errors - conn.close() + write_fixture_database(filepath) if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) - c2 = sqlite3.connect(extra_filepath) - c2.executescript(extra_sql) - c2.close() + if extra_sql == EXTRA_DATABASE_SQL: + write_extra_database(extra_filepath) + else: + from datasette.utils.sqlite import sqlite3 + + c2 = sqlite3.connect(extra_filepath) + c2.executescript(extra_sql) + c2.close() # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) @@ -279,29 +279,6 @@ def app_client_immutable_and_inspect_file(): yield client -def generate_compound_rows(num): - for a, b, c in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=3), num - ): - yield a, b, c, f"{a}-{b}-{c}" - - -def generate_sortable_rows(num): - rand = random.Random(42) - for a, b in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=2), num - ): - yield { - "pk1": a, - "pk2": b, - "content": f"{a}-{b}", - "sortable": rand.randint(-100, 100), - "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), - "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), - "text": rand.choice(["$null", "$blah"]), - } - - CONFIG = { "plugins": { "name-of-plugin": {"depth": "root"}, @@ -399,345 +376,6 @@ METADATA = { }, } -TABLES = ( - """ -CREATE TABLE simple_primary_key ( - id integer primary key, - content text -); - -CREATE TABLE primary_key_multiple_columns ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE primary_key_multiple_columns_explicit_label ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE compound_primary_key ( - pk1 varchar(30), - pk2 varchar(30), - content text, - PRIMARY KEY (pk1, pk2) -); - -INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); -INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); -INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); - -CREATE TABLE compound_three_primary_keys ( - pk1 varchar(30), - pk2 varchar(30), - pk3 varchar(30), - content text, - PRIMARY KEY (pk1, pk2, pk3) -); -CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); - -CREATE TABLE foreign_key_references ( - pk varchar(30) primary key, - foreign_key_with_label integer, - foreign_key_with_blank_label integer, - foreign_key_with_no_label varchar(30), - foreign_key_compound_pk1 varchar(30), - foreign_key_compound_pk2 varchar(30), - FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) - FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) -); - -CREATE TABLE sortable ( - pk1 varchar(30), - pk2 varchar(30), - content text, - sortable integer, - sortable_with_nulls real, - sortable_with_nulls_2 real, - text text, - PRIMARY KEY (pk1, pk2) -); - -CREATE TABLE no_primary_key ( - content text, - a text, - b text, - c text -); - -CREATE TABLE [123_starts_with_digits] ( - content text -); - -CREATE VIEW paginated_view AS - SELECT - content, - '- ' || content || ' -' AS content_extra - FROM no_primary_key; - -CREATE TABLE "Table With Space In Name" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "table/with/slashes.csv" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "complex_foreign_keys" ( - pk varchar(30) primary key, - f1 integer, - f2 integer, - f3 integer, - FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) -); - -CREATE TABLE "custom_foreign_key_label" ( - pk varchar(30) primary key, - foreign_key_with_custom_label text, - FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) -); - -CREATE TABLE tags ( - tag TEXT PRIMARY KEY -); - -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text, - [name with . and spaces] text -); - -CREATE TABLE searchable_tags ( - searchable_id integer, - tag text, - PRIMARY KEY (searchable_id, tag), - FOREIGN KEY (searchable_id) REFERENCES searchable(pk), - FOREIGN KEY (tag) REFERENCES tags(tag) -); - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); - -INSERT INTO tags VALUES ("canine"); -INSERT INTO tags VALUES ("feline"); - -INSERT INTO searchable_tags (searchable_id, tag) VALUES - (1, "feline"), - (2, "canine") -; - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); -INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); - -CREATE TABLE [select] ( - [group] text, - [having] text, - [and] text, - [json] text -); -INSERT INTO [select] VALUES ('group', 'having', 'and', - '{"href": "http://example.com/", "label":"Example"}' -); - -CREATE TABLE infinity ( - value REAL -); -INSERT INTO infinity VALUES - (1e999), - (-1e999), - (1.5) -; - -CREATE TABLE facet_cities ( - id integer primary key, - name text -); -INSERT INTO facet_cities (id, name) VALUES - (1, 'San Francisco'), - (2, 'Los Angeles'), - (3, 'Detroit'), - (4, 'Memnonia') -; - -CREATE TABLE facetable ( - pk integer primary key, - created text, - planet_int integer, - on_earth integer, - state text, - _city_id integer, - _neighborhood text, - tags text, - complex_array text, - distinct_some_null, - n text, - FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) -); -INSERT INTO facetable - (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) -VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) -; - -CREATE TABLE binary_data ( - data BLOB -); - --- Many 2 Many demo: roadside attractions! - -CREATE TABLE roadside_attractions ( - pk integer primary key, - name text, - address text, - url text, - latitude real, - longitude real -); -INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", - 37.0167, -122.0024 -); -INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", - 37.3184, -121.9511 -); -INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, - 37.5793, -122.3442 -); -INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", - 37.0414, -122.0725 -); - -CREATE TABLE attraction_characteristic ( - pk integer primary key, - name text -); -INSERT INTO attraction_characteristic VALUES ( - 1, "Museum" -); -INSERT INTO attraction_characteristic VALUES ( - 2, "Paranormal" -); - -CREATE TABLE roadside_attraction_characteristics ( - attraction_id INTEGER REFERENCES roadside_attractions(pk), - characteristic_id INTEGER REFERENCES attraction_characteristic(pk) -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 1, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 2, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 3, 1 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 1 -); - -INSERT INTO simple_primary_key VALUES (1, 'hello'); -INSERT INTO simple_primary_key VALUES (2, 'world'); -INSERT INTO simple_primary_key VALUES (3, ''); -INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); -INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); - -INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); -INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); - -INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); -INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); - -INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); -INSERT INTO custom_foreign_key_label VALUES (1, 1); - -INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); - -CREATE VIEW simple_view AS - SELECT content, upper(content) AS upper_content FROM simple_primary_key; - -CREATE VIEW searchable_view AS - SELECT * from searchable; - -CREATE VIEW searchable_view_configured_by_metadata AS - SELECT * from searchable; - -""" - + "\n".join( - [ - 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( - i=i + 1 - ) - for i in range(201) - ] - ) - + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' - + "\n".join( - [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) - for a, b, c, content in generate_compound_rows(1001) - ] - ) - + "\n".join(["""INSERT INTO sortable VALUES ( - "{pk1}", "{pk2}", "{content}", {sortable}, - {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); - """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) -) -TABLE_PARAMETERIZED_SQL = [ - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (null);", []), -] - -EXTRA_DATABASE_SQL = """ -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text -); - -CREATE VIEW searchable_view AS SELECT * FROM searchable; - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS3 (text1, text2, content="searchable"); -INSERT INTO "searchable_fts" (rowid, text1, text2) - SELECT rowid, text1, text2 FROM searchable; -""" - def assert_permissions_checked(datasette, actions): # actions is a list of "action" or (action, resource) tuples @@ -819,12 +457,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename ) else: pathlib.Path(db_filename).unlink() - conn = sqlite3.connect(db_filename) - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - conn.close() + write_fixture_database(db_filename) print(f"Test tables written to {db_filename}") if metadata: with open(metadata, "w") as fp: @@ -851,9 +484,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename ) else: pathlib.Path(extra_db_filename).unlink() - conn = sqlite3.connect(extra_db_filename) - conn.executescript(EXTRA_DATABASE_SQL) - conn.close() + write_extra_database(extra_db_filename) print(f"Test tables written to {extra_db_filename}") diff --git a/tests/test_docs.py b/tests/test_docs.py index b94a6f23..396ba1a2 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -3,6 +3,7 @@ Tests to ensure certain things are documented. """ from datasette import app, utils +import datasette.fixtures # noqa: F401 from datasette.app import Datasette from datasette.filters import Filters from pathlib import Path @@ -95,20 +96,17 @@ def test_table_filters_are_documented(documented_table_filters, subtests): @pytest.fixture(scope="session") -def documented_fns(): - internals_rst = (docs_path / "internals.rst").read_text() - # Any line that starts .. _internals_utils_X - lines = internals_rst.split("\n") - prefix = ".. _internals_utils_" - return { - line.split(prefix)[1].split(":")[0] for line in lines if line.startswith(prefix) - } +def documented_labels(): + labels = set() + for filename in docs_path.glob("*.rst"): + labels.update(get_labels(filename.name)) + return labels -def test_functions_marked_with_documented_are_documented(documented_fns, subtests): +def test_functions_marked_with_documented_are_documented(documented_labels, subtests): for fn in utils.functions_marked_as_documented: with subtests.test(fn=fn.__name__): - assert fn.__name__ in documented_fns + assert fn._datasette_docs_label in documented_labels def test_rst_heading_underlines_match_title_length(): diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py new file mode 100644 index 00000000..45f9854b --- /dev/null +++ b/tests/test_fixtures.py @@ -0,0 +1,49 @@ +from datasette.fixtures import ( + populate_extra_database, + populate_fixture_database, + write_extra_database, + write_fixture_database, +) +from datasette.utils.sqlite import sqlite3 + + +def count(conn, table): + return conn.execute(f"select count(*) from [{table}]").fetchone()[0] + + +def test_populate_fixture_database(): + conn = sqlite3.connect(":memory:") + try: + populate_fixture_database(conn) + assert count(conn, "facetable") == 15 + assert count(conn, "compound_three_primary_keys") == 1001 + assert count(conn, "binary_data") == 3 + finally: + conn.close() + + +def test_write_fixture_database(tmp_path): + db_path = tmp_path / "fixtures.db" + write_fixture_database(db_path) + conn = sqlite3.connect(db_path) + try: + assert count(conn, "sortable") == 201 + finally: + conn.close() + + +def test_extra_database_helpers(tmp_path): + conn = sqlite3.connect(":memory:") + try: + populate_extra_database(conn) + assert count(conn, "searchable") == 2 + finally: + conn.close() + + db_path = tmp_path / "extra.db" + write_extra_database(db_path) + conn = sqlite3.connect(db_path) + try: + assert count(conn, "searchable") == 2 + finally: + conn.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 7ebd57f3..d87f577a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,7 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import ( make_app_client, - TABLES, TEMP_PLUGIN_SECRET_FILE, PLUGINS_DIR, TestClient as _TestClient, @@ -9,6 +8,7 @@ from .fixtures import ( from click.testing import CliRunner from datasette.app import Datasette from datasette import cli, hookimpl +from datasette.fixtures import TABLES from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.permissions import PermissionSQL, Action diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 51e40ad1..ceeb646d 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1,6 +1,7 @@ from datasette.utils import detect_json1 from datasette.utils.sqlite import sqlite_version -from .fixtures import generate_compound_rows, generate_sortable_rows, make_app_client +from datasette.fixtures import generate_compound_rows, generate_sortable_rows +from .fixtures import make_app_client import json import pytest import urllib