remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-05-27 12:34:37 +02:00
Code Issues Projects Releases Packages Wiki Activity

datasette.fixtures module, closes #2733

Browse source 
https://gist.github.com/simonw/613be79094d491dd08f45e05f4f70691
This commit is contained in:
Simon Willison 2026-05-21 23:05:37 -07:00
commit 1000d50220
10 changed files with 545 additions and 407 deletions
Download patch file Download diff file Expand all files Collapse all files

415
datasette/fixtures.py Normal file
View file

@ -0,0 +1,415 @@
from datasette.utils.sqlite import sqlite3
from datasette.utils import documented
import itertools
import random
import string
__all__ = [
"EXTRA_DATABASE_SQL",
"TABLES",
"TABLE_PARAMETERIZED_SQL",
"generate_compound_rows",
"generate_sortable_rows",
"populate_extra_database",
"populate_fixture_database",
"write_extra_database",
"write_fixture_database",
]
def generate_compound_rows(num):
"""Generate rows for the compound_three_primary_keys fixture table."""
for a, b, c in itertools.islice(
itertools.product(string.ascii_lowercase, repeat=3), num
):
yield a, b, c, f"{a}-{b}-{c}"
def generate_sortable_rows(num):
"""Generate rows for the sortable fixture table."""
rand = random.Random(42)
for a, b in itertools.islice(
itertools.product(string.ascii_lowercase, repeat=2), num
):
yield {
"pk1": a,
"pk2": b,
"content": f"{a}-{b}",
"sortable": rand.randint(-100, 100),
"sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]),
"sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]),
"text": rand.choice(["$null", "$blah"]),
}
TABLES = (
"""
CREATE TABLE simple_primary_key (
id integer primary key,
content text
);
CREATE TABLE primary_key_multiple_columns (
id varchar(30) primary key,
content text,
content2 text
);
CREATE TABLE primary_key_multiple_columns_explicit_label (
id varchar(30) primary key,
content text,
content2 text
);
CREATE TABLE compound_primary_key (
pk1 varchar(30),
pk2 varchar(30),
content text,
PRIMARY KEY (pk1, pk2)
);
INSERT INTO compound_primary_key VALUES ('a', 'b', 'c');
INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c');
INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO');
CREATE TABLE compound_three_primary_keys (
pk1 varchar(30),
pk2 varchar(30),
pk3 varchar(30),
content text,
PRIMARY KEY (pk1, pk2, pk3)
);
CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
CREATE TABLE foreign_key_references (
pk varchar(30) primary key,
foreign_key_with_label integer,
foreign_key_with_blank_label integer,
foreign_key_with_no_label varchar(30),
foreign_key_compound_pk1 varchar(30),
foreign_key_compound_pk2 varchar(30),
FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id),
FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id),
FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id)
FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2)
);
CREATE TABLE sortable (
pk1 varchar(30),
pk2 varchar(30),
content text,
sortable integer,
sortable_with_nulls real,
sortable_with_nulls_2 real,
text text,
PRIMARY KEY (pk1, pk2)
);
CREATE TABLE no_primary_key (
content text,
a text,
b text,
c text
);
CREATE TABLE [123_starts_with_digits] (
content text
);
CREATE VIEW paginated_view AS
SELECT
content,
'- ' || content || ' -' AS content_extra
FROM no_primary_key;
CREATE TABLE "Table With Space In Name" (
pk varchar(30) primary key,
content text
);
CREATE TABLE "table/with/slashes.csv" (
pk varchar(30) primary key,
content text
);
CREATE TABLE "complex_foreign_keys" (
pk varchar(30) primary key,
f1 integer,
f2 integer,
f3 integer,
FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id),
FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id),
FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id)
);
CREATE TABLE "custom_foreign_key_label" (
pk varchar(30) primary key,
foreign_key_with_custom_label text,
FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id)
);
CREATE TABLE tags (
tag TEXT PRIMARY KEY
);
CREATE TABLE searchable (
pk integer primary key,
text1 text,
text2 text,
[name with . and spaces] text
);
CREATE TABLE searchable_tags (
searchable_id integer,
tag text,
PRIMARY KEY (searchable_id, tag),
FOREIGN KEY (searchable_id) REFERENCES searchable(pk),
FOREIGN KEY (tag) REFERENCES tags(tag)
);
INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther');
INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma');
INSERT INTO tags VALUES ("canine");
INSERT INTO tags VALUES ("feline");
INSERT INTO searchable_tags (searchable_id, tag) VALUES
(1, "feline"),
(2, "canine")
;
CREATE VIRTUAL TABLE "searchable_fts"
USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk");
INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild');
CREATE TABLE [select] (
[group] text,
[having] text,
[and] text,
[json] text
);
INSERT INTO [select] VALUES ('group', 'having', 'and',
'{"href": "http://example.com/", "label":"Example"}'
);
CREATE TABLE infinity (
value REAL
);
INSERT INTO infinity VALUES
(1e999),
(-1e999),
(1.5)
;
CREATE TABLE facet_cities (
id integer primary key,
name text
);
INSERT INTO facet_cities (id, name) VALUES
(1, 'San Francisco'),
(2, 'Los Angeles'),
(3, 'Detroit'),
(4, 'Memnonia')
;
CREATE TABLE facetable (
pk integer primary key,
created text,
planet_int integer,
on_earth integer,
state text,
_city_id integer,
_neighborhood text,
tags text,
complex_array text,
distinct_some_null,
n text,
FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
);
INSERT INTO facetable
(created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n)
VALUES
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'),
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'),
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null),
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null),
("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null),
("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null),
("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null)
;
CREATE TABLE binary_data (
data BLOB
);
-- Many 2 Many demo: roadside attractions!
CREATE TABLE roadside_attractions (
pk integer primary key,
name text,
address text,
url text,
latitude real,
longitude real
);
INSERT INTO roadside_attractions VALUES (
1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/",
37.0167, -122.0024
);
INSERT INTO roadside_attractions VALUES (
2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/",
37.3184, -121.9511
);
INSERT INTO roadside_attractions VALUES (
3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null,
37.5793, -122.3442
);
INSERT INTO roadside_attractions VALUES (
4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/",
37.0414, -122.0725
);
CREATE TABLE attraction_characteristic (
pk integer primary key,
name text
);
INSERT INTO attraction_characteristic VALUES (
1, "Museum"
);
INSERT INTO attraction_characteristic VALUES (
2, "Paranormal"
);
CREATE TABLE roadside_attraction_characteristics (
attraction_id INTEGER REFERENCES roadside_attractions(pk),
characteristic_id INTEGER REFERENCES attraction_characteristic(pk)
);
INSERT INTO roadside_attraction_characteristics VALUES (
1, 2
);
INSERT INTO roadside_attraction_characteristics VALUES (
2, 2
);
INSERT INTO roadside_attraction_characteristics VALUES (
4, 2
);
INSERT INTO roadside_attraction_characteristics VALUES (
3, 1
);
INSERT INTO roadside_attraction_characteristics VALUES (
4, 1
);
INSERT INTO simple_primary_key VALUES (1, 'hello');
INSERT INTO simple_primary_key VALUES (2, 'world');
INSERT INTO simple_primary_key VALUES (3, '');
INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO');
INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC');
INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world');
INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2');
INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b');
INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null);
INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1);
INSERT INTO custom_foreign_key_label VALUES (1, 1);
INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey');
CREATE VIEW simple_view AS
SELECT content, upper(content) AS upper_content FROM simple_primary_key;
CREATE VIEW searchable_view AS
SELECT * from searchable;
CREATE VIEW searchable_view_configured_by_metadata AS
SELECT * from searchable;
"""
+ "\n".join(
[
'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format(
i=i + 1
)
for i in range(201)
]
)
+ '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n'
+ "\n".join(
[
'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format(
a=a, b=b, c=c, content=content
)
for a, b, c, content in generate_compound_rows(1001)
]
)
+ "\n".join(["""INSERT INTO sortable VALUES (
"{pk1}", "{pk2}", "{content}", {sortable},
{sortable_with_nulls}, {sortable_with_nulls_2}, "{text}");
""".format(**row).replace("None", "null") for row in generate_sortable_rows(201)])
)
TABLE_PARAMETERIZED_SQL = [
("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]),
("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]),
("insert into binary_data (data) values (null);", []),
]
EXTRA_DATABASE_SQL = """
CREATE TABLE searchable (
pk integer primary key,
text1 text,
text2 text
);
CREATE VIEW searchable_view AS SELECT * FROM searchable;
INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog');
INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel');
CREATE VIRTUAL TABLE "searchable_fts"
USING FTS3 (text1, text2, content="searchable");
INSERT INTO "searchable_fts" (rowid, text1, text2)
SELECT rowid, text1, text2 FROM searchable;
"""
@documented(label="datasette_fixtures_populate_fixture_database")
def populate_fixture_database(conn):
"""Populate a SQLite connection with Datasette's test fixture tables."""
conn.executescript(TABLES)
for sql, params in TABLE_PARAMETERIZED_SQL:
with conn:
conn.execute(sql, params)
def populate_extra_database(conn):
"""Populate a SQLite connection with the extra database used in tests."""
conn.executescript(EXTRA_DATABASE_SQL)
def write_fixture_database(db_filename):
"""Write Datasette's test fixture tables to a SQLite database file."""
conn = sqlite3.connect(db_filename)
try:
populate_fixture_database(conn)
finally:
conn.close()
def write_extra_database(db_filename):
"""Write the extra test database tables to a SQLite database file."""
conn = sqlite3.connect(db_filename)
try:
populate_extra_database(conn)
finally:
conn.close()

12
datasette/utils/__init__.py
View file

@ -155,9 +155,15 @@ Column = namedtuple(
functions_marked_as_documented = []
def documented(fn):
functions_marked_as_documented.append(fn)
return fn
def documented(fn=None, *, label=None):
def decorate(fn):
fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__)
functions_marked_as_documented.append(fn)
return fn
if fn is None:
return decorate
return decorate(fn)
@documented

1
docs/changelog.rst
View file

@ -13,6 +13,7 @@ Unreleased
- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`)
- Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`)
- New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`)
- New documented :ref:`datasette.fixtures.populate_fixture_database(conn) <datasette_fixtures_populate_fixture_database>` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites.
.. _v1_0_a29:

42
docs/testing_plugins.rst
View file

@ -82,6 +82,48 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar
If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request.
.. _testing_plugins_datasette_fixtures_database:
Using Datasette's fixtures database
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Datasette's own test suite uses a SQLite database with tables that exercise features such as compound primary keys, foreign keys, sortable columns, facets, full-text search and binary data.
You can use those same tables in your plugin tests using the ``populate_fixture_database(conn)`` helper in ``datasette.fixtures``:
Be aware that future Datasette releases may change details of these tables, so try not to rely on their exact structure in your own tests.
.. _datasette_fixtures_populate_fixture_database:
``populate_fixture_database(conn)``
Populates an existing SQLite connection with the fixture tables.
For an in-memory test database:
.. code-block:: python
from datasette.app import Datasette
from datasette.fixtures import populate_fixture_database
import pytest
import pytest_asyncio
@pytest_asyncio.fixture
async def datasette():
datasette = Datasette()
db = datasette.add_memory_database("fixtures")
await db.execute_write_fn(populate_fixture_database)
await datasette.invoke_startup()
return datasette
@pytest.mark.asyncio
async def test_facetable(datasette):
response = await datasette.client.get(
"/fixtures/facetable.json?_shape=array"
)
assert response.status_code == 200
.. _testing_plugins_autoclose:
Automatic cleanup of Datasette instances

9
tests/conftest.py
View file

@ -72,7 +72,7 @@ async def ds_client():
"num_sql_threads": 1,
},
)
from .fixtures import TABLES, TABLE_PARAMETERIZED_SQL
from datasette.fixtures import populate_fixture_database
# Use a unique memory_name to avoid collisions between different
# Datasette instances in the same process, but use "fixtures" for routing
@ -82,10 +82,7 @@ async def ds_client():
def prepare(conn):
if not conn.execute("select count(*) from sqlite_master").fetchone()[0]:
conn.executescript(TABLES)
for sql, params in TABLE_PARAMETERIZED_SQL:
with conn:
conn.execute(sql, params)
populate_fixture_database(conn)
await db.execute_write_fn(prepare)
await ds.invoke_startup()
@ -264,8 +261,6 @@ from .fixtures import ( # noqa: E402, F401
app_client_with_cors,
app_client_with_dot,
app_client_with_trace,
generate_compound_rows,
generate_sortable_rows,
make_app_client,
TEMP_PLUGIN_SECRET_FILE,
)

401
tests/fixtures.py
View file

@ -1,15 +1,16 @@
from datasette.app import Datasette
from datasette.utils.sqlite import sqlite3
from datasette.fixtures import (
EXTRA_DATABASE_SQL,
write_extra_database,
write_fixture_database,
)
from datasette.utils.testing import TestClient
import click
import contextlib
import itertools
import json
import os
import pathlib
import pytest
import random
import string
import tempfile
import textwrap
@ -128,19 +129,18 @@ def make_app_client(
else:
files = [filepath]
immutables = []
conn = sqlite3.connect(filepath)
conn.executescript(TABLES)
for sql, params in TABLE_PARAMETERIZED_SQL:
with conn:
conn.execute(sql, params)
# Close the connection to avoid "too many open files" errors
conn.close()
write_fixture_database(filepath)
if extra_databases is not None:
for extra_filename, extra_sql in extra_databases.items():
extra_filepath = os.path.join(tmpdir, extra_filename)
c2 = sqlite3.connect(extra_filepath)
c2.executescript(extra_sql)
c2.close()
if extra_sql == EXTRA_DATABASE_SQL:
write_extra_database(extra_filepath)
else:
from datasette.utils.sqlite import sqlite3
c2 = sqlite3.connect(extra_filepath)
c2.executescript(extra_sql)
c2.close()
# Insert at start to help test /-/databases ordering:
files.insert(0, extra_filepath)
os.chdir(os.path.dirname(filepath))
@ -279,29 +279,6 @@ def app_client_immutable_and_inspect_file():
yield client
def generate_compound_rows(num):
for a, b, c in itertools.islice(
itertools.product(string.ascii_lowercase, repeat=3), num
):
yield a, b, c, f"{a}-{b}-{c}"
def generate_sortable_rows(num):
rand = random.Random(42)
for a, b in itertools.islice(
itertools.product(string.ascii_lowercase, repeat=2), num
):
yield {
"pk1": a,
"pk2": b,
"content": f"{a}-{b}",
"sortable": rand.randint(-100, 100),
"sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]),
"sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]),
"text": rand.choice(["$null", "$blah"]),
}
CONFIG = {
"plugins": {
"name-of-plugin": {"depth": "root"},
@ -399,345 +376,6 @@ METADATA = {
},
}
TABLES = (
"""
CREATE TABLE simple_primary_key (
id integer primary key,
content text
);
CREATE TABLE primary_key_multiple_columns (
id varchar(30) primary key,
content text,
content2 text
);
CREATE TABLE primary_key_multiple_columns_explicit_label (
id varchar(30) primary key,
content text,
content2 text
);
CREATE TABLE compound_primary_key (
pk1 varchar(30),
pk2 varchar(30),
content text,
PRIMARY KEY (pk1, pk2)
);
INSERT INTO compound_primary_key VALUES ('a', 'b', 'c');
INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c');
INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO');
CREATE TABLE compound_three_primary_keys (
pk1 varchar(30),
pk2 varchar(30),
pk3 varchar(30),
content text,
PRIMARY KEY (pk1, pk2, pk3)
);
CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
CREATE TABLE foreign_key_references (
pk varchar(30) primary key,
foreign_key_with_label integer,
foreign_key_with_blank_label integer,
foreign_key_with_no_label varchar(30),
foreign_key_compound_pk1 varchar(30),
foreign_key_compound_pk2 varchar(30),
FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id),
FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id),
FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id)
FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2)
);
CREATE TABLE sortable (
pk1 varchar(30),
pk2 varchar(30),
content text,
sortable integer,
sortable_with_nulls real,
sortable_with_nulls_2 real,
text text,
PRIMARY KEY (pk1, pk2)
);
CREATE TABLE no_primary_key (
content text,
a text,
b text,
c text
);
CREATE TABLE [123_starts_with_digits] (
content text
);
CREATE VIEW paginated_view AS
SELECT
content,
'- ' || content || ' -' AS content_extra
FROM no_primary_key;
CREATE TABLE "Table With Space In Name" (
pk varchar(30) primary key,
content text
);
CREATE TABLE "table/with/slashes.csv" (
pk varchar(30) primary key,
content text
);
CREATE TABLE "complex_foreign_keys" (
pk varchar(30) primary key,
f1 integer,
f2 integer,
f3 integer,
FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id),
FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id),
FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id)
);
CREATE TABLE "custom_foreign_key_label" (
pk varchar(30) primary key,
foreign_key_with_custom_label text,
FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id)
);
CREATE TABLE tags (
tag TEXT PRIMARY KEY
);
CREATE TABLE searchable (
pk integer primary key,
text1 text,
text2 text,
[name with . and spaces] text
);
CREATE TABLE searchable_tags (
searchable_id integer,
tag text,
PRIMARY KEY (searchable_id, tag),
FOREIGN KEY (searchable_id) REFERENCES searchable(pk),
FOREIGN KEY (tag) REFERENCES tags(tag)
);
INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther');
INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma');
INSERT INTO tags VALUES ("canine");
INSERT INTO tags VALUES ("feline");
INSERT INTO searchable_tags (searchable_id, tag) VALUES
(1, "feline"),
(2, "canine")
;
CREATE VIRTUAL TABLE "searchable_fts"
USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk");
INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild');
CREATE TABLE [select] (
[group] text,
[having] text,
[and] text,
[json] text
);
INSERT INTO [select] VALUES ('group', 'having', 'and',
'{"href": "http://example.com/", "label":"Example"}'
);
CREATE TABLE infinity (
value REAL
);
INSERT INTO infinity VALUES
(1e999),
(-1e999),
(1.5)
;
CREATE TABLE facet_cities (
id integer primary key,
name text
);
INSERT INTO facet_cities (id, name) VALUES
(1, 'San Francisco'),
(2, 'Los Angeles'),
(3, 'Detroit'),
(4, 'Memnonia')
;
CREATE TABLE facetable (
pk integer primary key,
created text,
planet_int integer,
on_earth integer,
state text,
_city_id integer,
_neighborhood text,
tags text,
complex_array text,
distinct_some_null,
n text,
FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
);
INSERT INTO facetable
(created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n)
VALUES
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'),
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'),
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null),
("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null),
("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null),
("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null),
("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null),
("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null),
("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null)
;
CREATE TABLE binary_data (
data BLOB
);
-- Many 2 Many demo: roadside attractions!
CREATE TABLE roadside_attractions (
pk integer primary key,
name text,
address text,
url text,
latitude real,
longitude real
);
INSERT INTO roadside_attractions VALUES (
1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/",
37.0167, -122.0024
);
INSERT INTO roadside_attractions VALUES (
2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/",
37.3184, -121.9511
);
INSERT INTO roadside_attractions VALUES (
3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null,
37.5793, -122.3442
);
INSERT INTO roadside_attractions VALUES (
4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/",
37.0414, -122.0725
);
CREATE TABLE attraction_characteristic (
pk integer primary key,
name text
);
INSERT INTO attraction_characteristic VALUES (
1, "Museum"
);
INSERT INTO attraction_characteristic VALUES (
2, "Paranormal"
);
CREATE TABLE roadside_attraction_characteristics (
attraction_id INTEGER REFERENCES roadside_attractions(pk),
characteristic_id INTEGER REFERENCES attraction_characteristic(pk)
);
INSERT INTO roadside_attraction_characteristics VALUES (
1, 2
);
INSERT INTO roadside_attraction_characteristics VALUES (
2, 2
);
INSERT INTO roadside_attraction_characteristics VALUES (
4, 2
);
INSERT INTO roadside_attraction_characteristics VALUES (
3, 1
);
INSERT INTO roadside_attraction_characteristics VALUES (
4, 1
);
INSERT INTO simple_primary_key VALUES (1, 'hello');
INSERT INTO simple_primary_key VALUES (2, 'world');
INSERT INTO simple_primary_key VALUES (3, '');
INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO');
INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC');
INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world');
INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2');
INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b');
INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null);
INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1);
INSERT INTO custom_foreign_key_label VALUES (1, 1);
INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey');
CREATE VIEW simple_view AS
SELECT content, upper(content) AS upper_content FROM simple_primary_key;
CREATE VIEW searchable_view AS
SELECT * from searchable;
CREATE VIEW searchable_view_configured_by_metadata AS
SELECT * from searchable;
"""
+ "\n".join(
[
'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format(
i=i + 1
)
for i in range(201)
]
)
+ '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n'
+ "\n".join(
[
'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format(
a=a, b=b, c=c, content=content
)
for a, b, c, content in generate_compound_rows(1001)
]
)
+ "\n".join(["""INSERT INTO sortable VALUES (
"{pk1}", "{pk2}", "{content}", {sortable},
{sortable_with_nulls}, {sortable_with_nulls_2}, "{text}");
""".format(**row).replace("None", "null") for row in generate_sortable_rows(201)])
)
TABLE_PARAMETERIZED_SQL = [
("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]),
("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]),
("insert into binary_data (data) values (null);", []),
]
EXTRA_DATABASE_SQL = """
CREATE TABLE searchable (
pk integer primary key,
text1 text,
text2 text
);
CREATE VIEW searchable_view AS SELECT * FROM searchable;
INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog');
INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel');
CREATE VIRTUAL TABLE "searchable_fts"
USING FTS3 (text1, text2, content="searchable");
INSERT INTO "searchable_fts" (rowid, text1, text2)
SELECT rowid, text1, text2 FROM searchable;
"""
def assert_permissions_checked(datasette, actions):
# actions is a list of "action" or (action, resource) tuples
@ -819,12 +457,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename
)
else:
pathlib.Path(db_filename).unlink()
conn = sqlite3.connect(db_filename)
conn.executescript(TABLES)
for sql, params in TABLE_PARAMETERIZED_SQL:
with conn:
conn.execute(sql, params)
conn.close()
write_fixture_database(db_filename)
print(f"Test tables written to {db_filename}")
if metadata:
with open(metadata, "w") as fp:
@ -851,9 +484,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename
)
else:
pathlib.Path(extra_db_filename).unlink()
conn = sqlite3.connect(extra_db_filename)
conn.executescript(EXTRA_DATABASE_SQL)
conn.close()
write_extra_database(extra_db_filename)
print(f"Test tables written to {extra_db_filename}")

18
tests/test_docs.py
View file

@ -3,6 +3,7 @@ Tests to ensure certain things are documented.
"""
from datasette import app, utils
import datasette.fixtures # noqa: F401
from datasette.app import Datasette
from datasette.filters import Filters
from pathlib import Path
@ -95,20 +96,17 @@ def test_table_filters_are_documented(documented_table_filters, subtests):
@pytest.fixture(scope="session")
def documented_fns():
internals_rst = (docs_path / "internals.rst").read_text()
# Any line that starts .. _internals_utils_X
lines = internals_rst.split("\n")
prefix = ".. _internals_utils_"
return {
line.split(prefix)[1].split(":")[0] for line in lines if line.startswith(prefix)
}
def documented_labels():
labels = set()
for filename in docs_path.glob("*.rst"):
labels.update(get_labels(filename.name))
return labels
def test_functions_marked_with_documented_are_documented(documented_fns, subtests):
def test_functions_marked_with_documented_are_documented(documented_labels, subtests):
for fn in utils.functions_marked_as_documented:
with subtests.test(fn=fn.__name__):
assert fn.__name__ in documented_fns
assert fn._datasette_docs_label in documented_labels
def test_rst_heading_underlines_match_title_length():

49
tests/test_fixtures.py Normal file
View file

@ -0,0 +1,49 @@
from datasette.fixtures import (
populate_extra_database,
populate_fixture_database,
write_extra_database,
write_fixture_database,
)
from datasette.utils.sqlite import sqlite3
def count(conn, table):
return conn.execute(f"select count(*) from [{table}]").fetchone()[0]
def test_populate_fixture_database():
conn = sqlite3.connect(":memory:")
try:
populate_fixture_database(conn)
assert count(conn, "facetable") == 15
assert count(conn, "compound_three_primary_keys") == 1001
assert count(conn, "binary_data") == 3
finally:
conn.close()
def test_write_fixture_database(tmp_path):
db_path = tmp_path / "fixtures.db"
write_fixture_database(db_path)
conn = sqlite3.connect(db_path)
try:
assert count(conn, "sortable") == 201
finally:
conn.close()
def test_extra_database_helpers(tmp_path):
conn = sqlite3.connect(":memory:")
try:
populate_extra_database(conn)
assert count(conn, "searchable") == 2
finally:
conn.close()
db_path = tmp_path / "extra.db"
write_extra_database(db_path)
conn = sqlite3.connect(db_path)
try:
assert count(conn, "searchable") == 2
finally:
conn.close()

2
tests/test_plugins.py
View file

@ -1,7 +1,6 @@
from bs4 import BeautifulSoup as Soup
from .fixtures import (
make_app_client,
TABLES,
TEMP_PLUGIN_SECRET_FILE,
PLUGINS_DIR,
TestClient as _TestClient,
@ -9,6 +8,7 @@ from .fixtures import (
from click.testing import CliRunner
from datasette.app import Datasette
from datasette import cli, hookimpl
from datasette.fixtures import TABLES
from datasette.filters import FilterArguments
from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
from datasette.permissions import PermissionSQL, Action

3
tests/test_table_api.py
View file

@ -1,6 +1,7 @@
from datasette.utils import detect_json1
from datasette.utils.sqlite import sqlite_version
from .fixtures import generate_compound_rows, generate_sortable_rows, make_app_client
from datasette.fixtures import generate_compound_rows, generate_sortable_rows
from .fixtures import make_app_client
import json
import pytest
import urllib