diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 23eeb571..c09582f7 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -76,7 +76,7 @@

{{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

-

{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

+

{% if table.count is none %}Many rows{% elif table.count_truncated %}>{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

{% endif %} {% endfor %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index e06ef94e..b7b776ab 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -48,7 +48,7 @@ {% if count or human_description_en %}

- {% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows + {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %} diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 238b9bb3..e503ed35 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -32,6 +32,8 @@ class Context: "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + if f.name.startswith("_"): + continue from_extra = bool(f.metadata.get("from_extra")) if from_extra: help_text = cls._extra_description(f.name) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0d35b87f..46a90c4c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -230,7 +230,6 @@ class DatabaseView(View): database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, - count_limit=db.count_limit, allow_download=datasette.setting("allow_download") and not db.is_mutable and not db.is_memory, @@ -267,7 +266,11 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of stored query objects"}) @@ -295,7 +298,6 @@ class DatabaseContext(Context): editable: bool = field( metadata={"help": "Boolean indicating if the database is editable"} ) - count_limit: int = field(metadata={"help": "The maximum number of rows to count"}) allow_download: bool = field( metadata={"help": "Boolean indicating if database download is allowed"} ) @@ -365,7 +367,11 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) named_parameter_values: dict = field( metadata={"help": "Dictionary of parameter names/values"} ) @@ -430,6 +436,9 @@ async def get_tables(datasette, request, db, allowed_dict): "columns": table_columns, "primary_keys": await db.primary_keys(table), "count": table_counts[table], + "count_truncated": _table_count_truncated( + datasette, db, table, table_counts[table] + ), "hidden": table in hidden_table_names, "fts_table": await db.fts_table(table), "foreign_keys": all_foreign_keys[table], @@ -440,6 +449,18 @@ async def get_tables(datasette, request, db, allowed_dict): return tables +def _table_count_truncated(datasette, db, table, count): + if count != db.count_limit + 1: + return False + if not db.is_mutable and datasette.inspect_data: + try: + datasette.inspect_data[db.name]["tables"][table]["count"] + return False + except KeyError: + pass + return True + + async def database_download(request, datasette): from datasette.resources import DatabaseResource diff --git a/datasette/views/table.py b/datasette/views/table.py index a6769242..38a69f5f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -113,6 +113,11 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + count_truncated: bool = field( + metadata={ + "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." + } + ) rows: list = field( metadata={ "help": "The rows for this page, as a list of dictionaries mapping column name to value" @@ -185,11 +190,6 @@ class TableContext(Context): top_table: callable = field( metadata={"help": "Async function rendering the top_table plugin slot"} ) - count_limit: int = field( - metadata={ - "help": "The maximum number of rows Datasette will count before showing an approximation" - } - ) table_page_data: dict = field( metadata={"help": "JSON data used by JavaScript on the table page"} ) @@ -1391,7 +1391,6 @@ class TableFragmentView(BaseView): path_with_replaced_args=path_with_replaced_args, fix_path=self.ds.urls.path, settings=self.ds.settings_dict(), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1843,7 +1842,6 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -2276,6 +2274,9 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: + data["count_truncated"] = _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, data.get("count") + ) data.update(extra_context_from_filters) # filter_columns combine the columns we know are available # in the table with any additional columns (such as rowid) @@ -2341,6 +2342,24 @@ async def table_view_data( return data, rows[:page_size], columns, expanded_columns, sql, next_url +def _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, count +): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 7cb4d8f0..f688433d 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -127,8 +127,8 @@ class CountExtra(Extra): pass if context.count_sql and count is None and not context.nocount: - count_sql_limited = ( - f"select count(*) from (select * {context.from_sql} limit 10001)" + count_sql_limited = "select count(*) from (select * {} limit {})".format( + context.from_sql, context.db.count_limit + 1 ) try: count_rows = list( diff --git a/docs/template_context.rst b/docs/template_context.rst index f94ecd48..b27f2f79 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -100,9 +100,6 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``attached_databases`` - ``list`` List of names of attached databases -``count_limit`` - ``int`` - The maximum number of rows to count - ``database`` - ``str`` The name of the database @@ -152,7 +149,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re Dictionary mapping table names to their column lists ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Callable to render the top_database slot @@ -241,7 +238,7 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie Dictionary of table name to list of column names ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Callable to render the top_query slot @@ -280,12 +277,12 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. ``count`` - ``int`` Total count of rows matching these filters -``count_limit`` - ``int`` - The maximum number of rows Datasette will count before showing an approximation - ``count_sql`` - ``str`` SQL query used to calculate the total count +``count_truncated`` - ``bool`` + True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit. + ``custom_table_templates`` - ``list`` Custom template names considered for this table diff --git a/tests/test_template_context.py b/tests/test_template_context.py index b970cb8a..8d78bf77 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -21,6 +21,7 @@ def test_documented_fields(): @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) + _internal: str = field() count: int = field(metadata={"help": "How many there are"}) fields = DemoContext.documented_fields() @@ -165,7 +166,11 @@ async def test_template_context_matches_documented_contract( # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context documented = documented_context_keys(page_name) - actual = set(await get_template_context(context_ds, path)) + actual = { + key + for key in await get_template_context(context_ds, path) + if not key.startswith("_") + } undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( @@ -178,6 +183,33 @@ async def test_template_context_matches_documented_contract( ) +@pytest.mark.asyncio +async def test_count_truncated_replaces_count_limit_context_key(context_ds): + db = context_ds.databases["fixtures"] + previous_count_limit = db.count_limit + previous_cached_table_counts = db._cached_table_counts + db.count_limit = 10 + db._cached_table_counts = None + try: + table_context = await get_template_context(context_ds, "/fixtures/facetable") + assert table_context["count"] == 11 + assert table_context["count_truncated"] is True + assert "count_limit" not in table_context + + database_context = await get_template_context(context_ds, "/fixtures") + facetable = next( + table + for table in database_context["tables"] + if table["name"] == "facetable" + ) + assert facetable["count"] == 11 + assert facetable["count_truncated"] is True + assert "count_limit" not in database_context + finally: + db.count_limit = previous_count_limit + db._cached_table_counts = previous_cached_table_counts + + def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name)