diff --git a/README.md b/README.md index 2e41ea6d..cefeed8e 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,6 @@ http://localhost:8001/History/downloads.json?_shape=objects will return that dat useful for development --cors Enable CORS by serving Access-Control-Allow- Origin: * - --page_size INTEGER Page size - default is 100 --load-extension PATH Path to a SQLite extension to load --inspect-file TEXT Path to JSON file created using "datasette inspect" @@ -123,8 +122,9 @@ http://localhost:8001/History/downloads.json?_shape=objects will return that dat --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --limit LIMIT Set a limit using limitname:integer - datasette.readthedocs.io/en/latest/limits.html + --config CONFIG Set config option using configname:value + datasette.readthedocs.io/en/latest/config.html + --help-config Show available config options --help Show this message and exit. ## metadata.json @@ -213,13 +213,14 @@ If you have docker installed you can use `datasette package` to create a new Doc Both publish and package accept an `extra_options` argument option, which will affect how the resulting application is executed. For example, say you want to increase the SQL time limit for a particular container: - datasette package parlgov.db --extra-options="--limit sql_time_limit_ms:2500 --page_size=10" + datasette package parlgov.db \ + --extra-options="--config sql_time_limit_ms:2500 --config default_page_size:10" The resulting container will run the application with those options. Here's example output for the package command: - $ datasette package parlgov.db --extra-options="--limit sql_time_limit_ms:2500 --page_size=10" + $ datasette package parlgov.db --extra-options="--config sql_time_limit_ms:2500" Sending build context to Docker daemon 4.459MB Step 1/7 : FROM python:3 ---> 79e1dc9af1c1 @@ -238,7 +239,7 @@ Here's example output for the package command: Step 6/7 : EXPOSE 8001 ---> Using cache ---> 8e83844b0fed - Step 7/7 : CMD datasette serve parlgov.db --port 8001 --inspect-file inspect-data.json --limit sql_time_limit_ms:2500 --page_size=10 + Step 7/7 : CMD datasette serve parlgov.db --port 8001 --inspect-file inspect-data.json --config sql_time_limit_ms:2500 ---> Using cache ---> 1bd380ea8af3 Successfully built 1bd380ea8af3 diff --git a/datasette/app.py b/datasette/app.py index 4b420811..e4ce6622 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,3 +1,4 @@ +import collections import hashlib import itertools import json @@ -45,12 +46,32 @@ pm.add_hookspecs(hookspecs) pm.load_setuptools_entrypoints("datasette") -DEFAULT_LIMITS = { - "max_returned_rows": 1000, - "sql_time_limit_ms": 1000, - "default_facet_size": 30, - "facet_time_limit_ms": 200, - "facet_suggest_time_limit_ms": 50, +ConfigOption = collections.namedtuple( + "ConfigOption", ("name", "default", "help") +) +CONFIG_OPTIONS = ( + ConfigOption("default_page_size", 100, """ + Default page size for the table view + """.strip()), + ConfigOption("max_returned_rows", 1000, """ + Maximum rows that can be returned from a table or custom query + """.strip()), + ConfigOption("sql_time_limit_ms", 1000, """ + Time limit for a SQL query in milliseconds + """.strip()), + ConfigOption("default_facet_size", 30, """ + Number of values to return for requested facets + """.strip()), + ConfigOption("facet_time_limit_ms", 200, """ + Time limit for calculating a requested facet + """.strip()), + ConfigOption("facet_suggest_time_limit_ms", 50, """ + Time limit for calculating a suggested facet + """.strip()), +) +DEFAULT_CONFIG = { + option.name: option.default + for option in CONFIG_OPTIONS } @@ -87,7 +108,6 @@ class Datasette: files, num_threads=3, cache_headers=True, - page_size=100, cors=False, inspect_data=None, metadata=None, @@ -95,13 +115,12 @@ class Datasette: template_dir=None, plugins_dir=None, static_mounts=None, - limits=None, + config=None, ): self.files = files self.num_threads = num_threads self.executor = futures.ThreadPoolExecutor(max_workers=num_threads) self.cache_headers = cache_headers - self.page_size = page_size self.cors = cors self._inspect = inspect_data self.metadata = metadata or {} @@ -110,9 +129,10 @@ class Datasette: self.template_dir = template_dir self.plugins_dir = plugins_dir self.static_mounts = static_mounts or [] - self.limits = dict(DEFAULT_LIMITS, **(limits or {})) - self.max_returned_rows = self.limits["max_returned_rows"] - self.sql_time_limit_ms = self.limits["sql_time_limit_ms"] + self.config = dict(DEFAULT_CONFIG, **(config or {})) + self.max_returned_rows = self.config["max_returned_rows"] + self.sql_time_limit_ms = self.config["sql_time_limit_ms"] + self.page_size = self.config["default_page_size"] # Execute plugins in constructor, to ensure they are available # when the rest of `datasette inspect` executes if self.plugins_dir: @@ -443,8 +463,8 @@ class Datasette: "/-/plugins", ) app.add_route( - JsonDataView.as_view(self, "limits.json", lambda: self.limits), - "/-/limits", + JsonDataView.as_view(self, "config.json", lambda: self.config), + "/-/config", ) app.add_route( DatabaseView.as_view(self), "/" diff --git a/datasette/cli.py b/datasette/cli.py index f4818b75..7dc03195 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -1,11 +1,12 @@ import click +from click import formatting from click_default_group import DefaultGroup import json import os import shutil from subprocess import call, check_output import sys -from .app import Datasette, DEFAULT_LIMITS +from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS from .utils import temporary_docker_directory, temporary_heroku_directory @@ -24,8 +25,8 @@ class StaticMount(click.ParamType): return path, dirpath -class Limit(click.ParamType): - name = "limit" +class Config(click.ParamType): + name = "config" def convert(self, value, param, ctx): ok = True @@ -39,7 +40,7 @@ class Limit(click.ParamType): '"{}" should be of format name:integer'.format(value), param, ctx ) - if name not in DEFAULT_LIMITS: + if name not in DEFAULT_CONFIG: self.fail("{} is not a valid limit".format(name), param, ctx) return name, int(intvalue) @@ -384,7 +385,6 @@ def package( @click.option( "--cors", is_flag=True, help="Enable CORS by serving Access-Control-Allow-Origin: *" ) -@click.option("--page_size", default=100, help="Page size - default is 100") @click.option( "sqlite_extensions", "--load-extension", @@ -419,11 +419,16 @@ def package( multiple=True, ) @click.option( - "--limit", - type=Limit(), - help="Set a limit using limitname:integer datasette.readthedocs.io/en/latest/limits.html", + "--config", + type=Config(), + help="Set config option using configname:value datasette.readthedocs.io/en/latest/config.html", multiple=True, ) +@click.option( + "--help-config", + is_flag=True, + help="Show available config options", +) def serve( files, host, @@ -431,16 +436,27 @@ def serve( debug, reload, cors, - page_size, sqlite_extensions, inspect_file, metadata, template_dir, plugins_dir, static, - limit, + config, + help_config, ): """Serve up specified SQLite database files with a web UI""" + if help_config: + formatter = formatting.HelpFormatter() + with formatter.section("Config options"): + formatter.write_dl([ + (option.name, '{} (default={})'.format( + option.help, option.default + )) + for option in CONFIG_OPTIONS + ]) + click.echo(formatter.getvalue()) + sys.exit(0) if reload: import hupper @@ -461,14 +477,13 @@ def serve( files, cache_headers=not debug and not reload, cors=cors, - page_size=page_size, inspect_data=inspect_data, metadata=metadata_data, sqlite_extensions=sqlite_extensions, template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - limits=dict(limit), + config=dict(config), ) # Force initial hashing/table counting ds.inspect() diff --git a/datasette/views/table.py b/datasette/views/table.py index 20917983..07e3cd59 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -500,7 +500,7 @@ class TableView(RowTableShared): return await self.custom_sql(request, name, hash, sql, editable=True) extra_args = {} - # Handle ?_page_size=500 + # Handle ?_size=500 page_size = request.raw_args.get("_size") if page_size: if page_size == "max": @@ -539,7 +539,7 @@ class TableView(RowTableShared): ) # facets support - facet_size = self.ds.limits["default_facet_size"] + facet_size = self.ds.config["default_facet_size"] metadata_facets = table_metadata.get("facets", []) facets = metadata_facets[:] try: @@ -563,7 +563,7 @@ class TableView(RowTableShared): facet_rows = await self.execute( name, facet_sql, params, truncate=False, - custom_time_limit=self.ds.limits["facet_time_limit_ms"], + custom_time_limit=self.ds.config["facet_time_limit_ms"], ) facet_results_values = [] facet_results[column] = { @@ -668,7 +668,7 @@ class TableView(RowTableShared): distinct_values = await self.execute( name, suggested_facet_sql, from_sql_params, truncate=False, - custom_time_limit=self.ds.limits["facet_suggest_time_limit_ms"], + custom_time_limit=self.ds.config["facet_suggest_time_limit_ms"], ) num_distinct_values = len(distinct_values) if ( diff --git a/docs/limits.rst b/docs/config.rst similarity index 68% rename from docs/limits.rst rename to docs/config.rst index ccc0555d..adeac557 100644 --- a/docs/limits.rst +++ b/docs/config.rst @@ -1,7 +1,17 @@ -Limits +Config ====== -To prevent rogue, long-running queries from making a Datasette instance inaccessible to other users, Datasette imposes some limits on the SQL that you can execute. +Datasette provides a number of configuration options. These can be set using the ``--config name:value`` option to ``datasette serve``. + +To prevent rogue, long-running queries from making a Datasette instance inaccessible to other users, Datasette imposes some limits on the SQL that you can execute. These are exposed as config options which you can over-ride. + +default_page_size +----------------- + +The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` querystring parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--config`` like so:: + + datasette mydatabase.db --config default_page_size:50 + sql_time_limit_ms ----------------- @@ -10,9 +20,9 @@ By default, queries have a time limit of one second. If a query takes longer tha If this time limit is too short for you, you can customize it using the ``sql_time_limit_ms`` limit - for example, to increase it to 3.5 seconds:: - datasette mydatabase.db --limit sql_time_limit_ms:3500 + datasette mydatabase.db --config sql_time_limit_ms:3500 -You can optionally set a lower time limit for an individual query using the ``_timelimit`` query string argument:: +You can optionally set a lower time limit for an individual query using the ``?_timelimit=100`` query string argument:: /my-database/my-table?qSpecies=44&_timelimit=100 @@ -25,21 +35,21 @@ Datasette returns a maximum of 1,000 rows of data at a time. If you execute a qu You can increase or decrease this limit like so:: - datasette mydatabase.db --limit max_returned_rows:2000 + datasette mydatabase.db --config max_returned_rows:2000 default_facet_size ------------------ The default number of unique rows returned by :ref:`facets` is 30. You can customize it like this:: - datasette mydatabase.db --limit default_facet_size:50 + datasette mydatabase.db --config default_facet_size:50 facet_time_limit_ms ------------------- This is the time limit Datasette allows for calculating a facet, which defaults to 200ms:: - datasette mydatabase.db --limit facet_time_limit_ms:1000 + datasette mydatabase.db --config facet_time_limit_ms:1000 facet_suggest_time_limit_ms --------------------------- @@ -48,4 +58,4 @@ When Datasette calculates suggested facets it needs to run a SQL query for every You can increase this time limit like so:: - datasette mydatabase.db --limit facet_suggest_time_limit_ms:500 + datasette mydatabase.db --config facet_suggest_time_limit_ms:500 diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 75ba3b31..d969e587 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -100,7 +100,6 @@ datasette serve options useful for development --cors Enable CORS by serving Access-Control-Allow- Origin: * - --page_size INTEGER Page size - default is 100 --load-extension PATH Path to a SQLite extension to load --inspect-file TEXT Path to JSON file created using "datasette inspect" @@ -110,6 +109,7 @@ datasette serve options --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --limit LIMIT Set a limit using limitname:integer - datasette.readthedocs.io/en/latest/limits.html + --config CONFIG Set config option using configname:value + datasette.readthedocs.io/en/latest/config.html + --help-config Show available config options --help Show this message and exit. diff --git a/docs/index.rst b/docs/index.rst index 23cb6225..9d85f6f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,7 +22,7 @@ Contents facets full_text_search metadata - limits + config custom_templates plugins changelog diff --git a/tests/fixtures.py b/tests/fixtures.py index 436fa447..f94be72b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -20,10 +20,10 @@ def app_client(sql_time_limit_ms=None, max_returned_rows=None): open(os.path.join(plugins_dir, 'my_plugin.py'), 'w').write(PLUGIN) ds = Datasette( [filepath], - page_size=50, metadata=METADATA, plugins_dir=plugins_dir, - limits={ + config={ + 'default_page_size': 50, 'max_returned_rows': max_returned_rows or 100, 'sql_time_limit_ms': sql_time_limit_ms or 200, } diff --git a/tests/test_api.py b/tests/test_api.py index 5ae8886a..aa77c269 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -897,12 +897,13 @@ def test_versions_json(app_client): assert 'fts_versions' in response.json['sqlite'] -def test_limits_json(app_client): +def test_config_json(app_client): response = app_client.get( - "/-/limits.json", + "/-/config.json", gather_request=False ) assert { + "default_page_size": 50, "default_facet_size": 30, "facet_suggest_time_limit_ms": 50, "facet_time_limit_ms": 200,