mirror of
https://github.com/simonw/datasette.git
synced 2025-12-10 16:51:24 +01:00
Renamed --limit to --config, added --help-config, closes #274
Removed the --page_size= argument to datasette serve in favour of:
datasette serve --config default_page_size:50 mydb.db
Added new help section:
$ datasette --help-config
Config options:
default_page_size Default page size for the table view
(default=100)
max_returned_rows Maximum rows that can be returned from a table
or custom query (default=1000)
sql_time_limit_ms Time limit for a SQL query in milliseconds
(default=1000)
default_facet_size Number of values to return for requested facets
(default=30)
facet_time_limit_ms Time limit for calculating a requested facet
(default=200)
facet_suggest_time_limit_ms Time limit for calculating a suggested facet
(default=50)
This commit is contained in:
Simon Willison
parent
918de9403e
commit
f6183ff5fa
10 changed files with 142 additions and 95 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import collections
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
|
|
@ -45,12 +46,32 @@ pm.add_hookspecs(hookspecs)
|
|||
pm.load_setuptools_entrypoints("datasette")
|
||||
|
||||
|
||||
DEFAULT_LIMITS = {
|
||||
"max_returned_rows": 1000,
|
||||
"sql_time_limit_ms": 1000,
|
||||
"default_facet_size": 30,
|
||||
"facet_time_limit_ms": 200,
|
||||
"facet_suggest_time_limit_ms": 50,
|
||||
ConfigOption = collections.namedtuple(
|
||||
"ConfigOption", ("name", "default", "help")
|
||||
)
|
||||
CONFIG_OPTIONS = (
|
||||
ConfigOption("default_page_size", 100, """
|
||||
Default page size for the table view
|
||||
""".strip()),
|
||||
ConfigOption("max_returned_rows", 1000, """
|
||||
Maximum rows that can be returned from a table or custom query
|
||||
""".strip()),
|
||||
ConfigOption("sql_time_limit_ms", 1000, """
|
||||
Time limit for a SQL query in milliseconds
|
||||
""".strip()),
|
||||
ConfigOption("default_facet_size", 30, """
|
||||
Number of values to return for requested facets
|
||||
""".strip()),
|
||||
ConfigOption("facet_time_limit_ms", 200, """
|
||||
Time limit for calculating a requested facet
|
||||
""".strip()),
|
||||
ConfigOption("facet_suggest_time_limit_ms", 50, """
|
||||
Time limit for calculating a suggested facet
|
||||
""".strip()),
|
||||
)
|
||||
DEFAULT_CONFIG = {
|
||||
option.name: option.default
|
||||
for option in CONFIG_OPTIONS
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -87,7 +108,6 @@ class Datasette:
|
|||
files,
|
||||
num_threads=3,
|
||||
cache_headers=True,
|
||||
page_size=100,
|
||||
cors=False,
|
||||
inspect_data=None,
|
||||
metadata=None,
|
||||
|
|
@ -95,13 +115,12 @@ class Datasette:
|
|||
template_dir=None,
|
||||
plugins_dir=None,
|
||||
static_mounts=None,
|
||||
limits=None,
|
||||
config=None,
|
||||
):
|
||||
self.files = files
|
||||
self.num_threads = num_threads
|
||||
self.executor = futures.ThreadPoolExecutor(max_workers=num_threads)
|
||||
self.cache_headers = cache_headers
|
||||
self.page_size = page_size
|
||||
self.cors = cors
|
||||
self._inspect = inspect_data
|
||||
self.metadata = metadata or {}
|
||||
|
|
@ -110,9 +129,10 @@ class Datasette:
|
|||
self.template_dir = template_dir
|
||||
self.plugins_dir = plugins_dir
|
||||
self.static_mounts = static_mounts or []
|
||||
self.limits = dict(DEFAULT_LIMITS, **(limits or {}))
|
||||
self.max_returned_rows = self.limits["max_returned_rows"]
|
||||
self.sql_time_limit_ms = self.limits["sql_time_limit_ms"]
|
||||
self.config = dict(DEFAULT_CONFIG, **(config or {}))
|
||||
self.max_returned_rows = self.config["max_returned_rows"]
|
||||
self.sql_time_limit_ms = self.config["sql_time_limit_ms"]
|
||||
self.page_size = self.config["default_page_size"]
|
||||
# Execute plugins in constructor, to ensure they are available
|
||||
# when the rest of `datasette inspect` executes
|
||||
if self.plugins_dir:
|
||||
|
|
@ -443,8 +463,8 @@ class Datasette:
|
|||
"/-/plugins<as_json:(\.json)?$>",
|
||||
)
|
||||
app.add_route(
|
||||
JsonDataView.as_view(self, "limits.json", lambda: self.limits),
|
||||
"/-/limits<as_json:(\.json)?$>",
|
||||
JsonDataView.as_view(self, "config.json", lambda: self.config),
|
||||
"/-/config<as_json:(\.json)?$>",
|
||||
)
|
||||
app.add_route(
|
||||
DatabaseView.as_view(self), "/<db_name:[^/\.]+?><as_json:(\.jsono?)?$>"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import click
|
||||
from click import formatting
|
||||
from click_default_group import DefaultGroup
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from subprocess import call, check_output
|
||||
import sys
|
||||
from .app import Datasette, DEFAULT_LIMITS
|
||||
from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS
|
||||
from .utils import temporary_docker_directory, temporary_heroku_directory
|
||||
|
||||
|
||||
|
|
@ -24,8 +25,8 @@ class StaticMount(click.ParamType):
|
|||
return path, dirpath
|
||||
|
||||
|
||||
class Limit(click.ParamType):
|
||||
name = "limit"
|
||||
class Config(click.ParamType):
|
||||
name = "config"
|
||||
|
||||
def convert(self, value, param, ctx):
|
||||
ok = True
|
||||
|
|
@ -39,7 +40,7 @@ class Limit(click.ParamType):
|
|||
'"{}" should be of format name:integer'.format(value),
|
||||
param, ctx
|
||||
)
|
||||
if name not in DEFAULT_LIMITS:
|
||||
if name not in DEFAULT_CONFIG:
|
||||
self.fail("{} is not a valid limit".format(name), param, ctx)
|
||||
return name, int(intvalue)
|
||||
|
||||
|
|
@ -384,7 +385,6 @@ def package(
|
|||
@click.option(
|
||||
"--cors", is_flag=True, help="Enable CORS by serving Access-Control-Allow-Origin: *"
|
||||
)
|
||||
@click.option("--page_size", default=100, help="Page size - default is 100")
|
||||
@click.option(
|
||||
"sqlite_extensions",
|
||||
"--load-extension",
|
||||
|
|
@ -419,11 +419,16 @@ def package(
|
|||
multiple=True,
|
||||
)
|
||||
@click.option(
|
||||
"--limit",
|
||||
type=Limit(),
|
||||
help="Set a limit using limitname:integer datasette.readthedocs.io/en/latest/limits.html",
|
||||
"--config",
|
||||
type=Config(),
|
||||
help="Set config option using configname:value datasette.readthedocs.io/en/latest/config.html",
|
||||
multiple=True,
|
||||
)
|
||||
@click.option(
|
||||
"--help-config",
|
||||
is_flag=True,
|
||||
help="Show available config options",
|
||||
)
|
||||
def serve(
|
||||
files,
|
||||
host,
|
||||
|
|
@ -431,16 +436,27 @@ def serve(
|
|||
debug,
|
||||
reload,
|
||||
cors,
|
||||
page_size,
|
||||
sqlite_extensions,
|
||||
inspect_file,
|
||||
metadata,
|
||||
template_dir,
|
||||
plugins_dir,
|
||||
static,
|
||||
limit,
|
||||
config,
|
||||
help_config,
|
||||
):
|
||||
"""Serve up specified SQLite database files with a web UI"""
|
||||
if help_config:
|
||||
formatter = formatting.HelpFormatter()
|
||||
with formatter.section("Config options"):
|
||||
formatter.write_dl([
|
||||
(option.name, '{} (default={})'.format(
|
||||
option.help, option.default
|
||||
))
|
||||
for option in CONFIG_OPTIONS
|
||||
])
|
||||
click.echo(formatter.getvalue())
|
||||
sys.exit(0)
|
||||
if reload:
|
||||
import hupper
|
||||
|
||||
|
|
@ -461,14 +477,13 @@ def serve(
|
|||
files,
|
||||
cache_headers=not debug and not reload,
|
||||
cors=cors,
|
||||
page_size=page_size,
|
||||
inspect_data=inspect_data,
|
||||
metadata=metadata_data,
|
||||
sqlite_extensions=sqlite_extensions,
|
||||
template_dir=template_dir,
|
||||
plugins_dir=plugins_dir,
|
||||
static_mounts=static,
|
||||
limits=dict(limit),
|
||||
config=dict(config),
|
||||
)
|
||||
# Force initial hashing/table counting
|
||||
ds.inspect()
|
||||
|
|
|
|||
|
|
@ -500,7 +500,7 @@ class TableView(RowTableShared):
|
|||
return await self.custom_sql(request, name, hash, sql, editable=True)
|
||||
|
||||
extra_args = {}
|
||||
# Handle ?_page_size=500
|
||||
# Handle ?_size=500
|
||||
page_size = request.raw_args.get("_size")
|
||||
if page_size:
|
||||
if page_size == "max":
|
||||
|
|
@ -539,7 +539,7 @@ class TableView(RowTableShared):
|
|||
)
|
||||
|
||||
# facets support
|
||||
facet_size = self.ds.limits["default_facet_size"]
|
||||
facet_size = self.ds.config["default_facet_size"]
|
||||
metadata_facets = table_metadata.get("facets", [])
|
||||
facets = metadata_facets[:]
|
||||
try:
|
||||
|
|
@ -563,7 +563,7 @@ class TableView(RowTableShared):
|
|||
facet_rows = await self.execute(
|
||||
name, facet_sql, params,
|
||||
truncate=False,
|
||||
custom_time_limit=self.ds.limits["facet_time_limit_ms"],
|
||||
custom_time_limit=self.ds.config["facet_time_limit_ms"],
|
||||
)
|
||||
facet_results_values = []
|
||||
facet_results[column] = {
|
||||
|
|
@ -668,7 +668,7 @@ class TableView(RowTableShared):
|
|||
distinct_values = await self.execute(
|
||||
name, suggested_facet_sql, from_sql_params,
|
||||
truncate=False,
|
||||
custom_time_limit=self.ds.limits["facet_suggest_time_limit_ms"],
|
||||
custom_time_limit=self.ds.config["facet_suggest_time_limit_ms"],
|
||||
)
|
||||
num_distinct_values = len(distinct_values)
|
||||
if (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue