Compare commits

...

1 commit

Author SHA1 Message Date
Claude
1a7b027b60
Make setting(), settings_dict(), and plugin_config() async
This change makes all settings and plugin config access asynchronous in
preparation for a future plugin hook that will allow fetching settings
from a database or via HTTP.

Changes:
- Datasette.setting() is now async
- Datasette.settings_dict() is now async
- Datasette.plugin_config() is now async
- Added _setting_sync() for contexts that cannot await (threads, sync hooks)
- Added _plugin_config_sync() for contexts that cannot await
- Updated all callers throughout the codebase to use await
- Updated tests to work with async methods
2025-12-30 16:21:16 +00:00
13 changed files with 90 additions and 76 deletions

View file

@ -472,15 +472,15 @@ class Datasette:
self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {}))
self.renderers = {} # File extension -> (renderer, can_render) functions
self.version_note = version_note
if self.setting("num_sql_threads") == 0:
if self._setting_sync("num_sql_threads") == 0:
self.executor = None
else:
self.executor = futures.ThreadPoolExecutor(
max_workers=self.setting("num_sql_threads")
max_workers=self._setting_sync("num_sql_threads")
)
self.max_returned_rows = self.setting("max_returned_rows")
self.sql_time_limit_ms = self.setting("sql_time_limit_ms")
self.page_size = self.setting("default_page_size")
self.max_returned_rows = self._setting_sync("max_returned_rows")
self.sql_time_limit_ms = self._setting_sync("sql_time_limit_ms")
self.page_size = self._setting_sync("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
@ -790,12 +790,16 @@ class Datasette:
new_databases.pop(name)
self.databases = new_databases
def setting(self, key):
async def setting(self, key):
return self._settings.get(key, None)
def settings_dict(self):
def _setting_sync(self, key):
# Synchronous access for contexts that cannot await (threads, sync callbacks)
return self._settings.get(key, None)
async def settings_dict(self):
# Returns a fully resolved settings dictionary, useful for templates
return {option.name: self.setting(option.name) for option in SETTINGS}
return {option.name: await self.setting(option.name) for option in SETTINGS}
def _metadata_recursive_update(self, orig, updated):
if not isinstance(orig, dict) or not isinstance(updated, dict):
@ -919,7 +923,7 @@ class Datasette:
def get_internal_database(self):
return self._internal_database
def plugin_config(self, plugin_name, database=None, table=None, fallback=True):
async def plugin_config(self, plugin_name, database=None, table=None, fallback=True):
"""Return config for plugin, falling back from specified database/table"""
if database is None and table is None:
config = self._plugin_config_top(plugin_name)
@ -928,6 +932,15 @@ class Datasette:
return resolve_env_secrets(config, os.environ)
def _plugin_config_sync(self, plugin_name, database=None, table=None, fallback=True):
"""Synchronous access for contexts that cannot await (threads, sync callbacks)"""
if database is None and table is None:
config = self._plugin_config_top(plugin_name)
else:
config = self._plugin_config_nested(plugin_name, database, table, fallback)
return resolve_env_secrets(config, os.environ)
def _plugin_config_top(self, plugin_name):
"""Returns any top-level plugin configuration for the specified plugin."""
return ((self.config or {}).get("plugins") or {}).get(plugin_name)
@ -1003,8 +1016,8 @@ class Datasette:
conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
else:
conn.execute("SELECT load_extension(?)", [extension])
if self.setting("cache_size_kb"):
conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
if self._setting_sync("cache_size_kb"):
conn.execute(f"PRAGMA cache_size=-{self._setting_sync('cache_size_kb')}")
# pylint: disable=no-member
if database != INTERNAL_DB_NAME:
pm.hook.prepare_connection(conn=conn, database=database, datasette=self)
@ -1517,7 +1530,7 @@ class Datasette:
def absolute_url(self, request, path):
url = urllib.parse.urljoin(request.url, path)
if url.startswith("http://") and self.setting("force_https_urls"):
if url.startswith("http://") and self._setting_sync("force_https_urls"):
url = "https://" + url[len("http://") :]
return url
@ -1632,7 +1645,7 @@ class Datasette:
]
def _threads(self):
if self.setting("num_sql_threads") == 0:
if self._setting_sync("num_sql_threads") == 0:
return {"num_threads": 0, "threads": []}
threads = list(threading.enumerate())
d = {
@ -1790,13 +1803,13 @@ class Datasette:
"extra_js_urls": await self._asset_urls(
"extra_js_urls", template, context, request, view_name
),
"base_url": self.setting("base_url"),
"base_url": await self.setting("base_url"),
"csrftoken": request.scope["csrftoken"] if request else lambda: "",
"datasette_version": __version__,
},
**extra_template_vars,
}
if request and request.args.get("_context") and self.setting("template_debug"):
if request and request.args.get("_context") and await self.setting("template_debug"):
return "<pre>{}</pre>".format(
escape(json.dumps(template_context, default=repr, indent=4))
)
@ -2110,7 +2123,7 @@ class Datasette:
),
send_csrf_failed=custom_csrf_error,
)
if self.setting("trace_debug"):
if self._setting_sync("trace_debug"):
asgi = AsgiTracer(asgi)
asgi = AsgiLifespan(asgi)
asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup])
@ -2135,7 +2148,7 @@ class DatasetteRouter:
async def route_path(self, scope, receive, send, path):
# Strip off base_url if present before routing
base_url = self.ds.setting("base_url")
base_url = await self.ds.setting("base_url")
if base_url != "/" and path.startswith(base_url):
path = "/" + path[len(base_url) :]
scope = dict(scope, route_path=path)
@ -2151,7 +2164,7 @@ class DatasetteRouter:
scope_modifications = {}
# Apply force_https_urls, if set
if (
self.ds.setting("force_https_urls")
await self.ds.setting("force_https_urls")
and scope["type"] == "http"
and scope.get("scheme") != "https"
):

View file

@ -41,7 +41,7 @@ async def default_allow_sql_check(
unless explicitly allowed by config or other rules.
"""
if action == "execute-sql":
if not datasette.setting("default_allow_sql"):
if not await datasette.setting("default_allow_sql"):
return PermissionSQL.deny(reason="default_allow_sql is false")
return None

View file

@ -18,7 +18,7 @@ from datasette import hookimpl
@hookimpl(specname="actor_from_request")
def actor_from_signed_api_token(datasette: "Datasette", request) -> Optional[dict]:
async def actor_from_signed_api_token(datasette: "Datasette", request) -> Optional[dict]:
"""
Authenticate requests using signed API tokens (dstok_ prefix).
@ -33,10 +33,10 @@ def actor_from_signed_api_token(datasette: "Datasette", request) -> Optional[dic
prefix = "dstok_"
# Check if tokens are enabled
if not datasette.setting("allow_signed_tokens"):
if not await datasette.setting("allow_signed_tokens"):
return None
max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl")
max_signed_tokens_ttl = await datasette.setting("max_signed_tokens_ttl")
# Get authorization header
authorization = request.headers.get("authorization")

View file

@ -100,9 +100,9 @@ class Facet:
# [('_foo', 'bar'), ('_foo', '2'), ('empty', '')]
return urllib.parse.parse_qsl(self.request.query_string, keep_blank_values=True)
def get_facet_size(self):
facet_size = self.ds.setting("default_facet_size")
max_returned_rows = self.ds.setting("max_returned_rows")
async def get_facet_size(self):
facet_size = await self.ds.setting("default_facet_size")
max_returned_rows = await self.ds.setting("max_returned_rows")
table_facet_size = None
if self.table:
config_facet_size = (
@ -154,7 +154,7 @@ class ColumnFacet(Facet):
async def suggest(self):
row_count = await self.get_row_count()
columns = await self.get_columns(self.sql, self.params)
facet_size = self.get_facet_size()
facet_size = await self.get_facet_size()
suggested_facets = []
already_enabled = [c["config"]["simple"] for c in self.get_configs()]
for column in columns:
@ -179,7 +179,7 @@ class ColumnFacet(Facet):
suggested_facet_sql,
self.params,
truncate=False,
custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"),
custom_time_limit=await self.ds.setting("facet_suggest_time_limit_ms"),
)
num_distinct_values = len(distinct_values)
if (
@ -222,7 +222,7 @@ class ColumnFacet(Facet):
qs_pairs = self.get_querystring_pairs()
facet_size = self.get_facet_size()
facet_size = await self.get_facet_size()
for source_and_config in self.get_configs():
config = source_and_config["config"]
source = source_and_config["source"]
@ -242,7 +242,7 @@ class ColumnFacet(Facet):
facet_sql,
self.params,
truncate=False,
custom_time_limit=self.ds.setting("facet_time_limit_ms"),
custom_time_limit=await self.ds.setting("facet_time_limit_ms"),
)
facet_results_values = []
facet_results.append(
@ -333,7 +333,7 @@ class ArrayFacet(Facet):
suggested_facet_sql,
self.params,
truncate=False,
custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"),
custom_time_limit=await self.ds.setting("facet_suggest_time_limit_ms"),
log_sql_errors=False,
)
types = tuple(r[0] for r in results.rows)
@ -352,7 +352,7 @@ class ArrayFacet(Facet):
).format(column=escape_sqlite(column), sql=self.sql),
self.params,
truncate=False,
custom_time_limit=self.ds.setting(
custom_time_limit=await self.ds.setting(
"facet_suggest_time_limit_ms"
),
log_sql_errors=False,
@ -384,7 +384,7 @@ class ArrayFacet(Facet):
facet_results = []
facets_timed_out = []
facet_size = self.get_facet_size()
facet_size = await self.get_facet_size()
for source_and_config in self.get_configs():
config = source_and_config["config"]
source = source_and_config["source"]
@ -420,7 +420,7 @@ class ArrayFacet(Facet):
facet_sql,
self.params,
truncate=False,
custom_time_limit=self.ds.setting("facet_time_limit_ms"),
custom_time_limit=await self.ds.setting("facet_time_limit_ms"),
)
facet_results_values = []
facet_results.append(
@ -491,7 +491,7 @@ class DateFacet(Facet):
suggested_facet_sql,
self.params,
truncate=False,
custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"),
custom_time_limit=await self.ds.setting("facet_suggest_time_limit_ms"),
log_sql_errors=False,
)
values = tuple(r[0] for r in results.rows)
@ -518,7 +518,7 @@ class DateFacet(Facet):
facet_results = []
facets_timed_out = []
args = dict(self.get_querystring_pairs())
facet_size = self.get_facet_size()
facet_size = await self.get_facet_size()
for source_and_config in self.get_configs():
config = source_and_config["config"]
source = source_and_config["source"]
@ -539,7 +539,7 @@ class DateFacet(Facet):
facet_sql,
self.params,
truncate=False,
custom_time_limit=self.ds.setting("facet_time_limit_ms"),
custom_time_limit=await self.ds.setting("facet_time_limit_ms"),
)
facet_results_values = []
facet_results.append(

View file

@ -10,7 +10,7 @@ class Urls:
if not isinstance(path, PrefixedUrlString):
if path.startswith("/"):
path = path[1:]
path = self.ds.setting("base_url") + path
path = self.ds._setting_sync("base_url") + path
if format is not None:
path = path_with_format(path=path, format=format)
return PrefixedUrlString(path)

View file

@ -374,7 +374,7 @@ class DataView(BaseView):
if key not in ("_labels", "_facet", "_size")
]
+ [("_size", "max")],
"settings": self.ds.settings_dict(),
"settings": await self.ds.settings_dict(),
},
}
if "metadata" not in context:
@ -385,7 +385,7 @@ class DataView(BaseView):
ttl = request.args.get("_ttl", None)
if ttl is None or not ttl.isdigit():
ttl = self.ds.setting("default_cache_ttl")
ttl = await self.ds.setting("default_cache_ttl")
return self.set_response_headers(r, ttl)
@ -428,7 +428,7 @@ async def stream_csv(datasette, fetch_data, request, database):
request = Request(new_scope, receive)
if stream:
# Some quick soundness checks
if not datasette.setting("allow_csv_stream"):
if not await datasette.setting("allow_csv_stream"):
raise BadRequest("CSV streaming is disabled")
if request.args.get("_next"):
raise BadRequest("_next not allowed for CSV streaming")
@ -477,7 +477,7 @@ async def stream_csv(datasette, fetch_data, request, database):
async def stream_fn(r):
nonlocal data, trace
limited_writer = LimitedWriter(r, datasette.setting("max_csv_mb"))
limited_writer = LimitedWriter(r, await datasette.setting("max_csv_mb"))
if trace:
await limited_writer.write(preamble)
writer = csv.writer(EscapeHtmlWriter(limited_writer))

View file

@ -183,7 +183,7 @@ class DatabaseView(View):
show_hidden=request.args.get("_show_hidden"),
editable=True,
count_limit=db.count_limit,
allow_download=datasette.setting("allow_download")
allow_download=await datasette.setting("allow_download")
and not db.is_mutable
and not db.is_memory,
attached_databases=attached_databases,
@ -390,7 +390,7 @@ async def database_download(request, datasette):
if db.is_memory:
raise DatasetteError("Cannot download in-memory databases", status=404)
if not datasette.setting("allow_download") or db.is_mutable:
if not await datasette.setting("allow_download") or db.is_mutable:
raise Forbidden("Database download is forbidden")
if not db.path:
raise DatasetteError("Cannot download database", status=404)
@ -1190,7 +1190,7 @@ async def _table_columns(datasette, database_name):
async def display_rows(datasette, database, request, rows, columns):
display_rows = []
truncate_cells = datasette.setting("truncate_cells_html")
truncate_cells = await datasette.setting("truncate_cells_html")
for row in rows:
display_row = []
for column, value in zip(columns, row):

View file

@ -620,8 +620,8 @@ class CreateTokenView(BaseView):
name = "create_token"
has_json_alternate = False
def check_permission(self, request):
if not self.ds.setting("allow_signed_tokens"):
async def check_permission(self, request):
if not await self.ds.setting("allow_signed_tokens"):
raise Forbidden("Signed tokens are not enabled for this Datasette instance")
if not request.actor:
raise Forbidden("You must be logged in to create a token")
@ -635,7 +635,7 @@ class CreateTokenView(BaseView):
)
async def shared(self, request):
self.check_permission(request)
await self.check_permission(request)
# Build list of databases and tables the user has permission to view
db_page = await self.ds.allowed_resources("view-database", request.actor)
allowed_databases = [r async for r in db_page.all()]
@ -681,13 +681,13 @@ class CreateTokenView(BaseView):
}
async def get(self, request):
self.check_permission(request)
await self.check_permission(request)
return await self.render(
["create_token.html"], request, await self.shared(request)
)
async def post(self, request):
self.check_permission(request)
await self.check_permission(request)
post = await request.post_vars()
errors = []
expires_after = None

View file

@ -196,7 +196,7 @@ async def display_columns_and_rows(
}
cell_rows = []
base_url = datasette.setting("base_url")
base_url = await datasette.setting("base_url")
for row in rows:
cells = []
# Unless we are a view, the first column is a link - either to the rowid
@ -389,7 +389,7 @@ class TableInsertView(BaseView):
return _errors(['"rows" must be a list of dictionaries'])
# Does this exceed max_insert_rows?
max_insert_rows = self.ds.setting("max_insert_rows")
max_insert_rows = await self.ds.setting("max_insert_rows")
if len(rows) > max_insert_rows:
return _errors(
["Too many rows, maximum allowed is {}".format(max_insert_rows)]
@ -771,7 +771,7 @@ async def table_view(datasette, request):
# Cache TTL header
ttl = request.args.get("_ttl", None)
if ttl is None or not ttl.isdigit():
ttl = datasette.setting("default_cache_ttl")
ttl = await datasette.setting("default_cache_ttl")
if datasette.cache_headers and response.status == 200:
ttl = int(ttl)
@ -919,11 +919,11 @@ async def table_view_traced(datasette, request):
append_querystring=append_querystring,
path_with_replaced_args=path_with_replaced_args,
fix_path=datasette.urls.path,
settings=datasette.settings_dict(),
settings=await datasette.settings_dict(),
# TODO: review up all of these hacks:
alternate_url_json=alternate_url_json,
datasette_allow_facet=(
"true" if datasette.setting("allow_facet") else "false"
"true" if await datasette.setting("allow_facet") else "false"
),
is_sortable=any(c["sortable"] for c in data["display_columns"]),
allow_execute_sql=await datasette.allowed(
@ -1377,8 +1377,8 @@ async def table_view_data(
suggested_facets = []
# Calculate suggested facets
if (
datasette.setting("suggest_facets")
and datasette.setting("allow_facet")
await datasette.setting("suggest_facets")
and await datasette.setting("allow_facet")
and not _next
and not nofacet
and not nosuggest
@ -1390,7 +1390,7 @@ async def table_view_data(
return suggested_facets
# Faceting
if not datasette.setting("allow_facet") and any(
if not await datasette.setting("allow_facet") and any(
arg.startswith("_facet") for arg in request.args
):
raise BadRequest("_facet= is not allowed")
@ -1477,7 +1477,7 @@ async def table_view_data(
results.description,
rows,
link_column=not is_view,
truncate_cells=datasette.setting("truncate_cells_html"),
truncate_cells=await datasette.setting("truncate_cells_html"),
sortable_columns=sortable_columns,
request=request,
)

View file

@ -27,7 +27,7 @@ def prepare_connection(conn, database, datasette):
def prepare_connection_args():
return 'database={}, datasette.plugin_config("name-of-plugin")={}'.format(
database, datasette.plugin_config("name-of-plugin")
database, datasette._plugin_config_sync("name-of-plugin")
)
conn.create_function("prepare_connection_args", 0, prepare_connection_args)
@ -84,7 +84,7 @@ def extra_body_script(
"template": template,
"database": database,
"table": table,
"config": datasette.plugin_config(
"config": await datasette.plugin_config(
"name-of-plugin",
database=database,
table=table,
@ -113,7 +113,7 @@ def render_cell(row, value, column, table, database, datasette, request):
"column": column,
"table": table,
"database": database,
"config": datasette.plugin_config(
"config": await datasette.plugin_config(
"name-of-plugin",
database=database,
table=table,
@ -453,8 +453,8 @@ def skip_csrf(scope):
@hookimpl
def register_actions(datasette):
extras_old = datasette.plugin_config("datasette-register-permissions") or {}
extras_new = datasette.plugin_config("datasette-register-actions") or {}
extras_old = datasette._plugin_config_sync("datasette-register-permissions") or {}
extras_new = datasette._plugin_config_sync("datasette-register-actions") or {}
actions = [
Action(

View file

@ -168,7 +168,7 @@ def table_actions(datasette, database, table, actor, request):
@hookimpl
def register_routes(datasette):
config = datasette.plugin_config("register-route-demo")
config = datasette._plugin_config_sync("register-route-demo")
if not config:
return
path = config["path"]

View file

@ -47,8 +47,9 @@ def test_sign_unsign(datasette, value, namespace):
("allow_csv_stream", True),
),
)
def test_datasette_setting(datasette, setting, expected):
assert datasette.setting(setting) == expected
@pytest.mark.asyncio
async def test_datasette_setting(datasette, setting, expected):
assert await datasette.setting(setting) == expected
@pytest.mark.asyncio

View file

@ -224,26 +224,26 @@ async def test_hook_render_cell_async(ds_client, path):
@pytest.mark.asyncio
async def test_plugin_config(ds_client):
assert {"depth": "table"} == ds_client.ds.plugin_config(
assert {"depth": "table"} == await ds_client.ds.plugin_config(
"name-of-plugin", database="fixtures", table="sortable"
)
assert {"depth": "database"} == ds_client.ds.plugin_config(
assert {"depth": "database"} == await ds_client.ds.plugin_config(
"name-of-plugin", database="fixtures", table="unknown_table"
)
assert {"depth": "database"} == ds_client.ds.plugin_config(
assert {"depth": "database"} == await ds_client.ds.plugin_config(
"name-of-plugin", database="fixtures"
)
assert {"depth": "root"} == ds_client.ds.plugin_config(
assert {"depth": "root"} == await ds_client.ds.plugin_config(
"name-of-plugin", database="unknown_database"
)
assert {"depth": "root"} == ds_client.ds.plugin_config("name-of-plugin")
assert None is ds_client.ds.plugin_config("unknown-plugin")
assert {"depth": "root"} == await ds_client.ds.plugin_config("name-of-plugin")
assert None is await ds_client.ds.plugin_config("unknown-plugin")
@pytest.mark.asyncio
async def test_plugin_config_env(ds_client, monkeypatch):
monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT")
assert ds_client.ds.plugin_config("env-plugin") == {"foo": "FROM_ENVIRONMENT"}
assert await ds_client.ds.plugin_config("env-plugin") == {"foo": "FROM_ENVIRONMENT"}
@pytest.mark.asyncio
@ -252,13 +252,13 @@ async def test_plugin_config_env_from_config(monkeypatch):
datasette = Datasette(
config={"plugins": {"env-plugin": {"setting": {"$env": "FOO_ENV"}}}}
)
assert datasette.plugin_config("env-plugin") == {"setting": "FROM_ENVIRONMENT_2"}
assert await datasette.plugin_config("env-plugin") == {"setting": "FROM_ENVIRONMENT_2"}
@pytest.mark.asyncio
async def test_plugin_config_env_from_list(ds_client):
os.environ["FOO_ENV"] = "FROM_ENVIRONMENT"
assert [{"in_a_list": "FROM_ENVIRONMENT"}] == ds_client.ds.plugin_config(
assert [{"in_a_list": "FROM_ENVIRONMENT"}] == await ds_client.ds.plugin_config(
"env-plugin-list"
)
del os.environ["FOO_ENV"]
@ -268,7 +268,7 @@ async def test_plugin_config_env_from_list(ds_client):
async def test_plugin_config_file(ds_client):
with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp:
fp.write("FROM_FILE")
assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin")
assert {"foo": "FROM_FILE"} == await ds_client.ds.plugin_config("file-plugin")
os.remove(TEMP_PLUGIN_SECRET_FILE)