Fix remaining pytest warnings (#2928)

- Close in-memory database connections, including reads opened on threads
- Close completed and partial file upload file handles
- Close SQLite connections and file handles owned by tests
This commit is contained in:
Simon Willison 2026-09-16 14:50:06 -07:00 • committed by GitHub
commit 266eaddb73
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 340 additions and 139 deletions

View file

@ -94,8 +94,9 @@ class Database:
# These are used when in non-threaded mode:
self._read_connection = None
self._write_connection = None
# This is used to track all file connections so they can be closed
self._all_file_connections = []
# Track file and memory connections, including reads on worker threads,
# so close() can release all of them from the calling thread.
self._all_connections = []
if not is_temp_disk:
self.mode = mode
@ -146,9 +147,12 @@ class Database:
)
if not write:
conn.execute("PRAGMA query_only=1")
self._all_connections.append(conn)
return conn
if self.is_memory:
return sqlite3.connect(":memory:", uri=True)
conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False)
self._all_connections.append(conn)
return conn
# mode=ro or immutable=1?
if self.is_mutable:
@ -165,7 +169,7 @@ class Database:
conn = sqlite3.connect(
f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
)
self._all_file_connections.append(conn)
self._all_connections.append(conn)
if self.is_temp_disk and not self._wal_enabled:
conn.execute("PRAGMA journal_mode=WAL")
self._wal_enabled = True
@ -202,13 +206,13 @@ class Database:
except Exception: # noqa: BLE001, S110
# Shutdown teardown - a failed pending write must not block close()
pass
# Close anything still tracked in _all_file_connections
for connection in self._all_file_connections:
# Close anything still tracked in _all_connections
for connection in self._all_connections:
try:
connection.close()
except Exception: # noqa: BLE001, S110
pass
self._all_file_connections = []
self._all_connections = []
# Drop per-thread cached read connections we can reach
try:
delattr(connections, self._thread_local_id)
@ -324,9 +328,9 @@ class Database:
finally:
isolated_connection.close()
try:
self._all_file_connections.remove(isolated_connection)
self._all_connections.remove(isolated_connection)
except ValueError:
# Was probably a memory connection
# May already have been cleared by close().
pass
if self.ds.executor is None:
@ -491,9 +495,9 @@ class Database:
finally:
isolated_connection.close()
try:
self._all_file_connections.remove(isolated_connection)
self._all_connections.remove(isolated_connection)
except ValueError:
# Was probably a memory connection
# May already have been cleared by close().
pass
except Exception as e: # noqa: BLE001
# Write thread must survive any task failure or the database wedges

View file

@ -358,6 +358,13 @@ class MultipartParser:
self.buffer.extend(chunk)
self._process()
def close(self) -> None:
"""Discard completed uploads and any file still being received."""
if self.current_file is not None:
self.current_file.close()
self.current_file = None
self.form_data.close()
def _process(self) -> None:
"""Process buffered data."""
while True:
@ -577,6 +584,9 @@ class MultipartParser:
def _finish_part(self) -> None:
"""Finalize current part and add to form data."""
if self.current_name is None:
if self.current_file is not None:
self.current_file.close()
self.current_file = None
return
if self.current_filename is not None:
@ -722,29 +732,50 @@ async def parse_form_data(
batch_target = 64 * 1024
batch = bytearray()
async def run_parser(fn, *args):
# Cancellation must not close files while a worker is using them.
task = asyncio.create_task(asyncio.to_thread(fn, *args))
try:
return await asyncio.shield(task)
except asyncio.CancelledError as cancelled:
try:
while not task.done():
try:
await asyncio.shield(task)
except asyncio.CancelledError:
continue
task.result()
finally:
raise cancelled
async def flush_batch() -> None:
if batch:
data = bytes(batch)
batch.clear()
await asyncio.to_thread(parser.feed, data)
await run_parser(parser.feed, data)
while True:
message = await receive()
message_type = message.get("type")
if message_type == "http.disconnect":
raise MultipartParseError("Client disconnected during request body")
if message_type is not None and message_type != "http.request":
continue
chunk = message.get("body", b"")
if chunk:
batch.extend(chunk)
if len(batch) >= batch_target:
await flush_batch()
if not message.get("more_body", False):
break
try:
while True:
message = await receive()
message_type = message.get("type")
if message_type == "http.disconnect":
raise MultipartParseError("Client disconnected during request body")
if message_type is not None and message_type != "http.request":
continue
chunk = message.get("body", b"")
if chunk:
batch.extend(chunk)
if len(batch) >= batch_target:
await flush_batch()
if not message.get("more_body", False):
break
await flush_batch()
return await asyncio.to_thread(parser.finalize)
await flush_batch()
return await run_parser(parser.finalize)
except BaseException:
# No FormData is returned to the caller to take ownership on failure.
await asyncio.to_thread(parser.close)
raise
else:
raise MultipartParseError(