mirror of
https://github.com/simonw/datasette.git
synced 2026-11-16 15:18:30 +01:00
Fix remaining pytest warnings (#2928)
- Close in-memory database connections, including reads opened on threads - Close completed and partial file upload file handles - Close SQLite connections and file handles owned by tests
This commit is contained in:
parent
926c6ed2cb
commit
266eaddb73
9 changed files with 340 additions and 139 deletions
|
|
@ -94,8 +94,9 @@ class Database:
|
|||
# These are used when in non-threaded mode:
|
||||
self._read_connection = None
|
||||
self._write_connection = None
|
||||
# This is used to track all file connections so they can be closed
|
||||
self._all_file_connections = []
|
||||
# Track file and memory connections, including reads on worker threads,
|
||||
# so close() can release all of them from the calling thread.
|
||||
self._all_connections = []
|
||||
if not is_temp_disk:
|
||||
self.mode = mode
|
||||
|
||||
|
|
@ -146,9 +147,12 @@ class Database:
|
|||
)
|
||||
if not write:
|
||||
conn.execute("PRAGMA query_only=1")
|
||||
self._all_connections.append(conn)
|
||||
return conn
|
||||
if self.is_memory:
|
||||
return sqlite3.connect(":memory:", uri=True)
|
||||
conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False)
|
||||
self._all_connections.append(conn)
|
||||
return conn
|
||||
|
||||
# mode=ro or immutable=1?
|
||||
if self.is_mutable:
|
||||
|
|
@ -165,7 +169,7 @@ class Database:
|
|||
conn = sqlite3.connect(
|
||||
f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
|
||||
)
|
||||
self._all_file_connections.append(conn)
|
||||
self._all_connections.append(conn)
|
||||
if self.is_temp_disk and not self._wal_enabled:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
self._wal_enabled = True
|
||||
|
|
@ -202,13 +206,13 @@ class Database:
|
|||
except Exception: # noqa: BLE001, S110
|
||||
# Shutdown teardown - a failed pending write must not block close()
|
||||
pass
|
||||
# Close anything still tracked in _all_file_connections
|
||||
for connection in self._all_file_connections:
|
||||
# Close anything still tracked in _all_connections
|
||||
for connection in self._all_connections:
|
||||
try:
|
||||
connection.close()
|
||||
except Exception: # noqa: BLE001, S110
|
||||
pass
|
||||
self._all_file_connections = []
|
||||
self._all_connections = []
|
||||
# Drop per-thread cached read connections we can reach
|
||||
try:
|
||||
delattr(connections, self._thread_local_id)
|
||||
|
|
@ -324,9 +328,9 @@ class Database:
|
|||
finally:
|
||||
isolated_connection.close()
|
||||
try:
|
||||
self._all_file_connections.remove(isolated_connection)
|
||||
self._all_connections.remove(isolated_connection)
|
||||
except ValueError:
|
||||
# Was probably a memory connection
|
||||
# May already have been cleared by close().
|
||||
pass
|
||||
|
||||
if self.ds.executor is None:
|
||||
|
|
@ -491,9 +495,9 @@ class Database:
|
|||
finally:
|
||||
isolated_connection.close()
|
||||
try:
|
||||
self._all_file_connections.remove(isolated_connection)
|
||||
self._all_connections.remove(isolated_connection)
|
||||
except ValueError:
|
||||
# Was probably a memory connection
|
||||
# May already have been cleared by close().
|
||||
pass
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Write thread must survive any task failure or the database wedges
|
||||
|
|
|
|||
|
|
@ -358,6 +358,13 @@ class MultipartParser:
|
|||
self.buffer.extend(chunk)
|
||||
self._process()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Discard completed uploads and any file still being received."""
|
||||
if self.current_file is not None:
|
||||
self.current_file.close()
|
||||
self.current_file = None
|
||||
self.form_data.close()
|
||||
|
||||
def _process(self) -> None:
|
||||
"""Process buffered data."""
|
||||
while True:
|
||||
|
|
@ -577,6 +584,9 @@ class MultipartParser:
|
|||
def _finish_part(self) -> None:
|
||||
"""Finalize current part and add to form data."""
|
||||
if self.current_name is None:
|
||||
if self.current_file is not None:
|
||||
self.current_file.close()
|
||||
self.current_file = None
|
||||
return
|
||||
|
||||
if self.current_filename is not None:
|
||||
|
|
@ -722,29 +732,50 @@ async def parse_form_data(
|
|||
batch_target = 64 * 1024
|
||||
batch = bytearray()
|
||||
|
||||
async def run_parser(fn, *args):
|
||||
# Cancellation must not close files while a worker is using them.
|
||||
task = asyncio.create_task(asyncio.to_thread(fn, *args))
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
except asyncio.CancelledError as cancelled:
|
||||
try:
|
||||
while not task.done():
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
task.result()
|
||||
finally:
|
||||
raise cancelled
|
||||
|
||||
async def flush_batch() -> None:
|
||||
if batch:
|
||||
data = bytes(batch)
|
||||
batch.clear()
|
||||
await asyncio.to_thread(parser.feed, data)
|
||||
await run_parser(parser.feed, data)
|
||||
|
||||
while True:
|
||||
message = await receive()
|
||||
message_type = message.get("type")
|
||||
if message_type == "http.disconnect":
|
||||
raise MultipartParseError("Client disconnected during request body")
|
||||
if message_type is not None and message_type != "http.request":
|
||||
continue
|
||||
chunk = message.get("body", b"")
|
||||
if chunk:
|
||||
batch.extend(chunk)
|
||||
if len(batch) >= batch_target:
|
||||
await flush_batch()
|
||||
if not message.get("more_body", False):
|
||||
break
|
||||
try:
|
||||
while True:
|
||||
message = await receive()
|
||||
message_type = message.get("type")
|
||||
if message_type == "http.disconnect":
|
||||
raise MultipartParseError("Client disconnected during request body")
|
||||
if message_type is not None and message_type != "http.request":
|
||||
continue
|
||||
chunk = message.get("body", b"")
|
||||
if chunk:
|
||||
batch.extend(chunk)
|
||||
if len(batch) >= batch_target:
|
||||
await flush_batch()
|
||||
if not message.get("more_body", False):
|
||||
break
|
||||
|
||||
await flush_batch()
|
||||
return await asyncio.to_thread(parser.finalize)
|
||||
await flush_batch()
|
||||
return await run_parser(parser.finalize)
|
||||
except BaseException:
|
||||
# No FormData is returned to the caller to take ownership on failure.
|
||||
await asyncio.to_thread(parser.close)
|
||||
raise
|
||||
|
||||
else:
|
||||
raise MultipartParseError(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue