mirror of
https://github.com/simonw/datasette.git
synced 2026-10-27 21:57:06 +01:00
d1dbb40362Tools for profiling Datasette's request/response cycle in-process with Python 3.15's profiling.sampling, plus the patches proposed by the study (one per optimisation, and ALL-combined.diff). The patches are not applied to the code. A report with measurements follows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01F8mEgRS8swZfYqwqrVMeo5
62 lines
2.1 KiB
Diff
62 lines
2.1 KiB
Diff
diff --git a/datasette/views/base.py b/datasette/views/base.py
|
|
index 7262edb..1d66290 100644
|
|
--- a/datasette/views/base.py
|
|
+++ b/datasette/views/base.py
|
|
@@ -18,6 +18,31 @@ from datasette.utils.asgi import (
|
|
Response,
|
|
)
|
|
|
|
+# Streamed CSV rows are collected into chunks of at least this many characters
|
|
+# so each ASGI message carries many rows rather than one
|
|
+CSV_CHUNK_SIZE = 64 * 1024
|
|
+
|
|
+
|
|
+class _ChunkedWriter:
|
|
+ def __init__(self, writer, chunk_size=CSV_CHUNK_SIZE):
|
|
+ self.writer = writer
|
|
+ self.chunk_size = chunk_size
|
|
+ self.parts = []
|
|
+ self.size = 0
|
|
+
|
|
+ async def write(self, content):
|
|
+ self.parts.append(content)
|
|
+ self.size += len(content)
|
|
+ if self.size >= self.chunk_size:
|
|
+ await self.flush()
|
|
+
|
|
+ async def flush(self):
|
|
+ chunk = "".join(self.parts)
|
|
+ self.parts = []
|
|
+ self.size = 0
|
|
+ if chunk:
|
|
+ await self.writer.write(chunk)
|
|
+
|
|
|
|
class DatasetteError(Exception):
|
|
def __init__(
|
|
@@ -239,6 +264,7 @@ async def stream_csv(datasette, fetch_data, request, database):
|
|
|
|
async def stream_fn(r):
|
|
nonlocal data, trace
|
|
+ r = _ChunkedWriter(r)
|
|
limited_writer = LimitedWriter(r, datasette.setting("max_csv_mb"))
|
|
if trace:
|
|
await limited_writer.write(preamble)
|
|
@@ -316,13 +342,17 @@ async def stream_csv(datasette, fetch_data, request, database):
|
|
else:
|
|
new_row.append(cell)
|
|
await writer.writerow(new_row)
|
|
+ # Send each page before fetching the next one
|
|
+ await r.flush()
|
|
except Exception as ex: # noqa: BLE001
|
|
# Streaming CSV: report the error into the response body and stop
|
|
sys.stderr.write(f"Caught this error: {ex}\n")
|
|
sys.stderr.flush()
|
|
await r.write(str(ex))
|
|
+ await r.flush()
|
|
return
|
|
await limited_writer.write(postamble)
|
|
+ await r.flush()
|
|
|
|
headers = {}
|
|
if datasette.cors:
|