mirror of
https://github.com/simonw/datasette.git
synced 2026-10-19 07:47:00 +02:00
92 lines
3.8 KiB
Python
92 lines
3.8 KiB
Python
"""Exercise real HTTP sockets and WAL files under a small OS descriptor limit."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="Requires RLIMIT_NOFILE")
|
|
def test_500_wal_databases_with_http_and_writes_under_256_fds():
|
|
script = r"""
|
|
import asyncio, concurrent.futures, json, resource, socket, sqlite3, tempfile, threading, time, urllib.request
|
|
from pathlib import Path
|
|
import psutil
|
|
import uvicorn
|
|
from datasette.app import Datasette
|
|
|
|
async def main():
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
paths = []
|
|
for i in range(500):
|
|
path = str(Path(directory) / f"db{i}.db")
|
|
conn = sqlite3.connect(path)
|
|
conn.execute("pragma journal_mode=wal")
|
|
conn.execute("create table t(id integer primary key)")
|
|
conn.close()
|
|
paths.append(path)
|
|
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
resource.setrlimit(resource.RLIMIT_NOFILE, (256, hard))
|
|
ds = Datasette(paths)
|
|
await ds.invoke_startup()
|
|
process = psutil.Process()
|
|
baseline_rss = process.memory_info().rss
|
|
peak = {"fds": 0, "rss": 0, "writers": 0}
|
|
stopped = threading.Event()
|
|
def monitor():
|
|
while not stopped.wait(0.002):
|
|
peak["fds"] = max(peak["fds"], process.num_fds())
|
|
peak["rss"] = max(peak["rss"], process.memory_info().rss)
|
|
peak["writers"] = max(peak["writers"], ds._write_budget.snapshot()["writers"])
|
|
watcher = threading.Thread(target=monitor)
|
|
watcher.start()
|
|
sock = socket.socket()
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
server = uvicorn.Server(uvicorn.Config(ds.app(), log_level="error"))
|
|
thread = threading.Thread(target=lambda: server.run(sockets=[sock]))
|
|
thread.start()
|
|
try:
|
|
deadline = time.monotonic() + 10
|
|
while not server.started:
|
|
assert time.monotonic() < deadline
|
|
await asyncio.sleep(0.01)
|
|
def request(i):
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/db{i}/-/query.json?sql=select+count(*)+from+t", timeout=20) as response:
|
|
assert response.status == 200
|
|
assert json.load(response)["ok"]
|
|
def reads():
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as clients:
|
|
list(clients.map(request, range(500)))
|
|
async def writes():
|
|
for start in range(0, 500, 16):
|
|
await asyncio.gather(*(ds.get_database(f"db{i}").execute_write("insert into t values (1)") for i in range(start, min(start + 16, 500))))
|
|
await asyncio.gather(asyncio.to_thread(reads), writes())
|
|
for i in range(500):
|
|
assert (await ds.get_database(f"db{i}").execute("select count(*) from t")).single_value() == 1
|
|
assert ds._read_pool.stats["peak_open"] <= 32
|
|
assert ds._write_budget.snapshot()["stats"]["peak_writers"] <= 8
|
|
print(json.dumps({"peak": peak, "baseline_rss": baseline_rss, "writes": 500, "http_reads": 500}), flush=True)
|
|
finally:
|
|
server.should_exit = True
|
|
await asyncio.to_thread(thread.join, 15)
|
|
assert not thread.is_alive()
|
|
stopped.set()
|
|
watcher.join()
|
|
sock.close()
|
|
ds.close()
|
|
asyncio.run(main())
|
|
"""
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", script],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=90,
|
|
check=False,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
measurements = json.loads(result.stdout.strip().splitlines()[-1])
|
|
assert measurements["peak"]["fds"] < 256
|
|
assert measurements["peak"]["writers"] <= 8
|
|
print(measurements)
|