--crossdb option for joining across databases (#1232)

* Test for cross-database join, refs #283
* Warn if --crossdb used with more than 10 DBs, refs #283
* latest.datasette.io demo of --crossdb joins, refs #283
* Show attached databases on /_memory page, refs #283
* Documentation for cross-database queries, refs #283

tests/fixtures.py

@@ -105,6 +105,7 @@ def make_app_client(
     static_mounts=None,
     template_dir=None,
     metadata=None,
+    crossdb=False,
 ):
     with tempfile.TemporaryDirectory() as tmpdir:
         filepath = os.path.join(tmpdir, filename)
@@ -149,6 +150,7 @@ def make_app_client(
             inspect_data=inspect_data,
             static_mounts=static_mounts,
             template_dir=template_dir,
+            crossdb=crossdb,
         )
         ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n))))
         yield TestClient(ds)
@@ -180,6 +182,15 @@ def app_client_two_attached_databases():
         yield client
 
 
+@pytest.fixture(scope="session")
+def app_client_two_attached_databases_crossdb_enabled():
+    with make_app_client(
+        extra_databases={"extra database.db": EXTRA_DATABASE_SQL},
+        crossdb=True,
+    ) as client:
+        yield client
+
+
 @pytest.fixture(scope="session")
 def app_client_conflicting_database_names():
     with make_app_client(
@@ -750,7 +761,12 @@ def assert_permissions_checked(datasette, actions):
     default=False,
     help="Delete and recreate database if it exists",
 )
-def cli(db_filename, metadata, plugins_path, recreate):
+@click.option(
+    "--extra-db-filename",
+    type=click.Path(file_okay=True, dir_okay=False),
+    help="Write out second test DB to this file",
+)
+def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename):
     """Write out the fixtures database used by Datasette's test suite"""
     if metadata and not metadata.endswith(".json"):
         raise click.ClickException("Metadata should end with .json")
@@ -784,6 +800,17 @@ def cli(db_filename, metadata, plugins_path, recreate):
             newpath = path / filepath.name
             newpath.write_text(filepath.open().read())
             print(f"  Wrote plugin: {newpath}")
+    if extra_db_filename:
+        if pathlib.Path(extra_db_filename).exists():
+            if not recreate:
+                raise click.ClickException(
+                    f"{extra_db_filename} already exists, use --recreate to reset it"
+                )
+            else:
+                pathlib.Path(extra_db_filename).unlink()
+        conn = sqlite3.connect(extra_db_filename)
+        conn.executescript(EXTRA_DATABASE_SQL)
+        print(f"Test tables written to {extra_db_filename}")
 
 
 if __name__ == "__main__":

tests/test_cli.py

@@ -147,6 +147,7 @@ def test_metadata_yaml():
         get=None,
         help_config=False,
         pdb=False,
+        crossdb=False,
         open_browser=False,
         create=False,
         ssl_keyfile=None,

tests/test_crossdb.py

@@ -0,0 +1,75 @@
+from datasette.cli import cli
+from click.testing import CliRunner
+import urllib
+import sqlite3
+from .fixtures import app_client_two_attached_databases_crossdb_enabled
+
+
+def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled):
+    app_client = app_client_two_attached_databases_crossdb_enabled
+    sql = """
+    select
+      'extra database' as db,
+      pk,
+      text1,
+      text2
+    from
+      [extra database].searchable
+    union all
+    select
+      'fixtures' as db,
+      pk,
+      text1,
+      text2
+    from
+      fixtures.searchable
+    """
+    response = app_client.get(
+        "/_memory.json?" + urllib.parse.urlencode({"sql": sql, "_shape": "array"})
+    )
+    assert response.status == 200
+    assert response.json == [
+        {"db": "extra database", "pk": 1, "text1": "barry cat", "text2": "terry dog"},
+        {"db": "extra database", "pk": 2, "text1": "terry dog", "text2": "sara weasel"},
+        {"db": "fixtures", "pk": 1, "text1": "barry cat", "text2": "terry dog"},
+        {"db": "fixtures", "pk": 2, "text1": "terry dog", "text2": "sara weasel"},
+    ]
+
+
+def test_crossdb_warning_if_too_many_databases(tmp_path_factory):
+    db_dir = tmp_path_factory.mktemp("dbs")
+    dbs = []
+    for i in range(11):
+        path = str(db_dir / "db_{}.db".format(i))
+        conn = sqlite3.connect(path)
+        conn.execute("vacuum")
+        dbs.append(path)
+    runner = CliRunner(mix_stderr=False)
+    result = runner.invoke(
+        cli,
+        [
+            "serve",
+            "--crossdb",
+            "--get",
+            "/",
+        ]
+        + dbs,
+        catch_exceptions=False,
+    )
+    assert (
+        "Warning: --crossdb only works with the first 10 attached databases"
+        in result.stderr
+    )
+
+
+def test_crossdb_attached_database_list_display(
+    app_client_two_attached_databases_crossdb_enabled,
+):
+    app_client = app_client_two_attached_databases_crossdb_enabled
+    response = app_client.get("/_memory")
+    for fragment in (
+        "databases are attached to this connection",
+        "<li><strong>fixtures</strong> - ",
+        "<li><strong>extra database</strong> - ",
+    ):
+        assert fragment in response.text

tests/test_internals_database.py

@@ -4,7 +4,7 @@ Tests for the datasette.database.Database class
 from datasette.database import Database, Results, MultipleValues
 from datasette.utils.sqlite import sqlite3, supports_generated_columns
 from datasette.utils import Column
-from .fixtures import app_client
+from .fixtures import app_client, app_client_two_attached_databases_crossdb_enabled
 import pytest
 import time
 import uuid
@@ -466,6 +466,15 @@ def test_is_mutable(app_client):
     assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False
 
 
+@pytest.mark.asyncio
+async def test_attached_databases(app_client_two_attached_databases_crossdb_enabled):
+    database = app_client_two_attached_databases_crossdb_enabled.ds.get_database(
+        "_memory"
+    )
+    attached = await database.attached_databases()
+    assert {a.name for a in attached} == {"extra database", "fixtures"}
+
+
 @pytest.mark.asyncio
 async def test_database_memory_name(app_client):
     ds = app_client.ds