First working -d based Datasette Library

Refs #417

First proof-of-concept for Datasette Library. Run like this:

    datasette -d ~/Library

Uses a new plugin hook - available_databases()

BUT... I don't think this is quite the way I want to go.

datasette/app.py

@@ -151,6 +151,7 @@ class Datasette:
         memory=False,
         config=None,
         version_note=None,
+        extra_serve_options=None,
     ):
         immutables = immutables or []
         self.files = tuple(files) + tuple(immutables)
@@ -159,7 +160,8 @@ class Datasette:
             self.files = [MEMORY]
         elif memory:
             self.files = (MEMORY,) + self.files
-        self.databases = {}
+        self.extra_serve_options = extra_serve_options or {}
+        self._databases = {}
         self.inspect_data = inspect_data
         for file in self.files:
             path = file
@@ -171,7 +173,7 @@ class Datasette:
             db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory)
             if db.name in self.databases:
                 raise Exception("Multiple files with same stem: {}".format(db.name))
-            self.databases[db.name] = db
+            self._databases[db.name] = db
         self.cache_headers = cache_headers
         self.cors = cors
         self._metadata = metadata or {}
@@ -201,6 +203,14 @@ class Datasette:
                     # Plugin already registered
                     pass
 
+    @property
+    def databases(self):
+        databases = dict(self._databases)
+        # pylint: disable=no-member
+        for pairs in pm.hook.available_databases(datasette=self):
+            databases.update(pairs)
+        return databases
+
     async def run_sanity_checks(self):
         # Only one check right now, for Spatialite
         for database_name, database in self.databases.items():
@@ -470,20 +480,7 @@ class Datasette:
         def in_thread():
             conn = getattr(connections, db_name, None)
             if not conn:
-                db = self.databases[db_name]
-                if db.is_memory:
-                    conn = sqlite3.connect(":memory:")
-                else:
-                    # mode=ro or immutable=1?
-                    if db.is_mutable:
-                        qs = "mode=ro"
-                    else:
-                        qs = "immutable=1"
-                    conn = sqlite3.connect(
-                        "file:{}?{}".format(db.path, qs),
-                        uri=True,
-                        check_same_thread=False,
-                    )
+                conn = self.databases[db_name].connect()
                 self.prepare_connection(conn)
                 setattr(connections, db_name, conn)
             return fn(conn)

datasette/cli.py

@@ -186,7 +186,7 @@ def package(
     install,
     spatialite,
     version_note,
-    **extra_metadata
+    **extra_metadata,
 ):
     "Package specified SQLite files into a new datasette Docker container"
     if not shutil.which("docker"):
@@ -220,6 +220,13 @@ def package(
         call(args)
 
 
+def extra_serve_options(serve):
+    for options in pm.hook.extra_serve_options():
+        for option in reversed(options):
+            serve = option(serve)
+    return serve
+
+
 @cli.command()
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
 @click.option(
@@ -286,6 +293,7 @@ def package(
 )
 @click.option("--version-note", help="Additional note to show on /-/versions")
 @click.option("--help-config", is_flag=True, help="Show available config options")
+@extra_serve_options
 def serve(
     files,
     immutable,
@@ -304,6 +312,7 @@ def serve(
     config,
     version_note,
     help_config,
+    **extra_serve_options,
 ):
     """Serve up specified SQLite database files with a web UI"""
     if help_config:
@@ -350,6 +359,7 @@ def serve(
         config=dict(config),
         memory=memory,
         version_note=version_note,
+        extra_serve_options=extra_serve_options,
     )
     # Run async sanity checks - but only if we're not under pytest
     asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks())

datasette/database.py

@@ -14,15 +14,19 @@ from .inspect import inspect_hash
 
 
 class Database:
-    def __init__(self, ds, path=None, is_mutable=False, is_memory=False):
+    def __init__(
+        self, ds, path=None, name=None, is_mutable=False, is_memory=False, comment=None
+    ):
         self.ds = ds
+        self._name = name
         self.path = path
         self.is_mutable = is_mutable
         self.is_memory = is_memory
         self.hash = None
         self.cached_size = None
         self.cached_table_counts = None
-        if not self.is_mutable:
+        self.comment = comment
+        if not self.is_mutable and path is not None:
             p = Path(path)
             self.hash = inspect_hash(p)
             self.cached_size = p.stat().st_size
@@ -33,9 +37,21 @@ class Database:
                     for key, value in self.ds.inspect_data[self.name]["tables"].items()
                 }
 
+    def connect(self):
+        if self.is_memory:
+            return sqlite3.connect(":memory:")
+        # mode=ro or immutable=1?
+        if self.is_mutable:
+            qs = "mode=ro"
+        else:
+            qs = "immutable=1"
+        return sqlite3.connect(
+            "file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False
+        )
+
     @property
     def size(self):
-        if self.is_memory:
+        if self.is_memory or self.path is None:
             return 0
         if self.cached_size is not None:
             return self.cached_size
@@ -71,6 +87,8 @@ class Database:
 
     @property
     def name(self):
+        if self._name:
+            return self._name
         if self.is_memory:
             return ":memory:"
         else:

datasette/hookspecs.py

@@ -58,3 +58,13 @@ def register_output_renderer(datasette):
 @hookspec
 def register_facet_classes():
     "Register Facet subclasses"
+
+
+@hookspec
+def extra_serve_options():
+    "Return list of extra click.option decorators to be applied to 'datasette serve'"
+
+
+@hookspec
+def available_databases(datasette):
+    "Return list of (name, database) pairs to be added to the available databases"

datasette/plugins.py

@@ -8,6 +8,7 @@ DEFAULT_PLUGINS = (
     "datasette.publish.now",
     "datasette.publish.cloudrun",
     "datasette.facets",
+    "datasette.serve_dir",
 )
 
 pm = pluggy.PluginManager("datasette")

datasette/serve_dir.py

@@ -0,0 +1,76 @@
+from datasette import hookimpl
+from pathlib import Path
+from .database import Database
+from .utils import escape_sqlite
+import click
+
+
+@hookimpl
+def extra_serve_options():
+    return [
+        click.option(
+            "-d",
+            "--dir",
+            type=click.Path(exists=True, file_okay=False, dir_okay=True),
+            help="Directories to scan for SQLite databases",
+            multiple=True,
+        ),
+        click.option(
+            "--scan",
+            is_flag=True,
+            help="Continually scan directories for new database files",
+        ),
+    ]
+
+
+cached_results = None
+
+
+@hookimpl
+def available_databases(datasette):
+    global cached_results
+    if cached_results is not None:
+        return cached_results
+    i = 0
+    counts = {name: 0 for name in datasette._databases}
+    results = []
+    for directory in datasette.extra_serve_options.get("dir") or []:
+        for filepath in Path(directory).glob("**/*"):
+            if is_sqlite(filepath):
+                name = filepath.stem
+                if name in counts:
+                    new_name = "{}_{}".format(name, counts[name] + 1)
+                    counts[name] += 1
+                    name = new_name
+                try:
+                    database = Database(datasette, str(filepath), comment=str(filepath))
+                    conn = database.connect()
+                    result = conn.execute(
+                        "select name from sqlite_master where type = 'table'"
+                    )
+                    table_names = [r[0] for r in result]
+                    for table_name in table_names:
+                        conn.execute(
+                            "PRAGMA table_info({});".format(escape_sqlite(table_name))
+                        )
+                except Exception as e:
+                    print("Could not open {}".format(filepath))
+                    print("  " + str(e))
+                else:
+                    results.append((name, database))
+
+    cached_results = results
+    return results
+
+
+magic = b"SQLite format 3\x00"
+
+
+def is_sqlite(path):
+    if not path.is_file():
+        return False
+    try:
+        with open(path, "rb") as fp:
+            return fp.read(len(magic)) == magic
+    except PermissionError:
+        return False

datasette/templates/index.html

@@ -11,6 +11,7 @@
 
 {% for database in databases %}
     <h2 style="padding-left: 10px; border-left: 10px solid #{{ database.color }}"><a href="{{ database.path }}">{{ database.name }}</a></h2>
+    {% if database.comment %}<p style="color: #aaa">{{ database.comment }}</p>{% endif %}
     <p>
         {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%}
         {% if database.hidden_tables_count -%}

datasette/views/base.py

@@ -203,6 +203,8 @@ class DataView(BaseView):
                 hash = None
         else:
             name = db_name
+        if "%" in name:
+            name = urllib.parse.unquote_plus(name)
         # Verify the hash
         try:
             db = self.ds.databases[name]

datasette/views/database.py

@@ -47,6 +47,7 @@ class DatabaseView(DataView):
             {
                 "database": database,
                 "size": db.size,
+                "comment": db.comment,
                 "tables": tables,
                 "hidden_count": len([t for t in tables if t["hidden"]]),
                 "views": views,

datasette/views/index.py

@@ -79,6 +79,7 @@ class IndexView(BaseView):
                 {
                     "name": name,
                     "hash": db.hash,
+                    "comment": db.comment,
                     "color": db.hash[:6]
                     if db.hash
                     else hashlib.md5(name.encode("utf8")).hexdigest()[:6],

docs/changelog.rst

@@ -51,7 +51,7 @@ Two new plugins take advantage of this hook:
 New plugin hook: extra_template_vars
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The :ref:`plugin_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github <https://github.com/simonw/datasette-auth-github>`__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 <https://github.com/simonw/datasette/issues/540>`__).
+The :ref:`plugin_hook_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github <https://github.com/simonw/datasette-auth-github>`__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 <https://github.com/simonw/datasette/issues/540>`__).
 
 Secret plugin configuration options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/datasette-serve-help.txt

@@ -23,4 +23,6 @@ Options:
                             datasette.readthedocs.io/en/latest/config.html
   --version-note TEXT       Additional note to show on /-/versions
   --help-config             Show available config options
+  -d, --dir DIRECTORY       Directories to scan for SQLite databases
+  --scan                    Continually scan directories for new database files
   --help                    Show this message and exit.

docs/plugins.rst

@@ -812,3 +812,67 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
                 await app(scope, recieve, wrapped_send)
             return add_x_databases_header
         return wrap_with_databases_header
+
+.. _plugin_hook_extra_serve_options:
+
+extra_serve_options()
+~~~~~~~~~~~~~~~~~~~~~
+
+Add extra Click options to the ``datasette serve`` command. Options you add here will be displayed in ``datasette serve --help`` and their values will be available to your plugin anywhere it can access the ``datasette`` object by reading from ``datasette.extra_serve_options``.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import click
+
+    @hookimpl
+    def extra_serve_options():
+        return [
+            click.option(
+                "--my-plugin-paths",
+                type=click.Path(exists=True, file_okay=False, dir_okay=True),
+                help="Directories to use with my-plugin",
+                multiple=True,
+            ),
+            click.option(
+                "--my-plugin-enable",
+                is_flag=True,
+                help="Enable functionality from my-plugin",
+            ),
+        ]
+
+Your other plugin hooks can then access these settings like so:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def extra_template_vars(datasette):
+        return {
+            "my_plugin_paths": datasette.extra_serve_options.get("my_plugin_paths") or []
+        }
+
+Be careful not to define an option which clashes with a Datasette default option, or with options provided by another plugin. For this reason we recommend using a common prefix for your plugin, as shown above.
+
+.. _plugin_hook_available_databases:
+
+available_databases(datasette)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Return a list of ``(name, database)`` pairs to be added to the available databases.
+
+``name`` should be a string. ``database`` should be a ``datasette.database.Database`` instance.
+
+This allows plugins to make databases available from new sources.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette.database import Database
+
+    @hookimpl
+    def available_databases(datasette):
+        return [
+            ("hardcoded_database", Database(datasette, "/mnt/hard_coded.db"))
+        ]

tests/fixtures.py

@@ -108,6 +108,7 @@ def make_app_client(
     inspect_data=None,
     static_mounts=None,
     template_dir=None,
+    extra_serve_options=None,
 ):
     with tempfile.TemporaryDirectory() as tmpdir:
         filepath = os.path.join(tmpdir, filename)
@@ -151,6 +152,7 @@ def make_app_client(
             inspect_data=inspect_data,
             static_mounts=static_mounts,
             template_dir=template_dir,
+            extra_serve_options=extra_serve_options,
         )
         ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n))))
         client = TestClient(ds.app())
@@ -215,6 +217,11 @@ def app_client_with_dot():
     yield from make_app_client(filename="fixtures.dot.db")
 
 
+@pytest.fixture(scope="session")
+def app_client_with_space():
+    yield from make_app_client(filename="fixtures with space.db")
+
+
 @pytest.fixture(scope="session")
 def app_client_with_cors():
     yield from make_app_client(cors=True)
@@ -314,6 +321,8 @@ METADATA = {
 
 PLUGIN1 = """
 from datasette import hookimpl
+from datasette.database import Database
+from datasette.utils import sqlite3
 import base64
 import pint
 import json
@@ -386,9 +395,24 @@ def extra_template_vars(template, database, table, view_name, request, datasette
     return {
         "extra_template_vars": json.dumps({
             "template": template,
-            "scope_path": request.scope["path"]
+            "scope_path": request.scope["path"],
+            "extra_serve_options": datasette.extra_serve_options,
         }, default=lambda b: b.decode("utf8"))
     }
+
+
+class SpecialDatabase(Database):
+    def connect(self):
+        db = sqlite3.connect(":memory:")
+        db.executescript("CREATE TABLE foo (id integer primary key, bar text)")
+        db.executescript("INSERT INTO foo (id, bar) VALUES (1, 'hello')")
+        return db
+
+@hookimpl
+def available_databases(datasette):
+    return [
+        ("special", SpecialDatabase(datasette, name="special")),
+    ]
 """
 
 PLUGIN2 = """

tests/test_api.py

@@ -9,6 +9,7 @@ from .fixtures import (  # noqa
     app_client_two_attached_databases_one_immutable,
     app_client_with_cors,
     app_client_with_dot,
+    app_client_with_space,
     generate_compound_rows,
     generate_sortable_rows,
     make_app_client,
@@ -23,7 +24,7 @@ def test_homepage(app_client):
     response = app_client.get("/.json")
     assert response.status == 200
     assert "application/json; charset=utf-8" == response.headers["content-type"]
-    assert response.json.keys() == {"fixtures": 0}.keys()
+    assert {"fixtures", "special"} == set(response.json.keys())
     d = response.json["fixtures"]
     assert d["name"] == "fixtures"
     assert d["tables_count"] == 24
@@ -517,19 +518,45 @@ def test_no_files_uses_memory_database(app_client_no_files):
     assert response.status == 200
     assert {
         ":memory:": {
+            "name": ":memory:",
             "hash": None,
+            "comment": None,
             "color": "f7935d",
+            "path": "/:memory:",
+            "tables_and_views_truncated": [],
+            "tables_and_views_more": False,
+            "tables_count": 0,
+            "table_rows_sum": 0,
+            "show_table_row_counts": False,
             "hidden_table_rows_sum": 0,
             "hidden_tables_count": 0,
-            "name": ":memory:",
-            "show_table_row_counts": False,
-            "path": "/:memory:",
-            "table_rows_sum": 0,
-            "tables_count": 0,
-            "tables_and_views_more": False,
-            "tables_and_views_truncated": [],
             "views_count": 0,
-        }
+        },
+        "special": {
+            "name": "special",
+            "hash": None,
+            "comment": None,
+            "color": "0bd650",
+            "path": "/special",
+            "tables_and_views_truncated": [
+                {
+                    "name": "foo",
+                    "columns": ["id", "bar"],
+                    "primary_keys": ["id"],
+                    "count": 1,
+                    "hidden": False,
+                    "fts_table": None,
+                    "num_relationships_for_sorting": 0,
+                }
+            ],
+            "tables_and_views_more": False,
+            "tables_count": 1,
+            "table_rows_sum": 1,
+            "show_table_row_counts": True,
+            "hidden_table_rows_sum": 0,
+            "hidden_tables_count": 0,
+            "views_count": 0,
+        },
     } == response.json
     # Try that SQL query
     response = app_client_no_files.get(
@@ -544,6 +571,11 @@ def test_database_page_for_database_with_dot_in_name(app_client_with_dot):
     assert 200 == response.status
 
 
+def test_database_page_for_database_with_space_in_name(app_client_with_space):
+    response = app_client_with_space.get("/fixtures%20with%20space.json")
+    assert 200 == response.status
+
+
 def test_custom_sql(app_client):
     response = app_client.get(
         "/fixtures.json?sql=select+content+from+simple_primary_key&_shape=objects"
@@ -1164,8 +1196,10 @@ def test_unit_filters(app_client):
 def test_databases_json(app_client_two_attached_databases_one_immutable):
     response = app_client_two_attached_databases_one_immutable.get("/-/databases.json")
     databases = response.json
-    assert 2 == len(databases)
-    extra_database, fixtures_database = databases
+    assert 3 == len(databases)
+    by_name = {database["name"]: database for database in databases}
+    extra_database = by_name["extra_database"]
+    fixtures_database = by_name["fixtures"]
     assert "extra_database" == extra_database["name"]
     assert None == extra_database["hash"]
     assert True == extra_database["is_mutable"]

tests/test_cli.py

@@ -9,7 +9,6 @@ def test_inspect_cli(app_client):
     runner = CliRunner()
     result = runner.invoke(cli, ["inspect", "fixtures.db"])
     data = json.loads(result.output)
-    assert ["fixtures"] == list(data.keys())
     database = data["fixtures"]
     assert "fixtures.db" == database["file"]
     assert isinstance(database["hash"], str)
@@ -28,7 +27,7 @@ def test_inspect_cli_writes_to_file(app_client):
     )
     assert 0 == result.exit_code, result.output
     data = json.load(open("foo.json"))
-    assert ["fixtures"] == list(data.keys())
+    assert {"fixtures", "special"} == set(data.keys())
 
 
 def test_serve_with_inspect_file_prepopulates_table_counts_cache():

tests/test_html.py

@@ -28,6 +28,7 @@ def test_homepage(app_client_two_attached_databases):
     assert [
         {"href": "/extra_database", "text": "extra_database"},
         {"href": "/fixtures", "text": "fixtures"},
+        {"href": "/special", "text": "special"},
     ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")]
     # The first attached database should show count text and attached tables
     h2 = soup.select("h2")[0]

tests/test_plugins.py

@@ -188,7 +188,7 @@ def test_plugins_extra_body_script(app_client, path, expected_extra_body_script)
 
 def test_plugins_asgi_wrapper(app_client):
     response = app_client.get("/fixtures")
-    assert "fixtures" == response.headers["x-databases"]
+    assert "fixtures, special" == response.headers["x-databases"]
 
 
 def test_plugins_extra_template_vars(restore_working_directory):
@@ -203,6 +203,7 @@ def test_plugins_extra_template_vars(restore_working_directory):
         assert {
             "template": "show_json.html",
             "scope_path": "/-/metadata",
+            "extra_serve_options": {},
         } == extra_template_vars
         extra_template_vars_from_awaitable = json.loads(
             Soup(response.body, "html.parser")
@@ -214,3 +215,32 @@ def test_plugins_extra_template_vars(restore_working_directory):
             "awaitable": True,
             "scope_path": "/-/metadata",
         } == extra_template_vars_from_awaitable
+
+
+def test_extra_serve_options_available_on_datasette(restore_working_directory):
+    for client in make_app_client(
+        template_dir=str(pathlib.Path(__file__).parent / "test_templates"),
+        extra_serve_options={"foo": "bar"},
+    ):
+        response = client.get("/-/metadata")
+        assert response.status == 200
+        extra_template_vars = json.loads(
+            Soup(response.body, "html.parser").select("pre.extra_template_vars")[0].text
+        )
+        assert {"foo": "bar"} == extra_template_vars["extra_serve_options"]
+
+
+def test_plugins_available_databases(app_client):
+    response = app_client.get("/-/databases.json")
+    assert 200 == response.status
+    assert {
+        "name": "special",
+        "path": None,
+        "size": 0,
+        "is_mutable": False,
+        "is_memory": False,
+        "hash": None,
+    } in response.json
+    assert [{"id": 1, "bar": "hello"}] == app_client.get(
+        "/special/foo.json?_shape=array"
+    ).json