Verify SQLite DBs with check_connection

datasette/app.py

@@ -7,6 +7,7 @@ import os
 import re
 import sys
 import threading
+import time
 import traceback
 import urllib.parse
 from concurrent import futures
@@ -31,6 +32,7 @@ from .utils import (
     escape_css_string,
     escape_sqlite,
     format_bytes,
+    is_valid_sqlite,
     module_from_path,
     sqlite3,
     to_css_class,
@@ -149,6 +151,7 @@ class Datasette:
     def __init__(
         self,
         files,
+        dirs=None,
         immutables=None,
         cache_headers=True,
         cors=False,
@@ -163,6 +166,7 @@ class Datasette:
         version_note=None,
     ):
         immutables = immutables or []
+        self.dirs = dirs or []
         self.files = tuple(files) + tuple(immutables)
         self.immutables = set(immutables)
         if not self.files:
@@ -182,6 +186,11 @@ class Datasette:
             if db.name in self.databases:
                 raise Exception("Multiple files with same stem: {}".format(db.name))
             self.add_database(db.name, db)
+        if dirs:
+            self.scan_dirs_thread = threading.Thread(
+                target=self.scan_dirs, name="scan-dirs", daemon=True
+            )
+            self.scan_dirs_thread.start()
         self.cache_headers = cache_headers
         self.cors = cors
         self._metadata = metadata or {}
@@ -217,6 +226,27 @@ class Datasette:
     def remove_database(self, name):
         self.databases.pop(name)
 
+    def scan_dirs(self):
+        # Recurse through self.dirs looking for new SQLite DBs. Runs in a thread.
+        while True:
+            current_filepaths = {
+                d.path for d in list(self.databases.values()) if d.path is not None
+            }
+            for dir in self.dirs:
+                for filepath in Path(dir).glob("**/*.db"):
+                    if str(filepath) in current_filepaths:
+                        continue
+                    print(filepath)
+                    if is_valid_sqlite(filepath):
+                        self.add_database(
+                            str(filepath)
+                            .replace("../", "")
+                            .replace("/", "_")
+                            .replace(".db", ""),
+                            Database(self, str(filepath), is_mutable=True),
+                        )
+            time.sleep(10)
+
     def config(self, key):
         return self._config.get(key, None)
 

datasette/cli.py

@@ -232,6 +232,13 @@ def package(
 
 @cli.command()
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
+@click.option(
+    "-d",
+    "--dir",
+    type=click.Path(exists=True),
+    help="Directories to scan for SQLite files to serve",
+    multiple=True,
+)
 @click.option(
     "-i",
     "--immutable",
@@ -310,6 +317,7 @@ def package(
 @click.option("--help-config", is_flag=True, help="Show available config options")
 def serve(
     files,
+    dir,
     immutable,
     host,
     port,
@@ -361,6 +369,7 @@ def serve(
     )
     ds = Datasette(
         files,
+        dir,
         immutables=immutable,
         cache_headers=not debug and not reload,
         cors=cors,

datasette/utils/__init__.py

@@ -588,6 +588,28 @@ def to_css_class(s):
     return "-".join(bits)
 
 
+SQLITE_MAGIC = b"SQLite format 3\x00"
+
+
+def is_valid_sqlite(path):
+    if not path.is_file():
+        return False
+    try:
+        with open(path, "rb") as fp:
+            has_magic = fp.read(len(SQLITE_MAGIC)) == SQLITE_MAGIC
+    except PermissionError:
+        return False
+    if not has_magic:
+        return False
+    # Check we can run `select * from sqlite_master`
+    try:
+        conn = sqlite3.connect(str(path))
+        check_connection(conn)
+    except Exception:
+        return False
+    return True
+
+
 def link_or_copy(src, dst):
     # Intended for use in populating a temp directory. We link if possible,
     # but fall back to copying if the temp directory is on a different device

datasette/views/index.py

@@ -23,7 +23,10 @@ class IndexView(BaseView):
 
     async def get(self, request, as_format):
         databases = []
-        for name, db in self.ds.databases.items():
+        # Using list() here because scan_dirs() running in a thread might
+        # modify self.ds.databases while we are iterating it, which could
+        # cause 'RuntimeError: OrderedDict mutated during iteration'
+        for name, db in list(self.ds.databases.items()):
             table_names = await db.table_names()
             hidden_table_names = set(await db.hidden_table_names())
             views = await db.view_names()

docs/datasette-serve-help.txt

@@ -5,6 +5,7 @@ Usage: datasette serve [OPTIONS] [FILES]...
   Serve up specified SQLite database files with a web UI
 
 Options:
+  -d, --dir PATH            Directories to scan for SQLite files to serve
   -i, --immutable PATH      Database files to open in immutable mode
   -h, --host TEXT           Host for server. Defaults to 127.0.0.1 which means only
                             connections from the local machine will be allowed. Use