Deploy this as a preview

* Ability to deploy additional branch demos, closes #1442
* Only run tests before deploy on main branch
* Documentation for continuous deployment

.github/workflows/deploy-latest.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - query-info
 
 jobs:
   deploy:
@@ -29,6 +30,7 @@ jobs:
         python -m pip install -e .[docs]
         python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
@@ -50,6 +52,8 @@ jobs:
       run: |-
         gcloud config set run/region us-central1
         gcloud config set project datasette-222320
+        export SUFFIX="-${GITHUB_REF#refs/heads/}"
+        export SUFFIX=${SUFFIX#-main}
         datasette publish cloudrun fixtures.db extra_database.db \
             -m fixtures.json \
             --plugins-dir=plugins \
@@ -57,7 +61,10 @@ jobs:
             --version-note=$GITHUB_SHA \
             --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \
             --install=pysqlite3-binary \
-            --service=datasette-latest
+            --service "datasette-latest$SUFFIX"
+    - name: Deploy to docs as well (only for main)
+      if: ${{ github.ref == 'refs/heads/main' }}
+      run: |-
         # Deploy docs.db to a different service
         datasette publish cloudrun docs.db \
             --branch=$GITHUB_SHA \

datasette/app.py

@@ -200,7 +200,7 @@ class Datasette:
         plugins_dir=None,
         static_mounts=None,
         memory=False,
-        config=None,
+        settings=None,
         secret=None,
         version_note=None,
         config_dir=None,
@@ -277,9 +277,9 @@ class Datasette:
         self.static_mounts = static_mounts or []
         if config_dir and (config_dir / "config.json").exists():
             raise StartupError("config.json should be renamed to settings.json")
-        if config_dir and (config_dir / "settings.json").exists() and not config:
+        if config_dir and (config_dir / "settings.json").exists() and not settings:
-            config = json.loads((config_dir / "settings.json").read_text())
+            settings = json.loads((config_dir / "settings.json").read_text())
-        self._settings = dict(DEFAULT_SETTINGS, **(config or {}))
+        self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
         self.executor = futures.ThreadPoolExecutor(
@@ -419,8 +419,8 @@ class Datasette:
     def setting(self, key):
         return self._settings.get(key, None)
 
-    def config_dict(self):
+    def settings_dict(self):
-        # Returns a fully resolved config dictionary, useful for templates
+        # Returns a fully resolved settings dictionary, useful for templates
         return {option.name: self.setting(option.name) for option in SETTINGS}
 
     def _metadata_recursive_update(self, orig, updated):

datasette/cli.py

@@ -51,7 +51,7 @@ class Config(click.ParamType):
         name, value = config.split(":", 1)
         if name not in DEFAULT_SETTINGS:
             self.fail(
-                f"{name} is not a valid option (--help-config to see all)",
+                f"{name} is not a valid option (--help-settings to see all)",
                 param,
                 ctx,
             )
@@ -84,7 +84,7 @@ class Setting(CompositeParamType):
         name, value = config
         if name not in DEFAULT_SETTINGS:
             self.fail(
-                f"{name} is not a valid option (--help-config to see all)",
+                f"{name} is not a valid option (--help-settings to see all)",
                 param,
                 ctx,
             )
@@ -408,7 +408,7 @@ def uninstall(packages, yes):
     help="Run an HTTP GET request against this path, print results and exit",
 )
 @click.option("--version-note", help="Additional note to show on /-/versions")
-@click.option("--help-config", is_flag=True, help="Show available config options")
+@click.option("--help-settings", is_flag=True, help="Show available settings")
 @click.option("--pdb", is_flag=True, help="Launch debugger on any errors")
 @click.option(
     "-o",
@@ -456,7 +456,7 @@ def serve(
     root,
     get,
     version_note,
-    help_config,
+    help_settings,
     pdb,
     open_browser,
     create,
@@ -466,9 +466,9 @@ def serve(
     return_instance=False,
 ):
     """Serve up specified SQLite database files with a web UI"""
-    if help_config:
+    if help_settings:
         formatter = formatting.HelpFormatter()
-        with formatter.section("Config options"):
+        with formatter.section("Settings"):
             formatter.write_dl(
                 [
                     (option.name, f"{option.help} (default={option.default})")
@@ -495,14 +495,14 @@ def serve(
     if metadata:
         metadata_data = parse_metadata(metadata.read())
 
-    combined_config = {}
+    combined_settings = {}
     if config:
         click.echo(
             "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead",
             err=True,
         )
-        combined_config.update(config)
+        combined_settings.update(config)
-    combined_config.update(settings)
+    combined_settings.update(settings)
 
     kwargs = dict(
         immutables=immutable,
@@ -514,7 +514,7 @@ def serve(
         template_dir=template_dir,
         plugins_dir=plugins_dir,
         static_mounts=static,
-        config=combined_config,
+        settings=combined_settings,
         memory=memory,
         secret=secret,
         version_note=version_note,

datasette/templates/query.html

@@ -67,6 +67,8 @@
     </p>
 </form>
 
+extra_column_info: {{ extra_column_info }}
+
 {% if display_rows %}
 <p class="export-links">This data as {% for name, url in renderers.items() %}<a href="{{ url }}">{{ name }}</a>{{ ", " if not loop.last }}{% endfor %}, <a href="{{ url_csv }}">CSV</a></p>
 <div class="table-wrapper"><table class="rows-and-columns">

datasette/templates/table.html

@@ -201,7 +201,7 @@
                 CSV options:
                 <label><input type="checkbox" name="_dl"> download file</label>
                 {% if expandable_columns %}<label><input type="checkbox" name="_labels" checked> expand labels</label>{% endif %}
-                {% if next_url and config.allow_csv_stream %}<label><input type="checkbox" name="_stream"> stream all rows</label>{% endif %}
+                {% if next_url and settings.allow_csv_stream %}<label><input type="checkbox" name="_stream"> stream all rows</label>{% endif %}
                 <input type="submit" value="Export CSV">
                 {% for key, value in url_csv_hidden_args %}
                     <input type="hidden" name="{{ key }}" value="{{ value }}">

datasette/utils/__init__.py

@@ -1089,3 +1089,75 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
+
+
+def columns_for_query(conn, sql, params=None):
+    """
+    Given a SQLite connection ``conn`` and a SQL query ``sql``, returns a list of
+    ``(table_name, column_name)`` pairs corresponding to the columns that would be
+    returned by that SQL query.
+
+    Each pair indicates the source table and column for the returned column, or
+    ``(None, None)`` if no table and column could be derived (e.g. for "select 1")
+    """
+    if sql.lower().strip().startswith("explain"):
+        return []
+    opcodes = conn.execute("explain " + sql, params).fetchall()
+    table_rootpage_by_register = {
+        r["p1"]: r["p2"] for r in opcodes if r["opcode"] == "OpenRead"
+    }
+    print(f"{table_rootpage_by_register=}")
+    names_and_types_by_rootpage = dict(
+        [
+            (r[0], (r[1], r[2]))
+            for r in conn.execute(
+                "select rootpage, name, type from sqlite_master where rootpage in ({})".format(
+                    ", ".join(map(str, table_rootpage_by_register.values()))
+                )
+            )
+        ]
+    )
+    print(f"{names_and_types_by_rootpage=}")
+    columns_by_column_register = {}
+    for opcode_row in opcodes:
+        if opcode_row["opcode"] in ("Rowid", "Column"):
+            addr, opcode, table_id, cid, column_register, p4, p5, comment = opcode_row
+            print(f"{table_id=} {cid=} {column_register=}")
+            table = None
+            try:
+                table = names_and_types_by_rootpage[
+                    table_rootpage_by_register[table_id]
+                ][0]
+                columns_by_column_register[column_register] = (table, cid)
+            except KeyError as e:
+                print("  KeyError")
+                print("   ", e)
+                print(
+                    "    table = names_and_types_by_rootpage[table_rootpage_by_register[table_id]][0]"
+                )
+                print(
+                    f"    {names_and_types_by_rootpage=} {table_rootpage_by_register=} {table_id=}"
+                )
+                print("    columns_by_column_register[column_register] = (table, cid)")
+                print(f"    {column_register=} = ({table=}, {cid=})")
+                pass
+    result_row = [dict(r) for r in opcodes if r["opcode"] == "ResultRow"][0]
+    result_registers = list(
+        range(result_row["p1"], result_row["p1"] + result_row["p2"])
+    )
+    print(f"{result_registers=}")
+    print(f"{columns_by_column_register=}")
+    all_column_names = {}
+    for (table, _) in names_and_types_by_rootpage.values():
+        table_xinfo = conn.execute("pragma table_xinfo({})".format(table)).fetchall()
+        for column_info in table_xinfo:
+            all_column_names[(table, column_info["cid"])] = column_info["name"]
+    print(f"{all_column_names=}")
+    final_output = []
+    for register in result_registers:
+        try:
+            table, cid = columns_by_column_register[register]
+            final_output.append((table, all_column_names[table, cid]))
+        except KeyError:
+            final_output.append((None, None))
+    return final_output

datasette/views/base.py

@@ -614,7 +614,7 @@ class DataView(BaseView):
                     ]
                     + [("_size", "max")],
                     "datasette_version": __version__,
-                    "config": self.ds.config_dict(),
+                    "settings": self.ds.settings_dict(),
                 },
             }
             if "metadata" not in context:

datasette/views/database.py

@@ -10,6 +10,7 @@ import markupsafe
 from datasette.utils import (
     await_me_maybe,
     check_visibility,
+    columns_for_query,
     derive_named_parameters,
     to_css_class,
     validate_sql_select,
@@ -248,6 +249,8 @@ class QueryView(DataView):
 
         query_error = None
 
+        extra_column_info = None
+
         # Execute query - as write or as read
         if write:
             if request.method == "POST":
@@ -334,6 +337,12 @@ class QueryView(DataView):
                     database, sql, params_for_query, truncate=True, **extra_args
                 )
                 columns = [r[0] for r in results.description]
+
+                # Try to figure out extra column information
+                db = self.ds.get_database(database)
+                extra_column_info = await db.execute_fn(
+                    lambda conn: columns_for_query(conn, sql, params_for_query)
+                )
             except sqlite3.DatabaseError as e:
                 query_error = e
                 results = None
@@ -456,12 +465,13 @@ class QueryView(DataView):
                 "canned_query": canned_query,
                 "edit_sql_url": edit_sql_url,
                 "metadata": metadata,
-                "config": self.ds.config_dict(),
+                "settings": self.ds.settings_dict(),
                 "request": request,
                 "show_hide_link": show_hide_link,
                 "show_hide_text": show_hide_text,
                 "show_hide_hidden": markupsafe.Markup(show_hide_hidden),
                 "hide_sql": hide_sql,
+                "extra_column_info": extra_column_info,
             }
 
         return (

docs/contributing.rst

@@ -202,6 +202,17 @@ For added productivity, you can use use `sphinx-autobuild <https://pypi.org/proj
 
 Now browse to ``http://localhost:8000/`` to view the documentation. Any edits you make should be instantly reflected in your browser.
 
+.. _contributing_continuous_deployment:
+
+Continuously deployed demo instances
+------------------------------------
+
+The demo instance at `latest.datasette.io <https://latest.datasette.io/>`__ is re-deployed automatically to Google Cloud Run for every push to ``main`` that passes the test suite. This is implemented by the GitHub Actions workflow at `.github/workflows/deploy-latest.yml <https://github.com/simonw/datasette/blob/main/.github/workflows/deploy-latest.yml>`__.
+
+Specific branches can also be set to automatically deploy by adding them to the ``on: push: branches`` block at the top of the workflow YAML file. Branches configured in this way will be deployed to a new Cloud Run service whether or not their tests pass.
+
+The Cloud Run URL for a branch demo can be found in the GitHub Actions logs.
+
 .. _contributing_release:
 
 Release process

docs/datasette-serve-help.txt

@@ -32,7 +32,7 @@ Options:
   --get TEXT                Run an HTTP GET request against this path, print results and
                             exit
   --version-note TEXT       Additional note to show on /-/versions
-  --help-config             Show available config options
+  --help-settings           Show available settings
   --pdb                     Launch debugger on any errors
   -o, --open                Open Datasette in your web browser
   --create                  Create database files if they do not exist

tests/fixtures.py

@@ -99,7 +99,7 @@ def make_app_client(
     max_returned_rows=None,
     cors=False,
     memory=False,
-    config=None,
+    settings=None,
     filename="fixtures.db",
     is_immutable=False,
     extra_databases=None,
@@ -129,7 +129,7 @@ def make_app_client(
                 # Insert at start to help test /-/databases ordering:
                 files.insert(0, extra_filepath)
         os.chdir(os.path.dirname(filepath))
-        config = config or {}
+        settings = settings or {}
         for key, value in {
             "default_page_size": 50,
             "max_returned_rows": max_returned_rows or 100,
@@ -138,8 +138,8 @@ def make_app_client(
             # errors when running the full test suite:
             "num_sql_threads": 1,
         }.items():
-            if key not in config:
+            if key not in settings:
-                config[key] = value
+                settings[key] = value
         ds = Datasette(
             files,
             immutables=immutables,
@@ -147,7 +147,7 @@ def make_app_client(
             cors=cors,
             metadata=metadata or METADATA,
             plugins_dir=PLUGINS_DIR,
-            config=config,
+            settings=settings,
             inspect_data=inspect_data,
             static_mounts=static_mounts,
             template_dir=template_dir,
@@ -171,7 +171,7 @@ def app_client_no_files():
 
 @pytest.fixture(scope="session")
 def app_client_base_url_prefix():
-    with make_app_client(config={"base_url": "/prefix/"}) as client:
+    with make_app_client(settings={"base_url": "/prefix/"}) as client:
         yield client
 
 
@@ -210,13 +210,13 @@ def app_client_two_attached_databases_one_immutable():
 
 @pytest.fixture(scope="session")
 def app_client_with_hash():
-    with make_app_client(config={"hash_urls": True}, is_immutable=True) as client:
+    with make_app_client(settings={"hash_urls": True}, is_immutable=True) as client:
         yield client
 
 
 @pytest.fixture(scope="session")
 def app_client_with_trace():
-    with make_app_client(config={"trace_debug": True}, is_immutable=True) as client:
+    with make_app_client(settings={"trace_debug": True}, is_immutable=True) as client:
         yield client
 
 
@@ -234,13 +234,13 @@ def app_client_returned_rows_matches_page_size():
 
 @pytest.fixture(scope="session")
 def app_client_larger_cache_size():
-    with make_app_client(config={"cache_size_kb": 2500}) as client:
+    with make_app_client(settings={"cache_size_kb": 2500}) as client:
         yield client
 
 
 @pytest.fixture(scope="session")
 def app_client_csv_max_mb_one():
-    with make_app_client(config={"max_csv_mb": 1}) as client:
+    with make_app_client(settings={"max_csv_mb": 1}) as client:
         yield client
 
 

tests/test_api.py

@@ -1711,14 +1711,14 @@ def test_suggested_facets(app_client):
 
 
 def test_allow_facet_off():
-    with make_app_client(config={"allow_facet": False}) as client:
+    with make_app_client(settings={"allow_facet": False}) as client:
         assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status
         # Should not suggest any facets either:
         assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
 
 
 def test_suggest_facets_off():
-    with make_app_client(config={"suggest_facets": False}) as client:
+    with make_app_client(settings={"suggest_facets": False}) as client:
         # Now suggested_facets should be []
         assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
 
@@ -1883,7 +1883,7 @@ def test_config_cache_size(app_client_larger_cache_size):
 
 
 def test_config_force_https_urls():
-    with make_app_client(config={"force_https_urls": True}) as client:
+    with make_app_client(settings={"force_https_urls": True}) as client:
         response = client.get("/fixtures/facetable.json?_size=3&_facet=state")
         assert response.json["next_url"].startswith("https://")
         assert response.json["facet_results"]["state"]["results"][0][
@@ -1921,7 +1921,7 @@ def test_custom_query_with_unicode_characters(app_client):
 
 @pytest.mark.parametrize("trace_debug", (True, False))
 def test_trace(trace_debug):
-    with make_app_client(config={"trace_debug": trace_debug}) as client:
+    with make_app_client(settings={"trace_debug": trace_debug}) as client:
         response = client.get("/fixtures/simple_primary_key.json?_trace=1")
         assert response.status == 200
 

tests/test_cli.py

@@ -5,6 +5,7 @@ from .fixtures import (
     EXPECTED_PLUGINS,
 )
 import asyncio
+from datasette.app import SETTINGS
 from datasette.plugins import DEFAULT_PLUGINS
 from datasette.cli import cli, serve
 from datasette.version import __version__
@@ -147,7 +148,7 @@ def test_metadata_yaml():
         root=False,
         version_note=None,
         get=None,
-        help_config=False,
+        help_settings=False,
         pdb=False,
         crossdb=False,
         open_browser=False,
@@ -291,3 +292,10 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename):
         cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))]
     )
     assert result2.exit_code == 0, result2.output
+
+
+def test_help_settings():
+    runner = CliRunner()
+    result = runner.invoke(cli, ["--help-settings"])
+    for setting in SETTINGS:
+        assert setting.name in result.output

tests/test_custom_pages.py

@@ -14,7 +14,7 @@ def custom_pages_client():
 @pytest.fixture(scope="session")
 def custom_pages_client_with_base_url():
     with make_app_client(
-        template_dir=TEST_TEMPLATE_DIRS, config={"base_url": "/prefix/"}
+        template_dir=TEST_TEMPLATE_DIRS, settings={"base_url": "/prefix/"}
     ) as client:
         yield client
 

tests/test_facets.py

@@ -351,7 +351,7 @@ async def test_json_array_with_blanks_and_nulls():
 
 @pytest.mark.asyncio
 async def test_facet_size():
-    ds = Datasette([], memory=True, config={"max_returned_rows": 50})
+    ds = Datasette([], memory=True, settings={"max_returned_rows": 50})
     db = ds.add_database(Database(ds, memory_name="test_facet_size"))
     await db.execute_write(
         "create table neighbourhoods(city text, neighbourhood text)", block=True

tests/test_html.py

@@ -214,7 +214,7 @@ def test_definition_sql(path, expected_definition_sql, app_client):
 
 
 def test_table_cell_truncation():
-    with make_app_client(config={"truncate_cells_html": 5}) as client:
+    with make_app_client(settings={"truncate_cells_html": 5}) as client:
         response = client.get("/fixtures/facetable")
         assert response.status == 200
         table = Soup(response.body, "html.parser").find("table")
@@ -239,7 +239,7 @@ def test_table_cell_truncation():
 
 
 def test_row_page_does_not_truncate():
-    with make_app_client(config={"truncate_cells_html": 5}) as client:
+    with make_app_client(settings={"truncate_cells_html": 5}) as client:
         response = client.get("/fixtures/facetable/1")
         assert response.status == 200
         table = Soup(response.body, "html.parser").find("table")
@@ -1072,7 +1072,9 @@ def test_database_download_disallowed_for_memory():
 
 
 def test_allow_download_off():
-    with make_app_client(is_immutable=True, config={"allow_download": False}) as client:
+    with make_app_client(
+        is_immutable=True, settings={"allow_download": False}
+    ) as client:
         response = client.get("/fixtures")
         soup = Soup(response.body, "html.parser")
         assert not len(soup.findAll("a", {"href": re.compile(r"\.db$")}))
@@ -1486,7 +1488,7 @@ def test_query_error(app_client):
 
 
 def test_config_template_debug_on():
-    with make_app_client(config={"template_debug": True}) as client:
+    with make_app_client(settings={"template_debug": True}) as client:
         response = client.get("/fixtures/facetable?_context=1")
         assert response.status == 200
         assert response.text.startswith("<pre>{")
@@ -1500,7 +1502,7 @@ def test_config_template_debug_off(app_client):
 
 def test_debug_context_includes_extra_template_vars():
     # https://github.com/simonw/datasette/issues/693
-    with make_app_client(config={"template_debug": True}) as client:
+    with make_app_client(settings={"template_debug": True}) as client:
         response = client.get("/fixtures/facetable?_context=1")
         # scope_path is added by PLUGIN1
         assert "scope_path" in response.text
@@ -1744,7 +1746,7 @@ def test_facet_more_links(
     expected_ellipses_url,
 ):
     with make_app_client(
-        config={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
+        settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
     ) as client:
         response = client.get(path)
         soup = Soup(response.body, "html.parser")