From 2c9af739c204fabbbf489e01a7695d9b579872a1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Nov 2022 22:31:11 -0800 Subject: [PATCH 01/64] Test for datasette plugin registration, refs #4 --- setup.py | 11 ++++++++++- tests/test_datasette_plugin.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/test_datasette_plugin.py diff --git a/setup.py b/setup.py index 79c2226..b1c1aee 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,15 @@ setup( "console_scripts": ["dclient = dclient.cli:cli"], }, install_requires=["click", "httpx", "appdirs"], - extras_require={"test": ["pytest", "pytest-httpx", "cogapp", "pytest-mock"]}, + extras_require={ + "test": [ + "pytest", + "pytest-asyncio", + "pytest-httpx", + "cogapp", + "pytest-mock", + "datasette", + ] + }, python_requires=">=3.7", ) diff --git a/tests/test_datasette_plugin.py b/tests/test_datasette_plugin.py new file mode 100644 index 0000000..05d3c80 --- /dev/null +++ b/tests/test_datasette_plugin.py @@ -0,0 +1,23 @@ +from click.testing import CliRunner +from datasette.app import Datasette +from datasette.cli import cli +import pytest + + +def test_plugin_help(): + runner = CliRunner() + result = runner.invoke(cli, ["client", "--help"]) + assert result.exit_code == 0 + assert "Usage: cli client" in result.output + bits = result.output.split("Commands:") + assert "alias" in bits[1] + assert "query" in bits[1] + + +@pytest.mark.asyncio +async def test_plugin_installed(): + ds = Datasette(memory=True) + await ds.invoke_startup() + response = await ds.client.get("/-/plugins.json") + assert response.status_code == 200 + assert any(p["name"] == "dclient" for p in response.json()) From aea4d3fd500f8031e573cb6f3517372475c6ab08 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Nov 2022 22:35:34 -0800 Subject: [PATCH 02/64] Test on Python 3.11, upgrade workflows --- .github/workflows/publish.yml | 15 ++++++++------- .github/workflows/test.yml | 15 ++++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 394c938..25fe3f9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,20 +4,22 @@ on: release: types: [created] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: '**/setup.py' - name: Install dependencies run: | pip install -e '.[test]' @@ -28,13 +30,12 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.11" cache: pip - cache-dependency-path: '**/setup.py' - name: Install dependencies run: | pip install setuptools wheel twine build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64cb615..1709693 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,29 +1,26 @@ name: Test -on: - push: - workflow_dispatch: +on: [push, pull_request] + +permissions: + contents: read jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - if: hashFiles('setup.py') - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: '**/setup.py' - name: Install dependencies - if: hashFiles('setup.py') run: | pip install -e '.[test]' - name: Run tests - if: hashFiles('setup.py') run: | pytest From ca301dcbe36cabc6d88a35a39c70bf18d2927df7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Nov 2022 22:59:12 -0800 Subject: [PATCH 03/64] cache-dependency-path: setup.py --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1709693..06d2822 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | pip install -e '.[test]' From d59c680ffd7de865a7217c05bd38e003ca7c074d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Nov 2022 23:00:33 -0800 Subject: [PATCH 04/64] cache-dependency-path: setup.py --- .github/workflows/publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 25fe3f9..d8c8a9f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | pip install -e '.[test]' @@ -36,6 +37,7 @@ jobs: with: python-version: "3.11" cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | pip install setuptools wheel twine build From c0d25a7512120ee54a0fd168122f4bee7e68f50a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 17 Jul 2023 14:50:10 -0700 Subject: [PATCH 05/64] Fixed failing test --- tests/test_query.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_query.py b/tests/test_query.py index d5ad08c..d565750 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -88,7 +88,9 @@ def test_aliases(mocker, tmpdir, httpx_mock): # Should have hit https://example.com/foo.json url = httpx_mock.get_request().url - assert url == "https://example.com/foo.json?sql=select+11+%2A+3&_shape=objects" + assert url.host == "example.com" + assert url.path == "/foo.json" + assert dict(url.params) == {"sql": "select 11 * 3", "_shape": "objects"} # Remove alias result = runner.invoke(cli, ["alias", "remove", "invalid"]) From 055d6a6ffc858e56dda4d43c6f906b90239a4257 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 17 Jul 2023 15:00:46 -0700 Subject: [PATCH 06/64] Swap appdirs for click.get_app_dir(), change dir from dclient to io.datasette.dclient - closes #10 --- dclient/cli.py | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 8e64414..db5b510 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,12 +1,11 @@ import click import httpx import json -import appdirs import pathlib def get_config_dir(): - return pathlib.Path(appdirs.user_config_dir("dclient")) + return pathlib.Path(click.get_app_dir("io.datasette.dclient")) @click.group() diff --git a/setup.py b/setup.py index b1c1aee..7cc7600 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ setup( "datasette": ["client = dclient.plugin"], "console_scripts": ["dclient = dclient.cli:cli"], }, - install_requires=["click", "httpx", "appdirs"], + install_requires=["click", "httpx"], extras_require={ "test": [ "pytest", From 8a205d81a3d0d6525d59d69f0f45d82c4918dc31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 17 Jul 2023 16:23:53 -0700 Subject: [PATCH 07/64] dclient auth commands Closes #3 --- README.md | 24 ++++++++ dclient/cli.py | 121 +++++++++++++++++++++++++++++++++++++---- dclient/utils.py | 42 ++++++++++++++ tests/test_cli_auth.py | 41 ++++++++++++++ tests/test_query.py | 23 +++++++- tests/test_utils.py | 67 +++++++++++++++++++++++ 6 files changed, 305 insertions(+), 13 deletions(-) create mode 100644 dclient/utils.py create mode 100644 tests/test_cli_auth.py create mode 100644 tests/test_utils.py diff --git a/README.md b/README.md index baf7308..4c23541 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,30 @@ Options: ``` +## Authentication + +`dclient` can handle API tokens for Datasette instances that require authentication. + +You can pass an API token to `query` using `-t/--token` like this: + +```bash +dclient query https://latest.datasette.io/fixtures "select * from facetable" -t dstok_mytoken +``` +You can also store tokens for a specific URL prefix. To always use `dstok_mytoken` for any URL on the `https://latest.datasette.io/` instance you can run this: +```bash +dclient auth add https://latest.datasette.io/ +``` +Then paste in the token and hit enter when prompted to do so. + +To list which URLs you have set tokens for, run the `auth list` command: +```bash +dclient auth list +``` +To delete the token for a specific URL, run `auth remove`: +```bash +dclient auth remove https://latest.datasette.io/ +``` + ## Aliases You can assign an alias to a Datasette database using the `dclient alias` command: diff --git a/dclient/cli.py b/dclient/cli.py index db5b510..ac660f4 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -2,6 +2,7 @@ import click import httpx import json import pathlib +from .utils import token_for_url def get_config_dir(): @@ -17,7 +18,8 @@ def cli(): @cli.command() @click.argument("url") @click.argument("sql") -def query(url, sql): +@click.option("--token", "-t", help="API token") +def query(url, sql, token): """ Run a SQL query against a Datasette database URL @@ -29,17 +31,50 @@ def query(url, sql): url = aliases[url] if not url.endswith(".json"): url += ".json" - response = httpx.get(url, params={"sql": sql, "_shape": "objects"}) - if response.status_code != 200 or not response.json()["ok"]: - data = response.json() + if token is None: + # Maybe there's a token in auth.json? + token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + response = httpx.get(url, params={"sql": sql, "_shape": "objects"}, headers=headers) + if response.status_code != 200: + # Is it valid JSON? + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException( + "{} status code. Response was not valid JSON".format( + response.status_code + ) + ) bits = [] if data.get("title"): bits.append(data["title"]) if data.get("error"): bits.append(data["error"]) + raise click.ClickException( + "{} status code. {}".format(response.status_code, ": ".join(bits)) + ) + + # We should have JSON now + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException("Response was not valid JSON") + # ... but it may have a {"ok": false} error + if not data.get("ok"): + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + if not bits: + bits = [json.dumps(data)] raise click.ClickException(": ".join(bits)) - else: - click.echo(json.dumps(response.json()["rows"], indent=2)) + + # Output results + click.echo(json.dumps(response.json()["rows"], indent=2)) @cli.group() @@ -60,10 +95,10 @@ def list_(_json): click.echo(f"{alias} = {url}") -@alias.command() +@alias.command(name="add") @click.argument("name") @click.argument("url") -def add(name, url): +def alias_add(name, url): "Add an alias" config_dir = get_config_dir() config_dir.mkdir(parents=True, exist_ok=True) @@ -73,9 +108,9 @@ def add(name, url): aliases_file.write_text(json.dumps(aliases, indent=4)) -@alias.command() +@alias.command(name="remove") @click.argument("name") -def remove(name): +def alias_remove(name): "Remove an alias" config_dir = get_config_dir() aliases_file = config_dir / "aliases.json" @@ -87,9 +122,75 @@ def remove(name): raise click.ClickException("No such alias") +@cli.group() +def auth(): + "Manage authentication for different instances" + + +@auth.command(name="add") +@click.argument("alias_or_url") +@click.option("--token", prompt=True, hide_input=True) +def auth_add(alias_or_url, token): + """ + Add an authentication token for an alias or URL + + Example usage: + + \b + dclient auth add https://datasette.io/content + + Paste in the token when prompted. + """ + aliases_file = get_config_dir() / "aliases.json" + aliases = _load_aliases(aliases_file) + url = alias_or_url + if alias_or_url in aliases: + url = aliases[alias_or_url] + config_dir = get_config_dir() + config_dir.mkdir(parents=True, exist_ok=True) + auth_file = config_dir / "auth.json" + auths = _load_auths(auth_file) + auths[url] = token + auth_file.write_text(json.dumps(auths, indent=4)) + + +@auth.command(name="list") +def auth_list(): + "List stored API tokens" + auths_file = get_config_dir() / "auth.json" + click.echo("Tokens file: {}".format(auths_file)) + auths = _load_auths(auths_file) + if auths: + click.echo() + for url, token in auths.items(): + click.echo("{}:\t{}..".format(url, token[:1])) + + +@auth.command(name="remove") +@click.argument("alias_or_url") +def auth_remove(alias_or_url): + "Remove the API token for an alias or URL" + config_dir = get_config_dir() + auth_file = config_dir / "auth.json" + auths = _load_auths(auth_file) + try: + del auths[alias_or_url] + auth_file.write_text(json.dumps(auths, indent=4)) + except KeyError: + raise click.ClickException("No such URL or alias") + + def _load_aliases(aliases_file): if aliases_file.exists(): aliases = json.loads(aliases_file.read_text()) else: aliases = {} return aliases + + +def _load_auths(auth_file): + if auth_file.exists(): + auths = json.loads(auth_file.read_text()) + else: + auths = {} + return auths diff --git a/dclient/utils.py b/dclient/utils.py new file mode 100644 index 0000000..0a6576d --- /dev/null +++ b/dclient/utils.py @@ -0,0 +1,42 @@ +from urllib.parse import urlparse + + +def url_matches_prefix(url: str, prefix: str) -> bool: + url_parsed = urlparse(url) + prefix_parsed = urlparse(prefix) + + if ( + url_parsed.scheme != prefix_parsed.scheme + or url_parsed.netloc != prefix_parsed.netloc + ): + return False + + if url_parsed.path == prefix_parsed.path: + return True + + url_path = url_parsed.path + + # Add '/' to the end of the paths if they don't already end with '/' + prefix_path = prefix_parsed.path + + # Special treatment for /foo.json + if url_path.endswith(".json"): + if prefix_path + ".json" == url_path: + return True + + if not prefix_path.endswith("/"): + prefix_path += "/" + + return url_path.startswith(prefix_path) + + +def token_for_url(url: str, tokens: dict) -> str: + matches = [] + for prefix_url, token in tokens.items(): + if url_matches_prefix(url, prefix_url): + matches.append((prefix_url, token)) + # Sort by length of prefix_url, so that the longest match is first + matches.sort(key=lambda x: len(x[0]), reverse=True) + if matches: + return matches[0][1] + return None diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py new file mode 100644 index 0000000..ba30e22 --- /dev/null +++ b/tests/test_cli_auth.py @@ -0,0 +1,41 @@ +from click.testing import CliRunner +from dclient.cli import cli +import pathlib +import json + + +def test_auth(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + result = runner.invoke(cli, ["auth", "list"]) + assert result.exit_code == 0 + assert result.output.startswith("Tokens file:") + # Should only have one line + assert len([line for line in result.output.split("\n") if line.strip()]) == 1 + + # Now add a token + result2 = runner.invoke(cli, ["auth", "add", "https://example.com"], input="xyz\n") + assert result2.exit_code == 0 + + # Check the tokens file + auth_file = pathlib.Path(tmpdir) / "auth.json" + assert json.loads(auth_file.read_text()) == {"https://example.com": "xyz"} + + # auth list should show that now + result3 = runner.invoke(cli, ["auth", "list"]) + assert result3.output.startswith("Tokens file:") + assert "https://example.com" in result3.output + + # Remove should fail with an incorrect URL + result4 = runner.invoke(cli, ["auth", "remove", "https://example.com/foo"]) + assert result4.exit_code == 1 + assert result4.output == "Error: No such URL or alias\n" + + # Remove should work with the correct URL + result5 = runner.invoke(cli, ["auth", "remove", "https://example.com"]) + assert result5.exit_code == 0 + assert result5.output == "" + + # Check the tokens file + auth_file = pathlib.Path(tmpdir) / "auth.json" + assert json.loads(auth_file.read_text()) == {} diff --git a/tests/test_query.py b/tests/test_query.py index d565750..552acb0 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -2,6 +2,7 @@ from click.testing import CliRunner from dclient.cli import cli import json import pathlib +import pytest def test_query_error(httpx_mock): @@ -17,10 +18,14 @@ def test_query_error(httpx_mock): runner = CliRunner() result = runner.invoke(cli, ["query", "https://example.com", "hello"]) assert result.exit_code == 1 - assert result.output == "Error: Invalid SQL: Statement must be a SELECT\n" + assert ( + result.output + == "Error: 400 status code. Invalid SQL: Statement must be a SELECT\n" + ) -def test_query(httpx_mock): +@pytest.mark.parametrize("with_token", (False, True)) +def test_query(httpx_mock, with_token): httpx_mock.add_response( json={ "ok": True, @@ -37,10 +42,22 @@ def test_query(httpx_mock): status_code=200, ) runner = CliRunner() - result = runner.invoke(cli, ["query", "https://example.com", "hello"]) + args = ["query", "https://example.com", "hello"] + if with_token: + args.append("--token") + args.append("xyz") + result = runner.invoke(cli, args) assert result.exit_code == 0 assert json.loads(result.output) == [{"5 * 2": 10}] + # Check the request + request = httpx_mock.get_request() + assert str(request.url) == "https://example.com.json?sql=hello&_shape=objects" + if with_token: + assert request.headers["authorization"] == "Bearer xyz" + else: + assert "authorization" not in request.headers + def test_aliases(mocker, tmpdir, httpx_mock): mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..81a63e8 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,67 @@ +from dclient.utils import token_for_url, url_matches_prefix +import pytest + + +@pytest.mark.parametrize( + "url,prefix_url,expected", + ( + ("https://example.com/foo/bar", "https://example.com/foo", True), + ("https://example.com/foo/bar2", "https://example.com/foo/bar", False), + ("https://example.com/foo/bar/baz", "https://example.com/foo/bar", True), + ("https://example.com/foo/bar/baz", "https://example.com/foo", True), + ("https://example.com/foo.json", "https://example.com/foo", True), + # different scheme + ( + "http://example.com/foo/bar", + "https://example.com/foo", + False, + ), + # different netloc + ( + "https://example.org/foo/bar", + "https://example.com/foo", + False, + ), + # exactly the same + ("https://example.com/foo", "https://example.com/foo", True), + # trailing '/' + ( + "https://example.com/foo/bar", + "https://example.com/foo/bar/", + False, + ), + ( + "https://example.com/foo/bar/baz", + "https://example.com/foo/bar/", + True, + ), + ), +) +def test_url_matches_prefix(url, prefix_url, expected): + assert url_matches_prefix(url, prefix_url) == expected + + +@pytest.mark.parametrize( + "url,tokens,expected", + ( + ("https://foo.com/bar", {"https://foo.com": "foo"}, "foo"), + ("https://foo.com/bar", {"https://foo.com/baz": "baz"}, None), + ( + "https://foo.com/bar", + {"https://foo.com": "foo", "https://foo.com/bar": "bar"}, + "bar", + ), + ( + "https://foo.com/bar/baz", + { + "https://foo.com": "foo", + "https://foo.com/bar/baz": "baz", + "https://foo.com/bar": "bar", + }, + "baz", + ), + ), +) +def test_token_for_url(url, tokens, expected): + # Should always return longest matching of the available options + assert token_for_url(url, tokens) == expected From 823413a9a7821699ad86f808db64ef34bcf34ee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 17 Jul 2023 16:48:50 -0700 Subject: [PATCH 08/64] Documentation on ReadTheDocs using MyST Refs #7 --- Justfile | 37 +++++++ README.md | 222 ++++---------------------------------- docs/.gitignore | 1 + docs/Makefile | 23 ++++ docs/_templates/base.html | 6 ++ docs/aliases.md | 106 ++++++++++++++++++ docs/authentication.md | 127 ++++++++++++++++++++++ docs/conf.py | 173 +++++++++++++++++++++++++++++ docs/index.md | 93 ++++++++++++++++ docs/requirements.txt | 5 + 10 files changed, 591 insertions(+), 202 deletions(-) create mode 100644 Justfile create mode 100644 docs/.gitignore create mode 100644 docs/Makefile create mode 100644 docs/_templates/base.html create mode 100644 docs/aliases.md create mode 100644 docs/authentication.md create mode 100644 docs/conf.py create mode 100644 docs/index.md create mode 100644 docs/requirements.txt diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..b3cc38a --- /dev/null +++ b/Justfile @@ -0,0 +1,37 @@ +# Run tests and linters +@default: test lint + +# Install dependencies and test dependencies +@init: + pipenv run pip install -e '.[test]' + +# Run pytest with supplied options +@test *options: + pipenv run pytest {{options}} + +# Run linters +@lint: + echo "Linters..." + echo " Black" + pipenv run black . --check + echo " cog" + pipenv run cog --check README.md docs/*.md + echo " ruff" + pipenv run ruff . + +# Rebuild docs with cog +@cog: + pipenv run cog -r docs/*.md + +# Serve live docs on localhost:8000 +@docs: cog + cd docs && pipenv run make livehtml + +# Apply Black +@black: + pipenv run black . + +# Run automatic fixes +@fix: cog + pipenv run ruff . --fix + pipenv run black . diff --git a/README.md b/README.md index 4c23541..77996fb 100644 --- a/README.md +++ b/README.md @@ -10,218 +10,36 @@ A client CLI utility for [Datasette](https://datasette.io/) instances ## Installation Install this tool using `pip`: - - pip install dclient - +```bash +pip install dclient +``` If you want to install it in the same virtual environment as Datasette (to use it as a plugin) you can instead run: - - datasette install dclient - -## Standalone v.s. plugin - -Once installed you can use this tool like so: - - dclient --help - -If you also have Datasette installed in the same environment it will register itself as a command plugin. - -This means you can run any of these commands using `datasette client` instead, like this: - - datasette client --help - datasette client query https://latest.datasette.io/fixtures "select * from facetable limit 1" - -## Running queries - -You can run SQL queries against a Datasette instance like this: - +```bash +datasette install dclient ``` -$ dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" -``` -Output: -```json -[ - { - "pk": 1, - "created": "2019-01-14 08:00:00", - "planet_int": 1, - "on_earth": 1, - "state": "CA", - "_city_id": 1, - "_neighborhood": "Mission", - "tags": "[\"tag1\", \"tag2\"]", - "complex_array": "[{\"foo\": \"bar\"}]", - "distinct_some_null": "one", - "n": "n1" - } -] -``` - -### dclient query --help - -``` -Usage: dclient query [OPTIONS] URL SQL - - Run a SQL query against a Datasette database URL - - Returns a JSON array of objects - -Options: - --help Show this message and exit. - -``` - - -## Authentication - -`dclient` can handle API tokens for Datasette instances that require authentication. - -You can pass an API token to `query` using `-t/--token` like this: +## Running a query ```bash -dclient query https://latest.datasette.io/fixtures "select * from facetable" -t dstok_mytoken -``` -You can also store tokens for a specific URL prefix. To always use `dstok_mytoken` for any URL on the `https://latest.datasette.io/` instance you can run this: -```bash -dclient auth add https://latest.datasette.io/ -``` -Then paste in the token and hit enter when prompted to do so. - -To list which URLs you have set tokens for, run the `auth list` command: -```bash -dclient auth list -``` -To delete the token for a specific URL, run `auth remove`: -```bash -dclient auth remove https://latest.datasette.io/ +dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" ``` -## Aliases +## Documentation -You can assign an alias to a Datasette database using the `dclient alias` command: - - dclient alias add content https://datasette.io/content - -You can list aliases with `dclient alias list`: - - $ dclient alias list - content = https://datasette.io/content - -Once registered, you can pass an alias to commands such as `dclient query`: - - dclient query content "select * from news limit 1" - -### dclient alias --help - - -``` -Usage: dclient alias [OPTIONS] COMMAND [ARGS]... - - Manage aliases for different instances - -Options: - --help Show this message and exit. - -Commands: - add Add an alias - list List aliases - remove Remove an alias - -``` - - -### dclient alias list --help - - -``` -Usage: dclient alias list [OPTIONS] - - List aliases - -Options: - --json Output raw JSON - --help Show this message and exit. - -``` - - -### dclient alias add --help - - -``` -Usage: dclient alias add [OPTIONS] NAME URL - - Add an alias - -Options: - --help Show this message and exit. - -``` - - -### dclient alias remove --help - - -``` -Usage: dclient alias remove [OPTIONS] NAME - - Remove an alias - -Options: - --help Show this message and exit. - -``` - +Visit **[dclient.datasette.io](https://dclient.datasette.io)** for full documentation on using this tool. ## Development To contribute to this tool, first checkout the code. Then create a new virtual environment: - - cd dclient - python -m venv venv - source venv/bin/activate - +```bash +cd dclient +python -m venv venv +source venv/bin/activate +``` Now install the dependencies and test dependencies: - - pip install -e '.[test]' - +```bash +pip install -e '.[test]' +``` To run the tests: - - pytest +```bash +pytest +``` \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..e35d885 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +_build diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..a279768 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,23 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = sqlite-utils +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +livehtml: + sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) diff --git a/docs/_templates/base.html b/docs/_templates/base.html new file mode 100644 index 0000000..47a8bc3 --- /dev/null +++ b/docs/_templates/base.html @@ -0,0 +1,6 @@ +{%- extends "!base.html" %} + +{% block site_meta %} +{{ super() }} + +{% endblock %} diff --git a/docs/aliases.md b/docs/aliases.md new file mode 100644 index 0000000..6c0697f --- /dev/null +++ b/docs/aliases.md @@ -0,0 +1,106 @@ +# Aliases + +You can assign an alias to a Datasette database using the `dclient alias` command: + + dclient alias add content https://datasette.io/content + +You can list aliases with `dclient alias list`: + + $ dclient alias list + content = https://datasette.io/content + +Once registered, you can pass an alias to commands such as `dclient query`: + + dclient query content "select * from news limit 1" + +## dclient alias --help + +``` +Usage: dclient alias [OPTIONS] COMMAND [ARGS]... + + Manage aliases for different instances + +Options: + --help Show this message and exit. + +Commands: + add Add an alias + list List aliases + remove Remove an alias + +``` + + +## dclient alias list --help + + +``` +Usage: dclient alias list [OPTIONS] + + List aliases + +Options: + --json Output raw JSON + --help Show this message and exit. + +``` + + +## dclient alias add --help + + +``` +Usage: dclient alias add [OPTIONS] NAME URL + + Add an alias + +Options: + --help Show this message and exit. + +``` + + +## dclient alias remove --help + + +``` +Usage: dclient alias remove [OPTIONS] NAME + + Remove an alias + +Options: + --help Show this message and exit. + +``` + diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000..5f0bd18 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,127 @@ +# Authentication + +`dclient` can handle API tokens for Datasette instances that require authentication. + +You can pass an API token to `query` using `-t/--token` like this: + +```bash +dclient query https://latest.datasette.io/fixtures "select * from facetable" -t dstok_mytoken +``` + +A more convenient way to handle this is to store tokens to be used with different URL prefixes. + +## Using stored tokens + +To always use `dstok_mytoken` for any URL on the `https://latest.datasette.io/` instance you can run this: +```bash +dclient auth add https://latest.datasette.io/ +``` +Then paste in the token and hit enter when prompted to do so. + +To list which URLs you have set tokens for, run the `auth list` command: +```bash +dclient auth list +``` +To delete the token for a specific URL, run `auth remove`: +```bash +dclient auth remove https://latest.datasette.io/ +``` + + +## dclient auth --help + +``` +Usage: dclient auth [OPTIONS] COMMAND [ARGS]... + + Manage authentication for different instances + +Options: + --help Show this message and exit. + +Commands: + add Add an authentication token for an alias or URL + list List stored API tokens + remove Remove the API token for an alias or URL + +``` + + +## dclient auth add --help + + +``` +Usage: dclient auth add [OPTIONS] ALIAS_OR_URL + + Add an authentication token for an alias or URL + + Example usage: + + dclient auth add https://datasette.io/content + + Paste in the token when prompted. + +Options: + --token TEXT + --help Show this message and exit. + +``` + + +## dclient auth list --help + + +``` +Usage: dclient auth list [OPTIONS] + + List stored API tokens + +Options: + --help Show this message and exit. + +``` + + +## dclient auth remove --help + + +``` +Usage: dclient auth remove [OPTIONS] ALIAS_OR_URL + + Remove the API token for an alias or URL + +Options: + --help Show this message and exit. + +``` + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..019d32f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from subprocess import PIPE, Popen + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["myst_parser", "sphinx_copybutton"] +myst_enable_extensions = ["colon_fence"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = ['.rst', '.md'] + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "dclient" +copyright = "2023, Simon Willison" +author = "Simon Willison" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True) +git_version = pipe.stdout.read().decode("utf8") + +if git_version: + version = git_version.rsplit("-", 1)[0] + release = git_version +else: + version = "" + release = "" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. + +html_theme_options = {} +html_title = "dclient" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "dclient-doc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "dclient.tex", + "dclient documentation", + "Simon Willison", + "manual", + ) +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + master_doc, + "dclient", + "dclient documentation", + [author], + 1, + ) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "dclient", + "dclient documentation", + author, + "dclient", + " A client CLI utility for Datasette instances ", + "Miscellaneous", + ) +] diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..c18e4a8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,93 @@ +# dclient + +[![PyPI](https://img.shields.io/pypi/v/dclient.svg)](https://pypi.org/project/dclient/) +[![Changelog](https://img.shields.io/github/v/release/simonw/dclient?include_prereleases&label=changelog)](https://github.com/simonw/dclient/releases) +[![Tests](https://github.com/simonw/dclient/workflows/Test/badge.svg)](https://github.com/simonw/dclient/actions?query=workflow%3ATest) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/dclient/blob/master/LICENSE) + +A client CLI utility for [Datasette](https://datasette.io/) instances + +## Installation + +Install `dclient` using `pip` (or [pipx](https://pipxproject.github.io/pipx/): + +```bash +pip install dclient +``` + +### As a Datasette plugin + +If you also have Datasette installed in the same environment it will register itself as a command plugin. + +This means you can run any of these commands using `datasette client` instead, like this: +```bash +datasette client --help +datasette client query https://latest.datasette.io/fixtures "select * from facetable limit 1" +``` +You can install it into Datasette this way using: + +```bash +datasette install dclient +``` +## Running queries + +You can run SQL queries against a Datasette instance like this: + +```bash +dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" +``` +Output: +```json +[ + { + "pk": 1, + "created": "2019-01-14 08:00:00", + "planet_int": 1, + "on_earth": 1, + "state": "CA", + "_city_id": 1, + "_neighborhood": "Mission", + "tags": "[\"tag1\", \"tag2\"]", + "complex_array": "[{\"foo\": \"bar\"}]", + "distinct_some_null": "one", + "n": "n1" + } +] +``` + +### dclient query --help + +``` +Usage: dclient query [OPTIONS] URL SQL + + Run a SQL query against a Datasette database URL + + Returns a JSON array of objects + +Options: + -t, --token TEXT API token + --help Show this message and exit. + +``` + + + +## Contents + +```{toctree} +--- +maxdepth: 3 +--- +aliases +authentication +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..df90031 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +furo==2022.06.21 +sphinx-autobuild +sphinx-copybutton +myst-parser +cogapp From 20f0e18a805eeea06f8b92f8043843222cf643b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 17 Jul 2023 16:50:43 -0700 Subject: [PATCH 09/64] Ran Black --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 019d32f..13d491a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,7 +39,7 @@ templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # The master toctree document. master_doc = "index" From 41916a7be01ea791ffe3c50222b94bbf3ce84f54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 Jul 2023 19:25:22 -0700 Subject: [PATCH 10/64] Move making queries to own page, refs #7 --- docs/index.md | 59 +++++-------------------------------------------- docs/queries.md | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 54 deletions(-) create mode 100644 docs/queries.md diff --git a/docs/index.md b/docs/index.md index c18e4a8..90f9eba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,8 +17,10 @@ pip install dclient ### As a Datasette plugin -If you also have Datasette installed in the same environment it will register itself as a command plugin. - +If you also have Datasette installed in the same environment it will register itself as a command plugin: +```bash +datasette install dclient +``` This means you can run any of these commands using `datasette client` instead, like this: ```bash datasette client --help @@ -29,58 +31,6 @@ You can install it into Datasette this way using: ```bash datasette install dclient ``` -## Running queries - -You can run SQL queries against a Datasette instance like this: - -```bash -dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" -``` -Output: -```json -[ - { - "pk": 1, - "created": "2019-01-14 08:00:00", - "planet_int": 1, - "on_earth": 1, - "state": "CA", - "_city_id": 1, - "_neighborhood": "Mission", - "tags": "[\"tag1\", \"tag2\"]", - "complex_array": "[{\"foo\": \"bar\"}]", - "distinct_some_null": "one", - "n": "n1" - } -] -``` - -### dclient query --help - -``` -Usage: dclient query [OPTIONS] URL SQL - - Run a SQL query against a Datasette database URL - - Returns a JSON array of objects - -Options: - -t, --token TEXT API token - --help Show this message and exit. - -``` - - ## Contents @@ -88,6 +38,7 @@ Options: --- maxdepth: 3 --- +queries aliases authentication ``` diff --git a/docs/queries.md b/docs/queries.md new file mode 100644 index 0000000..fa3123e --- /dev/null +++ b/docs/queries.md @@ -0,0 +1,51 @@ +# Running queries + +You can run SQL queries against a Datasette instance like this: + +```bash +dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" +``` +Output: +```json +[ + { + "pk": 1, + "created": "2019-01-14 08:00:00", + "planet_int": 1, + "on_earth": 1, + "state": "CA", + "_city_id": 1, + "_neighborhood": "Mission", + "tags": "[\"tag1\", \"tag2\"]", + "complex_array": "[{\"foo\": \"bar\"}]", + "distinct_some_null": "one", + "n": "n1" + } +] +``` + +## dclient query --help + +``` +Usage: dclient query [OPTIONS] URL SQL + + Run a SQL query against a Datasette database URL + + Returns a JSON array of objects + +Options: + -t, --token TEXT API token + --help Show this message and exit. + +``` + From c847badc4ea719ac64fd280710d87f3eb7a39b79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 16:22:39 -0700 Subject: [PATCH 11/64] dclient insert command (#13) * Move making queries to own page, refs #7 * Documentation for the dclient insert feature, refs #8 * Implemented progress bar for insert, refs #8 * --pk option * --ignore and --replace for insert, refs #8 * --batch-size option, refs #8 * First insert test, using a mock - refs #8 * insert test that exercises against an in-memory Datasette instance, refs #8 * Insert tests now cover an error case, refs #8 * Tests for --ignore and --replace, refs #8 * Tests for different formats, refs #8 * Test for --no-detect-types * Support for --encoding, refs #8 --- README.md | 7 + dclient/cli.py | 222 +++++++++++++++++++++++++++- docs/authentication.md | 2 + docs/index.md | 1 + docs/inserting.md | 71 +++++++++ docs/queries.md | 2 +- setup.py | 4 +- tests/test_insert.py | 318 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 619 insertions(+), 8 deletions(-) create mode 100644 docs/inserting.md create mode 100644 tests/test_insert.py diff --git a/README.md b/README.md index 77996fb..511095c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,13 @@ A client CLI utility for [Datasette](https://datasette.io/) instances +## Things you can do with dclient + +- Run SQL queries against Datasette and returning the results as JSON +- Run queries against authenticated Datasette instances +- Create aliases and store authentication tokens for convenient access to Datasette +- Insert data into Datasette using the [insert API](https://docs.datasette.io/en/latest/json_api.html#the-json-write-api) (Datasette 1.0 alpha or higher) + ## Installation Install this tool using `pip`: diff --git a/dclient/cli.py b/dclient/cli.py index ac660f4..49ff338 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,7 +1,11 @@ import click import httpx +import io +import itertools import json import pathlib +from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar +import sys from .utils import token_for_url @@ -16,10 +20,10 @@ def cli(): @cli.command() -@click.argument("url") +@click.argument("url_or_alias") @click.argument("sql") @click.option("--token", "-t", help="API token") -def query(url, sql, token): +def query(url_or_alias, sql, token): """ Run a SQL query against a Datasette database URL @@ -27,9 +31,11 @@ def query(url, sql, token): """ aliases_file = get_config_dir() / "aliases.json" aliases = _load_aliases(aliases_file) - if url in aliases: - url = aliases[url] - if not url.endswith(".json"): + if url_or_alias in aliases: + url = aliases[url_or_alias] + else: + url = url_or_alias + if not url_or_alias.endswith(".json"): url += ".json" if token is None: # Maybe there's a token in auth.json? @@ -77,6 +83,159 @@ def query(url, sql, token): click.echo(json.dumps(response.json()["rows"], indent=2)) +@cli.command() +@click.argument("url_or_alias") +@click.argument("table") +@click.argument( + "filepath", type=click.Path("rb", readable=True, allow_dash=True, dir_okay=False) +) +@click.option("format_csv", "--csv", is_flag=True, help="Input is CSV") +@click.option("format_tsv", "--tsv", is_flag=True, help="Input is TSV") +@click.option("format_json", "--json", is_flag=True, help="Input is JSON") +@click.option("format_nl", "--nl", is_flag=True, help="Input is newline-delimited JSON") +@click.option("--encoding", help="Character encoding for CSV/TSV") +@click.option( + "--no-detect-types", is_flag=True, help="Don't detect column types for CSV/TSV" +) +@click.option( + "--replace", is_flag=True, help="Replace rows with a matching primary key" +) +@click.option("--ignore", is_flag=True, help="Ignore rows with a matching primary key") +@click.option("--create", is_flag=True, help="Create table if it does not exist") +@click.option( + "pks", + "--pk", + multiple=True, + help="Columns to use as the primary key when creating the table", +) +@click.option( + "--batch-size", type=int, default=100, help="Send rows in batches of this size" +) +@click.option("--token", "-t", help="API token") +@click.option("--silent", is_flag=True, help="Don't output progress") +def insert( + url_or_alias, + table, + filepath, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + replace, + ignore, + create, + pks, + batch_size, + token, + silent, +): + """ + Insert data into a remote Datasette instance + + Example usage: + + \b + dclient insert \\ + https://private.datasette.cloud/data \\ + mytable data.csv --pk id --create + """ + aliases_file = get_config_dir() / "aliases.json" + aliases = _load_aliases(aliases_file) + if url_or_alias in aliases: + url = aliases[url_or_alias] + else: + url = url_or_alias + + if token is None: + token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + + format = None + if format_csv: + format = Format.CSV + elif format_tsv: + format = Format.TSV + elif format_json: + format = Format.JSON + elif format_nl: + format = Format.NL + if format is None and filepath == "-": + raise click.ClickException( + "An explicit format is required - e.g. --csv " + "- when reading from standard input" + ) + + if filepath != "-": + file_size = pathlib.Path(filepath).stat().st_size + fp = open(filepath, "rb") + else: + fp = sys.stdin.buffer + file_size = None + + try: + rows, format = rows_from_file(fp, format=format, encoding=encoding) + except Exception as ex: + raise click.ClickException(str(ex)) + + if format in (Format.JSON, Format.NL): + # Disable progress bar - it can't handle these formats + file_size = None + no_detect_types = True + + first = True + + with progressbar( + length=file_size, + label="Inserting rows", + silent=silent or (file_size is None), + show_percent=True, + ) as bar: + bytes_so_far = 0 + for batch in _batches(rows, batch_size): + if file_size is not None: + try: + bytes_consumed_so_far = fp.tell() + new_bytes = bytes_consumed_so_far - bytes_so_far + bar.update(new_bytes) + bytes_so_far += new_bytes + except ValueError: + # File has likely been closed, so fp.tell() fails + pass + types = None + if first and not no_detect_types: + # Detect types on first batch + tracker = TypeTracker() + list(tracker.wrap(batch)) + types = tracker.types + # Convert types + for row in batch: + for key, value in row.items(): + if value is None: + continue + if types[key] == "integer": + if not value: + row[key] = None + else: + row[key] = int(value) + elif types[key] == "float": + if not value: + row[key] = None + else: + row[key] = float(value) + first = False + _insert_batch( + url=url, + table=table, + batch=batch, + token=token, + create=create, + pks=pks, + replace=replace, + ignore=ignore, + ) + + @cli.group() def alias(): "Manage aliases for different instances" @@ -194,3 +353,56 @@ def _load_auths(auth_file): else: auths = {} return auths + + +def _batches(iterable, size): + iterable = iter(iterable) + while True: + batch = list(itertools.islice(iterable, size)) + if not batch: + return + yield batch + + +def _insert_batch(*, url, table, batch, token, create, pks, replace, ignore): + if create: + data = { + "table": table, + "rows": batch, + } + if replace: + data["replace"] = True + if ignore: + data["ignore"] = True + if pks: + if len(pks) == 1: + data["pk"] = pks[0] + else: + data["pks"] = pks + url = "{}/-/create".format(url) + else: + data = { + "rows": batch, + } + if replace: + data["replace"] = True + if ignore: + data["ignore"] = True + url = "{}/{}/-/insert".format(url, table) + response = httpx.post( + url, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + json=data, + timeout=40.0, + ) + if str(response.status_code)[0] != "2": + # Is there an error we can show? + if "/json" in response.headers["content-type"]: + data = response.json() + if "errors" in data: + raise click.ClickException("\n".join(data["errors"])) + response.raise_for_status() + return response.json() diff --git a/docs/authentication.md b/docs/authentication.md index 5f0bd18..40e2dd7 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -1,3 +1,5 @@ +(authentication)= + # Authentication `dclient` can handle API tokens for Datasette instances that require authentication. diff --git a/docs/index.md b/docs/index.md index 90f9eba..b330d2e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,4 +41,5 @@ maxdepth: 3 queries aliases authentication +inserting ``` diff --git a/docs/inserting.md b/docs/inserting.md new file mode 100644 index 0000000..e664a29 --- /dev/null +++ b/docs/inserting.md @@ -0,0 +1,71 @@ +# Inserting data + +The `dclient insert` command can be used to insert data from a local file directly into a Datasette instance, via the [Write API](https://docs.datasette.io/en/latest/json_api.html#the-json-write-api) introduced in the Datasette 1.0 alphas. + +First you'll need to {ref}`authenticate ` with the instance. + +To insert data from a `data.csv` file into a table called `my_table`, creating that table if it does not exist: + +```bash +dclient insert https://my-private-space.datasette.cloud/data my_table data.csv --create +``` +You can also pipe data into standard input: +```bash +curl -s 'https://api.github.com/repos/simonw/dclient/issues' | \ + dclient insert \ + https://my-private-space.datasette.cloud/data \ + issues - --create +``` + +## Supported formats + +Data can be inserted from CSV, TSV, JSON or newline-delimited JSON files. + +The format of the file will be automatically detected. You can override this by using one of the following options: + +- `--csv` +- `--tsv` +- `--json` +- `--nl` for newline-delimited JSON + +Use `--encoding ` to specify the encoding of the file. The default is `utf-8`. + +### JSON + +JSON files should be formatted like this: +```json +[ + { + "id": 1 + "column1": "value1", + "column2": "value2" + }, + { + "id": 2 + "column1": "value1", + "column2": "value2" + } +] +``` +Newline-delimited files like this: +``` +{"id": 1, "column1": "value1", "column2": "value2"} +{"id": 2, "column1": "value1", "column2": "value2"} +``` + +### CSV and TSV + +CSV and TSV files should have a header row containing the names of the columns. + +By default, `dclient` will attempt to detect the types of the different columns in the CSV and TSV files - so if a column only ever contains numeric integers it will be stored as integers in the SQLite database. + +You can disable this and have every value treated as a string using `--no-detect-types`. + +### Other options + +- `--create` - create the table if it doesn't already exist +- `--replace` - replace any rows with a matching primary key +- `--ignore` - ignore any rows with a matching existing primary key +- `--pk id` - set a primary key (for if the table is being created) + +If you use `--create` a table will be created with rows to match the columns in your uploaded data - using the correctly detected types, unless you use `--no-detect-types` in which case every column will be of type `text`. diff --git a/docs/queries.md b/docs/queries.md index fa3123e..99d5ff3 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -37,7 +37,7 @@ cog.out( ) ]]] --> ``` -Usage: dclient query [OPTIONS] URL SQL +Usage: dclient query [OPTIONS] URL_OR_ALIAS SQL Run a SQL query against a Datasette database URL diff --git a/setup.py b/setup.py index 7cc7600..048546d 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ setup( "datasette": ["client = dclient.plugin"], "console_scripts": ["dclient = dclient.cli:cli"], }, - install_requires=["click", "httpx"], + install_requires=["click", "httpx", "sqlite-utils"], extras_require={ "test": [ "pytest", @@ -39,7 +39,7 @@ setup( "pytest-httpx", "cogapp", "pytest-mock", - "datasette", + "datasette>=1.0a2", ] }, python_requires=">=3.7", diff --git a/tests/test_insert.py b/tests/test_insert.py new file mode 100644 index 0000000..096ad32 --- /dev/null +++ b/tests/test_insert.py @@ -0,0 +1,318 @@ +import asyncio +from collections import namedtuple +from click.testing import CliRunner +from datasette.app import Datasette +from dclient.cli import cli +import httpx +import json +import pathlib +import pytest + + +@pytest.fixture +def assert_all_responses_were_requested() -> bool: + return False + + +@pytest.fixture +def non_mocked_hosts(): + # This ensures httpx-mock will not affect Datasette's own + # httpx calls made in the tests by datasette.client: + return ["localhost"] + + +def test_insert_mocked(httpx_mock, tmpdir): + httpx_mock.add_response( + json={ + "ok": True, + "database": "data", + "table": "table1", + "table_url": "http://datasette.example.com/data/table1", + "table_api_url": "http://datasette.example.com/data/table1.json", + "schema": "CREATE TABLE [table1] (...)", + "row_count": 100, + } + ) + path = pathlib.Path(tmpdir) / "data.csv" + path.write_text("a,b,c\n1,2,3\n") + runner = CliRunner() + result = runner.invoke( + cli, + [ + "insert", + "https://datasette.example.com/data", + "table1", + str(path), + "--csv", + "--token", + "x", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output == "Inserting rows\n" + request = httpx_mock.get_request() + assert request.headers["authorization"] == "Bearer x" + assert json.loads(request.read()) == {"rows": [{"a": 1, "b": 2, "c": 3}]} + + +SIMPLE_CSV = "a,b,c\n1,2,3\n" +SIMPLE_TSV = "a\tb\tc\n1\t2\t3\n" +SIMPLE_JSON = json.dumps( + [ + { + "a": 1, + "b": 2, + "c": 3, + } + ] +) +SIMPLE_JSON_NL = '{"a": 1, "b": 2, "c": 3}\n' +LATIN1_CSV = ( + b"date,name,latitude,longitude\n" + b"2020-01-01,Barra da Lagoa,-27.574,-48.422\n" + b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" + b"2020-04-05,Salta,-24.793:-65.408" +) + + +InsertTest = namedtuple( + "InsertTest", + ( + "input_data", + "cmd_args", + "table_exists", + "expected_output", + "should_error", + "expected_table_json", + ), +) + + +def make_format_test(content, arg): + return InsertTest( + input_data=content, + # Using --silent to force no display of progress bar, since it won't + # be shown for the JSON formats anyway + cmd_args=["--silent", "--create"] + ([arg] if arg is not None else []), + table_exists=False, + expected_output="", + should_error=False, + expected_table_json=[{"rowid": 1, "a": 1, "b": 2, "c": 3}], + ) + + +@pytest.mark.parametrize( + "input_data,cmd_args,table_exists,expected_output,should_error,expected_table_json", + ( + # Auto-detect formats + make_format_test(SIMPLE_CSV, None), + make_format_test(SIMPLE_TSV, None), + make_format_test(SIMPLE_JSON, None), + make_format_test(SIMPLE_JSON_NL, None), + # Explicit formats + make_format_test(SIMPLE_CSV, "--csv"), + make_format_test(SIMPLE_TSV, "--tsv"), + make_format_test(SIMPLE_JSON, "--json"), + make_format_test(SIMPLE_JSON_NL, "--nl"), + # No --create option should error: + InsertTest( + input_data=SIMPLE_CSV, + cmd_args=[], + table_exists=False, + expected_output="Inserting rows\nError: Table not found: table1\n", + should_error=True, + expected_table_json=None, + ), + # --no-detect-types + InsertTest( + input_data=SIMPLE_CSV, + cmd_args=["--no-detect-types", "--create"], + table_exists=False, + expected_output="Inserting rows\n", + should_error=False, + expected_table_json=[{"rowid": 1, "a": "1", "b": "2", "c": "3"}], + ), + # --encoding - without it this should error: + InsertTest( + input_data=LATIN1_CSV, + cmd_args=["--no-detect-types", "--create", "--csv"], + table_exists=False, + expected_output="Inserting rows\n", + should_error=True, + expected_table_json=None, + ), + # --encoding - with it this should work: + InsertTest( + input_data=LATIN1_CSV, + cmd_args=[ + "--no-detect-types", + "--create", + "--encoding", + "latin-1", + "--csv", + ], + table_exists=False, + expected_output="Inserting rows\n", + should_error=False, + expected_table_json=[ + { + "rowid": 1, + "date": "2020-01-01", + "name": "Barra da Lagoa", + "latitude": "-27.574", + "longitude": "-48.422", + }, + { + "rowid": 2, + "date": "2020-03-04", + "name": "São Paulo", + "latitude": "-23.561", + "longitude": "-46.645", + }, + { + "rowid": 3, + "date": "2020-04-05", + "name": "Salta", + "latitude": "-24.793:-65.408", + "longitude": None, + }, + ], + ), + # Existing table, conflicting pk + InsertTest( + input_data=SIMPLE_CSV, + cmd_args=[], + table_exists=True, + expected_output="Inserting rows\nUNIQUE constraint failed: table1.a\nError: UNIQUE constraint failed: table1.a\n", + should_error=True, + expected_table_json=[{"a": 1, "b": 2, "c": 3}, {"a": 4, "b": 5, "c": 6}], + ), + # Existing table, --replace + InsertTest( + input_data="a,b,c\n1,2,4\n", + cmd_args=["--replace"], + table_exists=True, + expected_output="Inserting rows\n", + should_error=False, + expected_table_json=[{"a": 1, "b": 2, "c": 4}, {"a": 4, "b": 5, "c": 6}], + ), + # Existing table, --ignore + InsertTest( + input_data="a,b,c\n1,2,4\n", + cmd_args=["--ignore"], + table_exists=True, + expected_output="Inserting rows\n", + should_error=False, + expected_table_json=[{"a": 1, "b": 2, "c": 3}, {"a": 4, "b": 5, "c": 6}], + ), + ), +) +def test_insert_against_datasette( + httpx_mock, + tmpdir, + input_data, + cmd_args, + table_exists, + expected_output, + should_error, + expected_table_json, +): + ds = Datasette( + metadata={ + "permissions": { + "create-table": {"id": "*"}, + "insert-row": {"id": "*"}, + "update-row": {"id": "*"}, + } + } + ) + db = ds.add_memory_database("data") + loop = asyncio.get_event_loop() + + # Drop all tables in the database each time, because in-memory + # databases persist in between test runs + drop_all_tables(db, loop) + + if table_exists: + + async def run_table_exists(): + await db.execute_write( + "create table table1 (a integer primary key, b integer, c integer)" + ) + await db.execute_write( + "insert into table1 (a, b, c) values (1, 2, 3), (4, 5, 6)" + ) + + loop.run_until_complete(run_table_exists()) + + token = ds.create_token("actor") + + # These are useful with pytest --pdb to see what happened + datasette_requests = [] + datasette_responses = [] + + def custom_response(request: httpx.Request): + # Need to run this in async loop, because dclient itself uses + # sync HTTPX and not async HTTPX + async def run(): + datasette_requests.append(request) + response = await ds.client.request( + request.method, + request.url.path, + json=json.loads(request.read()), + headers=request.headers, + ) + # Create a fresh response to avoid an error where stream has been consumed + response = httpx.Response( + status_code=response.status_code, + headers=response.headers, + content=response.content, + ) + datasette_responses.append(response) + return response + + return loop.run_until_complete(run()) + + httpx_mock.add_callback(custom_response) + + path = pathlib.Path(tmpdir) / "data.txt" + if isinstance(input_data, str): + path.write_text(input_data) + else: + path.write_bytes(input_data) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "insert", + "http://datasette.example.com/data", + "table1", + str(path), + "--token", + token, + ] + + cmd_args, + ) + if not should_error: + assert result.exit_code == 0 + else: + assert result.exit_code != 0 + assert result.output == expected_output + + if expected_table_json: + + async def fetch_table(): + response = await ds.client.get("/data/table1.json?_shape=array") + return response + + response = loop.run_until_complete(fetch_table()) + assert response.json() == expected_table_json + + +def drop_all_tables(db, loop): + async def run(): + for table in await db.table_names(): + await db.execute_write("drop table {}".format(table)) + + loop.run_until_complete(run()) From 6b7297205f86802d949f02925e499893dda6bf60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 16:23:30 -0700 Subject: [PATCH 12/64] Much of the functionality requires Datasette 1.0a2 or higher --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 511095c..135b175 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ [![Tests](https://github.com/simonw/dclient/workflows/Test/badge.svg)](https://github.com/simonw/dclient/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/dclient/blob/master/LICENSE) -A client CLI utility for [Datasette](https://datasette.io/) instances +A client CLI utility for [Datasette](https://datasette.io/) instances. + +Much of the functionality requires Datasette 1.0a2 or higher. ## Things you can do with dclient @@ -49,4 +51,4 @@ pip install -e '.[test]' To run the tests: ```bash pytest -``` \ No newline at end of file +``` From 829f27b8d1a0f6605fda914d7a716a8973256bfb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 16:24:05 -0700 Subject: [PATCH 13/64] Release 0.2 Refs #8, #13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 048546d..de07043 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup import os -VERSION = "0.1a2" +VERSION = "0.2" def get_long_description(): From ce625925405fd1d30461d8a83279ac0768a15af9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 21:33:46 -0700 Subject: [PATCH 14/64] Demonstrate aliases directly in the README --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 135b175..ec1925a 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,14 @@ datasette install dclient ```bash dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" ``` - +To shorten that, create an alias: +```bash +dclient alias add fixtures https://latest.datasette.io/fixtures +``` +Then run it like this instead: +```bash +dclient query fixtures "select * from facetable limit 1" +``` ## Documentation Visit **[dclient.datasette.io](https://dclient.datasette.io)** for full documentation on using this tool. From a17b7f6e7d66fc1b407d21feb58eeb5ae0fcd530 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 21:35:38 -0700 Subject: [PATCH 15/64] Add dclient insert --help to insert docs, refs #8 --- docs/inserting.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/inserting.md b/docs/inserting.md index e664a29..e544aa6 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -69,3 +69,46 @@ You can disable this and have every value treated as a string using `--no-detect - `--pk id` - set a primary key (for if the table is being created) If you use `--create` a table will be created with rows to match the columns in your uploaded data - using the correctly detected types, unless you use `--no-detect-types` in which case every column will be of type `text`. + +## dclient insert --help + +``` +Usage: dclient insert [OPTIONS] URL_OR_ALIAS TABLE FILEPATH + + Insert data into a remote Datasette instance + + Example usage: + + dclient insert \ + https://private.datasette.cloud/data \ + mytable data.csv --pk id --create + +Options: + --csv Input is CSV + --tsv Input is TSV + --json Input is JSON + --nl Input is newline-delimited JSON + --encoding TEXT Character encoding for CSV/TSV + --no-detect-types Don't detect column types for CSV/TSV + --replace Replace rows with a matching primary key + --ignore Ignore rows with a matching primary key + --create Create table if it does not exist + --pk TEXT Columns to use as the primary key when creating the + table + --batch-size INTEGER Send rows in batches of this size + -t, --token TEXT API token + --silent Don't output progress + --help Show this message and exit. + +``` + From 66a3cdd3bee51ad894a549c352bcdfc6d2cb855f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 21:52:33 -0700 Subject: [PATCH 16/64] Remove query -t shortcut - so I can use it for table later, refs #16 --- dclient/cli.py | 2 +- docs/queries.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 49ff338..baebde8 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -22,7 +22,7 @@ def cli(): @cli.command() @click.argument("url_or_alias") @click.argument("sql") -@click.option("--token", "-t", help="API token") +@click.option("--token", help="API token") def query(url_or_alias, sql, token): """ Run a SQL query against a Datasette database URL diff --git a/docs/queries.md b/docs/queries.md index 99d5ff3..6ce1504 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -44,8 +44,8 @@ Usage: dclient query [OPTIONS] URL_OR_ALIAS SQL Returns a JSON array of objects Options: - -t, --token TEXT API token - --help Show this message and exit. + --token TEXT API token + --help Show this message and exit. ``` From 59129824e352b6b6242217cf289624b8faafeb65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Feb 2024 19:56:56 -0800 Subject: [PATCH 17/64] Update for Datasette 1.0 alphas to pass tests --- tests/test_insert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_insert.py b/tests/test_insert.py index 096ad32..293a025 100644 --- a/tests/test_insert.py +++ b/tests/test_insert.py @@ -219,7 +219,7 @@ def test_insert_against_datasette( expected_table_json, ): ds = Datasette( - metadata={ + config={ "permissions": { "create-table": {"id": "*"}, "insert-row": {"id": "*"}, From 43b685078040601461896e228a2b295ca89bd374 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Feb 2024 17:24:51 -0800 Subject: [PATCH 18/64] --interval option, refs #19 --- dclient/cli.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index baebde8..87d7bea 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,11 +1,10 @@ import click import httpx -import io -import itertools import json import pathlib from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar import sys +import time from .utils import token_for_url @@ -111,6 +110,9 @@ def query(url_or_alias, sql, token): @click.option( "--batch-size", type=int, default=100, help="Send rows in batches of this size" ) +@click.option( + "--interval", type=float, default=None, help="Send batch at least every X seconds" +) @click.option("--token", "-t", help="API token") @click.option("--silent", is_flag=True, help="Don't output progress") def insert( @@ -128,6 +130,7 @@ def insert( create, pks, batch_size, + interval, token, silent, ): @@ -192,7 +195,7 @@ def insert( show_percent=True, ) as bar: bytes_so_far = 0 - for batch in _batches(rows, batch_size): + for batch in _batches(rows, batch_size, interval=interval): if file_size is not None: try: bytes_consumed_so_far = fp.tell() @@ -355,13 +358,22 @@ def _load_auths(auth_file): return auths -def _batches(iterable, size): +def _batches(iterable, size, interval=None): iterable = iter(iterable) + last_yield_time = time.time() while True: - batch = list(itertools.islice(iterable, size)) + batch = [] + for _ in range(size): + try: + batch.append(next(iterable)) + except StopIteration: + break + if interval is not None and time.time() - last_yield_time >= interval: + break if not batch: return yield batch + last_yield_time = time.time() def _insert_batch(*, url, table, batch, token, create, pks, replace, ignore): From 37423fe77281162bf9002c3a6ea1995d83dba65b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Feb 2024 19:58:29 -0800 Subject: [PATCH 19/64] Drop Python 3.7, add 3.12 --- .github/workflows/publish.yml | 12 ++++++------ .github/workflows/test.yml | 6 +++--- setup.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d8c8a9f..dd58c96 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,11 +12,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip @@ -31,11 +31,11 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" cache: pip cache-dependency-path: setup.py - name: Install dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06d2822..27792b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,11 +10,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/setup.py b/setup.py index de07043..4bc9486 100644 --- a/setup.py +++ b/setup.py @@ -42,5 +42,5 @@ setup( "datasette>=1.0a2", ] }, - python_requires=">=3.7", + python_requires=">=3.8", ) From 5a0a81b8da9b09308d0073db2de231014401ecf7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 24 Feb 2024 17:44:06 -0800 Subject: [PATCH 20/64] Create .readthedocs.yaml --- .readthedocs.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..070a74c --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,17 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +formats: + - pdf + - epub + +python: + install: + - requirements: docs/requirements.txt From 44d7e2bf9e60222f57da2545c7bb0fb8b0120768 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 11:05:13 -0800 Subject: [PATCH 21/64] Docs for streaming and --interval, closes #21 --- dclient/cli.py | 2 +- docs/inserting.md | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 87d7bea..009fb9e 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -111,7 +111,7 @@ def query(url_or_alias, sql, token): "--batch-size", type=int, default=100, help="Send rows in batches of this size" ) @click.option( - "--interval", type=float, default=None, help="Send batch at least every X seconds" + "--interval", type=float, default=10, help="Send batch at least every X seconds" ) @click.option("--token", "-t", help="API token") @click.option("--silent", is_flag=True, help="Don't output progress") diff --git a/docs/inserting.md b/docs/inserting.md index e544aa6..fb01d47 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -7,7 +7,9 @@ First you'll need to {ref}`authenticate ` with the instance. To insert data from a `data.csv` file into a table called `my_table`, creating that table if it does not exist: ```bash -dclient insert https://my-private-space.datasette.cloud/data my_table data.csv --create +dclient insert \ + https://my-private-space.datasette.cloud/data \ + my_table data.csv --create ``` You can also pipe data into standard input: ```bash @@ -17,6 +19,26 @@ curl -s 'https://api.github.com/repos/simonw/dclient/issues' | \ issues - --create ``` +## Streaming data + +`dclient insert` works for streaming data as well. + +If you have a log file containing newline-delimited JSON you can tail it and send it to a Datasette instance like this: +```bash +tail -f log.jsonl | \ + dclient insert https://my-private-space.datasette.cloud/data logs - --nl +``` +When reading from standard input (filename `-`) you are required to specify the format. In this example that's `--nl` for newline-delimited JSON. `--csv` and `--tsv` are supported for streaming as well, but `--json` is not. + +In streaming mode records default to being sent to the server every 100 records or every 10 seconds, whichever comes first. You can adjust these values using the `--batch-size` and `--interval` settings. For example, here's how to send every 10 records or if 5 seconds has passed since the last time data was sent to the server: + +```bash +tail -f log.jsonl | dclient insert \ + https://my-private-space.datasette.cloud/data logs - --nl --create \ + --batch-size 10 \ + --interval 5 +``` + ## Supported formats Data can be inserted from CSV, TSV, JSON or newline-delimited JSON files. @@ -106,6 +128,7 @@ Options: --pk TEXT Columns to use as the primary key when creating the table --batch-size INTEGER Send rows in batches of this size + --interval FLOAT Send batch at least every X seconds -t, --token TEXT API token --silent Don't output progress --help Show this message and exit. From 8c82f9a57254420ca1994bfa3346b83e38543d7c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 11:10:53 -0800 Subject: [PATCH 22/64] Rename 'datasette client' to 'datasette dt', closes #18 Refs https://github.com/simonw/datasette/issues/2153#issuecomment-1691763427 --- dclient/plugin.py | 2 +- docs/index.md | 6 +++--- tests/test_datasette_plugin.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dclient/plugin.py b/dclient/plugin.py index 7990c7f..1915482 100644 --- a/dclient/plugin.py +++ b/dclient/plugin.py @@ -5,4 +5,4 @@ from datasette import hookimpl def register_commands(cli): from .cli import cli as dclient_cli - cli.add_command(dclient_cli, name="client") + cli.add_command(dclient_cli, name="dc") diff --git a/docs/index.md b/docs/index.md index b330d2e..00897bd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,10 +21,10 @@ If you also have Datasette installed in the same environment it will register it ```bash datasette install dclient ``` -This means you can run any of these commands using `datasette client` instead, like this: +This means you can run any of these commands using `datasette dt` instead, like this: ```bash -datasette client --help -datasette client query https://latest.datasette.io/fixtures "select * from facetable limit 1" +datasette dc --help +datasette dc query https://latest.datasette.io/fixtures "select * from facetable limit 1" ``` You can install it into Datasette this way using: diff --git a/tests/test_datasette_plugin.py b/tests/test_datasette_plugin.py index 05d3c80..ec74d2d 100644 --- a/tests/test_datasette_plugin.py +++ b/tests/test_datasette_plugin.py @@ -6,9 +6,9 @@ import pytest def test_plugin_help(): runner = CliRunner() - result = runner.invoke(cli, ["client", "--help"]) + result = runner.invoke(cli, ["dc", "--help"]) assert result.exit_code == 0 - assert "Usage: cli client" in result.output + assert "Usage: cli dc" in result.output bits = result.output.split("Commands:") assert "alias" in bits[1] assert "query" in bits[1] From f0058258b22afc3b795cebda344884ce83a00d31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 11:13:35 -0800 Subject: [PATCH 23/64] cog --check as part of CI --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27792b1..bbd465f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,3 +25,6 @@ jobs: - name: Run tests run: | pytest + - name: Check if cog needs to run + run: | + cog --check docs/*.md From 081aeb32a284937669e293b34f61614b1af79c8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 11:24:46 -0800 Subject: [PATCH 24/64] Example usage in --help, closes #22 --- dclient/cli.py | 56 ++++++++++++++++++++++++++++++++++++++---- docs/aliases.md | 16 ++++++++++++ docs/authentication.md | 8 ++++++ docs/queries.md | 6 +++++ 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 009fb9e..04f4681 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -27,6 +27,13 @@ def query(url_or_alias, sql, token): Run a SQL query against a Datasette database URL Returns a JSON array of objects + + Example usage: + + \b + dclient query \\ + https://datasette.io/content \\ + 'select * from news limit 10' """ aliases_file = get_config_dir() / "aliases.json" aliases = _load_aliases(aliases_file) @@ -247,7 +254,14 @@ def alias(): @alias.command(name="list") @click.option("_json", "--json", is_flag=True, help="Output raw JSON") def list_(_json): - "List aliases" + """ + List aliases + + Example usage: + + \b + dclient aliases list + """ aliases_file = get_config_dir() / "aliases.json" aliases = _load_aliases(aliases_file) if _json: @@ -261,7 +275,18 @@ def list_(_json): @click.argument("name") @click.argument("url") def alias_add(name, url): - "Add an alias" + """ + Add an alias + + Example usage: + + \b + dclient alias add content https://datasette.io/content + + Then: + + dclient query content 'select * from news limit 3' + """ config_dir = get_config_dir() config_dir.mkdir(parents=True, exist_ok=True) aliases_file = config_dir / "aliases.json" @@ -273,7 +298,14 @@ def alias_add(name, url): @alias.command(name="remove") @click.argument("name") def alias_remove(name): - "Remove an alias" + """ + Remove an alias + + Example usage: + + \b + dclient alias remove content + """ config_dir = get_config_dir() aliases_file = config_dir / "aliases.json" aliases = _load_aliases(aliases_file) @@ -318,7 +350,14 @@ def auth_add(alias_or_url, token): @auth.command(name="list") def auth_list(): - "List stored API tokens" + """ + List stored API tokens + + Example usage: + + \b + dclient auth list + """ auths_file = get_config_dir() / "auth.json" click.echo("Tokens file: {}".format(auths_file)) auths = _load_auths(auths_file) @@ -331,7 +370,14 @@ def auth_list(): @auth.command(name="remove") @click.argument("alias_or_url") def auth_remove(alias_or_url): - "Remove the API token for an alias or URL" + """ + Remove the API token for an alias or URL + + Example usage: + + \b + dclient auth remove https://datasette.io/content + """ config_dir = get_config_dir() auth_file = config_dir / "auth.json" auths = _load_auths(auth_file) diff --git a/docs/aliases.md b/docs/aliases.md index 6c0697f..3bb6f7c 100644 --- a/docs/aliases.md +++ b/docs/aliases.md @@ -56,6 +56,10 @@ Usage: dclient alias list [OPTIONS] List aliases + Example usage: + + dclient aliases list + Options: --json Output raw JSON --help Show this message and exit. @@ -78,6 +82,14 @@ Usage: dclient alias add [OPTIONS] NAME URL Add an alias + Example usage: + + dclient alias add content https://datasette.io/content + + Then: + + dclient query content 'select * from news limit 3' + Options: --help Show this message and exit. @@ -99,6 +111,10 @@ Usage: dclient alias remove [OPTIONS] NAME Remove an alias + Example usage: + + dclient alias remove content + Options: --help Show this message and exit. diff --git a/docs/authentication.md b/docs/authentication.md index 40e2dd7..e360fa0 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -101,6 +101,10 @@ Usage: dclient auth list [OPTIONS] List stored API tokens + Example usage: + + dclient auth list + Options: --help Show this message and exit. @@ -122,6 +126,10 @@ Usage: dclient auth remove [OPTIONS] ALIAS_OR_URL Remove the API token for an alias or URL + Example usage: + + dclient auth remove https://datasette.io/content + Options: --help Show this message and exit. diff --git a/docs/queries.md b/docs/queries.md index 6ce1504..7dfd295 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -43,6 +43,12 @@ Usage: dclient query [OPTIONS] URL_OR_ALIAS SQL Returns a JSON array of objects + Example usage: + + dclient query \ + https://datasette.io/content \ + 'select * from news limit 10' + Options: --token TEXT API token --help Show this message and exit. From 5303067da391c9b9f3d43ee09dcc99c88b7b152a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 11:36:27 -0800 Subject: [PATCH 25/64] Release 0.3 Refs #8, #16, #18, #19, #21, #22 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4bc9486..309569e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup import os -VERSION = "0.2" +VERSION = "0.3" def get_long_description(): From 0e40a80c84910189b5b6e605ca441c6d35183da6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 11:38:22 -0800 Subject: [PATCH 26/64] Deploy future packages with PyPI trusted actions --- .github/workflows/publish.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dd58c96..c56f512 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,11 +40,9 @@ jobs: cache-dependency-path: setup.py - name: Install dependencies run: | - pip install setuptools wheel twine build - - name: Publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + pip install setuptools wheel build + - name: Build run: | python -m build - twine upload dist/* + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 From adfa6cce14a18c21d136c1425ca94863365d3041 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 20:49:54 -0800 Subject: [PATCH 27/64] insert --alter support, closes #23 --- dclient/cli.py | 9 ++++++++- docs/inserting.md | 2 ++ tests/test_insert.py | 22 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/dclient/cli.py b/dclient/cli.py index 04f4681..fc9323f 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -108,6 +108,7 @@ def query(url_or_alias, sql, token): ) @click.option("--ignore", is_flag=True, help="Ignore rows with a matching primary key") @click.option("--create", is_flag=True, help="Create table if it does not exist") +@click.option("--alter", is_flag=True, help="Alter table to add any missing columns") @click.option( "pks", "--pk", @@ -135,6 +136,7 @@ def insert( replace, ignore, create, + alter, pks, batch_size, interval, @@ -240,6 +242,7 @@ def insert( batch=batch, token=token, create=create, + alter=alter, pks=pks, replace=replace, ignore=ignore, @@ -422,7 +425,7 @@ def _batches(iterable, size, interval=None): last_yield_time = time.time() -def _insert_batch(*, url, table, batch, token, create, pks, replace, ignore): +def _insert_batch(*, url, table, batch, token, create, alter, pks, replace, ignore): if create: data = { "table": table, @@ -432,6 +435,8 @@ def _insert_batch(*, url, table, batch, token, create, pks, replace, ignore): data["replace"] = True if ignore: data["ignore"] = True + if alter: + data["alter"] = True if pks: if len(pks) == 1: data["pk"] = pks[0] @@ -446,6 +451,8 @@ def _insert_batch(*, url, table, batch, token, create, pks, replace, ignore): data["replace"] = True if ignore: data["ignore"] = True + if alter: + data["alter"] = True url = "{}/{}/-/insert".format(url, table) response = httpx.post( url, diff --git a/docs/inserting.md b/docs/inserting.md index fb01d47..af6bb97 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -88,6 +88,7 @@ You can disable this and have every value treated as a string using `--no-detect - `--create` - create the table if it doesn't already exist - `--replace` - replace any rows with a matching primary key - `--ignore` - ignore any rows with a matching existing primary key +- `--alter` - alter table to add any columns that are missing - `--pk id` - set a primary key (for if the table is being created) If you use `--create` a table will be created with rows to match the columns in your uploaded data - using the correctly detected types, unless you use `--no-detect-types` in which case every column will be of type `text`. @@ -125,6 +126,7 @@ Options: --replace Replace rows with a matching primary key --ignore Ignore rows with a matching primary key --create Create table if it does not exist + --alter Alter table to add any missing columns --pk TEXT Columns to use as the primary key when creating the table --batch-size INTEGER Send rows in batches of this size diff --git a/tests/test_insert.py b/tests/test_insert.py index 293a025..c849a67 100644 --- a/tests/test_insert.py +++ b/tests/test_insert.py @@ -206,6 +206,27 @@ def make_format_test(content, arg): should_error=False, expected_table_json=[{"a": 1, "b": 2, "c": 3}, {"a": 4, "b": 5, "c": 6}], ), + # Existing table without alter should fail + InsertTest( + input_data="a,b,c,d\n1,2,4,5\n", + cmd_args=["--ignore"], + table_exists=True, + expected_output="Inserting rows\nError: Row 0 has invalid columns: d\n", + should_error=True, + expected_table_json=[{"a": 1, "b": 2, "c": 3}, {"a": 4, "b": 5, "c": 6}], + ), + # Existing table with --alter should work + InsertTest( + input_data="a,b,c,d\n1,2,4,5\n", + cmd_args=["--replace", "--alter"], + table_exists=True, + expected_output="Inserting rows\n", + should_error=False, + expected_table_json=[ + {"a": 1, "b": 2, "c": 4, "d": 5}, + {"a": 4, "b": 5, "c": 6, "d": None}, + ], + ), ), ) def test_insert_against_datasette( @@ -224,6 +245,7 @@ def test_insert_against_datasette( "create-table": {"id": "*"}, "insert-row": {"id": "*"}, "update-row": {"id": "*"}, + "alter-table": {"id": "*"}, } } ) From 20851f6ed653e401cb8e382d099edabb78460c3f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 20:52:15 -0800 Subject: [PATCH 28/64] -v/--verbose option, closes #25 --- dclient/cli.py | 28 ++++++++++++++++++++++++---- docs/inserting.md | 1 + docs/queries.md | 5 +++-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index fc9323f..aae4910 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -4,8 +4,10 @@ import json import pathlib from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar import sys +import textwrap import time from .utils import token_for_url +import urllib def get_config_dir(): @@ -22,7 +24,8 @@ def cli(): @click.argument("url_or_alias") @click.argument("sql") @click.option("--token", help="API token") -def query(url_or_alias, sql, token): +@click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") +def query(url_or_alias, sql, token, verbose): """ Run a SQL query against a Datasette database URL @@ -49,7 +52,11 @@ def query(url_or_alias, sql, token): headers = {} if token: headers["Authorization"] = f"Bearer {token}" - response = httpx.get(url, params={"sql": sql, "_shape": "objects"}, headers=headers) + params = {"sql": sql, "_shape": "objects"} + if verbose: + click.echo(url + "?" + urllib.parse.urlencode(params), err=True) + response = httpx.get(url, params=params, headers=headers) + if response.status_code != 200: # Is it valid JSON? try: @@ -123,6 +130,7 @@ def query(url_or_alias, sql, token): ) @click.option("--token", "-t", help="API token") @click.option("--silent", is_flag=True, help="Don't output progress") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request and response") def insert( url_or_alias, table, @@ -142,6 +150,7 @@ def insert( interval, token, silent, + verbose, ): """ Insert data into a remote Datasette instance @@ -246,6 +255,7 @@ def insert( pks=pks, replace=replace, ignore=ignore, + verbose=verbose, ) @@ -425,7 +435,9 @@ def _batches(iterable, size, interval=None): last_yield_time = time.time() -def _insert_batch(*, url, table, batch, token, create, alter, pks, replace, ignore): +def _insert_batch( + *, url, table, batch, token, create, alter, pks, replace, ignore, verbose +): if create: data = { "table": table, @@ -454,6 +466,9 @@ def _insert_batch(*, url, table, batch, token, create, alter, pks, replace, igno if alter: data["alter"] = True url = "{}/{}/-/insert".format(url, table) + if verbose: + click.echo("POST {}".format(url), err=True) + click.echo(textwrap.indent(json.dumps(data, indent=2), " "), err=True) response = httpx.post( url, headers={ @@ -463,6 +478,8 @@ def _insert_batch(*, url, table, batch, token, create, alter, pks, replace, igno json=data, timeout=40.0, ) + if verbose: + click.echo(str(response), err=True) if str(response.status_code)[0] != "2": # Is there an error we can show? if "/json" in response.headers["content-type"]: @@ -470,4 +487,7 @@ def _insert_batch(*, url, table, batch, token, create, alter, pks, replace, igno if "errors" in data: raise click.ClickException("\n".join(data["errors"])) response.raise_for_status() - return response.json() + response_data = response.json() + if verbose: + click.echo(textwrap.indent(json.dumps(response_data, indent=2), " "), err=True) + return response_data diff --git a/docs/inserting.md b/docs/inserting.md index af6bb97..316a3c0 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -133,6 +133,7 @@ Options: --interval FLOAT Send batch at least every X seconds -t, --token TEXT API token --silent Don't output progress + -v, --verbose Verbose output: show HTTP request and response --help Show this message and exit. ``` diff --git a/docs/queries.md b/docs/queries.md index 7dfd295..3bd671a 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -50,8 +50,9 @@ Usage: dclient query [OPTIONS] URL_OR_ALIAS SQL 'select * from news limit 10' Options: - --token TEXT API token - --help Show this message and exit. + --token TEXT API token + -v, --verbose Verbose output: show HTTP request + --help Show this message and exit. ``` From ea3dc64fb9c4c55daf2d1f5ed292c24612d25c0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 21:08:15 -0800 Subject: [PATCH 29/64] dclient actor command, closes #26 --- dclient/cli.py | 42 +++++++++++++++++++++++++++++++++++++++++- docs/authentication.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/dclient/cli.py b/dclient/cli.py index aae4910..a8b3376 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -130,7 +130,12 @@ def query(url_or_alias, sql, token, verbose): ) @click.option("--token", "-t", help="API token") @click.option("--silent", is_flag=True, help="Don't output progress") -@click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request and response") +@click.option( + "-v", + "--verbose", + is_flag=True, + help="Verbose output: show HTTP request and response", +) def insert( url_or_alias, table, @@ -259,6 +264,41 @@ def insert( ) +@cli.command() +@click.argument("url_or_alias") +@click.option("--token", help="API token") +def actor(url_or_alias, token): + """ + Show the actor represented by an API token + + Example usage: + + \b + dclient actor https://latest.datasette.io/fixtures + """ + aliases_file = get_config_dir() / "aliases.json" + aliases = _load_aliases(aliases_file) + if url_or_alias in aliases: + url = aliases[url_or_alias] + else: + url = url_or_alias + + if not (url.startswith("http://") or url.startswith("https://")): + raise click.ClickException("Invalid URL: " + url) + + if token is None: + token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + + url_bits = url.split("/") + url_bits[-1] = "-/actor.json" + actor_url = "/".join(url_bits) + response = httpx.get( + actor_url, headers={"Authorization": "Bearer {}".format(token)}, timeout=40.0 + ) + response.raise_for_status() + click.echo(json.dumps(response.json(), indent=4)) + + @cli.group() def alias(): "Manage aliases for different instances" diff --git a/docs/authentication.md b/docs/authentication.md index e360fa0..4e5f22f 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -28,7 +28,21 @@ To delete the token for a specific URL, run `auth remove`: ```bash dclient auth remove https://latest.datasette.io/ ``` +## Testing a token +The `dclient actor` command can be used to test a token, retrieving the actor that the token represents. +```bash +dclient actor https://latest.datasette.io/content +``` +The output looks like this: +```json +{ + "actor": { + "id": "root", + "token": "dstok" + } +} +``` ## dclient auth --help + +## dclient actor --help + + +``` +Usage: dclient actor [OPTIONS] URL_OR_ALIAS + + Show the actor represented by an API token + + Example usage: + + dclient actor https://latest.datasette.io/fixtures + +Options: + --token TEXT API token + --help Show this message and exit. + +``` + \ No newline at end of file From 0d4ab72b68a72bf25891aea80c15f8c986508bbe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 21:09:26 -0800 Subject: [PATCH 30/64] Release 0.4 Refs #23, #25, #26 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 309569e..e50775e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup import os -VERSION = "0.3" +VERSION = "0.4" def get_long_description(): From 1a0fe65015ec4a09b037ed723d43553ab47386ec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 Mar 2024 11:09:54 -0500 Subject: [PATCH 31/64] Upgrade Actions for publish of 0.4 --- .github/workflows/publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c56f512..cac9700 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,13 +23,16 @@ jobs: cache-dependency-path: setup.py - name: Install dependencies run: | - pip install -e '.[test]' + pip install '.[test]' - name: Run tests run: | pytest deploy: runs-on: ubuntu-latest needs: [test] + environment: release + permissions: + id-token: write steps: - uses: actions/checkout@v4 - name: Set up Python From a157065c3117020e415189f29c88b263c214dc79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Feb 2026 16:50:33 -0800 Subject: [PATCH 32/64] setup.py to pyproject.toml plus upgraded actions --- .github/workflows/publish.yml | 20 +++++++-------- .github/workflows/test.yml | 10 ++++---- .gitignore | 1 + pyproject.toml | 42 ++++++++++++++++++++++++++++++++ setup.py | 46 ----------------------------------- 5 files changed, 58 insertions(+), 61 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cac9700..967ae82 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,18 +12,18 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install '.[test]' + pip install . --group dev - name: Run tests run: | pytest @@ -34,16 +34,16 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: "3.12" + python-version: "3.13" cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install setuptools wheel build + pip install build - name: Build run: | python -m build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbd465f..ea123b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,18 +10,18 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev - name: Run tests run: | pytest diff --git a/.gitignore b/.gitignore index 53605b7..d07256d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv +uv.lock __pycache__/ *.py[cod] *$py.class diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..15d9c56 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "dclient" +version = "0.4" +description = "A client CLI utility for Datasette instances" +readme = "README.md" +authors = [{name = "Simon Willison"}] +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [ + "click", + "httpx", + "sqlite-utils", +] + +[project.urls] +Homepage = "https://github.com/simonw/dclient" +Issues = "https://github.com/simonw/dclient/issues" +CI = "https://github.com/simonw/dclient/actions" +Changelog = "https://github.com/simonw/dclient/releases" + +[project.scripts] +dclient = "dclient.cli:cli" + +[project.entry-points.datasette] +client = "dclient.plugin" + +[dependency-groups] +dev = [ + "pytest", + "pytest-asyncio", + "pytest-httpx", + "cogapp", + "pytest-mock", + "datasette>=1.0a2", +] + +[tool.uv.build-backend] +module-root = "" + +[build-system] +requires = ["uv_build>=0.9.18,<0.10.0"] +build-backend = "uv_build" diff --git a/setup.py b/setup.py deleted file mode 100644 index e50775e..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -from setuptools import setup -import os - -VERSION = "0.4" - - -def get_long_description(): - with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), - encoding="utf8", - ) as fp: - return fp.read() - - -setup( - name="dclient", - description="A client CLI utility for Datasette instances", - long_description=get_long_description(), - long_description_content_type="text/markdown", - author="Simon Willison", - url="https://github.com/simonw/dclient", - project_urls={ - "Issues": "https://github.com/simonw/dclient/issues", - "CI": "https://github.com/simonw/dclient/actions", - "Changelog": "https://github.com/simonw/dclient/releases", - }, - license="Apache License, Version 2.0", - version=VERSION, - packages=["dclient"], - entry_points={ - "datasette": ["client = dclient.plugin"], - "console_scripts": ["dclient = dclient.cli:cli"], - }, - install_requires=["click", "httpx", "sqlite-utils"], - extras_require={ - "test": [ - "pytest", - "pytest-asyncio", - "pytest-httpx", - "cogapp", - "pytest-mock", - "datasette>=1.0a2", - ] - }, - python_requires=">=3.8", -) From ab3aaf26f3a98ee32e39fd5970f0b3e9dc60693c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Feb 2026 21:43:23 -0800 Subject: [PATCH 33/64] Fixed asyncio pattern for tests --- tests/test_insert.py | 52 ++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/tests/test_insert.py b/tests/test_insert.py index c849a67..32fbceb 100644 --- a/tests/test_insert.py +++ b/tests/test_insert.py @@ -1,5 +1,6 @@ import asyncio from collections import namedtuple +from concurrent.futures import ThreadPoolExecutor from click.testing import CliRunner from datasette.app import Datasette from dclient.cli import cli @@ -14,6 +15,9 @@ def assert_all_responses_were_requested() -> bool: return False +pytestmark = pytest.mark.httpx_mock(assert_all_responses_were_requested=False) + + @pytest.fixture def non_mocked_hosts(): # This ensures httpx-mock will not affect Datasette's own @@ -120,7 +124,7 @@ def make_format_test(content, arg): input_data=SIMPLE_CSV, cmd_args=[], table_exists=False, - expected_output="Inserting rows\nError: Table not found: table1\n", + expected_output="Inserting rows\nError: Table not found\n", should_error=True, expected_table_json=None, ), @@ -229,7 +233,8 @@ def make_format_test(content, arg): ), ), ) -def test_insert_against_datasette( +@pytest.mark.asyncio +async def test_insert_against_datasette( httpx_mock, tmpdir, input_data, @@ -250,23 +255,19 @@ def test_insert_against_datasette( } ) db = ds.add_memory_database("data") - loop = asyncio.get_event_loop() # Drop all tables in the database each time, because in-memory # databases persist in between test runs - drop_all_tables(db, loop) + for table in await db.table_names(): + await db.execute_write("drop table {}".format(table)) if table_exists: - - async def run_table_exists(): - await db.execute_write( - "create table table1 (a integer primary key, b integer, c integer)" - ) - await db.execute_write( - "insert into table1 (a, b, c) values (1, 2, 3), (4, 5, 6)" - ) - - loop.run_until_complete(run_table_exists()) + await db.execute_write( + "create table table1 (a integer primary key, b integer, c integer)" + ) + await db.execute_write( + "insert into table1 (a, b, c) values (1, 2, 3), (4, 5, 6)" + ) token = ds.create_token("actor") @@ -275,8 +276,9 @@ def test_insert_against_datasette( datasette_responses = [] def custom_response(request: httpx.Request): - # Need to run this in async loop, because dclient itself uses - # sync HTTPX and not async HTTPX + # Need to run this in a new event loop, because dclient itself uses + # sync HTTPX and not async HTTPX, and the test's event loop is + # already running async def run(): datasette_requests.append(request) response = await ds.client.request( @@ -294,7 +296,8 @@ def test_insert_against_datasette( datasette_responses.append(response) return response - return loop.run_until_complete(run()) + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, run()).result() httpx_mock.add_callback(custom_response) @@ -323,18 +326,5 @@ def test_insert_against_datasette( assert result.output == expected_output if expected_table_json: - - async def fetch_table(): - response = await ds.client.get("/data/table1.json?_shape=array") - return response - - response = loop.run_until_complete(fetch_table()) + response = await ds.client.get("/data/table1.json?_shape=array") assert response.json() == expected_table_json - - -def drop_all_tables(db, loop): - async def run(): - for table in await db.table_names(): - await db.execute_write("drop table {}".format(table)) - - loop.run_until_complete(run()) From 32124a0bf36d7667afc2b7341f5da21b825d91c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Feb 2026 22:10:59 -0800 Subject: [PATCH 34/64] Support DATASETTE_URL and DATASETTE_TOKEN environment variables Allow configuring a base Datasette instance URL and API token via environment variables, so users can run commands like `dclient query data "select ..."` without specifying a full URL each time. DATASETTE_URL is combined with the argument as a path segment. DATASETTE_TOKEN is used as a lowest-priority fallback for auth. Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 56 +++++++------- docs/environment.md | 68 +++++++++++++++++ tests/test_env.py | 178 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+), 27 deletions(-) create mode 100644 docs/environment.md create mode 100644 tests/test_env.py diff --git a/dclient/cli.py b/dclient/cli.py index a8b3376..208238f 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,6 +1,7 @@ import click import httpx import json +import os import pathlib from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar import sys @@ -38,17 +39,10 @@ def query(url_or_alias, sql, token, verbose): https://datasette.io/content \\ 'select * from news limit 10' """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - if url_or_alias in aliases: - url = aliases[url_or_alias] - else: - url = url_or_alias - if not url_or_alias.endswith(".json"): + url = _resolve_url(url_or_alias) + token = _resolve_token(token, url) + if not url.endswith(".json"): url += ".json" - if token is None: - # Maybe there's a token in auth.json? - token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) headers = {} if token: headers["Authorization"] = f"Bearer {token}" @@ -167,15 +161,8 @@ def insert( https://private.datasette.cloud/data \\ mytable data.csv --pk id --create """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - if url_or_alias in aliases: - url = aliases[url_or_alias] - else: - url = url_or_alias - - if token is None: - token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + url = _resolve_url(url_or_alias) + token = _resolve_token(token, url) format = None if format_csv: @@ -276,18 +263,12 @@ def actor(url_or_alias, token): \b dclient actor https://latest.datasette.io/fixtures """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - if url_or_alias in aliases: - url = aliases[url_or_alias] - else: - url = url_or_alias + url = _resolve_url(url_or_alias) if not (url.startswith("http://") or url.startswith("https://")): raise click.ClickException("Invalid URL: " + url) - if token is None: - token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + token = _resolve_token(token, url) url_bits = url.split("/") url_bits[-1] = "-/actor.json" @@ -457,6 +438,27 @@ def _load_auths(auth_file): return auths +def _resolve_url(url_or_alias): + aliases = _load_aliases(get_config_dir() / "aliases.json") + if url_or_alias in aliases: + return aliases[url_or_alias] + if url_or_alias.startswith("http://") or url_or_alias.startswith("https://"): + return url_or_alias + base_url = os.environ.get("DATASETTE_URL") + if base_url: + return base_url.rstrip("/") + "/" + url_or_alias + return url_or_alias + + +def _resolve_token(token, url): + if token is not None: + return token + stored = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + if stored is not None: + return stored + return os.environ.get("DATASETTE_TOKEN") + + def _batches(iterable, size, interval=None): iterable = iter(iterable) last_yield_time = time.time() diff --git a/docs/environment.md b/docs/environment.md new file mode 100644 index 0000000..f597617 --- /dev/null +++ b/docs/environment.md @@ -0,0 +1,68 @@ +(environment-variables)= + +# Environment variables + +`dclient` supports two environment variables for convenient access to a Datasette instance without needing aliases or repeated URLs. + +## DATASETTE_URL + +Set this to the base URL of your Datasette instance: + +```bash +export DATASETTE_URL=https://my-instance.datasette.cloud +``` + +Then pass just the database name as the first argument to any command: + +```bash +dclient query data "select * from my_table limit 10" +``` + +This is equivalent to: + +```bash +dclient query https://my-instance.datasette.cloud/data "select * from my_table limit 10" +``` + +It works with all commands: + +```bash +dclient insert data my_table data.csv --csv +dclient actor data +``` + +Full URLs and aliases always take priority over `DATASETTE_URL`. If the argument starts with `http://` or `https://`, it is used as-is. If it matches an alias in `aliases.json`, the alias is used. + +## DATASETTE_TOKEN + +Set this to an API token: + +```bash +export DATASETTE_TOKEN=dstok_abc123 +``` + +The token will be used automatically for any request that doesn't have a more specific token configured. + +The precedence order for tokens is: + +1. `--token` CLI flag (highest priority) +2. Stored token from `auth.json` (matched by URL prefix) +3. `DATASETTE_TOKEN` environment variable (lowest priority) + +## Using both together + +These variables work well together for quick access to a single instance: + +```bash +export DATASETTE_URL=https://my-instance.datasette.cloud +export DATASETTE_TOKEN=dstok_abc123 + +# Query the "data" database +dclient query data "select * from my_table" + +# Insert into the "data" database +cat records.json | dclient insert data my_table - --json + +# Check your actor identity +dclient actor data +``` diff --git a/tests/test_env.py b/tests/test_env.py new file mode 100644 index 0000000..5afd593 --- /dev/null +++ b/tests/test_env.py @@ -0,0 +1,178 @@ +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib + + +QUERY_RESPONSE = { + "ok": True, + "database": "data", + "query_name": None, + "rows": [{"id": 1}], + "truncated": False, + "columns": ["id"], + "query": {"sql": "select 1", "params": {}}, + "error": None, + "private": False, + "allow_execute_sql": True, +} + + +# -- DATASETTE_TOKEN tests -- + + +def test_datasette_token_used_as_fallback(httpx_mock, mocker, tmpdir): + """DATASETTE_TOKEN is used when no --token flag and no auth.json match.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_TOKEN": "env-token-123"}) + result = runner.invoke(cli, ["query", "https://example.com", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.headers["authorization"] == "Bearer env-token-123" + + +def test_token_flag_overrides_datasette_token(httpx_mock, mocker, tmpdir): + """--token flag takes priority over DATASETTE_TOKEN.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_TOKEN": "env-token"}) + result = runner.invoke( + cli, ["query", "https://example.com", "select 1", "--token", "flag-token"] + ) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.headers["authorization"] == "Bearer flag-token" + + +def test_auth_json_overrides_datasette_token(httpx_mock, mocker, tmpdir): + """Stored auth.json token takes priority over DATASETTE_TOKEN.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + auth_file = pathlib.Path(tmpdir) / "auth.json" + auth_file.write_text(json.dumps({"https://example.com": "stored-token"})) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_TOKEN": "env-token"}) + result = runner.invoke(cli, ["query", "https://example.com", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.headers["authorization"] == "Bearer stored-token" + + +# -- DATASETTE_URL tests -- + + +def test_datasette_url_combines_with_database_name(httpx_mock, mocker, tmpdir): + """DATASETTE_URL + database name arg → combined URL.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_URL": "https://my-instance.datasette.cloud"}) + result = runner.invoke(cli, ["query", "data", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "my-instance.datasette.cloud" + assert request.url.path == "/data.json" + + +def test_datasette_url_with_trailing_slash(httpx_mock, mocker, tmpdir): + """DATASETTE_URL with trailing slash still works correctly.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_URL": "https://my-instance.datasette.cloud/"}) + result = runner.invoke(cli, ["query", "data", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.path == "/data.json" + + +def test_full_url_ignores_datasette_url(httpx_mock, mocker, tmpdir): + """A full URL argument is used as-is, ignoring DATASETTE_URL.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_URL": "https://should-be-ignored.com"}) + result = runner.invoke(cli, ["query", "https://other.example.com/db", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "other.example.com" + assert request.url.path == "/db.json" + + +def test_alias_takes_priority_over_datasette_url(httpx_mock, mocker, tmpdir): + """Alias match takes priority over DATASETTE_URL.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + aliases_file = pathlib.Path(tmpdir) / "aliases.json" + aliases_file.write_text(json.dumps({"myalias": "https://aliased.example.com/db"})) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner(env={"DATASETTE_URL": "https://should-be-ignored.com"}) + result = runner.invoke(cli, ["query", "myalias", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "aliased.example.com" + assert request.url.path == "/db.json" + + +# -- DATASETTE_URL with other commands -- + + +def test_datasette_url_with_insert(httpx_mock, mocker, tmpdir): + """DATASETTE_URL works with the insert command.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json={"ok": True}, status_code=200) + csv_path = pathlib.Path(tmpdir) / "data.csv" + csv_path.write_text("id,name\n1,hello\n") + runner = CliRunner( + env={ + "DATASETTE_URL": "https://my-instance.datasette.cloud", + "DATASETTE_TOKEN": "env-token", + } + ) + result = runner.invoke( + cli, + ["insert", "data", "my_table", str(csv_path), "--csv", "--create"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "my-instance.datasette.cloud" + assert "/-/create" in str(request.url.path) + assert request.headers["authorization"] == "Bearer env-token" + + +def test_datasette_url_with_actor(httpx_mock, mocker, tmpdir): + """DATASETTE_URL works with the actor command.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"actor": {"id": "root"}}, + status_code=200, + ) + runner = CliRunner( + env={ + "DATASETTE_URL": "https://my-instance.datasette.cloud", + "DATASETTE_TOKEN": "env-token", + } + ) + result = runner.invoke(cli, ["actor", "data"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "my-instance.datasette.cloud" + assert request.headers["authorization"] == "Bearer env-token" + + +# -- Both together -- + + +def test_datasette_url_and_token_together(httpx_mock, mocker, tmpdir): + """DATASETTE_URL and DATASETTE_TOKEN work together for a complete config.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner( + env={ + "DATASETTE_URL": "https://my-instance.datasette.cloud", + "DATASETTE_TOKEN": "env-token-456", + } + ) + result = runner.invoke(cli, ["query", "mydb", "select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "my-instance.datasette.cloud" + assert request.url.path == "/mydb.json" + assert request.headers["authorization"] == "Bearer env-token-456" From 166798c87cbb229a5a4e9d3cee5c79726b9f9e1e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Feb 2026 22:14:00 -0800 Subject: [PATCH 35/64] Follow redirects on query Means that the 1.0 feature where /db?sql is now /db/-/query?sql works --- dclient/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dclient/cli.py b/dclient/cli.py index 208238f..ab8d8c4 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -49,7 +49,7 @@ def query(url_or_alias, sql, token, verbose): params = {"sql": sql, "_shape": "objects"} if verbose: click.echo(url + "?" + urllib.parse.urlencode(params), err=True) - response = httpx.get(url, params=params, headers=headers) + response = httpx.get(url, params=params, headers=headers, follow_redirects=True) if response.status_code != 200: # Is it valid JSON? From 4cb77386c7b68b7343dfd6a2b3380a658874e58f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:56:27 -0800 Subject: [PATCH 36/64] _resolve_url falls back to DATASETTE_URL when no URL provided Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dclient/cli.py b/dclient/cli.py index ab8d8c4..05a17e9 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -439,6 +439,13 @@ def _load_auths(auth_file): def _resolve_url(url_or_alias): + if not url_or_alias: + base_url = os.environ.get("DATASETTE_URL") + if base_url: + return base_url.rstrip("/") + raise click.ClickException( + "No URL provided. Set DATASETTE_URL or pass a URL/alias." + ) aliases = _load_aliases(get_config_dir() / "aliases.json") if url_or_alias in aliases: return aliases[url_or_alias] From 1b4dc8597433704a2dd493b05f211813e2f7528f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:56:40 -0800 Subject: [PATCH 37/64] actor command: make URL optional, fix actor URL construction Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 05a17e9..91f92ce 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -252,7 +252,7 @@ def insert( @cli.command() -@click.argument("url_or_alias") +@click.argument("url_or_alias", default=None, required=False) @click.option("--token", help="API token") def actor(url_or_alias, token): """ @@ -270,9 +270,7 @@ def actor(url_or_alias, token): token = _resolve_token(token, url) - url_bits = url.split("/") - url_bits[-1] = "-/actor.json" - actor_url = "/".join(url_bits) + actor_url = url.rstrip("/") + "/-/actor.json" response = httpx.get( actor_url, headers={"Authorization": "Bearer {}".format(token)}, timeout=40.0 ) From d1dfdbf6dbbd204855d9df07901784171354e1bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:56:54 -0800 Subject: [PATCH 38/64] Add get command for authenticated GET requests Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/dclient/cli.py b/dclient/cli.py index 91f92ce..f2e6396 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -21,6 +21,41 @@ def cli(): "A client CLI utility for Datasette instances" +@cli.command() +@click.argument("path") +@click.option("--instance", default=None, help="Datasette URL or alias") +@click.option("--token", help="API token") +def get(path, instance, token): + """ + Make an authenticated GET request to a Datasette instance + + Example usage: + + \b + dclient get /-/plugins.json + dclient get /data/creatures.json --instance https://my.datasette.io + """ + url = _resolve_url(instance) + token = _resolve_token(token, url) + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + full_url = url.rstrip("/") + "/" + path.lstrip("/") + response = httpx.get( + full_url, + headers=headers, + follow_redirects=True, + timeout=30.0, + ) + if response.status_code != 200: + raise click.ClickException(f"{response.status_code} error for {full_url}") + # Pretty-print if JSON, otherwise raw + if "json" in response.headers.get("content-type", ""): + click.echo(json.dumps(response.json(), indent=2)) + else: + click.echo(response.text) + + @cli.command() @click.argument("url_or_alias") @click.argument("sql") From 8b98d971ddeb5f13cd1e52c4341cc7e16307a461 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:57:06 -0800 Subject: [PATCH 39/64] Add databases command to list databases on an instance Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dclient/cli.py b/dclient/cli.py index f2e6396..980d7bb 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -56,6 +56,34 @@ def get(path, instance, token): click.echo(response.text) +@cli.command() +@click.argument("url_or_alias", default=None, required=False) +@click.option("--token", help="API token") +def databases(url_or_alias, token): + """ + List databases available on a Datasette instance + + Example usage: + + \b + dclient databases https://latest.datasette.io + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(token, url) + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + response = httpx.get( + url.rstrip("/") + "/-/databases.json", + headers=headers, + follow_redirects=True, + timeout=30.0, + ) + if response.status_code != 200: + raise click.ClickException(f"{response.status_code} error") + click.echo(json.dumps(response.json(), indent=2)) + + @cli.command() @click.argument("url_or_alias") @click.argument("sql") From f47738495d50bc8814e2bd95b3177f26cac907e7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:57:18 -0800 Subject: [PATCH 40/64] Add tables command to list tables in a database Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dclient/cli.py b/dclient/cli.py index 980d7bb..d19a57e 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -84,6 +84,34 @@ def databases(url_or_alias, token): click.echo(json.dumps(response.json(), indent=2)) +@cli.command() +@click.argument("url_or_alias", default=None, required=False) +@click.option("--token", help="API token") +def tables(url_or_alias, token): + """ + List tables in a Datasette database + + Example usage: + + \b + dclient tables https://latest.datasette.io/fixtures + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(token, url) + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + response = httpx.get( + url.rstrip("/") + "/-/tables.json", + headers=headers, + follow_redirects=True, + timeout=30.0, + ) + if response.status_code != 200: + raise click.ClickException(f"{response.status_code} error") + click.echo(json.dumps(response.json(), indent=2)) + + @cli.command() @click.argument("url_or_alias") @click.argument("sql") From 7c9c0b4f5faeeada51058bd4799ee417af59d221 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:57:54 -0800 Subject: [PATCH 41/64] Spec for v2, refs #29 --- specs/dclient-v2.md | 408 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 specs/dclient-v2.md diff --git a/specs/dclient-v2.md b/specs/dclient-v2.md new file mode 100644 index 0000000..85dad50 --- /dev/null +++ b/specs/dclient-v2.md @@ -0,0 +1,408 @@ +# dclient v2 — Changes from v1 + +## Summary of changes + +1. Aliases point at Datasette instances, not database URLs +2. Instance is always a flag (`-i`/`--instance`), never a positional argument +3. `query` and `insert`/`upsert` take database as a required positional argument +4. Introspection commands (`tables`, `schema`, etc.) use `-d`/`--database` with defaults +5. Default instance and default database reduce typing +6. Bare SQL defaults to `query` command via `click-default-group`, using defaults + `-d` override +7. New commands: `databases`, `tables`, `schema`, `plugins` +8. Config and auth remain separate files, with a new config format +9. New `DATASETTE_DATABASE` environment variable + +--- + +## Config changes + +### config.json (replaces aliases.json) + +```json +{ + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://myapp.datasette.cloud", + "default_database": "main" + }, + "local": { + "url": "http://localhost:8001", + "default_database": null + } + } +} +``` + +### auth.json (same file, new key structure) + +Keys are now alias names instead of URLs: + +```json +{ + "prod": "dstok_abc123", + "local": "dstok_def456" +} +``` + +When no alias exists (using `-i` with a raw URL), tokens are looked up by URL with the existing prefix-matching logic as a fallback. + +### Migration + +On first run, if `config.json` doesn't exist but `aliases.json` does: + +- Parse each v1 alias URL. If it has a single path segment (e.g., `https://example.com/mydb`), split into instance URL + default database. Otherwise store the URL as-is. +- Migrate auth.json keys from URLs to alias names where a match exists, keep URL-keyed entries as fallbacks. +- Write new files, rename originals to `.bak`. + +### Environment variables + +| Variable | Purpose | Precedence | +|----------|---------|------------| +| `DATASETTE_URL` | Default instance URL (existing) | Below config default, above nothing | +| `DATASETTE_DATABASE` | Default database name (new) | Below config default, above auto-detect | +| `DATASETTE_TOKEN` | Auth token (existing) | Below stored token, above nothing | + +--- + +## Two ways to query: explicit and shortcut + +The design provides two distinct interfaces for running queries, optimized for different workflows. + +### Explicit: `dclient query ` + +Database is a required positional argument. No defaults involved. This is unambiguous and works well for zero-config usage and when switching between databases frequently. + +```bash +dclient query fixtures "select * from facetable limit 5" +dclient query analytics "select count(*) from events" -i staging +``` + +### Shortcut: `dclient ` + +When the first argument doesn't match any subcommand, `click-default-group` routes it to a default command that uses the configured default instance and default database. Override either with flags. + +```bash +dclient "select count(*) from users" # default instance + default db +dclient "select count(*) from users" -d analytics # override database +dclient "select count(*) from users" -i staging # override instance +``` + +This is the daily-driver mode for people with a configured default instance and database. + +### Implementation + +```python +from click_default_group import DefaultGroup + +@click.group(cls=DefaultGroup, default="default_query", default_if_no_args=False) +@click.version_option() +def cli(): + "A client CLI utility for Datasette instances" + +@cli.command() +@click.argument("database") +@click.argument("sql") +@click.option("-i", "--instance") +@click.option("--token") +@click.option("-v", "--verbose", is_flag=True) +# ... output format options ... +def query(database, sql, instance, token, verbose, **kwargs): + """Run a SQL query against a Datasette database + + Requires both a database name and a SQL string. + + Example: + + dclient query fixtures "select * from facetable limit 5" + """ + ... + +@cli.command(name="default_query", hidden=True) +@click.argument("sql") +@click.option("-i", "--instance") +@click.option("-d", "--database") +@click.option("--token") +@click.option("-v", "--verbose", is_flag=True) +# ... output format options ... +def default_query(sql, instance, database, token, verbose, **kwargs): + """Run a SQL query using default instance and database.""" + # Resolve instance and database from defaults/env vars + # Error if no default database can be resolved + ... +``` + +The `default_query` command is hidden from help output. Users see `query` in the help text; the bare-SQL shortcut just works without being documented as a separate command. + +--- + +## Instance resolution + +Every command that talks to a Datasette server accepts `-i, --instance TEXT`. Resolution order: + +1. `-i` flag (alias name or URL) +2. `config.default_instance` +3. `DATASETTE_URL` environment variable +4. Error + +If the value starts with `http://` or `https://`, use it as a URL directly. Otherwise look it up as an alias name in config. + +--- + +## Database resolution + +There are two modes of database resolution depending on the command. + +### Commands with required positional database + +`query`, `insert`, `upsert`: the database is always the first positional argument. No resolution logic, no defaults. You must name the database. + +### Commands with optional `-d` flag + +`tables`, `schema`, and the default query shortcut: resolve in order: + +1. `-d` flag +2. Instance's `default_database` from config +3. `DATASETTE_DATABASE` environment variable +4. Auto-detect if instance has exactly one (non-internal) database +5. Error with a helpful message listing available databases + +### Commands that don't need a database + +`databases`, `plugins`, `actor`, `alias`, `auth`: no database argument or flag. + +--- + +## New commands + +### `dclient databases` + +List databases on an instance. + +```bash +$ dclient databases +main +extra + +$ dclient databases --json +[{"name": "main", "tables_count": 12, ...}, ...] + +$ dclient databases -i https://latest.datasette.io +fixtures +``` + +Options: `-i`, `--json`. + +Implementation: `GET /.json` → `databases` key. + +### `dclient tables` + +List tables (and optionally views) in a database. + +```bash +$ dclient tables +facetable 15 rows +facet_cities 4 rows + +$ dclient tables -d analytics +events 1503 rows + +$ dclient tables --views --json +[{"name": "facetable", "columns": [...], "count": 15, ...}, ...] +``` + +Options: `-i`, `-d`, `--views`, `--views-only`, `--hidden`, `--json`. + +Implementation: `GET /.json` → `tables` and `views` keys. + +### `dclient schema` + +Show SQL schema for a database or a specific table. + +```bash +$ dclient schema +# all CREATE TABLE/VIEW statements for default database + +$ dclient schema -d analytics +# all schemas for the analytics database + +$ dclient schema facetable +# just that table's schema (in default database) + +$ dclient schema facetable -d analytics +# that table in a specific database +``` + +The optional table name is a positional argument. Options: `-i`, `-d`, `--json`. + +### `dclient plugins` + +List installed plugins on an instance. + +```bash +$ dclient plugins +datasette-files +datasette-auth-tokens + +$ dclient plugins --json +[{"name": "datasette-files", "version": "0.3.1", ...}, ...] +``` + +Options: `-i`, `--json`. + +Implementation: `GET /-/plugins.json`. + +--- + +## Changed commands + +### `dclient query` + +```bash +# v1 +dclient query https://datasette.io/content "select * from news" + +# v2: database is required positional, instance is a flag +dclient query content "select * from news" -i https://datasette.io +dclient query fixtures "select * from facetable" # uses default instance +``` + +Signature: `dclient query [-i instance] [--csv|--tsv|--nl|--table] [-o file] [--token TOKEN] [-v]` + +### `dclient insert` + +```bash +# v1 +dclient insert https://myapp.datasette.cloud/data mytable data.csv --csv + +# v2: database and table are required positionals, instance is a flag +dclient insert main mytable data.csv --csv -i myapp +dclient insert main mytable data.csv --csv # uses default instance +``` + +Signature: `dclient insert [-i instance] [--csv|--tsv|--json|--nl] [--create] [--replace] [--ignore] [--alter] [--pk col] [--batch-size N] [--interval N] [--token TOKEN] [-v] [--silent]` + +### `dclient upsert` + +New command. Same shape as `insert` but hits `/-/upsert`. + +Signature: `dclient upsert
[-i instance] [--csv|--tsv|--json|--nl] [--alter] [--pk col] [--batch-size N] [--interval N] [--token TOKEN] [-v] [--silent]` + +### `dclient alias` + +```bash +dclient alias add # url is the instance root, stored as-is +dclient alias remove +dclient alias list # shows * for default, (db: x) for default db +dclient alias default [name] # set/show default instance +dclient alias default --clear +dclient alias default-db [db] # set/show default database for an alias +dclient alias default-db --clear +``` + +**Tip: multiple aliases for the same instance.** You can create several aliases pointing at the same instance URL with different default databases. This gives you short names for databases you switch between frequently: + +```bash +dclient alias add prod https://myapp.datasette.cloud +dclient alias add prod-analytics https://myapp.datasette.cloud +dclient alias default-db prod main +dclient alias default-db prod-analytics analytics + +dclient tables -i prod # → main +dclient tables -i prod-analytics # → analytics +``` + +Auth tokens are resolved by alias name first, then by URL match as a fallback, so a token stored for either alias will work for both. + +### `dclient auth` + +```bash +dclient auth add # prompt for token +dclient auth add --token TOKEN +dclient auth remove +dclient auth list # shows which aliases have tokens, never values +dclient auth status [-i instance] # calls /-/actor.json to verify +``` + +### `dclient actor` + +Uses `-i` flag instead of positional URL: + +```bash +dclient actor # default instance +dclient actor -i prod +``` + +--- + +## Worked examples for the three usage modes + +### Mode 1: Zero config, public instance + +```bash +# Explicit database, instance as flag +dclient databases -i https://latest.datasette.io +dclient tables -i https://latest.datasette.io -d fixtures +dclient query fixtures "select * from facetable limit 3" -i https://latest.datasette.io + +# Or set env vars for a session +export DATASETTE_URL=https://latest.datasette.io +export DATASETTE_DATABASE=fixtures +dclient tables +dclient "select * from facetable limit 3" +``` + +### Mode 2: Single default instance + +```bash +# One-time setup +dclient alias add work https://myapp.datasette.cloud +dclient alias default work +dclient alias default-db work main +dclient auth add work + +# Daily use — bare SQL shortcut uses defaults +dclient tables +dclient "select count(*) from users" +dclient "select * from events" -d analytics + +# Explicit query when switching databases frequently +dclient query main "select count(*) from users" +dclient query analytics "select count(*) from events" + +# Insert always names the database +dclient insert main events data.csv --csv --create --pk id +dclient plugins +``` + +### Mode 3: Multiple aliases + +```bash +dclient alias add prod https://prod.datasette.cloud +dclient alias add staging https://staging.datasette.cloud +dclient alias default prod +dclient alias default-db prod main +dclient alias default-db staging main + +# Bare shortcut hits prod/main +dclient "select count(*) from users" + +# Explicit query — name the database, override instance with -i +dclient query main "select count(*) from users" -i staging +dclient query analytics "select * from events" -i staging + +# Insert always explicit +dclient insert main events data.csv --csv -i staging +``` + +--- + +## Not in this version (reserved) + +These command names are reserved for future work: + +- `dclient cloud` — Datasette Cloud integration +- `dclient files` — datasette-files management +- `dclient get`, `dclient rows`, `dclient update`, `dclient delete` — row-level CRUD +- `dclient create-table`, `dclient drop-table` — DDL +- `dclient queries` — canned queries \ No newline at end of file From ff95969d10924eab7fa7b3c1a2c068fa37565ca1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:59:23 -0800 Subject: [PATCH 42/64] Ran cog --- docs/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.md b/docs/authentication.md index 4e5f22f..caadba7 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -161,7 +161,7 @@ cog.out( ) ]]] --> ``` -Usage: dclient actor [OPTIONS] URL_OR_ALIAS +Usage: dclient actor [OPTIONS] [URL_OR_ALIAS] Show the actor represented by an API token From cf565e3eb6d9126e2bcce4750e92679eee462e4b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 14:59:23 -0800 Subject: [PATCH 43/64] Ran cog --- docs/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.md b/docs/authentication.md index 4e5f22f..caadba7 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -161,7 +161,7 @@ cog.out( ) ]]] --> ``` -Usage: dclient actor [OPTIONS] URL_OR_ALIAS +Usage: dclient actor [OPTIONS] [URL_OR_ALIAS] Show the actor represented by an API token From 3621f132632cd34ef8c7c0b11ec567d7164f55c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 15:21:52 -0800 Subject: [PATCH 44/64] v2 config system, CLI rewrite, and updated core commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace aliases.json with config.json storing instances with default_database. Instance resolution: -i flag → config default → DATASETTE_URL. Database resolution: -d flag → instance default_database → DATASETTE_DATABASE. Token resolution: --token → auth.json by alias → auth.json by URL → DATASETTE_TOKEN. Add click-default-group dependency for bare SQL shortcut support. Add DCLIENT_CONFIG_DIR env var to override config directory. Rewrite query, insert, get, and actor commands for v2 API where instance is always -i flag. Add upsert command sharing insert implementation. Add databases, tables, schema, plugins commands. Add default_query hidden command for bare SQL shortcut. Add alias default, alias default-db subcommands. Add auth status subcommand. refs #29 Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 834 +++++++++++++++++++++++++++++------------ docs/environment.md | 45 ++- docs/inserting.md | 69 +++- docs/queries.md | 32 +- pyproject.toml | 1 + tests/test_cli_auth.py | 16 +- tests/test_config.py | 249 ++++++++++++ tests/test_env.py | 69 +++- tests/test_insert.py | 8 +- tests/test_query.py | 38 +- 10 files changed, 1048 insertions(+), 313 deletions(-) create mode 100644 tests/test_config.py diff --git a/dclient/cli.py b/dclient/cli.py index d19a57e..3adfebc 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,4 +1,5 @@ import click +from click_default_group import DefaultGroup import httpx import json import os @@ -12,18 +13,128 @@ import urllib def get_config_dir(): + env = os.environ.get("DCLIENT_CONFIG_DIR") + if env: + return pathlib.Path(env) return pathlib.Path(click.get_app_dir("io.datasette.dclient")) -@click.group() +def _load_config(config_file): + if config_file.exists(): + return json.loads(config_file.read_text()) + return {"default_instance": None, "instances": {}} + + +def _save_config(config_file, config): + config_file.parent.mkdir(parents=True, exist_ok=True) + config_file.write_text(json.dumps(config, indent=4)) + + +def _load_auths(auth_file): + if auth_file.exists(): + auths = json.loads(auth_file.read_text()) + else: + auths = {} + return auths + + +def _resolve_instance(instance, config_file): + """Resolve instance: -i flag (alias or URL) → config default → DATASETTE_URL → error.""" + config = _load_config(config_file) + if instance: + # If it looks like a URL, use directly + if instance.startswith("http://") or instance.startswith("https://"): + return instance + # Otherwise look up as alias + if instance in config.get("instances", {}): + return config["instances"][instance]["url"] + raise click.ClickException( + f"Unknown instance: {instance}. Use a URL or configure an alias." + ) + # Try config default + default = config.get("default_instance") + if default and default in config.get("instances", {}): + return config["instances"][default]["url"] + # Try env var + env_url = os.environ.get("DATASETTE_URL") + if env_url: + return env_url.rstrip("/") + raise click.ClickException( + "No instance specified. Use -i, set a default instance, or set DATASETTE_URL." + ) + + +def _resolve_database(database, instance_alias, config_file): + """Resolve database: -d flag → instance default_database → DATASETTE_DATABASE → error.""" + if database: + return database + # Try instance's default_database from config + if instance_alias: + config = _load_config(config_file) + instances = config.get("instances", {}) + if instance_alias in instances: + default_db = instances[instance_alias].get("default_database") + if default_db: + return default_db + # Try env var + env_db = os.environ.get("DATASETTE_DATABASE") + if env_db: + return env_db + raise click.ClickException( + "No database specified. Use -d, set a default database, or set DATASETTE_DATABASE." + ) + + +def _instance_alias_for_url(url, config_file): + """Find the alias name for a given instance URL, if any.""" + config = _load_config(config_file) + for name, inst in config.get("instances", {}).items(): + if inst.get("url", "").rstrip("/") == url.rstrip("/"): + return name + return None + + +def _resolve_token(token, url, auth_file, config_file): + """Resolve token: --token flag → auth.json by alias → auth.json by URL → DATASETTE_TOKEN → None.""" + if token is not None: + return token + auths = _load_auths(auth_file) + # Try alias-based lookup + alias = _instance_alias_for_url(url, config_file) + if alias and alias in auths: + return auths[alias] + # Try URL-based prefix matching (fallback) + stored = token_for_url(url, auths) + if stored is not None: + return stored + return os.environ.get("DATASETTE_TOKEN") + + +@click.group(cls=DefaultGroup, default="default_query", default_if_no_args=False) @click.version_option() def cli(): "A client CLI utility for Datasette instances" +def _make_request(url, token, extra_path="", params=None): + """Make an authenticated GET request to a Datasette instance.""" + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + full_url = url.rstrip("/") + extra_path + response = httpx.get( + full_url, + headers=headers, + params=params, + follow_redirects=True, + timeout=30.0, + ) + return response + + @cli.command() @click.argument("path") -@click.option("--instance", default=None, help="Datasette URL or alias") +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") @click.option("--token", help="API token") def get(path, instance, token): """ @@ -33,23 +144,15 @@ def get(path, instance, token): \b dclient get /-/plugins.json - dclient get /data/creatures.json --instance https://my.datasette.io + dclient get /data/creatures.json -i https://my.datasette.io """ - url = _resolve_url(instance) - token = _resolve_token(token, url) - headers = {} - if token: - headers["Authorization"] = f"Bearer {token}" + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") full_url = url.rstrip("/") + "/" + path.lstrip("/") - response = httpx.get( - full_url, - headers=headers, - follow_redirects=True, - timeout=30.0, - ) + response = _make_request(url, token, "/" + path.lstrip("/")) if response.status_code != 200: raise click.ClickException(f"{response.status_code} error for {full_url}") - # Pretty-print if JSON, otherwise raw if "json" in response.headers.get("content-type", ""): click.echo(json.dumps(response.json(), indent=2)) else: @@ -57,93 +160,129 @@ def get(path, instance, token): @cli.command() -@click.argument("url_or_alias", default=None, required=False) +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("--json", "_json", is_flag=True, help="Output raw JSON") @click.option("--token", help="API token") -def databases(url_or_alias, token): +def databases(instance, _json, token): """ - List databases available on a Datasette instance + List databases on an instance Example usage: \b - dclient databases https://latest.datasette.io + dclient databases + dclient databases -i https://latest.datasette.io """ - url = _resolve_url(url_or_alias) - token = _resolve_token(token, url) - headers = {} - if token: - headers["Authorization"] = f"Bearer {token}" - response = httpx.get( - url.rstrip("/") + "/-/databases.json", - headers=headers, - follow_redirects=True, - timeout=30.0, - ) + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + response = _make_request(url, token, "/.json") if response.status_code != 200: raise click.ClickException(f"{response.status_code} error") - click.echo(json.dumps(response.json(), indent=2)) + data = response.json() + databases_data = data.get("databases", data if isinstance(data, list) else {}) + # Normalize: could be a dict {name: info} or a list [{name: ...}, ...] + if isinstance(databases_data, dict): + db_list = list(databases_data.values()) + else: + db_list = databases_data + if _json: + click.echo(json.dumps(db_list, indent=2)) + else: + for db in db_list: + name = db["name"] if isinstance(db, dict) else db + click.echo(name) @cli.command() -@click.argument("url_or_alias", default=None, required=False) +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("-d", "--database", default=None, help="Database name") +@click.option("--views", is_flag=True, help="Include views") +@click.option("--views-only", is_flag=True, help="Only show views") +@click.option("--hidden", is_flag=True, help="Include hidden tables") +@click.option("--json", "_json", is_flag=True, help="Output raw JSON") @click.option("--token", help="API token") -def tables(url_or_alias, token): +def tables(instance, database, views, views_only, hidden, _json, token): """ - List tables in a Datasette database + List tables in a database Example usage: \b - dclient tables https://latest.datasette.io/fixtures + dclient tables + dclient tables -d fixtures -i https://latest.datasette.io """ - url = _resolve_url(url_or_alias) - token = _resolve_token(token, url) - headers = {} - if token: - headers["Authorization"] = f"Bearer {token}" - response = httpx.get( - url.rstrip("/") + "/-/tables.json", - headers=headers, - follow_redirects=True, - timeout=30.0, - ) + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + instance_alias = _instance_alias_for_url(url, config_dir / "config.json") if not (instance and (instance.startswith("http://") or instance.startswith("https://"))) else None + if instance and not (instance.startswith("http://") or instance.startswith("https://")): + instance_alias = instance + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + db = _resolve_database(database, instance_alias, config_dir / "config.json") + response = _make_request(url, token, f"/{db}.json") if response.status_code != 200: raise click.ClickException(f"{response.status_code} error") - click.echo(json.dumps(response.json(), indent=2)) + data = response.json() + table_list = data.get("tables", []) + view_list = data.get("views", []) + if _json: + if views_only: + click.echo(json.dumps(view_list, indent=2)) + elif views: + click.echo(json.dumps(table_list + view_list, indent=2)) + else: + click.echo(json.dumps(table_list, indent=2)) + else: + items = [] + if not views_only: + for t in table_list: + if not hidden and t.get("hidden"): + continue + name = t["name"] if isinstance(t, dict) else t + count = t.get("count") if isinstance(t, dict) else None + if count is not None: + items.append(f"{name}\t{count} rows") + else: + items.append(name) + if views or views_only: + for v in view_list: + name = v["name"] if isinstance(v, dict) else v + items.append(name) + for item in items: + click.echo(item) @cli.command() -@click.argument("url_or_alias") +@click.argument("database") @click.argument("sql") +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") @click.option("--token", help="API token") @click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") -def query(url_or_alias, sql, token, verbose): +def query(database, sql, instance, token, verbose): """ - Run a SQL query against a Datasette database URL + Run a SQL query against a Datasette database - Returns a JSON array of objects + Requires both a database name and a SQL string. Example usage: \b - dclient query \\ - https://datasette.io/content \\ - 'select * from news limit 10' + dclient query fixtures "select * from facetable limit 5" + dclient query analytics "select count(*) from events" -i staging """ - url = _resolve_url(url_or_alias) - token = _resolve_token(token, url) - if not url.endswith(".json"): - url += ".json" + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + query_url = url.rstrip("/") + "/" + database + ".json" headers = {} if token: headers["Authorization"] = f"Bearer {token}" params = {"sql": sql, "_shape": "objects"} if verbose: - click.echo(url + "?" + urllib.parse.urlencode(params), err=True) - response = httpx.get(url, params=params, headers=headers, follow_redirects=True) + click.echo(query_url + "?" + urllib.parse.urlencode(params), err=True) + response = httpx.get(query_url, params=params, headers=headers, follow_redirects=True) if response.status_code != 200: - # Is it valid JSON? try: data = response.json() except json.JSONDecodeError: @@ -161,12 +300,10 @@ def query(url_or_alias, sql, token, verbose): "{} status code. {}".format(response.status_code, ": ".join(bits)) ) - # We should have JSON now try: data = response.json() except json.JSONDecodeError: raise click.ClickException("Response was not valid JSON") - # ... but it may have a {"ok": false} error if not data.get("ok"): bits = [] if data.get("title"): @@ -177,83 +314,17 @@ def query(url_or_alias, sql, token, verbose): bits = [json.dumps(data)] raise click.ClickException(": ".join(bits)) - # Output results click.echo(json.dumps(response.json()["rows"], indent=2)) -@cli.command() -@click.argument("url_or_alias") -@click.argument("table") -@click.argument( - "filepath", type=click.Path("rb", readable=True, allow_dash=True, dir_okay=False) -) -@click.option("format_csv", "--csv", is_flag=True, help="Input is CSV") -@click.option("format_tsv", "--tsv", is_flag=True, help="Input is TSV") -@click.option("format_json", "--json", is_flag=True, help="Input is JSON") -@click.option("format_nl", "--nl", is_flag=True, help="Input is newline-delimited JSON") -@click.option("--encoding", help="Character encoding for CSV/TSV") -@click.option( - "--no-detect-types", is_flag=True, help="Don't detect column types for CSV/TSV" -) -@click.option( - "--replace", is_flag=True, help="Replace rows with a matching primary key" -) -@click.option("--ignore", is_flag=True, help="Ignore rows with a matching primary key") -@click.option("--create", is_flag=True, help="Create table if it does not exist") -@click.option("--alter", is_flag=True, help="Alter table to add any missing columns") -@click.option( - "pks", - "--pk", - multiple=True, - help="Columns to use as the primary key when creating the table", -) -@click.option( - "--batch-size", type=int, default=100, help="Send rows in batches of this size" -) -@click.option( - "--interval", type=float, default=10, help="Send batch at least every X seconds" -) -@click.option("--token", "-t", help="API token") -@click.option("--silent", is_flag=True, help="Don't output progress") -@click.option( - "-v", - "--verbose", - is_flag=True, - help="Verbose output: show HTTP request and response", -) -def insert( - url_or_alias, - table, - filepath, - format_csv, - format_tsv, - format_json, - format_nl, - encoding, - no_detect_types, - replace, - ignore, - create, - alter, - pks, - batch_size, - interval, - token, - silent, - verbose, -): - """ - Insert data into a remote Datasette instance - - Example usage: - - \b - dclient insert \\ - https://private.datasette.cloud/data \\ - mytable data.csv --pk id --create - """ - url = _resolve_url(url_or_alias) - token = _resolve_token(token, url) +def _do_insert(database, table, filepath, format_csv, format_tsv, format_json, + format_nl, encoding, no_detect_types, replace, ignore, create, + alter, pks, batch_size, interval, token, silent, verbose, + instance, endpoint="insert"): + """Shared implementation for insert and upsert commands.""" + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") format = None if format_csv: @@ -283,11 +354,11 @@ def insert( raise click.ClickException(str(ex)) if format in (Format.JSON, Format.NL): - # Disable progress bar - it can't handle these formats file_size = None no_detect_types = True first = True + base_url = url.rstrip("/") + "/" + database with progressbar( length=file_size, @@ -304,15 +375,12 @@ def insert( bar.update(new_bytes) bytes_so_far += new_bytes except ValueError: - # File has likely been closed, so fp.tell() fails pass types = None if first and not no_detect_types: - # Detect types on first batch tracker = TypeTracker() list(tracker.wrap(batch)) types = tracker.types - # Convert types for row in batch: for key, value in row.items(): if value is None: @@ -329,7 +397,7 @@ def insert( row[key] = float(value) first = False _insert_batch( - url=url, + url=base_url, table=table, batch=batch, token=token, @@ -339,36 +407,244 @@ def insert( replace=replace, ignore=ignore, verbose=verbose, + endpoint=endpoint, ) +_insert_options = [ + click.argument("database"), + click.argument("table"), + click.argument( + "filepath", type=click.Path("rb", readable=True, allow_dash=True, dir_okay=False) + ), + click.option("-i", "--instance", default=None, help="Datasette instance URL or alias"), + click.option("format_csv", "--csv", is_flag=True, help="Input is CSV"), + click.option("format_tsv", "--tsv", is_flag=True, help="Input is TSV"), + click.option("format_json", "--json", is_flag=True, help="Input is JSON"), + click.option("format_nl", "--nl", is_flag=True, help="Input is newline-delimited JSON"), + click.option("--encoding", help="Character encoding for CSV/TSV"), + click.option( + "--no-detect-types", is_flag=True, help="Don't detect column types for CSV/TSV" + ), + click.option("--alter", is_flag=True, help="Alter table to add any missing columns"), + click.option( + "pks", "--pk", multiple=True, + help="Columns to use as the primary key when creating the table", + ), + click.option( + "--batch-size", type=int, default=100, help="Send rows in batches of this size" + ), + click.option( + "--interval", type=float, default=10, help="Send batch at least every X seconds" + ), + click.option("--token", "-t", help="API token"), + click.option("--silent", is_flag=True, help="Don't output progress"), + click.option( + "-v", "--verbose", is_flag=True, + help="Verbose output: show HTTP request and response", + ), +] + + +def _apply_options(options): + def decorator(func): + for option in reversed(options): + func = option(func) + return func + return decorator + + @cli.command() -@click.argument("url_or_alias", default=None, required=False) +@_apply_options(_insert_options) +@click.option("--replace", is_flag=True, help="Replace rows with a matching primary key") +@click.option("--ignore", is_flag=True, help="Ignore rows with a matching primary key") +@click.option("--create", is_flag=True, help="Create table if it does not exist") +def insert(database, table, filepath, instance, format_csv, format_tsv, format_json, + format_nl, encoding, no_detect_types, alter, pks, batch_size, interval, + token, silent, verbose, replace, ignore, create): + """ + Insert data into a remote Datasette instance + + Example usage: + + \b + dclient insert main mytable data.csv --csv -i myapp + dclient insert main mytable data.csv --csv --create --pk id + """ + _do_insert(database, table, filepath, format_csv, format_tsv, format_json, + format_nl, encoding, no_detect_types, replace, ignore, create, + alter, pks, batch_size, interval, token, silent, verbose, + instance, endpoint="insert") + + +@cli.command() +@_apply_options(_insert_options) +def upsert(database, table, filepath, instance, format_csv, format_tsv, format_json, + format_nl, encoding, no_detect_types, alter, pks, batch_size, interval, + token, silent, verbose): + """ + Upsert data into a remote Datasette instance + + Example usage: + + \b + dclient upsert main mytable data.csv --csv -i myapp + """ + _do_insert(database, table, filepath, format_csv, format_tsv, format_json, + format_nl, encoding, no_detect_types, False, False, False, + alter, pks, batch_size, interval, token, silent, verbose, + instance, endpoint="upsert") + + +@cli.command() +@click.argument("table_name", required=False, default=None) +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("-d", "--database", default=None, help="Database name") +@click.option("--json", "_json", is_flag=True, help="Output raw JSON") @click.option("--token", help="API token") -def actor(url_or_alias, token): +def schema(table_name, instance, database, _json, token): + """ + Show SQL schema for a database or specific table + + Example usage: + + \b + dclient schema + dclient schema facetable + dclient schema -d analytics + """ + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + instance_alias = _instance_alias_for_url(url, config_dir / "config.json") if not (instance and (instance.startswith("http://") or instance.startswith("https://"))) else None + if instance and not (instance.startswith("http://") or instance.startswith("https://")): + instance_alias = instance + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + db = _resolve_database(database, instance_alias, config_dir / "config.json") + if table_name: + response = _make_request(url, token, f"/{db}/{table_name}/-/schema.json") + else: + response = _make_request(url, token, f"/{db}/-/schema.json") + if response.status_code != 200: + raise click.ClickException(f"{response.status_code} error") + data = response.json() + if _json: + click.echo(json.dumps(data, indent=2)) + else: + click.echo(data.get("schema", "")) + + +@cli.command() +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("--json", "_json", is_flag=True, help="Output raw JSON") +@click.option("--token", help="API token") +def plugins(instance, _json, token): + """ + List installed plugins on an instance + + Example usage: + + \b + dclient plugins + dclient plugins -i https://latest.datasette.io + """ + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + response = _make_request(url, token, "/-/plugins.json") + if response.status_code != 200: + raise click.ClickException(f"{response.status_code} error") + data = response.json() + if _json: + click.echo(json.dumps(data, indent=2)) + else: + for plugin in data: + name = plugin["name"] if isinstance(plugin, dict) else plugin + click.echo(name) + + +@cli.command() +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("--token", help="API token") +def actor(instance, token): """ Show the actor represented by an API token Example usage: \b - dclient actor https://latest.datasette.io/fixtures + dclient actor + dclient actor -i prod """ - url = _resolve_url(url_or_alias) - - if not (url.startswith("http://") or url.startswith("https://")): - raise click.ClickException("Invalid URL: " + url) - - token = _resolve_token(token, url) - - actor_url = url.rstrip("/") + "/-/actor.json" - response = httpx.get( - actor_url, headers={"Authorization": "Bearer {}".format(token)}, timeout=40.0 - ) + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + response = _make_request(url, token, "/-/actor.json") response.raise_for_status() click.echo(json.dumps(response.json(), indent=4)) +@cli.command(name="default_query", hidden=True) +@click.argument("sql") +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("-d", "--database", default=None, help="Database name") +@click.option("--token", help="API token") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") +def default_query(sql, instance, database, token, verbose): + """Run a SQL query using default instance and database.""" + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + instance_alias = _instance_alias_for_url(url, config_dir / "config.json") if not (instance and (instance.startswith("http://") or instance.startswith("https://"))) else None + if instance and not (instance.startswith("http://") or instance.startswith("https://")): + instance_alias = instance + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + db = _resolve_database(database, instance_alias, config_dir / "config.json") + query_url = url.rstrip("/") + "/" + db + ".json" + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + params = {"sql": sql, "_shape": "objects"} + if verbose: + click.echo(query_url + "?" + urllib.parse.urlencode(params), err=True) + response = httpx.get(query_url, params=params, headers=headers, follow_redirects=True) + + if response.status_code != 200: + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException( + "{} status code. Response was not valid JSON".format( + response.status_code + ) + ) + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + raise click.ClickException( + "{} status code. {}".format(response.status_code, ": ".join(bits)) + ) + + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException("Response was not valid JSON") + if not data.get("ok"): + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + if not bits: + bits = [json.dumps(data)] + raise click.ClickException(": ".join(bits)) + + click.echo(json.dumps(response.json()["rows"], indent=2)) + + +# -- alias command group -- + + @cli.group() def alias(): "Manage aliases for different instances" @@ -376,22 +652,19 @@ def alias(): @alias.command(name="list") @click.option("_json", "--json", is_flag=True, help="Output raw JSON") -def list_(_json): - """ - List aliases - - Example usage: - - \b - dclient aliases list - """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) +def alias_list(_json): + """List aliases""" + config_file = get_config_dir() / "config.json" + config = _load_config(config_file) + instances = config.get("instances", {}) + default = config.get("default_instance") if _json: - click.echo(json.dumps(aliases, indent=2)) + click.echo(json.dumps(config, indent=2)) else: - for alias, url in aliases.items(): - click.echo(f"{alias} = {url}") + for name, inst in instances.items(): + marker = "* " if name == default else " " + db_info = f" (db: {inst['default_database']})" if inst.get("default_database") else "" + click.echo(f"{marker}{name} = {inst['url']}{db_info}") @alias.command(name="add") @@ -399,23 +672,19 @@ def list_(_json): @click.argument("url") def alias_add(name, url): """ - Add an alias + Add an alias for a Datasette instance Example usage: \b - dclient alias add content https://datasette.io/content - - Then: - - dclient query content 'select * from news limit 3' + dclient alias add prod https://myapp.datasette.cloud """ config_dir = get_config_dir() config_dir.mkdir(parents=True, exist_ok=True) - aliases_file = config_dir / "aliases.json" - aliases = _load_aliases(aliases_file) - aliases[name] = url - aliases_file.write_text(json.dumps(aliases, indent=4)) + config_file = config_dir / "config.json" + config = _load_config(config_file) + config["instances"][name] = {"url": url, "default_database": None} + _save_config(config_file, config) @alias.command(name="remove") @@ -427,18 +696,87 @@ def alias_remove(name): Example usage: \b - dclient alias remove content + dclient alias remove prod """ - config_dir = get_config_dir() - aliases_file = config_dir / "aliases.json" - aliases = _load_aliases(aliases_file) - if name in aliases: - del aliases[name] - aliases_file.write_text(json.dumps(aliases, indent=4)) + config_file = get_config_dir() / "config.json" + config = _load_config(config_file) + if name in config.get("instances", {}): + del config["instances"][name] + if config.get("default_instance") == name: + config["default_instance"] = None + _save_config(config_file, config) else: raise click.ClickException("No such alias") +@alias.command(name="default") +@click.argument("name", required=False, default=None) +@click.option("--clear", is_flag=True, help="Clear default instance") +def alias_default(name, clear): + """ + Set or show the default instance + + Example usage: + + \b + dclient alias default prod + dclient alias default + dclient alias default --clear + """ + config_file = get_config_dir() / "config.json" + config = _load_config(config_file) + if clear: + config["default_instance"] = None + _save_config(config_file, config) + elif name: + if name not in config.get("instances", {}): + raise click.ClickException(f"No such alias: {name}") + config["default_instance"] = name + _save_config(config_file, config) + else: + default = config.get("default_instance") + if default: + click.echo(default) + else: + click.echo("No default instance set") + + +@alias.command(name="default-db") +@click.argument("alias_name") +@click.argument("db", required=False, default=None) +@click.option("--clear", is_flag=True, help="Clear default database for this alias") +def alias_default_db(alias_name, db, clear): + """ + Set or show the default database for an alias + + Example usage: + + \b + dclient alias default-db prod main + dclient alias default-db prod + dclient alias default-db prod --clear + """ + config_file = get_config_dir() / "config.json" + config = _load_config(config_file) + if alias_name not in config.get("instances", {}): + raise click.ClickException(f"No such alias: {alias_name}") + if clear: + config["instances"][alias_name]["default_database"] = None + _save_config(config_file, config) + elif db: + config["instances"][alias_name]["default_database"] = db + _save_config(config_file, config) + else: + default_db = config["instances"][alias_name].get("default_database") + if default_db: + click.echo(default_db) + else: + click.echo(f"No default database set for {alias_name}") + + +# -- auth command group -- + + @cli.group() def auth(): "Manage authentication for different instances" @@ -454,20 +792,17 @@ def auth_add(alias_or_url, token): Example usage: \b - dclient auth add https://datasette.io/content + dclient auth add prod + dclient auth add https://datasette.io Paste in the token when prompted. """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - url = alias_or_url - if alias_or_url in aliases: - url = aliases[alias_or_url] config_dir = get_config_dir() config_dir.mkdir(parents=True, exist_ok=True) auth_file = config_dir / "auth.json" auths = _load_auths(auth_file) - auths[url] = token + # Store by alias name or URL as-is + auths[alias_or_url] = token auth_file.write_text(json.dumps(auths, indent=4)) @@ -486,8 +821,8 @@ def auth_list(): auths = _load_auths(auths_file) if auths: click.echo() - for url, token in auths.items(): - click.echo("{}:\t{}..".format(url, token[:1])) + for key, token in auths.items(): + click.echo("{}:\t{}..".format(key, token[:1])) @auth.command(name="remove") @@ -499,7 +834,7 @@ def auth_remove(alias_or_url): Example usage: \b - dclient auth remove https://datasette.io/content + dclient auth remove prod """ config_dir = get_config_dir() auth_file = config_dir / "auth.json" @@ -511,48 +846,76 @@ def auth_remove(alias_or_url): raise click.ClickException("No such URL or alias") -def _load_aliases(aliases_file): - if aliases_file.exists(): - aliases = json.loads(aliases_file.read_text()) - else: - aliases = {} - return aliases +@auth.command(name="status") +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("--token", help="API token") +def auth_status(instance, token): + """ + Verify authentication by calling /-/actor.json + + Example usage: + + \b + dclient auth status + dclient auth status -i prod + """ + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + response = _make_request(url, token, "/-/actor.json") + response.raise_for_status() + click.echo(json.dumps(response.json(), indent=4)) -def _load_auths(auth_file): - if auth_file.exists(): - auths = json.loads(auth_file.read_text()) - else: - auths = {} - return auths +# -- v1 → v2 migration -- -def _resolve_url(url_or_alias): - if not url_or_alias: - base_url = os.environ.get("DATASETTE_URL") - if base_url: - return base_url.rstrip("/") - raise click.ClickException( - "No URL provided. Set DATASETTE_URL or pass a URL/alias." - ) - aliases = _load_aliases(get_config_dir() / "aliases.json") - if url_or_alias in aliases: - return aliases[url_or_alias] - if url_or_alias.startswith("http://") or url_or_alias.startswith("https://"): - return url_or_alias - base_url = os.environ.get("DATASETTE_URL") - if base_url: - return base_url.rstrip("/") + "/" + url_or_alias - return url_or_alias +def _migrate_v1_to_v2(config_dir): + """Migrate v1 aliases.json + auth.json to v2 config.json + auth.json.""" + config_file = config_dir / "config.json" + aliases_file = config_dir / "aliases.json" + auth_file = config_dir / "auth.json" + if config_file.exists() or not aliases_file.exists(): + return + + aliases = json.loads(aliases_file.read_text()) if aliases_file.exists() else {} + old_auths = json.loads(auth_file.read_text()) if auth_file.exists() else {} + + config = {"default_instance": None, "instances": {}} + new_auths = {} + url_to_alias = {} + + for alias_name, alias_url in aliases.items(): + parsed = urllib.parse.urlparse(alias_url) + path_parts = [p for p in parsed.path.split("/") if p] + if len(path_parts) == 1: + # URL has a single path segment → instance URL + default database + instance_url = f"{parsed.scheme}://{parsed.netloc}" + default_db = path_parts[0] + else: + instance_url = alias_url + default_db = None + config["instances"][alias_name] = { + "url": instance_url, + "default_database": default_db, + } + url_to_alias[alias_url] = alias_name + + # Migrate auth keys from URLs to alias names + for url, token in old_auths.items(): + if url in url_to_alias: + new_auths[url_to_alias[url]] = token + else: + # Keep URL-keyed entries as fallbacks + new_auths[url] = token + + _save_config(config_file, config) + if new_auths or old_auths: + auth_file.rename(config_dir / "auth.json.bak") + auth_file.write_text(json.dumps(new_auths, indent=4)) + aliases_file.rename(config_dir / "aliases.json.bak") -def _resolve_token(token, url): - if token is not None: - return token - stored = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) - if stored is not None: - return stored - return os.environ.get("DATASETTE_TOKEN") def _batches(iterable, size, interval=None): @@ -574,7 +937,8 @@ def _batches(iterable, size, interval=None): def _insert_batch( - *, url, table, batch, token, create, alter, pks, replace, ignore, verbose + *, url, table, batch, token, create, alter, pks, replace, ignore, verbose, + endpoint="insert" ): if create: data = { @@ -603,7 +967,7 @@ def _insert_batch( data["ignore"] = True if alter: data["alter"] = True - url = "{}/{}/-/insert".format(url, table) + url = "{}/{}/-/{}".format(url, table, endpoint) if verbose: click.echo("POST {}".format(url), err=True) click.echo(textwrap.indent(json.dumps(data, indent=2), " "), err=True) diff --git a/docs/environment.md b/docs/environment.md index f597617..1ae7f81 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -2,37 +2,39 @@ # Environment variables -`dclient` supports two environment variables for convenient access to a Datasette instance without needing aliases or repeated URLs. +`dclient` supports several environment variables for convenient access to Datasette instances. ## DATASETTE_URL -Set this to the base URL of your Datasette instance: +Set this to the base URL of your Datasette instance. It is used as a fallback when no instance is specified via `-i` and no default instance is configured: ```bash export DATASETTE_URL=https://my-instance.datasette.cloud ``` -Then pass just the database name as the first argument to any command: +Then you can omit the `-i` flag: ```bash +dclient databases dclient query data "select * from my_table limit 10" ``` -This is equivalent to: +Aliases and the `-i` flag always take priority over `DATASETTE_URL`. + +## DATASETTE_DATABASE + +Set this to a default database name. It is used as a fallback when no database is specified via `-d` and the current instance has no `default_database` configured: ```bash -dclient query https://my-instance.datasette.cloud/data "select * from my_table limit 10" +export DATASETTE_DATABASE=data ``` -It works with all commands: +Then you can use the bare SQL shortcut: ```bash -dclient insert data my_table data.csv --csv -dclient actor data +dclient "select * from my_table limit 10" ``` -Full URLs and aliases always take priority over `DATASETTE_URL`. If the argument starts with `http://` or `https://`, it is used as-is. If it matches an alias in `aliases.json`, the alias is used. - ## DATASETTE_TOKEN Set this to an API token: @@ -46,23 +48,34 @@ The token will be used automatically for any request that doesn't have a more sp The precedence order for tokens is: 1. `--token` CLI flag (highest priority) -2. Stored token from `auth.json` (matched by URL prefix) +2. Stored token from `auth.json` (matched by alias name, then URL prefix) 3. `DATASETTE_TOKEN` environment variable (lowest priority) -## Using both together +## DCLIENT_CONFIG_DIR + +Override the config directory (default `~/.config/io.datasette.dclient` or platform equivalent): + +```bash +export DCLIENT_CONFIG_DIR=/path/to/config +``` + +This is useful for testing or running multiple configurations side by side. + +## Using them together These variables work well together for quick access to a single instance: ```bash export DATASETTE_URL=https://my-instance.datasette.cloud +export DATASETTE_DATABASE=data export DATASETTE_TOKEN=dstok_abc123 -# Query the "data" database -dclient query data "select * from my_table" +# Query +dclient "select * from my_table" -# Insert into the "data" database +# Insert cat records.json | dclient insert data my_table - --json # Check your actor identity -dclient actor data +dclient actor ``` diff --git a/docs/inserting.md b/docs/inserting.md index 316a3c0..17d81a7 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -4,19 +4,23 @@ The `dclient insert` command can be used to insert data from a local file direct First you'll need to {ref}`authenticate ` with the instance. -To insert data from a `data.csv` file into a table called `my_table`, creating that table if it does not exist: +To insert data from a `data.csv` file into a table called `my_table` in the `data` database: ```bash -dclient insert \ - https://my-private-space.datasette.cloud/data \ - my_table data.csv --create +dclient insert data my_table data.csv --create -i myapp ``` You can also pipe data into standard input: ```bash curl -s 'https://api.github.com/repos/simonw/dclient/issues' | \ - dclient insert \ - https://my-private-space.datasette.cloud/data \ - issues - --create + dclient insert data issues - --create -i myapp +``` + +## Upserting data + +The `dclient upsert` command works exactly like `insert` but uses the upsert endpoint, which will update existing rows with matching primary keys rather than raising an error. + +```bash +dclient upsert data my_table data.csv --csv -i myapp ``` ## Streaming data @@ -26,7 +30,7 @@ curl -s 'https://api.github.com/repos/simonw/dclient/issues' | \ If you have a log file containing newline-delimited JSON you can tail it and send it to a Datasette instance like this: ```bash tail -f log.jsonl | \ - dclient insert https://my-private-space.datasette.cloud/data logs - --nl + dclient insert data logs - --nl -i myapp ``` When reading from standard input (filename `-`) you are required to specify the format. In this example that's `--nl` for newline-delimited JSON. `--csv` and `--tsv` are supported for streaming as well, but `--json` is not. @@ -34,7 +38,7 @@ In streaming mode records default to being sent to the server every 100 records ```bash tail -f log.jsonl | dclient insert \ - https://my-private-space.datasette.cloud/data logs - --nl --create \ + data logs - --nl --create -i myapp \ --batch-size 10 \ --interval 5 ``` @@ -106,26 +110,65 @@ cog.out( ) ]]] --> ``` -Usage: dclient insert [OPTIONS] URL_OR_ALIAS TABLE FILEPATH +Usage: dclient insert [OPTIONS] DATABASE TABLE FILEPATH Insert data into a remote Datasette instance Example usage: - dclient insert \ - https://private.datasette.cloud/data \ - mytable data.csv --pk id --create + dclient insert main mytable data.csv --csv -i myapp + dclient insert main mytable data.csv --csv --create --pk id Options: + -i, --instance TEXT Datasette instance URL or alias --csv Input is CSV --tsv Input is TSV --json Input is JSON --nl Input is newline-delimited JSON --encoding TEXT Character encoding for CSV/TSV --no-detect-types Don't detect column types for CSV/TSV + --alter Alter table to add any missing columns + --pk TEXT Columns to use as the primary key when creating the + table + --batch-size INTEGER Send rows in batches of this size + --interval FLOAT Send batch at least every X seconds + -t, --token TEXT API token + --silent Don't output progress + -v, --verbose Verbose output: show HTTP request and response --replace Replace rows with a matching primary key --ignore Ignore rows with a matching primary key --create Create table if it does not exist + --help Show this message and exit. + +``` + + +## dclient upsert --help + +``` +Usage: dclient upsert [OPTIONS] DATABASE TABLE FILEPATH + + Upsert data into a remote Datasette instance + + Example usage: + + dclient upsert main mytable data.csv --csv -i myapp + +Options: + -i, --instance TEXT Datasette instance URL or alias + --csv Input is CSV + --tsv Input is TSV + --json Input is JSON + --nl Input is newline-delimited JSON + --encoding TEXT Character encoding for CSV/TSV + --no-detect-types Don't detect column types for CSV/TSV --alter Alter table to add any missing columns --pk TEXT Columns to use as the primary key when creating the table diff --git a/docs/queries.md b/docs/queries.md index 3bd671a..900f0ca 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -3,7 +3,7 @@ You can run SQL queries against a Datasette instance like this: ```bash -dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" +dclient query fixtures "select * from facetable limit 1" -i https://latest.datasette.io ``` Output: ```json @@ -24,6 +24,18 @@ Output: ] ``` +The `query` command takes a database name and SQL string as positional arguments. Use `-i` to specify the instance (alias or URL). If you have a default instance and default database configured, you can use the bare SQL shortcut instead: + +```bash +dclient "select * from facetable limit 1" +``` + +You can override just the database with `-d`: + +```bash +dclient "select * from counters" -d counters +``` + ## dclient query --help ``` -Usage: dclient query [OPTIONS] URL_OR_ALIAS SQL +Usage: dclient query [OPTIONS] DATABASE SQL - Run a SQL query against a Datasette database URL + Run a SQL query against a Datasette database - Returns a JSON array of objects + Requires both a database name and a SQL string. Example usage: - dclient query \ - https://datasette.io/content \ - 'select * from news limit 10' + dclient query fixtures "select * from facetable limit 5" + dclient query analytics "select count(*) from events" -i staging Options: - --token TEXT API token - -v, --verbose Verbose output: show HTTP request - --help Show this message and exit. + -i, --instance TEXT Datasette instance URL or alias + --token TEXT API token + -v, --verbose Verbose output: show HTTP request + --help Show this message and exit. ``` diff --git a/pyproject.toml b/pyproject.toml index 15d9c56..5bc7cfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ license = "Apache-2.0" requires-python = ">=3.10" dependencies = [ "click", + "click-default-group", "httpx", "sqlite-utils", ] diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py index ba30e22..60b8bb3 100644 --- a/tests/test_cli_auth.py +++ b/tests/test_cli_auth.py @@ -13,26 +13,26 @@ def test_auth(mocker, tmpdir): # Should only have one line assert len([line for line in result.output.split("\n") if line.strip()]) == 1 - # Now add a token - result2 = runner.invoke(cli, ["auth", "add", "https://example.com"], input="xyz\n") + # Now add a token (keys are now alias names or URLs) + result2 = runner.invoke(cli, ["auth", "add", "prod"], input="xyz\n") assert result2.exit_code == 0 # Check the tokens file auth_file = pathlib.Path(tmpdir) / "auth.json" - assert json.loads(auth_file.read_text()) == {"https://example.com": "xyz"} + assert json.loads(auth_file.read_text()) == {"prod": "xyz"} # auth list should show that now result3 = runner.invoke(cli, ["auth", "list"]) assert result3.output.startswith("Tokens file:") - assert "https://example.com" in result3.output + assert "prod" in result3.output - # Remove should fail with an incorrect URL - result4 = runner.invoke(cli, ["auth", "remove", "https://example.com/foo"]) + # Remove should fail with an incorrect key + result4 = runner.invoke(cli, ["auth", "remove", "nonexistent"]) assert result4.exit_code == 1 assert result4.output == "Error: No such URL or alias\n" - # Remove should work with the correct URL - result5 = runner.invoke(cli, ["auth", "remove", "https://example.com"]) + # Remove should work with the correct key + result5 = runner.invoke(cli, ["auth", "remove", "prod"]) assert result5.exit_code == 0 assert result5.output == "" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..f12bf85 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,249 @@ +"""Tests for the v2 config system: config.json format, instance resolution, database resolution.""" + +from dclient.cli import ( + _load_config, + _save_config, + _resolve_instance, + _resolve_database, + _resolve_token, + get_config_dir, +) +import json +import pathlib +import pytest + + +# -- Config loading/saving -- + + +def test_load_config_empty(tmpdir): + """Loading config when no file exists returns empty defaults.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config = _load_config(config_file) + assert config == {"default_instance": None, "instances": {}} + + +def test_load_config_existing(tmpdir): + """Loading config reads the JSON file.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://myapp.datasette.cloud", + "default_database": "main", + } + }, + } + ) + ) + config = _load_config(config_file) + assert config["default_instance"] == "prod" + assert config["instances"]["prod"]["url"] == "https://myapp.datasette.cloud" + assert config["instances"]["prod"]["default_database"] == "main" + + +def test_save_config(tmpdir): + """Saving config writes JSON to disk.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config = { + "default_instance": "local", + "instances": { + "local": { + "url": "http://localhost:8001", + "default_database": None, + } + }, + } + _save_config(config_file, config) + assert json.loads(config_file.read_text()) == config + + +# -- Instance resolution -- + + +def test_resolve_instance_from_flag(tmpdir): + """An explicit -i flag with a URL is used directly.""" + config_file = pathlib.Path(tmpdir) / "config.json" + url = _resolve_instance("https://example.com", config_file) + assert url == "https://example.com" + + +def test_resolve_instance_from_flag_alias(tmpdir): + """An explicit -i flag with an alias name resolves via config.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": None, + "instances": { + "prod": { + "url": "https://myapp.datasette.cloud", + "default_database": None, + } + }, + } + ) + ) + url = _resolve_instance("prod", config_file) + assert url == "https://myapp.datasette.cloud" + + +def test_resolve_instance_from_config_default(tmpdir): + """When no -i flag, uses config.default_instance.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://myapp.datasette.cloud", + "default_database": None, + } + }, + } + ) + ) + url = _resolve_instance(None, config_file) + assert url == "https://myapp.datasette.cloud" + + +def test_resolve_instance_from_env(tmpdir, monkeypatch): + """When no -i flag and no config default, falls back to DATASETTE_URL.""" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.setenv("DATASETTE_URL", "https://env.example.com") + url = _resolve_instance(None, config_file) + assert url == "https://env.example.com" + + +def test_resolve_instance_from_env_strips_trailing_slash(tmpdir, monkeypatch): + """DATASETTE_URL trailing slash is stripped.""" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.setenv("DATASETTE_URL", "https://env.example.com/") + url = _resolve_instance(None, config_file) + assert url == "https://env.example.com" + + +def test_resolve_instance_error(tmpdir, monkeypatch): + """When nothing is configured, raises an error.""" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.delenv("DATASETTE_URL", raising=False) + with pytest.raises(Exception, match="No instance specified"): + _resolve_instance(None, config_file) + + +def test_resolve_instance_unknown_alias(tmpdir): + """An -i flag with an unknown alias name raises an error.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text(json.dumps({"default_instance": None, "instances": {}})) + with pytest.raises(Exception, match="Unknown instance"): + _resolve_instance("nonexistent", config_file) + + +# -- Database resolution (optional -d flag mode) -- + + +def test_resolve_database_from_flag(tmpdir): + """An explicit -d flag is used directly.""" + config_file = pathlib.Path(tmpdir) / "config.json" + db = _resolve_database("mydb", None, config_file) + assert db == "mydb" + + +def test_resolve_database_from_instance_default(tmpdir): + """When no -d flag, uses the instance's default_database from config.""" + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://myapp.datasette.cloud", + "default_database": "main", + } + }, + } + ) + ) + db = _resolve_database(None, "prod", config_file) + assert db == "main" + + +def test_resolve_database_from_env(tmpdir, monkeypatch): + """When no -d flag and no instance default, falls back to DATASETTE_DATABASE.""" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.setenv("DATASETTE_DATABASE", "envdb") + db = _resolve_database(None, None, config_file) + assert db == "envdb" + + +def test_resolve_database_error(tmpdir, monkeypatch): + """When nothing is configured, raises an error.""" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.delenv("DATASETTE_DATABASE", raising=False) + with pytest.raises(Exception, match="No database specified"): + _resolve_database(None, None, config_file) + + +# -- Token resolution -- + + +def test_resolve_token_from_flag(tmpdir): + """An explicit --token flag is used directly.""" + auth_file = pathlib.Path(tmpdir) / "auth.json" + config_file = pathlib.Path(tmpdir) / "config.json" + token = _resolve_token("explicit-token", "https://example.com", auth_file, config_file) + assert token == "explicit-token" + + +def test_resolve_token_from_auth_by_alias(tmpdir): + """Auth token looked up by alias name.""" + auth_file = pathlib.Path(tmpdir) / "auth.json" + auth_file.write_text(json.dumps({"prod": "tok123"})) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": None, + "instances": { + "prod": { + "url": "https://myapp.datasette.cloud", + "default_database": None, + } + }, + } + ) + ) + token = _resolve_token(None, "https://myapp.datasette.cloud", auth_file, config_file) + assert token == "tok123" + + +def test_resolve_token_from_auth_by_url_fallback(tmpdir): + """Auth token falls back to URL prefix matching when no alias match.""" + auth_file = pathlib.Path(tmpdir) / "auth.json" + auth_file.write_text(json.dumps({"https://example.com": "url-tok"})) + config_file = pathlib.Path(tmpdir) / "config.json" + token = _resolve_token(None, "https://example.com/db", auth_file, config_file) + assert token == "url-tok" + + +def test_resolve_token_from_env(tmpdir, monkeypatch): + """Falls back to DATASETTE_TOKEN env var.""" + auth_file = pathlib.Path(tmpdir) / "auth.json" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.setenv("DATASETTE_TOKEN", "env-tok") + token = _resolve_token(None, "https://example.com", auth_file, config_file) + assert token == "env-tok" + + +def test_resolve_token_none(tmpdir, monkeypatch): + """Returns None when nothing is configured.""" + auth_file = pathlib.Path(tmpdir) / "auth.json" + config_file = pathlib.Path(tmpdir) / "config.json" + monkeypatch.delenv("DATASETTE_TOKEN", raising=False) + token = _resolve_token(None, "https://example.com", auth_file, config_file) + assert token is None diff --git a/tests/test_env.py b/tests/test_env.py index 5afd593..185611c 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -26,7 +26,9 @@ def test_datasette_token_used_as_fallback(httpx_mock, mocker, tmpdir): mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) runner = CliRunner(env={"DATASETTE_TOKEN": "env-token-123"}) - result = runner.invoke(cli, ["query", "https://example.com", "select 1"]) + result = runner.invoke( + cli, ["query", "data", "select 1", "-i", "https://example.com"] + ) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.headers["authorization"] == "Bearer env-token-123" @@ -38,7 +40,12 @@ def test_token_flag_overrides_datasette_token(httpx_mock, mocker, tmpdir): httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) runner = CliRunner(env={"DATASETTE_TOKEN": "env-token"}) result = runner.invoke( - cli, ["query", "https://example.com", "select 1", "--token", "flag-token"] + cli, + [ + "query", "data", "select 1", + "-i", "https://example.com", + "--token", "flag-token", + ], ) assert result.exit_code == 0 request = httpx_mock.get_request() @@ -52,7 +59,9 @@ def test_auth_json_overrides_datasette_token(httpx_mock, mocker, tmpdir): auth_file.write_text(json.dumps({"https://example.com": "stored-token"})) httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) runner = CliRunner(env={"DATASETTE_TOKEN": "env-token"}) - result = runner.invoke(cli, ["query", "https://example.com", "select 1"]) + result = runner.invoke( + cli, ["query", "data", "select 1", "-i", "https://example.com"] + ) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.headers["authorization"] == "Bearer stored-token" @@ -61,8 +70,8 @@ def test_auth_json_overrides_datasette_token(httpx_mock, mocker, tmpdir): # -- DATASETTE_URL tests -- -def test_datasette_url_combines_with_database_name(httpx_mock, mocker, tmpdir): - """DATASETTE_URL + database name arg → combined URL.""" +def test_datasette_url_used_as_instance(httpx_mock, mocker, tmpdir): + """DATASETTE_URL provides the instance when no -i flag.""" mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) runner = CliRunner(env={"DATASETTE_URL": "https://my-instance.datasette.cloud"}) @@ -84,12 +93,14 @@ def test_datasette_url_with_trailing_slash(httpx_mock, mocker, tmpdir): assert request.url.path == "/data.json" -def test_full_url_ignores_datasette_url(httpx_mock, mocker, tmpdir): - """A full URL argument is used as-is, ignoring DATASETTE_URL.""" +def test_explicit_instance_ignores_datasette_url(httpx_mock, mocker, tmpdir): + """An explicit -i flag is used, ignoring DATASETTE_URL.""" mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) runner = CliRunner(env={"DATASETTE_URL": "https://should-be-ignored.com"}) - result = runner.invoke(cli, ["query", "https://other.example.com/db", "select 1"]) + result = runner.invoke( + cli, ["query", "db", "select 1", "-i", "https://other.example.com"] + ) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.url.host == "other.example.com" @@ -97,13 +108,25 @@ def test_full_url_ignores_datasette_url(httpx_mock, mocker, tmpdir): def test_alias_takes_priority_over_datasette_url(httpx_mock, mocker, tmpdir): - """Alias match takes priority over DATASETTE_URL.""" + """Alias match via -i takes priority over DATASETTE_URL.""" mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) - aliases_file = pathlib.Path(tmpdir) / "aliases.json" - aliases_file.write_text(json.dumps({"myalias": "https://aliased.example.com/db"})) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": None, + "instances": { + "myalias": { + "url": "https://aliased.example.com", + "default_database": None, + } + }, + } + ) + ) httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) runner = CliRunner(env={"DATASETTE_URL": "https://should-be-ignored.com"}) - result = runner.invoke(cli, ["query", "myalias", "select 1"]) + result = runner.invoke(cli, ["query", "db", "select 1", "-i", "myalias"]) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.url.host == "aliased.example.com" @@ -150,7 +173,7 @@ def test_datasette_url_with_actor(httpx_mock, mocker, tmpdir): "DATASETTE_TOKEN": "env-token", } ) - result = runner.invoke(cli, ["actor", "data"]) + result = runner.invoke(cli, ["actor"]) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.url.host == "my-instance.datasette.cloud" @@ -176,3 +199,23 @@ def test_datasette_url_and_token_together(httpx_mock, mocker, tmpdir): assert request.url.host == "my-instance.datasette.cloud" assert request.url.path == "/mydb.json" assert request.headers["authorization"] == "Bearer env-token-456" + + +# -- DATASETTE_DATABASE tests -- + + +def test_datasette_database_with_default_query(httpx_mock, mocker, tmpdir): + """DATASETTE_DATABASE is used by default_query shortcut.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner( + env={ + "DATASETTE_URL": "https://my-instance.datasette.cloud", + "DATASETTE_DATABASE": "mydb", + } + ) + result = runner.invoke(cli, ["select 1"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "my-instance.datasette.cloud" + assert request.url.path == "/mydb.json" diff --git a/tests/test_insert.py b/tests/test_insert.py index 32fbceb..64f38e7 100644 --- a/tests/test_insert.py +++ b/tests/test_insert.py @@ -44,12 +44,14 @@ def test_insert_mocked(httpx_mock, tmpdir): cli, [ "insert", - "https://datasette.example.com/data", + "data", "table1", str(path), "--csv", "--token", "x", + "-i", + "https://datasette.example.com", ], catch_exceptions=False, ) @@ -311,11 +313,13 @@ async def test_insert_against_datasette( cli, [ "insert", - "http://datasette.example.com/data", + "data", "table1", str(path), "--token", token, + "-i", + "http://datasette.example.com", ] + cmd_args, ) diff --git a/tests/test_query.py b/tests/test_query.py index 552acb0..e273beb 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -16,7 +16,9 @@ def test_query_error(httpx_mock): status_code=400, ) runner = CliRunner() - result = runner.invoke(cli, ["query", "https://example.com", "hello"]) + result = runner.invoke( + cli, ["query", "content", "hello", "-i", "https://example.com"] + ) assert result.exit_code == 1 assert ( result.output @@ -42,17 +44,16 @@ def test_query(httpx_mock, with_token): status_code=200, ) runner = CliRunner() - args = ["query", "https://example.com", "hello"] + args = ["query", "content", "hello", "-i", "https://example.com"] if with_token: - args.append("--token") - args.append("xyz") + args.extend(["--token", "xyz"]) result = runner.invoke(cli, args) assert result.exit_code == 0 assert json.loads(result.output) == [{"5 * 2": 10}] # Check the request request = httpx_mock.get_request() - assert str(request.url) == "https://example.com.json?sql=hello&_shape=objects" + assert str(request.url) == "https://example.com/content.json?sql=hello&_shape=objects" if with_token: assert request.headers["authorization"] == "Bearer xyz" else: @@ -66,28 +67,32 @@ def test_aliases(mocker, tmpdir, httpx_mock): assert result.exit_code == 0 assert result.output == "" - result = runner.invoke(cli, ["alias", "add", "foo", "https://example.com/foo"]) + result = runner.invoke( + cli, ["alias", "add", "foo", "https://example.com"] + ) assert result.exit_code == 0 assert result.output == "" result = runner.invoke(cli, ["alias", "list"]) assert result.exit_code == 0 - assert result.output == "foo = https://example.com/foo\n" + assert "foo = https://example.com" in result.output # --json mode: result = runner.invoke(cli, ["alias", "list", "--json"]) assert result.exit_code == 0 - assert json.loads(result.output) == {"foo": "https://example.com/foo"} + data = json.loads(result.output) + assert data["instances"]["foo"]["url"] == "https://example.com" - # Check the aliases file - aliases_file = pathlib.Path(tmpdir) / "aliases.json" - assert json.loads(aliases_file.read_text()) == {"foo": "https://example.com/foo"} + # Check the config file + config_file = pathlib.Path(tmpdir) / "config.json" + config = json.loads(config_file.read_text()) + assert config["instances"]["foo"]["url"] == "https://example.com" # Try a query against that alias httpx_mock.add_response( json={ "ok": True, - "database": "foo", + "database": "mydb", "query_name": None, "rows": [{"11 * 3": 33}], "truncated": False, @@ -99,14 +104,14 @@ def test_aliases(mocker, tmpdir, httpx_mock): }, status_code=200, ) - result = runner.invoke(cli, ["query", "foo", "select 11 * 3"]) + result = runner.invoke(cli, ["query", "mydb", "select 11 * 3", "-i", "foo"]) assert result.exit_code == 0 assert json.loads(result.output) == [{"11 * 3": 33}] - # Should have hit https://example.com/foo.json + # Should have hit https://example.com/mydb.json url = httpx_mock.get_request().url assert url.host == "example.com" - assert url.path == "/foo.json" + assert url.path == "/mydb.json" assert dict(url.params) == {"sql": "select 11 * 3", "_shape": "objects"} # Remove alias @@ -117,4 +122,5 @@ def test_aliases(mocker, tmpdir, httpx_mock): result = runner.invoke(cli, ["alias", "remove", "foo"]) assert result.exit_code == 0 assert result.output == "" - assert json.loads(aliases_file.read_text()) == {} + config = json.loads(config_file.read_text()) + assert config["instances"] == {} From 92c0d2a5ba29f12d79dc181a3c56bf248c8e6cbb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 15:21:58 -0800 Subject: [PATCH 45/64] Tests and docs for v2 introspection commands and alias features Add test_commands_v2.py with 28 tests covering databases, tables, plugins, schema, default_query, upsert, auth status, actor, and get commands. Add test_alias_v2.py with 5 tests for alias default, alias default-db, and removing the default alias. Update aliases.md and authentication.md docs for v2 API. refs #29 Co-Authored-By: Claude Opus 4.6 --- docs/aliases.md | 111 ++++++-- docs/authentication.md | 88 +++++-- tests/test_alias_v2.py | 111 ++++++++ tests/test_commands_v2.py | 538 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 812 insertions(+), 36 deletions(-) create mode 100644 tests/test_alias_v2.py create mode 100644 tests/test_commands_v2.py diff --git a/docs/aliases.md b/docs/aliases.md index 3bb6f7c..8399085 100644 --- a/docs/aliases.md +++ b/docs/aliases.md @@ -1,17 +1,40 @@ # Aliases -You can assign an alias to a Datasette database using the `dclient alias` command: +You can assign an alias to a Datasette instance using the `dclient alias add` command: - dclient alias add content https://datasette.io/content + dclient alias add latest https://latest.datasette.io You can list aliases with `dclient alias list`: $ dclient alias list - content = https://datasette.io/content + latest = https://latest.datasette.io -Once registered, you can pass an alias to commands such as `dclient query`: +Once registered, you can pass an alias to commands using the `-i` flag: - dclient query content "select * from news limit 1" + dclient query fixtures "select * from news limit 1" -i latest + +## Default instance + +Set a default instance so you don't need `-i` every time: + + dclient alias default latest + +Now commands will use `latest` automatically: + + dclient databases + dclient tables -d fixtures + +## Default database + +Set a default database for an alias: + + dclient alias default-db latest fixtures + +Now you can run bare SQL queries directly: + + dclient "select * from facetable limit 5" + +This uses the default instance and default database. ## dclient alias --help @@ -56,10 +81,6 @@ Usage: dclient alias list [OPTIONS] List aliases - Example usage: - - dclient aliases list - Options: --json Output raw JSON --help Show this message and exit. @@ -80,15 +101,11 @@ cog.out( ``` Usage: dclient alias add [OPTIONS] NAME URL - Add an alias + Add an alias for a Datasette instance Example usage: - dclient alias add content https://datasette.io/content - - Then: - - dclient query content 'select * from news limit 3' + dclient alias add prod https://myapp.datasette.cloud Options: --help Show this message and exit. @@ -113,10 +130,66 @@ Usage: dclient alias remove [OPTIONS] NAME Example usage: - dclient alias remove content + dclient alias remove prod Options: --help Show this message and exit. ``` + +## dclient alias default --help + + +``` +Usage: dclient alias default [OPTIONS] [NAME] + + Set or show the default instance + + Example usage: + + dclient alias default prod + dclient alias default + dclient alias default --clear + +Options: + --clear Clear default instance + --help Show this message and exit. + +``` + + +## dclient alias default-db --help + + +``` +Usage: dclient alias default-db [OPTIONS] ALIAS_NAME [DB] + + Set or show the default database for an alias + + Example usage: + + dclient alias default-db prod main + dclient alias default-db prod + dclient alias default-db prod --clear + +Options: + --clear Clear default database for this alias + --help Show this message and exit. + +``` + diff --git a/docs/authentication.md b/docs/authentication.md index caadba7..cd3961c 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -4,35 +4,57 @@ `dclient` can handle API tokens for Datasette instances that require authentication. -You can pass an API token to `query` using `-t/--token` like this: +You can pass an API token to any command using `--token` like this: ```bash -dclient query https://latest.datasette.io/fixtures "select * from facetable" -t dstok_mytoken +dclient query fixtures "select * from facetable" --token dstok_mytoken -i latest ``` -A more convenient way to handle this is to store tokens to be used with different URL prefixes. +A more convenient way to handle this is to store tokens for your aliases. ## Using stored tokens -To always use `dstok_mytoken` for any URL on the `https://latest.datasette.io/` instance you can run this: +To store a token for an alias: ```bash -dclient auth add https://latest.datasette.io/ +dclient auth add latest ``` Then paste in the token and hit enter when prompted to do so. -To list which URLs you have set tokens for, run the `auth list` command: +Tokens can also be stored for direct URLs: +```bash +dclient auth add https://latest.datasette.io +``` + +To list which aliases/URLs you have set tokens for, run the `auth list` command: ```bash dclient auth list ``` -To delete the token for a specific URL, run `auth remove`: +To delete the token for a specific alias or URL, run `auth remove`: ```bash -dclient auth remove https://latest.datasette.io/ +dclient auth remove latest ``` + +## Token resolution order + +When making a request, dclient resolves the token in this order: + +1. `--token` CLI flag (highest priority) +2. Token stored by alias name in `auth.json` +3. Token stored by URL prefix in `auth.json` +4. `DATASETTE_TOKEN` environment variable (lowest priority) + ## Testing a token -The `dclient actor` command can be used to test a token, retrieving the actor that the token represents. +The `dclient auth status` command can be used to verify authentication by calling `/-/actor.json`: ```bash -dclient actor https://latest.datasette.io/content +dclient auth status +dclient auth status -i prod +``` + +The `dclient actor` command also shows the actor: +```bash +dclient actor +dclient actor -i prod ``` The output looks like this: ```json @@ -68,6 +90,7 @@ Commands: add Add an authentication token for an alias or URL list List stored API tokens remove Remove the API token for an alias or URL + status Verify authentication by calling /-/actor.json ``` @@ -89,7 +112,8 @@ Usage: dclient auth add [OPTIONS] ALIAS_OR_URL Example usage: - dclient auth add https://datasette.io/content + dclient auth add prod + dclient auth add https://datasette.io Paste in the token when prompted. @@ -142,7 +166,7 @@ Usage: dclient auth remove [OPTIONS] ALIAS_OR_URL Example usage: - dclient auth remove https://datasette.io/content + dclient auth remove prod Options: --help Show this message and exit. @@ -150,6 +174,34 @@ Options: ``` +## dclient auth status --help + + +``` +Usage: dclient auth status [OPTIONS] + + Verify authentication by calling /-/actor.json + + Example usage: + + dclient auth status + dclient auth status -i prod + +Options: + -i, --instance TEXT Datasette instance URL or alias + --token TEXT API token + --help Show this message and exit. + +``` + + ## dclient actor --help ``` -Usage: dclient actor [OPTIONS] [URL_OR_ALIAS] +Usage: dclient actor [OPTIONS] Show the actor represented by an API token Example usage: - dclient actor https://latest.datasette.io/fixtures + dclient actor + dclient actor -i prod Options: - --token TEXT API token - --help Show this message and exit. + -i, --instance TEXT Datasette instance URL or alias + --token TEXT API token + --help Show this message and exit. ``` - \ No newline at end of file + diff --git a/tests/test_alias_v2.py b/tests/test_alias_v2.py new file mode 100644 index 0000000..e93902d --- /dev/null +++ b/tests/test_alias_v2.py @@ -0,0 +1,111 @@ +"""Tests for v2 alias command: default, default-db subcommands.""" + +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib + + +def test_alias_default_workflow(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + + # Add an alias + result = runner.invoke(cli, ["alias", "add", "prod", "https://prod.example.com"]) + assert result.exit_code == 0 + + # No default yet + result = runner.invoke(cli, ["alias", "default"]) + assert result.exit_code == 0 + assert "No default instance set" in result.output + + # Set default + result = runner.invoke(cli, ["alias", "default", "prod"]) + assert result.exit_code == 0 + + # Show default + result = runner.invoke(cli, ["alias", "default"]) + assert result.exit_code == 0 + assert result.output.strip() == "prod" + + # List should show * marker + result = runner.invoke(cli, ["alias", "list"]) + assert result.exit_code == 0 + assert "* prod" in result.output + + # Clear default + result = runner.invoke(cli, ["alias", "default", "--clear"]) + assert result.exit_code == 0 + + result = runner.invoke(cli, ["alias", "default"]) + assert "No default instance set" in result.output + + +def test_alias_default_unknown_alias(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + result = runner.invoke(cli, ["alias", "default", "nonexistent"]) + assert result.exit_code == 1 + assert "No such alias" in result.output + + +def test_alias_default_db_workflow(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + + # Add an alias + result = runner.invoke(cli, ["alias", "add", "prod", "https://prod.example.com"]) + assert result.exit_code == 0 + + # No default database yet + result = runner.invoke(cli, ["alias", "default-db", "prod"]) + assert result.exit_code == 0 + assert "No default database set" in result.output + + # Set default database + result = runner.invoke(cli, ["alias", "default-db", "prod", "main"]) + assert result.exit_code == 0 + + # Show default database + result = runner.invoke(cli, ["alias", "default-db", "prod"]) + assert result.exit_code == 0 + assert result.output.strip() == "main" + + # List should show db info + result = runner.invoke(cli, ["alias", "list"]) + assert "(db: main)" in result.output + + # Clear default database + result = runner.invoke(cli, ["alias", "default-db", "prod", "--clear"]) + assert result.exit_code == 0 + + result = runner.invoke(cli, ["alias", "default-db", "prod"]) + assert "No default database set" in result.output + + +def test_alias_default_db_unknown_alias(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + result = runner.invoke(cli, ["alias", "default-db", "nonexistent", "main"]) + assert result.exit_code == 1 + assert "No such alias" in result.output + + +def test_alias_remove_clears_default(mocker, tmpdir): + """Removing the default alias also clears default_instance.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + + runner.invoke(cli, ["alias", "add", "prod", "https://prod.example.com"]) + runner.invoke(cli, ["alias", "default", "prod"]) + + # Verify it's set + config = json.loads((pathlib.Path(tmpdir) / "config.json").read_text()) + assert config["default_instance"] == "prod" + + # Remove alias + runner.invoke(cli, ["alias", "remove", "prod"]) + + config = json.loads((pathlib.Path(tmpdir) / "config.json").read_text()) + assert config["default_instance"] is None + assert "prod" not in config["instances"] diff --git a/tests/test_commands_v2.py b/tests/test_commands_v2.py new file mode 100644 index 0000000..2077a45 --- /dev/null +++ b/tests/test_commands_v2.py @@ -0,0 +1,538 @@ +"""Tests for v2 commands: databases, tables, plugins, schema, default_query, upsert.""" + +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib +import pytest + + +# -- databases command -- + + +def test_databases_json(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "databases": [ + {"name": "main", "tables_count": 12}, + {"name": "extra", "tables_count": 3}, + ] + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["databases", "-i", "https://example.com", "--json"] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 + assert data[0]["name"] == "main" + + +def test_databases_plain(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "databases": [ + {"name": "main", "tables_count": 12}, + {"name": "extra", "tables_count": 3}, + ] + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke(cli, ["databases", "-i", "https://example.com"]) + assert result.exit_code == 0 + assert "main\n" in result.output + assert "extra\n" in result.output + + +def test_databases_url(httpx_mock, mocker, tmpdir): + """databases command hits /.json on the instance.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"databases": [{"name": "db1"}]}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke(cli, ["databases", "-i", "https://example.com"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.path == "/.json" + + +# -- tables command -- + + +def test_tables_json(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "tables": [ + {"name": "facetable", "count": 15, "hidden": False}, + {"name": "facet_cities", "count": 4, "hidden": False}, + ], + "views": [], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["tables", "-i", "https://example.com", "-d", "fixtures", "--json"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 + + +def test_tables_plain(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "tables": [ + {"name": "facetable", "count": 15, "hidden": False}, + ], + "views": [], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["tables", "-i", "https://example.com", "-d", "fixtures"] + ) + assert result.exit_code == 0 + assert "facetable" in result.output + assert "15 rows" in result.output + + +def test_tables_url(httpx_mock, mocker, tmpdir): + """tables command hits /.json.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"tables": [], "views": []}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["tables", "-i", "https://example.com", "-d", "fixtures"] + ) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.path == "/fixtures.json" + + +def test_tables_with_views(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "tables": [{"name": "t1", "count": 5, "hidden": False}], + "views": [{"name": "v1"}], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["tables", "-i", "https://example.com", "-d", "db", "--views"] + ) + assert result.exit_code == 0 + assert "t1" in result.output + assert "v1" in result.output + + +def test_tables_views_only(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "tables": [{"name": "t1", "count": 5, "hidden": False}], + "views": [{"name": "v1"}], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["tables", "-i", "https://example.com", "-d", "db", "--views-only"], + ) + assert result.exit_code == 0 + assert "t1" not in result.output + assert "v1" in result.output + + +def test_tables_hidden(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "tables": [ + {"name": "visible", "count": 5, "hidden": False}, + {"name": "hidden_t", "count": 2, "hidden": True}, + ], + "views": [], + }, + status_code=200, + ) + runner = CliRunner() + # Without --hidden + result = runner.invoke( + cli, ["tables", "-i", "https://example.com", "-d", "db"] + ) + assert "visible" in result.output + assert "hidden_t" not in result.output + + # With --hidden + httpx_mock.add_response( + json={ + "tables": [ + {"name": "visible", "count": 5, "hidden": False}, + {"name": "hidden_t", "count": 2, "hidden": True}, + ], + "views": [], + }, + status_code=200, + ) + result = runner.invoke( + cli, + ["tables", "-i", "https://example.com", "-d", "db", "--hidden"], + ) + assert "visible" in result.output + assert "hidden_t" in result.output + + +# -- plugins command -- + + +def test_plugins_json(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json=[ + {"name": "datasette-files", "version": "0.3.1"}, + {"name": "datasette-auth-tokens", "version": "0.4"}, + ], + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["plugins", "-i", "https://example.com", "--json"] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 + + +def test_plugins_plain(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json=[ + {"name": "datasette-files", "version": "0.3.1"}, + {"name": "datasette-auth-tokens", "version": "0.4"}, + ], + status_code=200, + ) + runner = CliRunner() + result = runner.invoke(cli, ["plugins", "-i", "https://example.com"]) + assert result.exit_code == 0 + assert "datasette-files\n" in result.output + assert "datasette-auth-tokens\n" in result.output + + +def test_plugins_url(httpx_mock, mocker, tmpdir): + """plugins command hits /-/plugins.json.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json=[], status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["plugins", "-i", "https://example.com"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.path == "/-/plugins.json" + + +# -- schema command -- + + +def test_schema_all_tables(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + full_schema = ( + "CREATE TABLE users (id integer primary key, name text);\n" + "CREATE VIEW user_count AS SELECT count(*) FROM users;" + ) + httpx_mock.add_response( + json={"database": "main", "schema": full_schema}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["schema", "-i", "https://example.com", "-d", "main"] + ) + assert result.exit_code == 0 + assert "CREATE TABLE users" in result.output + assert "CREATE VIEW user_count" in result.output + request = httpx_mock.get_request() + assert request.url.path == "/main/-/schema.json" + + +def test_schema_specific_table(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "database": "main", + "table": "users", + "schema": "CREATE TABLE users (id integer primary key, name text)", + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["schema", "users", "-i", "https://example.com", "-d", "main"] + ) + assert result.exit_code == 0 + assert "CREATE TABLE users" in result.output + request = httpx_mock.get_request() + assert request.url.path == "/main/users/-/schema.json" + + +def test_schema_json(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + full_schema = "CREATE TABLE users (id integer primary key);" + httpx_mock.add_response( + json={"database": "main", "schema": full_schema}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["schema", "-i", "https://example.com", "-d", "main", "--json"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert "schema" in data + + +# -- default_query (bare SQL shortcut) -- + + +def test_default_query_with_defaults(httpx_mock, mocker, tmpdir): + """Bare SQL uses default instance + default database.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response( + json={ + "ok": True, + "rows": [{"count(*)": 42}], + "columns": ["count(*)"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke(cli, ["select count(*) from users"]) + assert result.exit_code == 0 + assert json.loads(result.output) == [{"count(*)": 42}] + request = httpx_mock.get_request() + assert request.url.host == "prod.example.com" + assert request.url.path == "/main.json" + + +def test_default_query_with_database_override(httpx_mock, mocker, tmpdir): + """Bare SQL with -d flag overrides the database.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response( + json={ + "ok": True, + "rows": [{"count(*)": 100}], + "columns": ["count(*)"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["select count(*) from events", "-d", "analytics"] + ) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.path == "/analytics.json" + + +def test_default_query_with_instance_override(httpx_mock, mocker, tmpdir): + """Bare SQL with -i flag overrides the instance.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + }, + "staging": { + "url": "https://staging.example.com", + "default_database": "main", + }, + }, + } + ) + ) + httpx_mock.add_response( + json={ + "ok": True, + "rows": [{"count(*)": 5}], + "columns": ["count(*)"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["select count(*) from users", "-i", "staging"] + ) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "staging.example.com" + + +def test_default_query_no_database_error(mocker, tmpdir): + """Bare SQL without a default database gives an error.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": None, + } + }, + } + ) + ) + runner = CliRunner() + result = runner.invoke(cli, ["select 1"]) + assert result.exit_code == 1 + assert "No database specified" in result.output + + +# -- upsert command -- + + +def test_upsert_mocked(httpx_mock, tmpdir, mocker): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "ok": True, + } + ) + path = pathlib.Path(tmpdir) / "data.csv" + path.write_text("a,b,c\n1,2,3\n") + runner = CliRunner() + result = runner.invoke( + cli, + [ + "upsert", + "data", + "table1", + str(path), + "--csv", + "--token", + "x", + "-i", + "https://datasette.example.com", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.headers["authorization"] == "Bearer x" + # Should hit /-/upsert endpoint + assert "/table1/-/upsert" in str(request.url) + assert json.loads(request.read()) == {"rows": [{"a": 1, "b": 2, "c": 3}]} + + +# -- auth status command -- + + +def test_auth_status(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"actor": {"id": "root"}}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["auth", "status", "-i", "https://example.com", "--token", "tok"] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert data["actor"]["id"] == "root" + request = httpx_mock.get_request() + assert request.url.path == "/-/actor.json" + + +# -- actor command -- + + +def test_actor_with_instance_flag(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"actor": {"id": "root"}}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["actor", "-i", "https://example.com", "--token", "tok"] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert data["actor"]["id"] == "root" + request = httpx_mock.get_request() + assert request.url.path == "/-/actor.json" + assert request.headers["authorization"] == "Bearer tok" + + +# -- get command -- + + +def test_get_command(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"hello": "world"}, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, ["get", "/-/plugins.json", "-i", "https://example.com"] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert data == {"hello": "world"} + request = httpx_mock.get_request() + assert request.url.path == "/-/plugins.json" From 742d46de0595027938d8bfea781f9bdf0f24f395 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 15:22:05 -0800 Subject: [PATCH 46/64] v1 to v2 config migration with tests Add _migrate_v1_to_v2() to convert aliases.json to config.json format. Single-path-segment URLs are split into instance URL + default_database. Auth keys are migrated from URLs to alias names. Original files are backed up as .bak. 7 tests covering simple migration, no path segment, auth migration, URL fallback for unmatched auth, skip if config exists, skip if no aliases, and multi-path-segment URLs. refs #29 Co-Authored-By: Claude Opus 4.6 --- tests/test_migration.py | 116 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tests/test_migration.py diff --git a/tests/test_migration.py b/tests/test_migration.py new file mode 100644 index 0000000..2e81f03 --- /dev/null +++ b/tests/test_migration.py @@ -0,0 +1,116 @@ +"""Tests for v1 → v2 config migration.""" + +from dclient.cli import _migrate_v1_to_v2 +import json +import pathlib + + +def test_migration_simple(tmpdir): + """Migrate a simple aliases.json with a database-in-URL alias.""" + config_dir = pathlib.Path(tmpdir) + aliases_file = config_dir / "aliases.json" + aliases_file.write_text( + json.dumps({"content": "https://datasette.io/content"}) + ) + + _migrate_v1_to_v2(config_dir) + + config = json.loads((config_dir / "config.json").read_text()) + assert config["instances"]["content"]["url"] == "https://datasette.io" + assert config["instances"]["content"]["default_database"] == "content" + assert config["default_instance"] is None + + # Original should be renamed + assert (config_dir / "aliases.json.bak").exists() + assert not (config_dir / "aliases.json").exists() + + +def test_migration_no_path_segment(tmpdir): + """Migrate an alias that has no database in the URL.""" + config_dir = pathlib.Path(tmpdir) + aliases_file = config_dir / "aliases.json" + aliases_file.write_text( + json.dumps({"local": "http://localhost:8001"}) + ) + + _migrate_v1_to_v2(config_dir) + + config = json.loads((config_dir / "config.json").read_text()) + assert config["instances"]["local"]["url"] == "http://localhost:8001" + assert config["instances"]["local"]["default_database"] is None + + +def test_migration_with_auth(tmpdir): + """Auth keys are migrated from URLs to alias names.""" + config_dir = pathlib.Path(tmpdir) + aliases_file = config_dir / "aliases.json" + aliases_file.write_text( + json.dumps({"content": "https://datasette.io/content"}) + ) + auth_file = config_dir / "auth.json" + auth_file.write_text( + json.dumps({"https://datasette.io/content": "tok123"}) + ) + + _migrate_v1_to_v2(config_dir) + + new_auths = json.loads((config_dir / "auth.json").read_text()) + assert "content" in new_auths + assert new_auths["content"] == "tok123" + + # Old auth should be backed up + assert (config_dir / "auth.json.bak").exists() + + +def test_migration_auth_url_fallback(tmpdir): + """Auth entries without matching aliases are kept as URL keys.""" + config_dir = pathlib.Path(tmpdir) + aliases_file = config_dir / "aliases.json" + aliases_file.write_text(json.dumps({})) + auth_file = config_dir / "auth.json" + auth_file.write_text( + json.dumps({"https://other.example.com": "tok456"}) + ) + + _migrate_v1_to_v2(config_dir) + + new_auths = json.loads((config_dir / "auth.json").read_text()) + assert new_auths["https://other.example.com"] == "tok456" + + +def test_migration_skips_if_config_exists(tmpdir): + """If config.json already exists, migration is skipped.""" + config_dir = pathlib.Path(tmpdir) + (config_dir / "config.json").write_text(json.dumps({"existing": True})) + (config_dir / "aliases.json").write_text(json.dumps({"foo": "https://bar.com"})) + + _migrate_v1_to_v2(config_dir) + + # config.json should be untouched + assert json.loads((config_dir / "config.json").read_text()) == {"existing": True} + # aliases.json should NOT be renamed + assert (config_dir / "aliases.json").exists() + + +def test_migration_skips_if_no_aliases(tmpdir): + """If aliases.json doesn't exist, migration is skipped.""" + config_dir = pathlib.Path(tmpdir) + + _migrate_v1_to_v2(config_dir) + + assert not (config_dir / "config.json").exists() + + +def test_migration_multi_path_segments(tmpdir): + """URL with multiple path segments stores URL as-is.""" + config_dir = pathlib.Path(tmpdir) + aliases_file = config_dir / "aliases.json" + aliases_file.write_text( + json.dumps({"deep": "https://example.com/a/b/c"}) + ) + + _migrate_v1_to_v2(config_dir) + + config = json.loads((config_dir / "config.json").read_text()) + assert config["instances"]["deep"]["url"] == "https://example.com/a/b/c" + assert config["instances"]["deep"]["default_database"] is None From 3f3221fdd3f23dacc243b3018919fae3ababde70 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 15:22:12 -0800 Subject: [PATCH 47/64] Update README and docs index for v2 Update examples to use v2 command syntax with -i flag and alias defaults. Add quick start section showing alias add/default/default-db workflow. Add introspection commands section. Add environment.md to docs toctree. Co-Authored-By: Claude Opus 4.6 --- README.md | 40 +++++++++++++++++++++++++++------------- docs/index.md | 10 +++------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ec1925a..2012f77 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,11 @@ Much of the functionality requires Datasette 1.0a2 or higher. ## Things you can do with dclient -- Run SQL queries against Datasette and returning the results as JSON +- Run SQL queries against Datasette and return the results as JSON +- Introspect databases, tables, plugins, and schema - Run queries against authenticated Datasette instances -- Create aliases and store authentication tokens for convenient access to Datasette -- Insert data into Datasette using the [insert API](https://docs.datasette.io/en/latest/json_api.html#the-json-write-api) (Datasette 1.0 alpha or higher) +- Create aliases and set default instances/databases for convenient access +- Insert and upsert data using the [write API](https://docs.datasette.io/en/latest/json_api.html#the-json-write-api) (Datasette 1.0 alpha or higher) ## Installation @@ -26,19 +27,32 @@ If you want to install it in the same virtual environment as Datasette (to use i ```bash datasette install dclient ``` -## Running a query +## Quick start + +Add an alias for a Datasette instance: +```bash +dclient alias add latest https://latest.datasette.io +dclient alias default latest +dclient alias default-db latest fixtures +``` +Now run queries directly: +```bash +dclient "select * from facetable limit 1" +``` +Or be explicit: +```bash +dclient query fixtures "select * from facetable limit 1" -i latest +``` + +## Introspection ```bash -dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" -``` -To shorten that, create an alias: -```bash -dclient alias add fixtures https://latest.datasette.io/fixtures -``` -Then run it like this instead: -```bash -dclient query fixtures "select * from facetable limit 1" +dclient databases +dclient tables +dclient plugins +dclient schema facetable ``` + ## Documentation Visit **[dclient.datasette.io](https://dclient.datasette.io)** for full documentation on using this tool. diff --git a/docs/index.md b/docs/index.md index 00897bd..dda58da 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,15 +21,10 @@ If you also have Datasette installed in the same environment it will register it ```bash datasette install dclient ``` -This means you can run any of these commands using `datasette dt` instead, like this: +This means you can run any of these commands using `datasette dc` instead, like this: ```bash datasette dc --help -datasette dc query https://latest.datasette.io/fixtures "select * from facetable limit 1" -``` -You can install it into Datasette this way using: - -```bash -datasette install dclient +datasette dc query fixtures "select * from facetable limit 1" -i latest ``` ## Contents @@ -42,4 +37,5 @@ queries aliases authentication inserting +environment ``` From 7b3a15027e710717f4e1976f90e5b72ceba15e76 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Feb 2026 15:25:57 -0800 Subject: [PATCH 48/64] Add instances command, improve error messages with actionable hints Add "dclient instances" command to list known instances from config, with --json support. Improve "No instance specified" and "No database specified" error messages to show the exact commands needed to fix the problem. Co-Authored-By: Claude Opus 4.6 --- dclient/cli.py | 34 ++++++++++++++++++++++-- tests/test_commands_v2.py | 54 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 3adfebc..48f249b 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -60,7 +60,10 @@ def _resolve_instance(instance, config_file): if env_url: return env_url.rstrip("/") raise click.ClickException( - "No instance specified. Use -i, set a default instance, or set DATASETTE_URL." + "No instance specified. Use -i , or configure a default:\n\n" + " dclient alias add \n" + " dclient alias default \n\n" + "Or set the DATASETTE_URL environment variable." ) @@ -81,7 +84,9 @@ def _resolve_database(database, instance_alias, config_file): if env_db: return env_db raise click.ClickException( - "No database specified. Use -d, set a default database, or set DATASETTE_DATABASE." + "No database specified. Use -d , or configure a default:\n\n" + " dclient alias default-db \n\n" + "Or set the DATASETTE_DATABASE environment variable." ) @@ -642,6 +647,31 @@ def default_query(sql, instance, database, token, verbose): click.echo(json.dumps(response.json()["rows"], indent=2)) +@cli.command() +@click.option("--json", "_json", is_flag=True, help="Output raw JSON") +def instances(_json): + """ + List known instances from the config + + Example usage: + + \b + dclient instances + dclient instances --json + """ + config_file = get_config_dir() / "config.json" + config = _load_config(config_file) + inst_map = config.get("instances", {}) + default = config.get("default_instance") + if _json: + click.echo(json.dumps(config, indent=2)) + else: + for name, inst in inst_map.items(): + marker = "* " if name == default else " " + db_info = f" (db: {inst['default_database']})" if inst.get("default_database") else "" + click.echo(f"{marker}{name} = {inst['url']}{db_info}") + + # -- alias command group -- diff --git a/tests/test_commands_v2.py b/tests/test_commands_v2.py index 2077a45..afc4e7c 100644 --- a/tests/test_commands_v2.py +++ b/tests/test_commands_v2.py @@ -536,3 +536,57 @@ def test_get_command(httpx_mock, mocker, tmpdir): assert data == {"hello": "world"} request = httpx_mock.get_request() assert request.url.path == "/-/plugins.json" + + +# -- instances command -- + + +def test_instances_plain(mocker, tmpdir): + config_dir = pathlib.Path(tmpdir) + mocker.patch("dclient.cli.get_config_dir", return_value=config_dir) + (config_dir / "config.json").write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": {"url": "https://prod.example.com", "default_database": "main"}, + "staging": {"url": "https://staging.example.com", "default_database": None}, + }, + } + ) + ) + runner = CliRunner() + result = runner.invoke(cli, ["instances"]) + assert result.exit_code == 0 + assert "* prod = https://prod.example.com (db: main)" in result.output + assert " staging = https://staging.example.com" in result.output + + +def test_instances_json(mocker, tmpdir): + config_dir = pathlib.Path(tmpdir) + mocker.patch("dclient.cli.get_config_dir", return_value=config_dir) + (config_dir / "config.json").write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": {"url": "https://prod.example.com", "default_database": "main"}, + }, + } + ) + ) + runner = CliRunner() + result = runner.invoke(cli, ["instances", "--json"]) + assert result.exit_code == 0 + data = json.loads(result.output) + assert "prod" in data["instances"] + assert data["default_instance"] == "prod" + + +def test_instances_empty(mocker, tmpdir): + config_dir = pathlib.Path(tmpdir) + mocker.patch("dclient.cli.get_config_dir", return_value=config_dir) + runner = CliRunner() + result = runner.invoke(cli, ["instances"]) + assert result.exit_code == 0 + assert result.output.strip() == "" From 4df076d1f0962ff6e0ecffe47fa9bd5324e4ea54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 15:00:15 -0800 Subject: [PATCH 49/64] Initial oauth flow for use with datasette-oauth --- dclient/cli.py | 372 +++++++++++++++++++++++++++++++++----- tests/test_cli_auth.py | 137 ++++++++++++++ tests/test_commands_v2.py | 44 ++--- tests/test_config.py | 9 +- tests/test_env.py | 11 +- tests/test_migration.py | 24 +-- tests/test_query.py | 8 +- 7 files changed, 503 insertions(+), 102 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 48f249b..91ccec5 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -153,7 +153,9 @@ def get(path, instance, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) full_url = url.rstrip("/") + "/" + path.lstrip("/") response = _make_request(url, token, "/" + path.lstrip("/")) if response.status_code != 200: @@ -180,7 +182,9 @@ def databases(instance, _json, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) response = _make_request(url, token, "/.json") if response.status_code != 200: raise click.ClickException(f"{response.status_code} error") @@ -219,10 +223,21 @@ def tables(instance, database, views, views_only, hidden, _json, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - instance_alias = _instance_alias_for_url(url, config_dir / "config.json") if not (instance and (instance.startswith("http://") or instance.startswith("https://"))) else None - if instance and not (instance.startswith("http://") or instance.startswith("https://")): + instance_alias = ( + _instance_alias_for_url(url, config_dir / "config.json") + if not ( + instance + and (instance.startswith("http://") or instance.startswith("https://")) + ) + else None + ) + if instance and not ( + instance.startswith("http://") or instance.startswith("https://") + ): instance_alias = instance - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) db = _resolve_database(database, instance_alias, config_dir / "config.json") response = _make_request(url, token, f"/{db}.json") if response.status_code != 200: @@ -277,7 +292,9 @@ def query(database, sql, instance, token, verbose): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) query_url = url.rstrip("/") + "/" + database + ".json" headers = {} if token: @@ -285,7 +302,9 @@ def query(database, sql, instance, token, verbose): params = {"sql": sql, "_shape": "objects"} if verbose: click.echo(query_url + "?" + urllib.parse.urlencode(params), err=True) - response = httpx.get(query_url, params=params, headers=headers, follow_redirects=True) + response = httpx.get( + query_url, params=params, headers=headers, follow_redirects=True + ) if response.status_code != 200: try: @@ -322,14 +341,35 @@ def query(database, sql, instance, token, verbose): click.echo(json.dumps(response.json()["rows"], indent=2)) -def _do_insert(database, table, filepath, format_csv, format_tsv, format_json, - format_nl, encoding, no_detect_types, replace, ignore, create, - alter, pks, batch_size, interval, token, silent, verbose, - instance, endpoint="insert"): +def _do_insert( + database, + table, + filepath, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + replace, + ignore, + create, + alter, + pks, + batch_size, + interval, + token, + silent, + verbose, + instance, + endpoint="insert", +): """Shared implementation for insert and upsert commands.""" config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) format = None if format_csv: @@ -420,20 +460,29 @@ _insert_options = [ click.argument("database"), click.argument("table"), click.argument( - "filepath", type=click.Path("rb", readable=True, allow_dash=True, dir_okay=False) + "filepath", + type=click.Path("rb", readable=True, allow_dash=True, dir_okay=False), + ), + click.option( + "-i", "--instance", default=None, help="Datasette instance URL or alias" ), - click.option("-i", "--instance", default=None, help="Datasette instance URL or alias"), click.option("format_csv", "--csv", is_flag=True, help="Input is CSV"), click.option("format_tsv", "--tsv", is_flag=True, help="Input is TSV"), click.option("format_json", "--json", is_flag=True, help="Input is JSON"), - click.option("format_nl", "--nl", is_flag=True, help="Input is newline-delimited JSON"), + click.option( + "format_nl", "--nl", is_flag=True, help="Input is newline-delimited JSON" + ), click.option("--encoding", help="Character encoding for CSV/TSV"), click.option( "--no-detect-types", is_flag=True, help="Don't detect column types for CSV/TSV" ), - click.option("--alter", is_flag=True, help="Alter table to add any missing columns"), click.option( - "pks", "--pk", multiple=True, + "--alter", is_flag=True, help="Alter table to add any missing columns" + ), + click.option( + "pks", + "--pk", + multiple=True, help="Columns to use as the primary key when creating the table", ), click.option( @@ -445,7 +494,9 @@ _insert_options = [ click.option("--token", "-t", help="API token"), click.option("--silent", is_flag=True, help="Don't output progress"), click.option( - "-v", "--verbose", is_flag=True, + "-v", + "--verbose", + is_flag=True, help="Verbose output: show HTTP request and response", ), ] @@ -456,17 +507,39 @@ def _apply_options(options): for option in reversed(options): func = option(func) return func + return decorator @cli.command() @_apply_options(_insert_options) -@click.option("--replace", is_flag=True, help="Replace rows with a matching primary key") +@click.option( + "--replace", is_flag=True, help="Replace rows with a matching primary key" +) @click.option("--ignore", is_flag=True, help="Ignore rows with a matching primary key") @click.option("--create", is_flag=True, help="Create table if it does not exist") -def insert(database, table, filepath, instance, format_csv, format_tsv, format_json, - format_nl, encoding, no_detect_types, alter, pks, batch_size, interval, - token, silent, verbose, replace, ignore, create): +def insert( + database, + table, + filepath, + instance, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + alter, + pks, + batch_size, + interval, + token, + silent, + verbose, + replace, + ignore, + create, +): """ Insert data into a remote Datasette instance @@ -476,17 +549,52 @@ def insert(database, table, filepath, instance, format_csv, format_tsv, format_j dclient insert main mytable data.csv --csv -i myapp dclient insert main mytable data.csv --csv --create --pk id """ - _do_insert(database, table, filepath, format_csv, format_tsv, format_json, - format_nl, encoding, no_detect_types, replace, ignore, create, - alter, pks, batch_size, interval, token, silent, verbose, - instance, endpoint="insert") + _do_insert( + database, + table, + filepath, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + replace, + ignore, + create, + alter, + pks, + batch_size, + interval, + token, + silent, + verbose, + instance, + endpoint="insert", + ) @cli.command() @_apply_options(_insert_options) -def upsert(database, table, filepath, instance, format_csv, format_tsv, format_json, - format_nl, encoding, no_detect_types, alter, pks, batch_size, interval, - token, silent, verbose): +def upsert( + database, + table, + filepath, + instance, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + alter, + pks, + batch_size, + interval, + token, + silent, + verbose, +): """ Upsert data into a remote Datasette instance @@ -495,10 +603,29 @@ def upsert(database, table, filepath, instance, format_csv, format_tsv, format_j \b dclient upsert main mytable data.csv --csv -i myapp """ - _do_insert(database, table, filepath, format_csv, format_tsv, format_json, - format_nl, encoding, no_detect_types, False, False, False, - alter, pks, batch_size, interval, token, silent, verbose, - instance, endpoint="upsert") + _do_insert( + database, + table, + filepath, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + False, + False, + False, + alter, + pks, + batch_size, + interval, + token, + silent, + verbose, + instance, + endpoint="upsert", + ) @cli.command() @@ -520,10 +647,21 @@ def schema(table_name, instance, database, _json, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - instance_alias = _instance_alias_for_url(url, config_dir / "config.json") if not (instance and (instance.startswith("http://") or instance.startswith("https://"))) else None - if instance and not (instance.startswith("http://") or instance.startswith("https://")): + instance_alias = ( + _instance_alias_for_url(url, config_dir / "config.json") + if not ( + instance + and (instance.startswith("http://") or instance.startswith("https://")) + ) + else None + ) + if instance and not ( + instance.startswith("http://") or instance.startswith("https://") + ): instance_alias = instance - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) db = _resolve_database(database, instance_alias, config_dir / "config.json") if table_name: response = _make_request(url, token, f"/{db}/{table_name}/-/schema.json") @@ -554,7 +692,9 @@ def plugins(instance, _json, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) response = _make_request(url, token, "/-/plugins.json") if response.status_code != 200: raise click.ClickException(f"{response.status_code} error") @@ -582,7 +722,9 @@ def actor(instance, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) response = _make_request(url, token, "/-/actor.json") response.raise_for_status() click.echo(json.dumps(response.json(), indent=4)) @@ -598,10 +740,21 @@ def default_query(sql, instance, database, token, verbose): """Run a SQL query using default instance and database.""" config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - instance_alias = _instance_alias_for_url(url, config_dir / "config.json") if not (instance and (instance.startswith("http://") or instance.startswith("https://"))) else None - if instance and not (instance.startswith("http://") or instance.startswith("https://")): + instance_alias = ( + _instance_alias_for_url(url, config_dir / "config.json") + if not ( + instance + and (instance.startswith("http://") or instance.startswith("https://")) + ) + else None + ) + if instance and not ( + instance.startswith("http://") or instance.startswith("https://") + ): instance_alias = instance - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) db = _resolve_database(database, instance_alias, config_dir / "config.json") query_url = url.rstrip("/") + "/" + db + ".json" headers = {} @@ -610,7 +763,9 @@ def default_query(sql, instance, database, token, verbose): params = {"sql": sql, "_shape": "objects"} if verbose: click.echo(query_url + "?" + urllib.parse.urlencode(params), err=True) - response = httpx.get(query_url, params=params, headers=headers, follow_redirects=True) + response = httpx.get( + query_url, params=params, headers=headers, follow_redirects=True + ) if response.status_code != 200: try: @@ -668,7 +823,11 @@ def instances(_json): else: for name, inst in inst_map.items(): marker = "* " if name == default else " " - db_info = f" (db: {inst['default_database']})" if inst.get("default_database") else "" + db_info = ( + f" (db: {inst['default_database']})" + if inst.get("default_database") + else "" + ) click.echo(f"{marker}{name} = {inst['url']}{db_info}") @@ -693,7 +852,11 @@ def alias_list(_json): else: for name, inst in instances.items(): marker = "* " if name == default else " " - db_info = f" (db: {inst['default_database']})" if inst.get("default_database") else "" + db_info = ( + f" (db: {inst['default_database']})" + if inst.get("default_database") + else "" + ) click.echo(f"{marker}{name} = {inst['url']}{db_info}") @@ -891,12 +1054,116 @@ def auth_status(instance, token): """ config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") - token = _resolve_token(token, url, config_dir / "auth.json", config_dir / "config.json") + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) response = _make_request(url, token, "/-/actor.json") response.raise_for_status() click.echo(json.dumps(response.json(), indent=4)) +# -- login command (OAuth device flow) -- + + +@cli.command() +@click.argument("alias_or_url", required=False, default=None) +@click.option("--scope", default=None, help="JSON scope array") +def login(alias_or_url, scope): + """ + Authenticate with a Datasette instance using OAuth + + Uses the OAuth device flow: opens a URL in your browser where you + approve access, then saves the resulting API token. + + Example usage: + + \b + dclient login https://simon.datasette.cloud/ + dclient login myalias + dclient login + """ + config_dir = get_config_dir() + config_dir.mkdir(parents=True, exist_ok=True) + config_file = config_dir / "config.json" + + if alias_or_url is None: + click.echo("Enter the URL of your Datasette instance, or an alias you have") + click.echo("already configured with 'dclient alias add'.\n") + alias_or_url = click.prompt("Instance URL or alias") + + # Resolve alias to URL if needed + if alias_or_url.startswith("http://") or alias_or_url.startswith("https://"): + url = alias_or_url + auth_key = alias_or_url + else: + url = _resolve_instance(alias_or_url, config_file) + auth_key = alias_or_url + + # Ensure trailing slash + if not url.endswith("/"): + url += "/" + + # Step 1: Request device code + device_url = url + "-/oauth/device" + data = {} + if scope: + data["scope"] = scope + response = httpx.post(device_url, data=data, timeout=30.0) + if response.status_code != 200: + raise click.ClickException( + f"Failed to start login flow: {response.status_code} from {device_url}" + ) + device_data = response.json() + device_code = device_data["device_code"] + user_code = device_data["user_code"] + verification_uri = device_data["verification_uri"] + interval = device_data.get("interval", 5) + + # Step 2: Show instructions + click.echo(f"\nOpen this URL in your browser:\n") + click.echo(f" {verification_uri}\n") + click.echo(f"Enter this code: {user_code}\n") + click.echo("Waiting for authorization...", nl=False) + + # Step 3: Poll for token + token_url = url + "-/oauth/token" + while True: + time.sleep(interval) + click.echo(".", nl=False) + token_response = httpx.post( + token_url, + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": device_code, + }, + timeout=30.0, + ) + token_data = token_response.json() + if "access_token" in token_data: + break + error = token_data.get("error") + if error == "authorization_pending": + continue + elif error == "access_denied": + click.echo() + raise click.ClickException("Authorization denied.") + elif error == "expired_token": + click.echo() + raise click.ClickException("Device code expired. Run login again.") + else: + click.echo() + raise click.ClickException(f"Unexpected error: {error}") + + # Step 4: Save token + click.echo() + access_token = token_data["access_token"] + auth_file = config_dir / "auth.json" + auths = _load_auths(auth_file) + auths[auth_key] = access_token + auth_file.write_text(json.dumps(auths, indent=4)) + click.echo(f"Login successful. Token saved for {auth_key}") + + # -- v1 → v2 migration -- @@ -947,7 +1214,6 @@ def _migrate_v1_to_v2(config_dir): aliases_file.rename(config_dir / "aliases.json.bak") - def _batches(iterable, size, interval=None): iterable = iter(iterable) last_yield_time = time.time() @@ -967,8 +1233,18 @@ def _batches(iterable, size, interval=None): def _insert_batch( - *, url, table, batch, token, create, alter, pks, replace, ignore, verbose, - endpoint="insert" + *, + url, + table, + batch, + token, + create, + alter, + pks, + replace, + ignore, + verbose, + endpoint="insert", ): if create: data = { diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py index 60b8bb3..ea17dd1 100644 --- a/tests/test_cli_auth.py +++ b/tests/test_cli_auth.py @@ -2,6 +2,7 @@ from click.testing import CliRunner from dclient.cli import cli import pathlib import json +import pytest def test_auth(mocker, tmpdir): @@ -39,3 +40,139 @@ def test_auth(mocker, tmpdir): # Check the tokens file auth_file = pathlib.Path(tmpdir) / "auth.json" assert json.loads(auth_file.read_text()) == {} + + +# -- login command (OAuth device flow) -- + +DEVICE_RESPONSE = { + "device_code": "devcode123", + "user_code": "ABCD-EFGH", + "verification_uri": "https://example.com/-/oauth/device/verify", + "expires_in": 900, + "interval": 0, +} + +TOKEN_SUCCESS = { + "access_token": "dstok_abc123", + "token_type": "bearer", + "expires_in": 3600, +} + + +def test_login_with_url(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 0 + assert "ABCD-EFGH" in result.output + assert "Login successful" in result.output + # Token should be saved + auth_file = pathlib.Path(tmpdir) / "auth.json" + auths = json.loads(auth_file.read_text()) + assert auths["https://example.com/"] == "dstok_abc123" + + +def test_login_adds_trailing_slash(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com"]) + assert result.exit_code == 0 + # Check that the device request went to the right URL + requests = httpx_mock.get_requests() + assert str(requests[0].url) == "https://example.com/-/oauth/device" + + +def test_login_with_alias(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + # Set up an alias first + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": None, + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": None, + } + }, + } + ) + ) + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login", "prod"]) + assert result.exit_code == 0 + assert "Login successful" in result.output + # Token should be saved by alias name + auth_file = pathlib.Path(tmpdir) / "auth.json" + auths = json.loads(auth_file.read_text()) + assert auths["prod"] == "dstok_abc123" + + +def test_login_interactive_prompt(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login"], input="https://example.com/\n") + assert result.exit_code == 0 + assert "Instance URL or alias" in result.output + assert "Login successful" in result.output + + +def test_login_access_denied(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json={"error": "access_denied"}, status_code=400) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 1 + assert "Authorization denied" in result.output + + +def test_login_expired_token(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json={"error": "expired_token"}, status_code=400) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 1 + assert "expired" in result.output + + +def test_login_pending_then_success(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + # First poll: pending + httpx_mock.add_response(json={"error": "authorization_pending"}, status_code=400) + # Second poll: success + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 0 + assert "Login successful" in result.output + auth_file = pathlib.Path(tmpdir) / "auth.json" + auths = json.loads(auth_file.read_text()) + assert auths["https://example.com/"] == "dstok_abc123" + + +def test_login_device_endpoint_error(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(status_code=403) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 1 + assert "Failed to start login flow" in result.output diff --git a/tests/test_commands_v2.py b/tests/test_commands_v2.py index afc4e7c..a4326b5 100644 --- a/tests/test_commands_v2.py +++ b/tests/test_commands_v2.py @@ -6,7 +6,6 @@ import json import pathlib import pytest - # -- databases command -- @@ -22,9 +21,7 @@ def test_databases_json(httpx_mock, mocker, tmpdir): status_code=200, ) runner = CliRunner() - result = runner.invoke( - cli, ["databases", "-i", "https://example.com", "--json"] - ) + result = runner.invoke(cli, ["databases", "-i", "https://example.com", "--json"]) assert result.exit_code == 0 data = json.loads(result.output) assert len(data) == 2 @@ -175,9 +172,7 @@ def test_tables_hidden(httpx_mock, mocker, tmpdir): ) runner = CliRunner() # Without --hidden - result = runner.invoke( - cli, ["tables", "-i", "https://example.com", "-d", "db"] - ) + result = runner.invoke(cli, ["tables", "-i", "https://example.com", "-d", "db"]) assert "visible" in result.output assert "hidden_t" not in result.output @@ -213,9 +208,7 @@ def test_plugins_json(httpx_mock, mocker, tmpdir): status_code=200, ) runner = CliRunner() - result = runner.invoke( - cli, ["plugins", "-i", "https://example.com", "--json"] - ) + result = runner.invoke(cli, ["plugins", "-i", "https://example.com", "--json"]) assert result.exit_code == 0 data = json.loads(result.output) assert len(data) == 2 @@ -262,9 +255,7 @@ def test_schema_all_tables(httpx_mock, mocker, tmpdir): status_code=200, ) runner = CliRunner() - result = runner.invoke( - cli, ["schema", "-i", "https://example.com", "-d", "main"] - ) + result = runner.invoke(cli, ["schema", "-i", "https://example.com", "-d", "main"]) assert result.exit_code == 0 assert "CREATE TABLE users" in result.output assert "CREATE VIEW user_count" in result.output @@ -372,9 +363,7 @@ def test_default_query_with_database_override(httpx_mock, mocker, tmpdir): status_code=200, ) runner = CliRunner() - result = runner.invoke( - cli, ["select count(*) from events", "-d", "analytics"] - ) + result = runner.invoke(cli, ["select count(*) from events", "-d", "analytics"]) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.url.path == "/analytics.json" @@ -410,9 +399,7 @@ def test_default_query_with_instance_override(httpx_mock, mocker, tmpdir): status_code=200, ) runner = CliRunner() - result = runner.invoke( - cli, ["select count(*) from users", "-i", "staging"] - ) + result = runner.invoke(cli, ["select count(*) from users", "-i", "staging"]) assert result.exit_code == 0 request = httpx_mock.get_request() assert request.url.host == "staging.example.com" @@ -528,9 +515,7 @@ def test_get_command(httpx_mock, mocker, tmpdir): status_code=200, ) runner = CliRunner() - result = runner.invoke( - cli, ["get", "/-/plugins.json", "-i", "https://example.com"] - ) + result = runner.invoke(cli, ["get", "/-/plugins.json", "-i", "https://example.com"]) assert result.exit_code == 0 data = json.loads(result.output) assert data == {"hello": "world"} @@ -549,8 +534,14 @@ def test_instances_plain(mocker, tmpdir): { "default_instance": "prod", "instances": { - "prod": {"url": "https://prod.example.com", "default_database": "main"}, - "staging": {"url": "https://staging.example.com", "default_database": None}, + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + }, + "staging": { + "url": "https://staging.example.com", + "default_database": None, + }, }, } ) @@ -570,7 +561,10 @@ def test_instances_json(mocker, tmpdir): { "default_instance": "prod", "instances": { - "prod": {"url": "https://prod.example.com", "default_database": "main"}, + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + }, }, } ) diff --git a/tests/test_config.py b/tests/test_config.py index f12bf85..d39a2ee 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,7 +12,6 @@ import json import pathlib import pytest - # -- Config loading/saving -- @@ -196,7 +195,9 @@ def test_resolve_token_from_flag(tmpdir): """An explicit --token flag is used directly.""" auth_file = pathlib.Path(tmpdir) / "auth.json" config_file = pathlib.Path(tmpdir) / "config.json" - token = _resolve_token("explicit-token", "https://example.com", auth_file, config_file) + token = _resolve_token( + "explicit-token", "https://example.com", auth_file, config_file + ) assert token == "explicit-token" @@ -218,7 +219,9 @@ def test_resolve_token_from_auth_by_alias(tmpdir): } ) ) - token = _resolve_token(None, "https://myapp.datasette.cloud", auth_file, config_file) + token = _resolve_token( + None, "https://myapp.datasette.cloud", auth_file, config_file + ) assert token == "tok123" diff --git a/tests/test_env.py b/tests/test_env.py index 185611c..67f2c3f 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -3,7 +3,6 @@ from dclient.cli import cli import json import pathlib - QUERY_RESPONSE = { "ok": True, "database": "data", @@ -42,9 +41,13 @@ def test_token_flag_overrides_datasette_token(httpx_mock, mocker, tmpdir): result = runner.invoke( cli, [ - "query", "data", "select 1", - "-i", "https://example.com", - "--token", "flag-token", + "query", + "data", + "select 1", + "-i", + "https://example.com", + "--token", + "flag-token", ], ) assert result.exit_code == 0 diff --git a/tests/test_migration.py b/tests/test_migration.py index 2e81f03..c720023 100644 --- a/tests/test_migration.py +++ b/tests/test_migration.py @@ -9,9 +9,7 @@ def test_migration_simple(tmpdir): """Migrate a simple aliases.json with a database-in-URL alias.""" config_dir = pathlib.Path(tmpdir) aliases_file = config_dir / "aliases.json" - aliases_file.write_text( - json.dumps({"content": "https://datasette.io/content"}) - ) + aliases_file.write_text(json.dumps({"content": "https://datasette.io/content"})) _migrate_v1_to_v2(config_dir) @@ -29,9 +27,7 @@ def test_migration_no_path_segment(tmpdir): """Migrate an alias that has no database in the URL.""" config_dir = pathlib.Path(tmpdir) aliases_file = config_dir / "aliases.json" - aliases_file.write_text( - json.dumps({"local": "http://localhost:8001"}) - ) + aliases_file.write_text(json.dumps({"local": "http://localhost:8001"})) _migrate_v1_to_v2(config_dir) @@ -44,13 +40,9 @@ def test_migration_with_auth(tmpdir): """Auth keys are migrated from URLs to alias names.""" config_dir = pathlib.Path(tmpdir) aliases_file = config_dir / "aliases.json" - aliases_file.write_text( - json.dumps({"content": "https://datasette.io/content"}) - ) + aliases_file.write_text(json.dumps({"content": "https://datasette.io/content"})) auth_file = config_dir / "auth.json" - auth_file.write_text( - json.dumps({"https://datasette.io/content": "tok123"}) - ) + auth_file.write_text(json.dumps({"https://datasette.io/content": "tok123"})) _migrate_v1_to_v2(config_dir) @@ -68,9 +60,7 @@ def test_migration_auth_url_fallback(tmpdir): aliases_file = config_dir / "aliases.json" aliases_file.write_text(json.dumps({})) auth_file = config_dir / "auth.json" - auth_file.write_text( - json.dumps({"https://other.example.com": "tok456"}) - ) + auth_file.write_text(json.dumps({"https://other.example.com": "tok456"})) _migrate_v1_to_v2(config_dir) @@ -105,9 +95,7 @@ def test_migration_multi_path_segments(tmpdir): """URL with multiple path segments stores URL as-is.""" config_dir = pathlib.Path(tmpdir) aliases_file = config_dir / "aliases.json" - aliases_file.write_text( - json.dumps({"deep": "https://example.com/a/b/c"}) - ) + aliases_file.write_text(json.dumps({"deep": "https://example.com/a/b/c"})) _migrate_v1_to_v2(config_dir) diff --git a/tests/test_query.py b/tests/test_query.py index e273beb..4531bb2 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -53,7 +53,9 @@ def test_query(httpx_mock, with_token): # Check the request request = httpx_mock.get_request() - assert str(request.url) == "https://example.com/content.json?sql=hello&_shape=objects" + assert ( + str(request.url) == "https://example.com/content.json?sql=hello&_shape=objects" + ) if with_token: assert request.headers["authorization"] == "Bearer xyz" else: @@ -67,9 +69,7 @@ def test_aliases(mocker, tmpdir, httpx_mock): assert result.exit_code == 0 assert result.output == "" - result = runner.invoke( - cli, ["alias", "add", "foo", "https://example.com"] - ) + result = runner.invoke(cli, ["alias", "add", "foo", "https://example.com"]) assert result.exit_code == 0 assert result.output == "" From ebb8268d51001ebca11b796f033243b5c4031104 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 15:00:49 -0800 Subject: [PATCH 50/64] Release 0.5a0 Refs #29 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5bc7cfa..c037918 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dclient" -version = "0.4" +version = "0.5a0" description = "A client CLI utility for Datasette instances" readme = "README.md" authors = [{name = "Simon Willison"}] From 9b03289e3ac68295aa28966eafb1341c1b9d956b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 15:05:43 -0800 Subject: [PATCH 51/64] Tests now use Datasette 1.0a25+ --- pyproject.toml | 2 +- tests/test_insert.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c037918..676ff9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dev = [ "pytest-httpx", "cogapp", "pytest-mock", - "datasette>=1.0a2", + "datasette>=1.0a25", ] [tool.uv.build-backend] diff --git a/tests/test_insert.py b/tests/test_insert.py index 64f38e7..aa28583 100644 --- a/tests/test_insert.py +++ b/tests/test_insert.py @@ -271,7 +271,7 @@ async def test_insert_against_datasette( "insert into table1 (a, b, c) values (1, 2, 3), (4, 5, 6)" ) - token = ds.create_token("actor") + token = await ds.create_token("actor") # These are useful with pytest --pdb to see what happened datasette_requests = [] From 9df1828d1da682f242b29b2146d71fafe5dbafc1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 15:25:06 -0800 Subject: [PATCH 52/64] Set defaults on login, if not already set --- dclient/cli.py | 49 ++++++++++++++++ tests/test_cli_auth.py | 125 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/dclient/cli.py b/dclient/cli.py index 91ccec5..cf081e0 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1163,6 +1163,55 @@ def login(alias_or_url, scope): auth_file.write_text(json.dumps(auths, indent=4)) click.echo(f"Login successful. Token saved for {auth_key}") + # Step 5: Set defaults if not already configured + config = _load_config(config_file) + default_alias = config.get("default_instance") + has_default_instance = default_alias is not None + has_default_db = bool( + config.get("instances", {}).get(default_alias or "", {}).get("default_database") + ) + if has_default_instance and has_default_db: + return + # Find alias for this instance, or use the auth_key (URL) as the instance key + instance_key = _instance_alias_for_url(url, config_file) or auth_key + # Ensure instance entry exists in config + if instance_key not in config.get("instances", {}): + config.setdefault("instances", {})[instance_key] = { + "url": url.rstrip("/"), + "default_database": None, + } + # Set as default instance if none configured + if not has_default_instance: + config["default_instance"] = instance_key + click.echo(f"Set default instance to {instance_key}") + # Query databases and set default database if none configured + if not has_default_db: + try: + db_response = _make_request(url, access_token, "/.json") + if db_response.status_code == 200: + db_data = db_response.json() + if isinstance(db_data, list): + databases_list = db_data + else: + databases_list = db_data.get("databases", []) + if isinstance(databases_list, dict): + databases_list = list(databases_list.values()) + db_names = [ + db["name"] if isinstance(db, dict) else db for db in databases_list + ] + if db_names: + if len(db_names) == 1: + default_db = db_names[0] + elif "data" in db_names: + default_db = "data" + else: + default_db = db_names[0] + config["instances"][instance_key]["default_database"] = default_db + click.echo(f"Set default database to {default_db}") + except Exception: + pass # Don't fail login if databases check fails + _save_config(config_file, config) + # -- v1 → v2 migration -- diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py index ea17dd1..39ba383 100644 --- a/tests/test_cli_auth.py +++ b/tests/test_cli_auth.py @@ -64,6 +64,7 @@ def test_login_with_url(httpx_mock, mocker, tmpdir): mocker.patch("dclient.cli.time.sleep") httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(json=[{"name": "data"}], status_code=200) runner = CliRunner() result = runner.invoke(cli, ["login", "https://example.com/"]) assert result.exit_code == 0 @@ -80,6 +81,7 @@ def test_login_adds_trailing_slash(httpx_mock, mocker, tmpdir): mocker.patch("dclient.cli.time.sleep") httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(json=[{"name": "data"}], status_code=200) runner = CliRunner() result = runner.invoke(cli, ["login", "https://example.com"]) assert result.exit_code == 0 @@ -108,6 +110,7 @@ def test_login_with_alias(httpx_mock, mocker, tmpdir): ) httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(json=[{"name": "data"}], status_code=200) runner = CliRunner() result = runner.invoke(cli, ["login", "prod"]) assert result.exit_code == 0 @@ -123,6 +126,7 @@ def test_login_interactive_prompt(httpx_mock, mocker, tmpdir): mocker.patch("dclient.cli.time.sleep") httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(json=[{"name": "data"}], status_code=200) runner = CliRunner() result = runner.invoke(cli, ["login"], input="https://example.com/\n") assert result.exit_code == 0 @@ -160,6 +164,7 @@ def test_login_pending_then_success(httpx_mock, mocker, tmpdir): httpx_mock.add_response(json={"error": "authorization_pending"}, status_code=400) # Second poll: success httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(json=[{"name": "data"}], status_code=200) runner = CliRunner() result = runner.invoke(cli, ["login", "https://example.com/"]) assert result.exit_code == 0 @@ -176,3 +181,123 @@ def test_login_device_endpoint_error(httpx_mock, mocker, tmpdir): result = runner.invoke(cli, ["login", "https://example.com/"]) assert result.exit_code == 1 assert "Failed to start login flow" in result.output + + +# -- login sets defaults -- + + +@pytest.mark.parametrize( + "databases_response,expected_db", + [ + ([{"name": "mydata", "is_mutable": True}], "mydata"), + ( + [ + {"name": "fixtures", "is_mutable": False}, + {"name": "data", "is_mutable": True}, + {"name": "extra", "is_mutable": True}, + ], + "data", + ), + ( + [ + {"name": "alpha", "is_mutable": True}, + {"name": "beta", "is_mutable": True}, + {"name": "gamma", "is_mutable": False}, + ], + "alpha", + ), + ], + ids=["single_db", "prefers_data", "first_when_no_data"], +) +def test_login_sets_defaults( + httpx_mock, mocker, tmpdir, databases_response, expected_db +): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(json=databases_response, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 0, result.output + config = json.loads((pathlib.Path(tmpdir) / "config.json").read_text()) + assert config["default_instance"] == "https://example.com/" + assert ( + config["instances"]["https://example.com/"]["default_database"] == expected_db + ) + + +def test_login_does_not_override_existing_defaults(httpx_mock, mocker, tmpdir): + """When defaults are already configured, login should not change them.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://other.example.com/"]) + assert result.exit_code == 0, result.output + config = json.loads(config_file.read_text()) + assert config["default_instance"] == "prod" + + +def test_login_with_alias_sets_defaults(httpx_mock, mocker, tmpdir): + """When logging in with an alias that has no default_database, set it.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": None, + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": None, + } + }, + } + ) + ) + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response( + json=[ + {"name": "fixtures", "is_mutable": False}, + {"name": "data", "is_mutable": True}, + {"name": "extra", "is_mutable": True}, + ], + status_code=200, + ) + runner = CliRunner() + result = runner.invoke(cli, ["login", "prod"]) + assert result.exit_code == 0, result.output + config = json.loads(config_file.read_text()) + assert config["default_instance"] == "prod" + assert config["instances"]["prod"]["default_database"] == "data" + + +def test_login_databases_error_still_succeeds(httpx_mock, mocker, tmpdir): + """If the databases check fails, login should still succeed.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + mocker.patch("dclient.cli.time.sleep") + httpx_mock.add_response(json=DEVICE_RESPONSE, status_code=200) + httpx_mock.add_response(json=TOKEN_SUCCESS, status_code=200) + httpx_mock.add_response(status_code=500) + runner = CliRunner() + result = runner.invoke(cli, ["login", "https://example.com/"]) + assert result.exit_code == 0, result.output + assert "Login successful" in result.output From 04e396b020d6835494f14b3528c42c113f4e9057 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 15:25:22 -0800 Subject: [PATCH 53/64] Release 0.5a1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 676ff9d..946fdd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dclient" -version = "0.5a0" +version = "0.5a1" description = "A client CLI utility for Datasette instances" readme = "README.md" authors = [{name = "Simon Willison"}] From 7a8643f72f297dbb0585fcc51186d715f169ab2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 20:43:18 -0800 Subject: [PATCH 54/64] Justfile now uses uv, docs gets a dependency group --- Justfile | 30 +++++++++++------------------- pyproject.toml | 13 ++++++++++++- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Justfile b/Justfile index b3cc38a..8904d06 100644 --- a/Justfile +++ b/Justfile @@ -1,37 +1,29 @@ # Run tests and linters @default: test lint -# Install dependencies and test dependencies -@init: - pipenv run pip install -e '.[test]' - # Run pytest with supplied options @test *options: - pipenv run pytest {{options}} + uv run pytest {{options}} # Run linters @lint: echo "Linters..." - echo " Black" - pipenv run black . --check echo " cog" - pipenv run cog --check README.md docs/*.md - echo " ruff" - pipenv run ruff . + uv run cog --check README.md docs/*.md + echo " ruff check" + uv run ruff check . + echo " ruff format" + uv run ruff format . --check # Rebuild docs with cog @cog: - pipenv run cog -r docs/*.md + uv run cog -r docs/*.md # Serve live docs on localhost:8000 @docs: cog - cd docs && pipenv run make livehtml + cd docs && uv run make livehtml -# Apply Black -@black: - pipenv run black . - -# Run automatic fixes +# Apply ruff fixes and formatting @fix: cog - pipenv run ruff . --fix - pipenv run black . + uv run ruff check . --fix + uv run ruff format . diff --git a/pyproject.toml b/pyproject.toml index 946fdd1..3028b99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dclient = "dclient.cli:cli" client = "dclient.plugin" [dependency-groups] -dev = [ +test = [ "pytest", "pytest-asyncio", "pytest-httpx", @@ -34,6 +34,17 @@ dev = [ "pytest-mock", "datasette>=1.0a25", ] +docs = [ + "furo", + "sphinx-autobuild", + "sphinx-copybutton", + "myst-parser", + "cogapp", +] +dev = [ + {include-group = "test"}, + {include-group = "docs"}, +] [tool.uv.build-backend] module-root = "" From 2f7f2a737899799bcd3ea5a3074a11f8b72cf1a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 20:44:39 -0800 Subject: [PATCH 55/64] Moved default commands to dclient default ..., closes #30 --- README.md | 4 +- dclient/cli.py | 95 ++++++++++++++++++++++++------------ docs/aliases.md | 87 ++------------------------------- docs/defaults.md | 106 +++++++++++++++++++++++++++++++++++++++++ docs/index.md | 1 + tests/test_alias_v2.py | 56 ++++++++++++++++------ 6 files changed, 220 insertions(+), 129 deletions(-) create mode 100644 docs/defaults.md diff --git a/README.md b/README.md index 2012f77..96b7764 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ datasette install dclient Add an alias for a Datasette instance: ```bash dclient alias add latest https://latest.datasette.io -dclient alias default latest -dclient alias default-db latest fixtures +dclient default instance latest +dclient default database latest fixtures ``` Now run queries directly: ```bash diff --git a/dclient/cli.py b/dclient/cli.py index cf081e0..fcc8047 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -53,8 +53,11 @@ def _resolve_instance(instance, config_file): ) # Try config default default = config.get("default_instance") - if default and default in config.get("instances", {}): - return config["instances"][default]["url"] + if default: + if default in config.get("instances", {}): + return config["instances"][default]["url"] + if default.startswith("http://") or default.startswith("https://"): + return default.rstrip("/") # Try env var env_url = os.environ.get("DATASETTE_URL") if env_url: @@ -62,7 +65,7 @@ def _resolve_instance(instance, config_file): raise click.ClickException( "No instance specified. Use -i , or configure a default:\n\n" " dclient alias add \n" - " dclient alias default \n\n" + " dclient default instance \n\n" "Or set the DATASETTE_URL environment variable." ) @@ -75,8 +78,13 @@ def _resolve_database(database, instance_alias, config_file): if instance_alias: config = _load_config(config_file) instances = config.get("instances", {}) - if instance_alias in instances: - default_db = instances[instance_alias].get("default_database") + key = instance_alias + if key not in instances and ( + key.startswith("http://") or key.startswith("https://") + ): + key = _instance_alias_for_url(key, config_file) + if key in instances: + default_db = instances[key].get("default_database") if default_db: return default_db # Try env var @@ -85,7 +93,7 @@ def _resolve_database(database, instance_alias, config_file): return env_db raise click.ClickException( "No database specified. Use -d , or configure a default:\n\n" - " dclient alias default-db \n\n" + " dclient default database \n\n" "Or set the DATASETTE_DATABASE environment variable." ) @@ -902,29 +910,56 @@ def alias_remove(name): raise click.ClickException("No such alias") -@alias.command(name="default") -@click.argument("name", required=False, default=None) +def _resolve_instance_key(alias_or_url, config): + instances = config.get("instances", {}) + if alias_or_url in instances: + return alias_or_url + if alias_or_url.startswith("http://") or alias_or_url.startswith("https://"): + normalized = alias_or_url.rstrip("/") + for name, inst in instances.items(): + if inst.get("url", "").rstrip("/") == normalized: + return name + raise click.ClickException(f"No such instance URL: {alias_or_url}") + raise click.ClickException(f"No such alias: {alias_or_url}") + + +# -- default command group -- + + +@cli.group() +def default(): + "Manage default instance and database" + + +@default.command(name="instance") +@click.argument("alias_or_url", required=False, default=None) @click.option("--clear", is_flag=True, help="Clear default instance") -def alias_default(name, clear): +def default_instance(alias_or_url, clear): """ Set or show the default instance Example usage: \b - dclient alias default prod - dclient alias default - dclient alias default --clear + dclient default instance prod + dclient default instance https://myapp.datasette.cloud + dclient default instance + dclient default instance --clear """ config_file = get_config_dir() / "config.json" config = _load_config(config_file) if clear: config["default_instance"] = None _save_config(config_file, config) - elif name: - if name not in config.get("instances", {}): - raise click.ClickException(f"No such alias: {name}") - config["default_instance"] = name + elif alias_or_url: + if alias_or_url.startswith("http://") or alias_or_url.startswith("https://"): + try: + key = _resolve_instance_key(alias_or_url, config) + except click.ClickException: + key = alias_or_url.rstrip("/") + else: + key = _resolve_instance_key(alias_or_url, config) + config["default_instance"] = key _save_config(config_file, config) else: default = config.get("default_instance") @@ -934,37 +969,37 @@ def alias_default(name, clear): click.echo("No default instance set") -@alias.command(name="default-db") -@click.argument("alias_name") +@default.command(name="database") +@click.argument("alias_or_url") @click.argument("db", required=False, default=None) -@click.option("--clear", is_flag=True, help="Clear default database for this alias") -def alias_default_db(alias_name, db, clear): +@click.option("--clear", is_flag=True, help="Clear default database for this instance") +def default_database(alias_or_url, db, clear): """ - Set or show the default database for an alias + Set or show the default database for an instance Example usage: \b - dclient alias default-db prod main - dclient alias default-db prod - dclient alias default-db prod --clear + dclient default database prod main + dclient default database https://myapp.datasette.cloud main + dclient default database prod + dclient default database prod --clear """ config_file = get_config_dir() / "config.json" config = _load_config(config_file) - if alias_name not in config.get("instances", {}): - raise click.ClickException(f"No such alias: {alias_name}") + instance_key = _resolve_instance_key(alias_or_url, config) if clear: - config["instances"][alias_name]["default_database"] = None + config["instances"][instance_key]["default_database"] = None _save_config(config_file, config) elif db: - config["instances"][alias_name]["default_database"] = db + config["instances"][instance_key]["default_database"] = db _save_config(config_file, config) else: - default_db = config["instances"][alias_name].get("default_database") + default_db = config["instances"][instance_key].get("default_database") if default_db: click.echo(default_db) else: - click.echo(f"No default database set for {alias_name}") + click.echo(f"No default database set for {instance_key}") # -- auth command group -- diff --git a/docs/aliases.md b/docs/aliases.md index 8399085..4c4255d 100644 --- a/docs/aliases.md +++ b/docs/aliases.md @@ -13,28 +13,7 @@ Once registered, you can pass an alias to commands using the `-i` flag: dclient query fixtures "select * from news limit 1" -i latest -## Default instance - -Set a default instance so you don't need `-i` every time: - - dclient alias default latest - -Now commands will use `latest` automatically: - - dclient databases - dclient tables -d fixtures - -## Default database - -Set a default database for an alias: - - dclient alias default-db latest fixtures - -Now you can run bare SQL queries directly: - - dclient "select * from facetable limit 5" - -This uses the default instance and default database. +See [Defaults](defaults.md) for default instance and default database settings. ## dclient alias --help @@ -137,59 +114,3 @@ Options: ``` - -## dclient alias default --help - - -``` -Usage: dclient alias default [OPTIONS] [NAME] - - Set or show the default instance - - Example usage: - - dclient alias default prod - dclient alias default - dclient alias default --clear - -Options: - --clear Clear default instance - --help Show this message and exit. - -``` - - -## dclient alias default-db --help - - -``` -Usage: dclient alias default-db [OPTIONS] ALIAS_NAME [DB] - - Set or show the default database for an alias - - Example usage: - - dclient alias default-db prod main - dclient alias default-db prod - dclient alias default-db prod --clear - -Options: - --clear Clear default database for this alias - --help Show this message and exit. - -``` - diff --git a/docs/defaults.md b/docs/defaults.md new file mode 100644 index 0000000..b53c334 --- /dev/null +++ b/docs/defaults.md @@ -0,0 +1,106 @@ +# Defaults + +Set a default instance so you don't need `-i` every time: + + dclient default instance latest + +Now commands will use `latest` automatically: + + dclient databases + dclient tables -d fixtures + +Set a default database for an alias or instance URL: + + dclient default database latest fixtures + dclient default database https://latest.datasette.io fixtures + +Now you can run bare SQL queries directly: + + dclient "select * from facetable limit 5" + +This uses the default instance and default database. + +## dclient default --help + +``` +Usage: dclient default [OPTIONS] COMMAND [ARGS]... + + Manage default instance and database + +Options: + --help Show this message and exit. + +Commands: + database Set or show the default database for an instance + instance Set or show the default instance + +``` + + +## dclient default instance --help + + +``` +Usage: dclient default instance [OPTIONS] [ALIAS_OR_URL] + + Set or show the default instance + + Example usage: + + dclient default instance prod + dclient default instance https://myapp.datasette.cloud + dclient default instance + dclient default instance --clear + +Options: + --clear Clear default instance + --help Show this message and exit. + +``` + + +## dclient default database --help + + +``` +Usage: dclient default database [OPTIONS] ALIAS_OR_URL [DB] + + Set or show the default database for an instance + + Example usage: + + dclient default database prod main + dclient default database https://myapp.datasette.cloud main + dclient default database prod + dclient default database prod --clear + +Options: + --clear Clear default database for this instance + --help Show this message and exit. + +``` + diff --git a/docs/index.md b/docs/index.md index dda58da..0b43234 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,7 @@ maxdepth: 3 --- queries aliases +defaults authentication inserting environment diff --git a/tests/test_alias_v2.py b/tests/test_alias_v2.py index e93902d..173382e 100644 --- a/tests/test_alias_v2.py +++ b/tests/test_alias_v2.py @@ -1,4 +1,4 @@ -"""Tests for v2 alias command: default, default-db subcommands.""" +"""Tests for v2 defaults command: instance and database subcommands.""" from click.testing import CliRunner from dclient.cli import cli @@ -15,16 +15,16 @@ def test_alias_default_workflow(mocker, tmpdir): assert result.exit_code == 0 # No default yet - result = runner.invoke(cli, ["alias", "default"]) + result = runner.invoke(cli, ["default", "instance"]) assert result.exit_code == 0 assert "No default instance set" in result.output # Set default - result = runner.invoke(cli, ["alias", "default", "prod"]) + result = runner.invoke(cli, ["default", "instance", "prod"]) assert result.exit_code == 0 # Show default - result = runner.invoke(cli, ["alias", "default"]) + result = runner.invoke(cli, ["default", "instance"]) assert result.exit_code == 0 assert result.output.strip() == "prod" @@ -34,17 +34,17 @@ def test_alias_default_workflow(mocker, tmpdir): assert "* prod" in result.output # Clear default - result = runner.invoke(cli, ["alias", "default", "--clear"]) + result = runner.invoke(cli, ["default", "instance", "--clear"]) assert result.exit_code == 0 - result = runner.invoke(cli, ["alias", "default"]) + result = runner.invoke(cli, ["default", "instance"]) assert "No default instance set" in result.output def test_alias_default_unknown_alias(mocker, tmpdir): mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) runner = CliRunner() - result = runner.invoke(cli, ["alias", "default", "nonexistent"]) + result = runner.invoke(cli, ["default", "instance", "nonexistent"]) assert result.exit_code == 1 assert "No such alias" in result.output @@ -58,16 +58,16 @@ def test_alias_default_db_workflow(mocker, tmpdir): assert result.exit_code == 0 # No default database yet - result = runner.invoke(cli, ["alias", "default-db", "prod"]) + result = runner.invoke(cli, ["default", "database", "prod"]) assert result.exit_code == 0 assert "No default database set" in result.output # Set default database - result = runner.invoke(cli, ["alias", "default-db", "prod", "main"]) + result = runner.invoke(cli, ["default", "database", "prod", "main"]) assert result.exit_code == 0 # Show default database - result = runner.invoke(cli, ["alias", "default-db", "prod"]) + result = runner.invoke(cli, ["default", "database", "prod"]) assert result.exit_code == 0 assert result.output.strip() == "main" @@ -76,17 +76,17 @@ def test_alias_default_db_workflow(mocker, tmpdir): assert "(db: main)" in result.output # Clear default database - result = runner.invoke(cli, ["alias", "default-db", "prod", "--clear"]) + result = runner.invoke(cli, ["default", "database", "prod", "--clear"]) assert result.exit_code == 0 - result = runner.invoke(cli, ["alias", "default-db", "prod"]) + result = runner.invoke(cli, ["default", "database", "prod"]) assert "No default database set" in result.output def test_alias_default_db_unknown_alias(mocker, tmpdir): mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) runner = CliRunner() - result = runner.invoke(cli, ["alias", "default-db", "nonexistent", "main"]) + result = runner.invoke(cli, ["default", "database", "nonexistent", "main"]) assert result.exit_code == 1 assert "No such alias" in result.output @@ -97,7 +97,7 @@ def test_alias_remove_clears_default(mocker, tmpdir): runner = CliRunner() runner.invoke(cli, ["alias", "add", "prod", "https://prod.example.com"]) - runner.invoke(cli, ["alias", "default", "prod"]) + runner.invoke(cli, ["default", "instance", "prod"]) # Verify it's set config = json.loads((pathlib.Path(tmpdir) / "config.json").read_text()) @@ -109,3 +109,31 @@ def test_alias_remove_clears_default(mocker, tmpdir): config = json.loads((pathlib.Path(tmpdir) / "config.json").read_text()) assert config["default_instance"] is None assert "prod" not in config["instances"] + + +def test_default_instance_accepts_url(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + + runner.invoke(cli, ["alias", "add", "prod", "https://prod.example.com"]) + result = runner.invoke(cli, ["default", "instance", "https://prod.example.com"]) + assert result.exit_code == 0 + + result = runner.invoke(cli, ["default", "instance"]) + assert result.exit_code == 0 + assert result.output.strip() == "prod" + + +def test_default_database_accepts_url(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + + runner.invoke(cli, ["alias", "add", "prod", "https://prod.example.com"]) + result = runner.invoke( + cli, ["default", "database", "https://prod.example.com", "main"] + ) + assert result.exit_code == 0 + + result = runner.invoke(cli, ["default", "database", "prod"]) + assert result.exit_code == 0 + assert result.output.strip() == "main" From f2aea1b108f46b812740647aa6624d759e2af015 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 20:56:22 -0800 Subject: [PATCH 56/64] --csv, --tsv, --nl, -t/--table output formats, closes #31 --- README.md | 6 ++- dclient/cli.py | 99 ++++++++++++++++++++++++++++++++++++++++++++--- docs/inserting.md | 4 +- docs/queries.md | 49 +++++++++++++++++++++++ 4 files changed, 150 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 96b7764..75225c6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Much of the functionality requires Datasette 1.0a2 or higher. ## Things you can do with dclient -- Run SQL queries against Datasette and return the results as JSON +- Run SQL queries against Datasette and return the results as JSON, CSV, TSV, or an ASCII table - Introspect databases, tables, plugins, and schema - Run queries against authenticated Datasette instances - Create aliases and set default instances/databases for convenient access @@ -43,6 +43,10 @@ Or be explicit: ```bash dclient query fixtures "select * from facetable limit 1" -i latest ``` +Output as a table with `-t`, or use `--csv`, `--tsv`, `--nl`: +```bash +dclient "select pk, state from facetable limit 3" -t +``` ## Introspection diff --git a/dclient/cli.py b/dclient/cli.py index fcc8047..36e2380 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,6 +1,8 @@ import click from click_default_group import DefaultGroup +import csv import httpx +import io import json import os import pathlib @@ -123,6 +125,83 @@ def _resolve_token(token, url, auth_file, config_file): return os.environ.get("DATASETTE_TOKEN") +def _output_rows(rows, fmt, columns=None): + """Output rows in the specified format. fmt is one of 'json', 'csv', 'tsv', 'nl', 'table'.""" + if fmt == "csv": + _output_csv(rows, columns) + elif fmt == "tsv": + _output_csv(rows, columns, delimiter="\t") + elif fmt == "nl": + for row in rows: + click.echo(json.dumps(row, default=str)) + elif fmt == "table": + _output_table(rows, columns) + else: + click.echo(json.dumps(rows, indent=2, default=str)) + + +def _output_csv(rows, columns=None, delimiter=","): + if not rows and not columns: + return + if columns is None: + columns = list(rows[0].keys()) if rows else [] + buf = io.StringIO() + writer = csv.writer(buf, delimiter=delimiter) + writer.writerow(columns) + for row in rows: + writer.writerow(str(row.get(col, "")) for col in columns) + click.echo(buf.getvalue(), nl=False) + + +def _output_table(rows, columns=None): + if not rows and not columns: + return + if columns is None: + columns = list(rows[0].keys()) if rows else [] + if not columns: + return + # Calculate column widths + widths = {col: len(str(col)) for col in columns} + for row in rows: + for col in columns: + widths[col] = max(widths[col], len(str(row.get(col, "")))) + # Header + header = " ".join(str(col).ljust(widths[col]) for col in columns) + click.echo(header) + # Separator + sep = " ".join("-" * widths[col] for col in columns) + click.echo(sep) + # Rows + for row in rows: + line = " ".join(str(row.get(col, "")).ljust(widths[col]) for col in columns) + click.echo(line) + + +def _determine_output_format(fmt_csv, fmt_tsv, fmt_nl, fmt_table): + if fmt_csv: + return "csv" + if fmt_tsv: + return "tsv" + if fmt_nl: + return "nl" + if fmt_table: + return "table" + return "json" + + +def output_format_options(f): + """Decorator that adds --csv, --tsv, --nl, --table options to a command.""" + f = click.option( + "fmt_table", "--table", "-t", is_flag=True, help="Output as ASCII table" + )(f) + f = click.option( + "fmt_nl", "--nl", is_flag=True, help="Output as newline-delimited JSON" + )(f) + f = click.option("fmt_tsv", "--tsv", is_flag=True, help="Output as TSV")(f) + f = click.option("fmt_csv", "--csv", is_flag=True, help="Output as CSV")(f) + return f + + @click.group(cls=DefaultGroup, default="default_query", default_if_no_args=False) @click.version_option() def cli(): @@ -286,7 +365,8 @@ def tables(instance, database, views, views_only, hidden, _json, token): @click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") @click.option("--token", help="API token") @click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") -def query(database, sql, instance, token, verbose): +@output_format_options +def query(database, sql, instance, token, verbose, fmt_csv, fmt_tsv, fmt_nl, fmt_table): """ Run a SQL query against a Datasette database @@ -346,7 +426,10 @@ def query(database, sql, instance, token, verbose): bits = [json.dumps(data)] raise click.ClickException(": ".join(bits)) - click.echo(json.dumps(response.json()["rows"], indent=2)) + rows = response.json()["rows"] + columns = response.json().get("columns") + fmt = _determine_output_format(fmt_csv, fmt_tsv, fmt_nl, fmt_table) + _output_rows(rows, fmt, columns) def _do_insert( @@ -499,7 +582,7 @@ _insert_options = [ click.option( "--interval", type=float, default=10, help="Send batch at least every X seconds" ), - click.option("--token", "-t", help="API token"), + click.option("--token", help="API token"), click.option("--silent", is_flag=True, help="Don't output progress"), click.option( "-v", @@ -744,7 +827,10 @@ def actor(instance, token): @click.option("-d", "--database", default=None, help="Database name") @click.option("--token", help="API token") @click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") -def default_query(sql, instance, database, token, verbose): +@output_format_options +def default_query( + sql, instance, database, token, verbose, fmt_csv, fmt_tsv, fmt_nl, fmt_table +): """Run a SQL query using default instance and database.""" config_dir = get_config_dir() url = _resolve_instance(instance, config_dir / "config.json") @@ -807,7 +893,10 @@ def default_query(sql, instance, database, token, verbose): bits = [json.dumps(data)] raise click.ClickException(": ".join(bits)) - click.echo(json.dumps(response.json()["rows"], indent=2)) + rows = response.json()["rows"] + columns = response.json().get("columns") + fmt = _determine_output_format(fmt_csv, fmt_tsv, fmt_nl, fmt_table) + _output_rows(rows, fmt, columns) @cli.command() diff --git a/docs/inserting.md b/docs/inserting.md index 17d81a7..4ac7d4c 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -132,7 +132,7 @@ Options: table --batch-size INTEGER Send rows in batches of this size --interval FLOAT Send batch at least every X seconds - -t, --token TEXT API token + --token TEXT API token --silent Don't output progress -v, --verbose Verbose output: show HTTP request and response --replace Replace rows with a matching primary key @@ -174,7 +174,7 @@ Options: table --batch-size INTEGER Send rows in batches of this size --interval FLOAT Send batch at least every X seconds - -t, --token TEXT API token + --token TEXT API token --silent Don't output progress -v, --verbose Verbose output: show HTTP request and response --help Show this message and exit. diff --git a/docs/queries.md b/docs/queries.md index 900f0ca..2db5cdb 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -36,6 +36,51 @@ You can override just the database with `-d`: dclient "select * from counters" -d counters ``` +## Output formats + +By default, results are returned as JSON. Use these flags to change the output format: + +- `--csv` — CSV +- `--tsv` — TSV +- `-t` / `--table` — ASCII table +- `--nl` — newline-delimited JSON (one JSON object per line) + +These flags work with both `dclient query` and the bare SQL shortcut. + +CSV output: + +```bash +dclient query fixtures "select * from facetable limit 2" -i latest --csv +``` +``` +pk,created,planet_int,on_earth,state,_city_id,_neighborhood +1,2019-01-14 08:00:00,1,1,CA,1,Mission +2,2019-01-15 08:00:00,1,1,CA,1,Dogpatch +``` + +ASCII table output: + +```bash +dclient query fixtures "select pk, state, _neighborhood from facetable limit 3" -i latest -t +``` +``` +pk state _neighborhood +-- ----- ------------- +1 CA Mission +2 CA Dogpatch +3 CA SOMA +``` + +Newline-delimited JSON, useful for piping into `jq` or other line-oriented tools: + +```bash +dclient query fixtures "select pk, state from facetable limit 2" -i latest --nl +``` +``` +{"pk": 1, "state": "CA"} +{"pk": 2, "state": "CA"} +``` + ## dclient query --help + +## dclient create-table --help + +``` +Usage: dclient create-table [OPTIONS] DATABASE TABLE_NAME + + Create a new empty table with an explicit schema + + Example usage: + + dclient create-table mydb dogs \ + --column id integer --column name text --pk id + +Options: + --column TEXT... Column definition: name type (e.g. --column id integer + --column name text) + --pk TEXT Column(s) to use as primary key + -i, --instance TEXT Datasette instance URL or alias + --token TEXT API token + -v, --verbose Verbose output: show HTTP request and response + --help Show this message and exit. + +``` + diff --git a/tests/test_create_table.py b/tests/test_create_table.py new file mode 100644 index 0000000..d3787de --- /dev/null +++ b/tests/test_create_table.py @@ -0,0 +1,203 @@ +"""Tests for the create-table command.""" + +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib + + +def test_create_table_basic(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={ + "ok": True, + "database": "mydb", + "table": "dogs", + "table_url": "http://example.com/mydb/dogs", + "table_api_url": "http://example.com/mydb/dogs.json", + "schema": "CREATE TABLE [dogs] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + }, + status_code=201, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "mydb", + "dogs", + "--column", + "id", + "integer", + "--column", + "name", + "text", + "--pk", + "id", + "-i", + "https://example.com", + "--token", + "tok", + ], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["ok"] is True + assert data["table"] == "dogs" + + # Verify request + request = httpx_mock.get_request() + assert request.url.path == "/mydb/-/create" + assert request.headers["authorization"] == "Bearer tok" + body = json.loads(request.read()) + assert body["table"] == "dogs" + assert body["columns"] == [ + {"name": "id", "type": "integer"}, + {"name": "name", "type": "text"}, + ] + assert body["pk"] == "id" + + +def test_create_table_compound_pk(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json={"ok": True}, status_code=201) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "mydb", + "events", + "--column", + "user_id", + "integer", + "-c", + "event_id", + "integer", + "--column", + "data", + "text", + "--pk", + "user_id", + "--pk", + "event_id", + "-i", + "https://example.com", + "--token", + "tok", + ], + ) + assert result.exit_code == 0, result.output + body = json.loads(httpx_mock.get_request().read()) + assert body["pks"] == ["user_id", "event_id"] + assert "pk" not in body + + +def test_create_table_no_pk(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response(json={"ok": True}, status_code=201) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "mydb", + "logs", + "--column", + "message", + "text", + "--column", + "level", + "integer", + "-i", + "https://example.com", + "--token", + "tok", + ], + ) + assert result.exit_code == 0, result.output + body = json.loads(httpx_mock.get_request().read()) + assert "pk" not in body + assert "pks" not in body + + +def test_create_table_no_columns_error(mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "mydb", + "empty", + "-i", + "https://example.com", + "--token", + "tok", + ], + ) + assert result.exit_code == 1 + assert "at least one --column" in result.output + + +def test_create_table_api_error(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + httpx_mock.add_response( + json={"ok": False, "errors": ["Table already exists: dogs"]}, + status_code=400, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "mydb", + "dogs", + "--column", + "id", + "integer", + "-i", + "https://example.com", + "--token", + "tok", + ], + ) + assert result.exit_code == 1 + assert "Table already exists" in result.output + + +def test_create_table_uses_default_instance(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response(json={"ok": True}, status_code=201) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "mydb", + "t1", + "--column", + "id", + "integer", + "--token", + "tok", + ], + ) + assert result.exit_code == 0, result.output + request = httpx_mock.get_request() + assert request.url.host == "prod.example.com" + assert request.url.path == "/mydb/-/create" diff --git a/tests/test_output_formats.py b/tests/test_output_formats.py new file mode 100644 index 0000000..b34a1e6 --- /dev/null +++ b/tests/test_output_formats.py @@ -0,0 +1,267 @@ +"""Tests for multiple output formats on the query and default_query commands.""" + +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib + +QUERY_RESPONSE = { + "ok": True, + "database": "fixtures", + "query_name": None, + "rows": [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Pancakes", "age": 3}, + ], + "truncated": False, + "columns": ["id", "name", "age"], + "query": {"sql": "select * from dogs", "params": {}}, + "error": None, + "private": False, + "allow_execute_sql": True, +} + + +def _mock_and_invoke(httpx_mock, extra_args=None): + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner() + args = ["query", "fixtures", "select * from dogs", "-i", "https://example.com"] + if extra_args: + args.extend(extra_args) + return runner.invoke(cli, args) + + +# -- query --csv -- + + +def test_query_csv(httpx_mock): + result = _mock_and_invoke(httpx_mock, ["--csv"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "id,name,age" + assert lines[1] == "1,Cleo,5" + assert lines[2] == "2,Pancakes,3" + + +# -- query --tsv -- + + +def test_query_tsv(httpx_mock): + result = _mock_and_invoke(httpx_mock, ["--tsv"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "id\tname\tage" + assert lines[1] == "1\tCleo\t5" + assert lines[2] == "2\tPancakes\t3" + + +# -- query --nl -- + + +def test_query_nl(httpx_mock): + result = _mock_and_invoke(httpx_mock, ["--nl"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert len(lines) == 2 + assert json.loads(lines[0]) == {"id": 1, "name": "Cleo", "age": 5} + assert json.loads(lines[1]) == {"id": 2, "name": "Pancakes", "age": 3} + + +# -- query --table -- + + +def test_query_table(httpx_mock): + result = _mock_and_invoke(httpx_mock, ["--table"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + # Should have a header row, a separator row, and 2 data rows + assert len(lines) == 4 + # Header should contain column names + assert "id" in lines[0] + assert "name" in lines[0] + assert "age" in lines[0] + # Data rows should contain values + assert "Cleo" in lines[2] + assert "Pancakes" in lines[3] + + +# -- query -t shortcut for --table -- + + +def test_query_table_shortcut(httpx_mock): + result = _mock_and_invoke(httpx_mock, ["-t"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert len(lines) == 4 + assert "Cleo" in lines[2] + + +# -- default JSON (no flag) stays the same -- + + +def test_query_default_json(httpx_mock): + result = _mock_and_invoke(httpx_mock) + assert result.exit_code == 0 + data = json.loads(result.output) + assert data == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Pancakes", "age": 3}, + ] + + +# -- default_query also supports output formats -- + + +def _mock_default_query(httpx_mock, mocker, tmpdir, extra_args=None): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response(json=QUERY_RESPONSE, status_code=200) + runner = CliRunner() + args = ["select * from dogs"] + if extra_args: + args.extend(extra_args) + return runner.invoke(cli, args) + + +def test_default_query_csv(httpx_mock, mocker, tmpdir): + result = _mock_default_query(httpx_mock, mocker, tmpdir, ["--csv"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "id,name,age" + assert lines[1] == "1,Cleo,5" + + +def test_default_query_table(httpx_mock, mocker, tmpdir): + result = _mock_default_query(httpx_mock, mocker, tmpdir, ["--table"]) + assert result.exit_code == 0 + assert "Cleo" in result.output + assert "Pancakes" in result.output + lines = result.output.strip().split("\n") + assert len(lines) == 4 + + +def test_default_query_nl(httpx_mock, mocker, tmpdir): + result = _mock_default_query(httpx_mock, mocker, tmpdir, ["--nl"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert json.loads(lines[0]) == {"id": 1, "name": "Cleo", "age": 5} + + +# -- edge cases -- + + +def test_query_csv_with_commas_in_values(httpx_mock): + httpx_mock.add_response( + json={ + "ok": True, + "rows": [{"name": "Smith, John", "note": 'He said "hi"'}], + "columns": ["name", "note"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "query", + "db", + "select * from t", + "-i", + "https://example.com", + "--csv", + ], + ) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "name,note" + # CSV should properly quote fields with commas/quotes + assert '"Smith, John"' in lines[1] + + +def test_query_table_empty_results(httpx_mock): + httpx_mock.add_response( + json={ + "ok": True, + "rows": [], + "columns": ["id", "name"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "query", + "db", + "select * from t", + "-i", + "https://example.com", + "--table", + ], + ) + assert result.exit_code == 0 + + +def test_query_csv_empty_results(httpx_mock): + httpx_mock.add_response( + json={ + "ok": True, + "rows": [], + "columns": ["id", "name"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "query", + "db", + "select * from t", + "-i", + "https://example.com", + "--csv", + ], + ) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "id,name" + assert len(lines) == 1 # header only, no data rows + + +def test_query_nl_empty_results(httpx_mock): + httpx_mock.add_response( + json={ + "ok": True, + "rows": [], + "columns": ["id", "name"], + }, + status_code=200, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "query", + "db", + "select * from t", + "-i", + "https://example.com", + "--nl", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == "" From 1340f68255b2ff7024eb7eabca63b2730bb63dc8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 21:04:25 -0800 Subject: [PATCH 58/64] Ran cog --- docs/inserting.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/inserting.md b/docs/inserting.md index ede3d90..e3a3039 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -240,13 +240,13 @@ Usage: dclient create-table [OPTIONS] DATABASE TABLE_NAME --column id integer --column name text --pk id Options: - --column TEXT... Column definition: name type (e.g. --column id integer - --column name text) - --pk TEXT Column(s) to use as primary key - -i, --instance TEXT Datasette instance URL or alias - --token TEXT API token - -v, --verbose Verbose output: show HTTP request and response - --help Show this message and exit. + -c, --column TEXT... Column definition: name type (e.g. --column id integer + --column name text) + --pk TEXT Column(s) to use as primary key + -i, --instance TEXT Datasette instance URL or alias + --token TEXT API token + -v, --verbose Verbose output: show HTTP request and response + --help Show this message and exit. ``` From 0b590d1a6fc30a052d4df4327a06a2bc3dc27c1b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 21:19:32 -0800 Subject: [PATCH 59/64] dclient rows command, closes #33 --- README.md | 5 + dclient/cli.py | 187 ++++++++++++++++- docs/queries.md | 117 ++++++++++- tests/test_commands_v2.py | 1 - tests/test_config.py | 1 - tests/test_rows.py | 410 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 717 insertions(+), 4 deletions(-) create mode 100644 tests/test_rows.py diff --git a/README.md b/README.md index 75225c6..a70a430 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Much of the functionality requires Datasette 1.0a2 or higher. ## Things you can do with dclient +- Browse table data with filtering, sorting, and pagination — no SQL required - Run SQL queries against Datasette and return the results as JSON, CSV, TSV, or an ASCII table - Introspect databases, tables, plugins, and schema - Run queries against authenticated Datasette instances @@ -47,6 +48,10 @@ Output as a table with `-t`, or use `--csv`, `--tsv`, `--nl`: ```bash dclient "select pk, state from facetable limit 3" -t ``` +Browse table rows without SQL: +```bash +dclient rows facetable -f state eq CA --sort _city_id -t +``` ## Introspection diff --git a/dclient/cli.py b/dclient/cli.py index b99e577..8b7dc72 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -359,6 +359,191 @@ def tables(instance, database, views, views_only, hidden, _json, token): click.echo(item) +# Convenience aliases for common filter operations. +# Any operation not listed here is passed through directly to Datasette, +# so plugins that add custom filter operations will work too. +FILTER_ALIASES = { + "eq": "exact", +} + + +@cli.command() +@click.argument("db_or_table") +@click.argument("table", required=False, default=None) +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("-d", "--database", default=None, help="Database name") +@click.option("--token", help="API token") +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + nargs=3, + help="Filter: column operation value (e.g. -f age gte 3)", +) +@click.option("--search", default=None, help="Full-text search query") +@click.option("--sort", default=None, help="Sort by column (ascending)") +@click.option("--sort-desc", default=None, help="Sort by column (descending)") +@click.option("--col", "columns", multiple=True, help="Include only these columns") +@click.option("--nocol", "nocolumns", multiple=True, help="Exclude these columns") +@click.option("--size", type=int, default=None, help="Number of rows per page") +@click.option("--limit", type=int, default=None, help="Maximum total rows to return") +@click.option("--all", "fetch_all", is_flag=True, help="Fetch all pages") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") +@output_format_options +def rows( + db_or_table, + table, + instance, + database, + token, + filters, + search, + sort, + sort_desc, + columns, + nocolumns, + size, + limit, + fetch_all, + verbose, + fmt_csv, + fmt_tsv, + fmt_nl, + fmt_table, +): + """ + Browse rows in a table with filtering and sorting + + If only one positional argument is given, it is treated as the table name + and the default database is used. Pass two arguments for database and table. + + Example usage: + + \b + dclient rows facet_cities + dclient rows fixtures facet_cities -i https://latest.datasette.io + dclient rows facet_cities -f id gte 3 --sort name -t + """ + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + + # Figure out database and table from positional args + if table is not None: + db = db_or_table + else: + # Only one positional arg — it's the table, resolve database from defaults + table = db_or_table + instance_alias = ( + _instance_alias_for_url(url, config_dir / "config.json") + if not ( + instance + and (instance.startswith("http://") or instance.startswith("https://")) + ) + else None + ) + if instance and not ( + instance.startswith("http://") or instance.startswith("https://") + ): + instance_alias = instance + db = _resolve_database(database, instance_alias, config_dir / "config.json") + + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) + + # Build query params + params = {"_shape": "objects"} + for col, op, val in filters: + datasette_op = FILTER_ALIASES.get(op, op) + params[f"{col}__{datasette_op}"] = val + if search: + params["_search"] = search + if sort: + params["_sort"] = sort + if sort_desc: + params["_sort_desc"] = sort_desc + for col in columns: + # httpx handles repeated keys if we use a list of tuples + pass + for col in nocolumns: + pass + if size: + params["_size"] = str(size) + + # Convert to list of tuples to support repeated keys (_col, _nocol) + param_items = list(params.items()) + for col in columns: + param_items.append(("_col", col)) + for col in nocolumns: + param_items.append(("_nocol", col)) + + # First request + table_url = url.rstrip("/") + "/" + db + "/" + table + ".json" + if verbose: + click.echo(table_url, err=True) + + all_rows = [] + col_names = None + total = 0 + next_page_url = None + first = True + + while True: + if first: + response = _make_request( + url, token, f"/{db}/{table}.json", params=param_items + ) + first = False + else: + # Follow next_url directly + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + response = httpx.get( + next_page_url, + headers=headers, + follow_redirects=True, + timeout=30.0, + ) + + if response.status_code != 200: + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException(f"{response.status_code} status code") + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + raise click.ClickException( + "{} status code. {}".format(response.status_code, ": ".join(bits)) + ) + + data = response.json() + page_rows = data.get("rows", []) + if col_names is None: + col_names = data.get("columns") + + if limit: + remaining = limit - total + page_rows = page_rows[:remaining] + + all_rows.extend(page_rows) + total += len(page_rows) + + if limit and total >= limit: + break + + next_page_url = data.get("next_url") + if not fetch_all or not next_page_url: + break + + fmt = _determine_output_format(fmt_csv, fmt_tsv, fmt_nl, fmt_table) + _output_rows(all_rows, fmt, col_names) + + @cli.command() @click.argument("database") @click.argument("sql") @@ -1320,7 +1505,7 @@ def login(alias_or_url, scope): interval = device_data.get("interval", 5) # Step 2: Show instructions - click.echo(f"\nOpen this URL in your browser:\n") + click.echo("\nOpen this URL in your browser:\n") click.echo(f" {verification_uri}\n") click.echo(f"Enter this code: {user_code}\n") click.echo("Waiting for authorization...", nl=False) diff --git a/docs/queries.md b/docs/queries.md index 2db5cdb..e9de25b 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -36,6 +36,121 @@ You can override just the database with `-d`: dclient "select * from counters" -d counters ``` +## Browsing rows + +The `dclient rows` command lets you browse table data without writing SQL: + +```bash +dclient rows fixtures facet_cities -i https://latest.datasette.io -t +``` +``` +id name +-- ------------- +3 Detroit +2 Los Angeles +4 Memnonia +1 San Francisco +``` + +If you have a default instance and database configured, you can just pass the table name: + +```bash +dclient rows facet_cities -t +``` + +### Filtering + +Use `-f` / `--filter` with three arguments: column, operation, value: + +```bash +dclient rows facet_cities -f id gte 3 -t +dclient rows facet_cities -f name eq Detroit +dclient rows facet_cities -f name contains M -f id gt 2 +``` + +The operation is passed directly to Datasette as a column filter suffix. Built-in Datasette operations include `exact`, `not`, `gt`, `gte`, `lt`, `lte`, `contains`, `like`, `startswith`, `endswith`, `glob`, `isnull`, `notnull`, and more. `eq` is a convenience alias for `exact`. Operations added by Datasette plugins will work too. + +### Sorting + +```bash +dclient rows dogs --sort age +dclient rows dogs --sort-desc age +``` + +### Column selection + +```bash +dclient rows dogs --col name --col age +dclient rows dogs --nocol id +``` + +### Search + +Full-text search (requires an FTS index on the table): + +```bash +dclient rows dogs --search "retriever" +``` + +### Pagination + +By default only one page of results is returned. Use `--all` to auto-paginate through all rows, and `--limit` to cap the total: + +```bash +dclient rows dogs --all +dclient rows dogs --all --limit 500 +dclient rows dogs --size 50 +``` + +### dclient rows --help + +``` +Usage: dclient rows [OPTIONS] DB_OR_TABLE [TABLE] + + Browse rows in a table with filtering and sorting + + If only one positional argument is given, it is treated as the table name and + the default database is used. Pass two arguments for database and table. + + Example usage: + + dclient rows facet_cities + dclient rows fixtures facet_cities -i https://latest.datasette.io + dclient rows facet_cities -f id gte 3 --sort name -t + +Options: + -i, --instance TEXT Datasette instance URL or alias + -d, --database TEXT Database name + --token TEXT API token + -f, --filter TEXT... Filter: column operation value (e.g. -f age gte 3) + --search TEXT Full-text search query + --sort TEXT Sort by column (ascending) + --sort-desc TEXT Sort by column (descending) + --col TEXT Include only these columns + --nocol TEXT Exclude these columns + --size INTEGER Number of rows per page + --limit INTEGER Maximum total rows to return + --all Fetch all pages + -v, --verbose Verbose output: show HTTP request + --csv Output as CSV + --tsv Output as TSV + --nl Output as newline-delimited JSON + -t, --table Output as ASCII table + --help Show this message and exit. + +``` + + ## Output formats By default, results are returned as JSON. Use these flags to change the output format: @@ -45,7 +160,7 @@ By default, results are returned as JSON. Use these flags to change the output f - `-t` / `--table` — ASCII table - `--nl` — newline-delimited JSON (one JSON object per line) -These flags work with both `dclient query` and the bare SQL shortcut. +These flags work with `dclient query`, `dclient rows`, and the bare SQL shortcut. CSV output: diff --git a/tests/test_commands_v2.py b/tests/test_commands_v2.py index a4326b5..8d21ce4 100644 --- a/tests/test_commands_v2.py +++ b/tests/test_commands_v2.py @@ -4,7 +4,6 @@ from click.testing import CliRunner from dclient.cli import cli import json import pathlib -import pytest # -- databases command -- diff --git a/tests/test_config.py b/tests/test_config.py index d39a2ee..65be345 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,6 @@ from dclient.cli import ( _resolve_instance, _resolve_database, _resolve_token, - get_config_dir, ) import json import pathlib diff --git a/tests/test_rows.py b/tests/test_rows.py new file mode 100644 index 0000000..33f8e0c --- /dev/null +++ b/tests/test_rows.py @@ -0,0 +1,410 @@ +"""Tests for the rows command.""" + +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib + +TABLE_RESPONSE = { + "ok": True, + "rows": [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Pancakes", "age": 3}, + {"id": 3, "name": "Fido", "age": 7}, + ], + "columns": ["id", "name", "age"], + "next": None, + "next_url": None, +} + + +def _invoke(httpx_mock, extra_args=None, response=None): + httpx_mock.add_response(json=response or TABLE_RESPONSE, status_code=200) + runner = CliRunner() + args = ["rows", "fixtures", "dogs", "-i", "https://example.com"] + if extra_args: + args.extend(extra_args) + return runner.invoke(cli, args) + + +# -- basic usage -- + + +def test_rows_default_json(httpx_mock): + result = _invoke(httpx_mock) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert len(data) == 3 + assert data[0]["name"] == "Cleo" + # Verify request URL + request = httpx_mock.get_request() + assert request.url.path == "/fixtures/dogs.json" + assert "_shape" in dict(request.url.params) + assert dict(request.url.params)["_shape"] == "objects" + + +def test_rows_table_format(httpx_mock): + result = _invoke(httpx_mock, ["-t"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert len(lines) == 5 # header + separator + 3 data rows + assert "Cleo" in lines[2] + assert "Pancakes" in lines[3] + + +def test_rows_csv(httpx_mock): + result = _invoke(httpx_mock, ["--csv"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "id,name,age" + assert lines[1] == "1,Cleo,5" + + +def test_rows_nl(httpx_mock): + result = _invoke(httpx_mock, ["--nl"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert json.loads(lines[0]) == {"id": 1, "name": "Cleo", "age": 5} + + +# -- single argument uses default database -- + + +def test_rows_single_arg_uses_default_database(httpx_mock, mocker, tmpdir): + """dclient rows tablename uses default database.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "data", + } + }, + } + ) + ) + httpx_mock.add_response(json=TABLE_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["rows", "dogs"]) + assert result.exit_code == 0, result.output + request = httpx_mock.get_request() + assert request.url.host == "prod.example.com" + assert request.url.path == "/data/dogs.json" + + +def test_rows_single_arg_no_default_database_errors(mocker, tmpdir): + """dclient rows tablename without default database gives error.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": None, + } + }, + } + ) + ) + runner = CliRunner() + result = runner.invoke(cli, ["rows", "dogs"]) + assert result.exit_code == 1 + assert "No database specified" in result.output + + +# -- filters -- + + +def test_rows_filter_eq(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "eq", "Cleo"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + params = dict(request.url.params) + assert params["name__exact"] == "Cleo" + + +def test_rows_filter_gt(httpx_mock): + result = _invoke(httpx_mock, ["--filter", "age", "gt", "3"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__gt"] == "3" + + +def test_rows_filter_gte(httpx_mock): + result = _invoke(httpx_mock, ["-f", "age", "gte", "5"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__gte"] == "5" + + +def test_rows_filter_lt(httpx_mock): + result = _invoke(httpx_mock, ["-f", "age", "lt", "5"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__lt"] == "5" + + +def test_rows_filter_lte(httpx_mock): + result = _invoke(httpx_mock, ["-f", "age", "lte", "5"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__lte"] == "5" + + +def test_rows_filter_not(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "not", "Cleo"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__not"] == "Cleo" + + +def test_rows_filter_contains(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "contains", "leo"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__contains"] == "leo" + + +def test_rows_filter_like(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "like", "%leo%"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__like"] == "%leo%" + + +def test_rows_filter_startswith(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "startswith", "Cl"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__startswith"] == "Cl" + + +def test_rows_filter_endswith(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "endswith", "eo"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__endswith"] == "eo" + + +def test_rows_filter_glob(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "glob", "C*"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__glob"] == "C*" + + +def test_rows_filter_isnull(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "isnull", "1"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__isnull"] == "1" + + +def test_rows_filter_notnull(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "notnull", "1"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__notnull"] == "1" + + +def test_rows_multiple_filters(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "eq", "Cleo", "-f", "age", "gte", "3"]) + assert result.exit_code == 0 + params = list(httpx_mock.get_request().url.params.multi_items()) + param_dict = {k: v for k, v in params} + assert param_dict["name__exact"] == "Cleo" + assert param_dict["age__gte"] == "3" + + +def test_rows_custom_filter_op_passthrough(httpx_mock): + """Unknown ops are passed through to Datasette, supporting plugin-added filters.""" + result = _invoke(httpx_mock, ["-f", "name", "custom_plugin_op", "x"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__custom_plugin_op"] == "x" + + +# -- sorting -- + + +def test_rows_sort(httpx_mock): + result = _invoke(httpx_mock, ["--sort", "age"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_sort"] == "age" + + +def test_rows_sort_desc(httpx_mock): + result = _invoke(httpx_mock, ["--sort-desc", "age"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_sort_desc"] == "age" + + +# -- column selection -- + + +def test_rows_col(httpx_mock): + result = _invoke(httpx_mock, ["--col", "name", "--col", "age"]) + assert result.exit_code == 0 + params = list(httpx_mock.get_request().url.params.multi_items()) + col_params = [v for k, v in params if k == "_col"] + assert col_params == ["name", "age"] + + +def test_rows_nocol(httpx_mock): + result = _invoke(httpx_mock, ["--nocol", "id"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_nocol"] == "id" + + +# -- search -- + + +def test_rows_search(httpx_mock): + result = _invoke(httpx_mock, ["--search", "pancakes"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_search"] == "pancakes" + + +# -- size -- + + +def test_rows_size(httpx_mock): + result = _invoke(httpx_mock, ["--size", "10"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_size"] == "10" + + +# -- limit -- + + +def test_rows_limit(httpx_mock): + response = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}, {"id": 3}], + "columns": ["id"], + "next": None, + } + result = _invoke(httpx_mock, ["--limit", "2"], response=response) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 + + +# -- pagination with --all -- + + +def test_rows_all_pagination(httpx_mock): + page1 = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}], + "columns": ["id"], + "next": "2", + "next_url": "https://example.com/fixtures/dogs.json?_next=2&_shape=objects", + } + page2 = { + "ok": True, + "rows": [{"id": 3}], + "columns": ["id"], + "next": None, + "next_url": None, + } + httpx_mock.add_response(json=page1, status_code=200) + httpx_mock.add_response(json=page2, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "fixtures", "dogs", "-i", "https://example.com", "--all"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 3 + assert [r["id"] for r in data] == [1, 2, 3] + + +def test_rows_all_with_limit(httpx_mock): + page1 = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}], + "columns": ["id"], + "next": "2", + "next_url": "https://example.com/fixtures/dogs.json?_next=2&_shape=objects", + } + page2 = { + "ok": True, + "rows": [{"id": 3}, {"id": 4}], + "columns": ["id"], + "next": None, + } + httpx_mock.add_response(json=page1, status_code=200) + httpx_mock.add_response(json=page2, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "rows", + "fixtures", + "dogs", + "-i", + "https://example.com", + "--all", + "--limit", + "3", + ], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 3 + + +def test_rows_no_all_ignores_next(httpx_mock): + """Without --all, pagination is not followed even if next is present.""" + response = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}], + "columns": ["id"], + "next": "2", + "next_url": "https://example.com/fixtures/dogs.json?_next=2&_shape=objects", + } + result = _invoke(httpx_mock, response=response) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 # only first page + + +# -- error handling -- + + +def test_rows_api_error(httpx_mock): + httpx_mock.add_response( + json={"ok": False, "error": "Table not found: dogs"}, + status_code=404, + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "fixtures", "dogs", "-i", "https://example.com"], + ) + assert result.exit_code == 1 + assert "Table not found" in result.output + + +# -- uses default instance -- + + +def test_rows_default_instance(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response(json=TABLE_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["rows", "data", "dogs"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "prod.example.com" + assert request.url.path == "/data/dogs.json" From e8d80a6495ecfebcd274eecc0a2d8fb7a3c98947 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 21:25:56 -0800 Subject: [PATCH 60/64] Logging in with OAuth docs --- docs/authentication.md | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/authentication.md b/docs/authentication.md index cd3961c..d83042a 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -12,6 +12,60 @@ dclient query fixtures "select * from facetable" --token dstok_mytoken -i latest A more convenient way to handle this is to store tokens for your aliases. +## Logging in with OAuth + +The easiest way to authenticate is using the `dclient login` command, which uses the OAuth device flow to obtain and store a token. + +This requires the Datasette instance to be running the [datasette-oauth](https://github.com/datasette/datasette-oauth) plugin with [device flow enabled](https://github.com/datasette/datasette-oauth?tab=readme-ov-file#plugin-configuration). + +```bash +dclient login https://my-datasette.example.com/ +dclient login myalias +dclient login +``` + +This will display a URL and a code. Open the URL in your browser, enter the code to approve access, and the resulting API token will be saved automatically. If you run `dclient login` without an argument, you will be prompted for the instance URL or alias. + +You can also pass a `--scope` option to request specific permissions: + +```bash +dclient login myalias --scope '[["view-instance"]]' +``` + +## dclient login --help + + +``` +Usage: dclient login [OPTIONS] [ALIAS_OR_URL] + + Authenticate with a Datasette instance using OAuth + + Uses the OAuth device flow: opens a URL in your browser where you approve + access, then saves the resulting API token. + + Example usage: + + dclient login https://simon.datasette.cloud/ + dclient login myalias + dclient login + +Options: + --scope TEXT JSON scope array + --help Show this message and exit. + +``` + + ## Using stored tokens To store a token for an alias: From 54df24db5e167206fcf8528370ece375df7b9675 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 26 Feb 2026 21:45:58 -0800 Subject: [PATCH 61/64] --read/--write options plus --token-only, closes #34 --- dclient/cli.py | 75 +++++++++++++- docs/authentication.md | 50 +++++++++- tests/test_cli_auth.py | 220 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 342 insertions(+), 3 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 8b7dc72..05dc98d 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1450,10 +1450,72 @@ def auth_status(instance, token): # -- login command (OAuth device flow) -- +READ_SCOPES = [ + "view-instance", + "view-table", + "view-database", + "view-query", + "execute-sql", +] +WRITE_SCOPES = [ + "insert-row", + "delete-row", + "update-row", + "create-table", + "alter-table", + "drop-table", +] + + +def _merge_scopes(scope, read_all, write_all, read, write): + """Merge --scope JSON with --read-all/--write-all/--read/--write options.""" + scopes = json.loads(scope) if scope else [] + if read_all: + for action in READ_SCOPES: + scopes.append([action]) + if write_all: + for action in READ_SCOPES + WRITE_SCOPES: + scopes.append([action]) + for target in read or []: + parts = target.split("/", 1) + if len(parts) == 1: + for action in READ_SCOPES: + scopes.append([action, parts[0]]) + else: + for action in READ_SCOPES: + scopes.append([action, parts[0], parts[1]]) + for target in write or []: + parts = target.split("/", 1) + if len(parts) == 1: + for action in READ_SCOPES + WRITE_SCOPES: + scopes.append([action, parts[0]]) + else: + for action in READ_SCOPES + WRITE_SCOPES: + scopes.append([action, parts[0], parts[1]]) + if scopes: + return json.dumps(scopes) + return None + + @cli.command() @click.argument("alias_or_url", required=False, default=None) @click.option("--scope", default=None, help="JSON scope array") -def login(alias_or_url, scope): +@click.option("--read-all", is_flag=True, help="Request instance-wide read access") +@click.option("--write-all", is_flag=True, help="Request instance-wide write access") +@click.option( + "--read", multiple=True, help="Request read access for a database or database/table" +) +@click.option( + "--write", + multiple=True, + help="Request write access for a database or database/table", +) +@click.option( + "--token-only", + is_flag=True, + help="Output the token to stdout instead of saving it", +) +def login(alias_or_url, scope, read_all, write_all, read, write, token_only): """ Authenticate with a Datasette instance using OAuth @@ -1466,7 +1528,13 @@ def login(alias_or_url, scope): dclient login https://simon.datasette.cloud/ dclient login myalias dclient login + dclient login --read-all + dclient login --write-all + dclient login --read db1 + dclient login --write db3/submissions + dclient login --read db1 --write db3/dogs """ + scope = _merge_scopes(scope, read_all, write_all, read, write) config_dir = get_config_dir() config_dir.mkdir(parents=True, exist_ok=True) config_file = config_dir / "config.json" @@ -1539,9 +1607,12 @@ def login(alias_or_url, scope): click.echo() raise click.ClickException(f"Unexpected error: {error}") - # Step 4: Save token + # Step 4: Save token (or print it) click.echo() access_token = token_data["access_token"] + if token_only: + click.echo(access_token) + return auth_file = config_dir / "auth.json" auths = _load_auths(auth_file) auths[auth_key] = access_token diff --git a/docs/authentication.md b/docs/authentication.md index d83042a..40a7700 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -26,12 +26,50 @@ dclient login This will display a URL and a code. Open the URL in your browser, enter the code to approve access, and the resulting API token will be saved automatically. If you run `dclient login` without an argument, you will be prompted for the instance URL or alias. -You can also pass a `--scope` option to request specific permissions: +### Requesting scoped tokens + +By default, `dclient login` requests an unrestricted token. You can request a token with limited permissions using the shorthand options: + +```bash +# Instance-wide read or write access +dclient login --read-all +dclient login --write-all + +# Database-level access +dclient login --read db1 +dclient login --write db3 + +# Table-level access +dclient login --read db1/dogs +dclient login --write db3/submissions + +# Mixed +dclient login --read db1 --write db3/dogs +``` + +`--read-all` grants: `view-instance`, `view-table`, `view-database`, `view-query`, `execute-sql`. + +`--write-all` grants all read scopes plus: `insert-row`, `delete-row`, `update-row`, `create-table`, `alter-table`, `drop-table`. + +`--read` and `--write` accept a database name or `database/table` and can be specified multiple times. `--write` implies read access for the same target. + +For advanced use, you can also pass raw scope JSON with `--scope`: ```bash dclient login myalias --scope '[["view-instance"]]' ``` +All scope options can be combined — the shorthand options append to whatever `--scope` provides. + +### Outputting the token directly + +Use `--token-only` to print the token to stdout instead of saving it. This is useful for creating debug tokens or piping them into other tools: + +```bash +dclient login --token-only --read mydb +dclient login --token-only --write-all +``` + ## dclient login --help