diff --git a/README.md b/README.md index 75225c6..a70a430 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Much of the functionality requires Datasette 1.0a2 or higher. ## Things you can do with dclient +- Browse table data with filtering, sorting, and pagination — no SQL required - Run SQL queries against Datasette and return the results as JSON, CSV, TSV, or an ASCII table - Introspect databases, tables, plugins, and schema - Run queries against authenticated Datasette instances @@ -47,6 +48,10 @@ Output as a table with `-t`, or use `--csv`, `--tsv`, `--nl`: ```bash dclient "select pk, state from facetable limit 3" -t ``` +Browse table rows without SQL: +```bash +dclient rows facetable -f state eq CA --sort _city_id -t +``` ## Introspection diff --git a/dclient/cli.py b/dclient/cli.py index b99e577..8b7dc72 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -359,6 +359,191 @@ def tables(instance, database, views, views_only, hidden, _json, token): click.echo(item) +# Convenience aliases for common filter operations. +# Any operation not listed here is passed through directly to Datasette, +# so plugins that add custom filter operations will work too. +FILTER_ALIASES = { + "eq": "exact", +} + + +@cli.command() +@click.argument("db_or_table") +@click.argument("table", required=False, default=None) +@click.option("-i", "--instance", default=None, help="Datasette instance URL or alias") +@click.option("-d", "--database", default=None, help="Database name") +@click.option("--token", help="API token") +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + nargs=3, + help="Filter: column operation value (e.g. -f age gte 3)", +) +@click.option("--search", default=None, help="Full-text search query") +@click.option("--sort", default=None, help="Sort by column (ascending)") +@click.option("--sort-desc", default=None, help="Sort by column (descending)") +@click.option("--col", "columns", multiple=True, help="Include only these columns") +@click.option("--nocol", "nocolumns", multiple=True, help="Exclude these columns") +@click.option("--size", type=int, default=None, help="Number of rows per page") +@click.option("--limit", type=int, default=None, help="Maximum total rows to return") +@click.option("--all", "fetch_all", is_flag=True, help="Fetch all pages") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") +@output_format_options +def rows( + db_or_table, + table, + instance, + database, + token, + filters, + search, + sort, + sort_desc, + columns, + nocolumns, + size, + limit, + fetch_all, + verbose, + fmt_csv, + fmt_tsv, + fmt_nl, + fmt_table, +): + """ + Browse rows in a table with filtering and sorting + + If only one positional argument is given, it is treated as the table name + and the default database is used. Pass two arguments for database and table. + + Example usage: + + \b + dclient rows facet_cities + dclient rows fixtures facet_cities -i https://latest.datasette.io + dclient rows facet_cities -f id gte 3 --sort name -t + """ + config_dir = get_config_dir() + url = _resolve_instance(instance, config_dir / "config.json") + + # Figure out database and table from positional args + if table is not None: + db = db_or_table + else: + # Only one positional arg — it's the table, resolve database from defaults + table = db_or_table + instance_alias = ( + _instance_alias_for_url(url, config_dir / "config.json") + if not ( + instance + and (instance.startswith("http://") or instance.startswith("https://")) + ) + else None + ) + if instance and not ( + instance.startswith("http://") or instance.startswith("https://") + ): + instance_alias = instance + db = _resolve_database(database, instance_alias, config_dir / "config.json") + + token = _resolve_token( + token, url, config_dir / "auth.json", config_dir / "config.json" + ) + + # Build query params + params = {"_shape": "objects"} + for col, op, val in filters: + datasette_op = FILTER_ALIASES.get(op, op) + params[f"{col}__{datasette_op}"] = val + if search: + params["_search"] = search + if sort: + params["_sort"] = sort + if sort_desc: + params["_sort_desc"] = sort_desc + for col in columns: + # httpx handles repeated keys if we use a list of tuples + pass + for col in nocolumns: + pass + if size: + params["_size"] = str(size) + + # Convert to list of tuples to support repeated keys (_col, _nocol) + param_items = list(params.items()) + for col in columns: + param_items.append(("_col", col)) + for col in nocolumns: + param_items.append(("_nocol", col)) + + # First request + table_url = url.rstrip("/") + "/" + db + "/" + table + ".json" + if verbose: + click.echo(table_url, err=True) + + all_rows = [] + col_names = None + total = 0 + next_page_url = None + first = True + + while True: + if first: + response = _make_request( + url, token, f"/{db}/{table}.json", params=param_items + ) + first = False + else: + # Follow next_url directly + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + response = httpx.get( + next_page_url, + headers=headers, + follow_redirects=True, + timeout=30.0, + ) + + if response.status_code != 200: + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException(f"{response.status_code} status code") + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + raise click.ClickException( + "{} status code. {}".format(response.status_code, ": ".join(bits)) + ) + + data = response.json() + page_rows = data.get("rows", []) + if col_names is None: + col_names = data.get("columns") + + if limit: + remaining = limit - total + page_rows = page_rows[:remaining] + + all_rows.extend(page_rows) + total += len(page_rows) + + if limit and total >= limit: + break + + next_page_url = data.get("next_url") + if not fetch_all or not next_page_url: + break + + fmt = _determine_output_format(fmt_csv, fmt_tsv, fmt_nl, fmt_table) + _output_rows(all_rows, fmt, col_names) + + @cli.command() @click.argument("database") @click.argument("sql") @@ -1320,7 +1505,7 @@ def login(alias_or_url, scope): interval = device_data.get("interval", 5) # Step 2: Show instructions - click.echo(f"\nOpen this URL in your browser:\n") + click.echo("\nOpen this URL in your browser:\n") click.echo(f" {verification_uri}\n") click.echo(f"Enter this code: {user_code}\n") click.echo("Waiting for authorization...", nl=False) diff --git a/docs/queries.md b/docs/queries.md index 2db5cdb..e9de25b 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -36,6 +36,121 @@ You can override just the database with `-d`: dclient "select * from counters" -d counters ``` +## Browsing rows + +The `dclient rows` command lets you browse table data without writing SQL: + +```bash +dclient rows fixtures facet_cities -i https://latest.datasette.io -t +``` +``` +id name +-- ------------- +3 Detroit +2 Los Angeles +4 Memnonia +1 San Francisco +``` + +If you have a default instance and database configured, you can just pass the table name: + +```bash +dclient rows facet_cities -t +``` + +### Filtering + +Use `-f` / `--filter` with three arguments: column, operation, value: + +```bash +dclient rows facet_cities -f id gte 3 -t +dclient rows facet_cities -f name eq Detroit +dclient rows facet_cities -f name contains M -f id gt 2 +``` + +The operation is passed directly to Datasette as a column filter suffix. Built-in Datasette operations include `exact`, `not`, `gt`, `gte`, `lt`, `lte`, `contains`, `like`, `startswith`, `endswith`, `glob`, `isnull`, `notnull`, and more. `eq` is a convenience alias for `exact`. Operations added by Datasette plugins will work too. + +### Sorting + +```bash +dclient rows dogs --sort age +dclient rows dogs --sort-desc age +``` + +### Column selection + +```bash +dclient rows dogs --col name --col age +dclient rows dogs --nocol id +``` + +### Search + +Full-text search (requires an FTS index on the table): + +```bash +dclient rows dogs --search "retriever" +``` + +### Pagination + +By default only one page of results is returned. Use `--all` to auto-paginate through all rows, and `--limit` to cap the total: + +```bash +dclient rows dogs --all +dclient rows dogs --all --limit 500 +dclient rows dogs --size 50 +``` + +### dclient rows --help + +``` +Usage: dclient rows [OPTIONS] DB_OR_TABLE [TABLE] + + Browse rows in a table with filtering and sorting + + If only one positional argument is given, it is treated as the table name and + the default database is used. Pass two arguments for database and table. + + Example usage: + + dclient rows facet_cities + dclient rows fixtures facet_cities -i https://latest.datasette.io + dclient rows facet_cities -f id gte 3 --sort name -t + +Options: + -i, --instance TEXT Datasette instance URL or alias + -d, --database TEXT Database name + --token TEXT API token + -f, --filter TEXT... Filter: column operation value (e.g. -f age gte 3) + --search TEXT Full-text search query + --sort TEXT Sort by column (ascending) + --sort-desc TEXT Sort by column (descending) + --col TEXT Include only these columns + --nocol TEXT Exclude these columns + --size INTEGER Number of rows per page + --limit INTEGER Maximum total rows to return + --all Fetch all pages + -v, --verbose Verbose output: show HTTP request + --csv Output as CSV + --tsv Output as TSV + --nl Output as newline-delimited JSON + -t, --table Output as ASCII table + --help Show this message and exit. + +``` + + ## Output formats By default, results are returned as JSON. Use these flags to change the output format: @@ -45,7 +160,7 @@ By default, results are returned as JSON. Use these flags to change the output f - `-t` / `--table` — ASCII table - `--nl` — newline-delimited JSON (one JSON object per line) -These flags work with both `dclient query` and the bare SQL shortcut. +These flags work with `dclient query`, `dclient rows`, and the bare SQL shortcut. CSV output: diff --git a/tests/test_commands_v2.py b/tests/test_commands_v2.py index a4326b5..8d21ce4 100644 --- a/tests/test_commands_v2.py +++ b/tests/test_commands_v2.py @@ -4,7 +4,6 @@ from click.testing import CliRunner from dclient.cli import cli import json import pathlib -import pytest # -- databases command -- diff --git a/tests/test_config.py b/tests/test_config.py index d39a2ee..65be345 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,6 @@ from dclient.cli import ( _resolve_instance, _resolve_database, _resolve_token, - get_config_dir, ) import json import pathlib diff --git a/tests/test_rows.py b/tests/test_rows.py new file mode 100644 index 0000000..33f8e0c --- /dev/null +++ b/tests/test_rows.py @@ -0,0 +1,410 @@ +"""Tests for the rows command.""" + +from click.testing import CliRunner +from dclient.cli import cli +import json +import pathlib + +TABLE_RESPONSE = { + "ok": True, + "rows": [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Pancakes", "age": 3}, + {"id": 3, "name": "Fido", "age": 7}, + ], + "columns": ["id", "name", "age"], + "next": None, + "next_url": None, +} + + +def _invoke(httpx_mock, extra_args=None, response=None): + httpx_mock.add_response(json=response or TABLE_RESPONSE, status_code=200) + runner = CliRunner() + args = ["rows", "fixtures", "dogs", "-i", "https://example.com"] + if extra_args: + args.extend(extra_args) + return runner.invoke(cli, args) + + +# -- basic usage -- + + +def test_rows_default_json(httpx_mock): + result = _invoke(httpx_mock) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert len(data) == 3 + assert data[0]["name"] == "Cleo" + # Verify request URL + request = httpx_mock.get_request() + assert request.url.path == "/fixtures/dogs.json" + assert "_shape" in dict(request.url.params) + assert dict(request.url.params)["_shape"] == "objects" + + +def test_rows_table_format(httpx_mock): + result = _invoke(httpx_mock, ["-t"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert len(lines) == 5 # header + separator + 3 data rows + assert "Cleo" in lines[2] + assert "Pancakes" in lines[3] + + +def test_rows_csv(httpx_mock): + result = _invoke(httpx_mock, ["--csv"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert lines[0] == "id,name,age" + assert lines[1] == "1,Cleo,5" + + +def test_rows_nl(httpx_mock): + result = _invoke(httpx_mock, ["--nl"]) + assert result.exit_code == 0 + lines = result.output.strip().split("\n") + assert json.loads(lines[0]) == {"id": 1, "name": "Cleo", "age": 5} + + +# -- single argument uses default database -- + + +def test_rows_single_arg_uses_default_database(httpx_mock, mocker, tmpdir): + """dclient rows tablename uses default database.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "data", + } + }, + } + ) + ) + httpx_mock.add_response(json=TABLE_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["rows", "dogs"]) + assert result.exit_code == 0, result.output + request = httpx_mock.get_request() + assert request.url.host == "prod.example.com" + assert request.url.path == "/data/dogs.json" + + +def test_rows_single_arg_no_default_database_errors(mocker, tmpdir): + """dclient rows tablename without default database gives error.""" + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": None, + } + }, + } + ) + ) + runner = CliRunner() + result = runner.invoke(cli, ["rows", "dogs"]) + assert result.exit_code == 1 + assert "No database specified" in result.output + + +# -- filters -- + + +def test_rows_filter_eq(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "eq", "Cleo"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + params = dict(request.url.params) + assert params["name__exact"] == "Cleo" + + +def test_rows_filter_gt(httpx_mock): + result = _invoke(httpx_mock, ["--filter", "age", "gt", "3"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__gt"] == "3" + + +def test_rows_filter_gte(httpx_mock): + result = _invoke(httpx_mock, ["-f", "age", "gte", "5"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__gte"] == "5" + + +def test_rows_filter_lt(httpx_mock): + result = _invoke(httpx_mock, ["-f", "age", "lt", "5"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__lt"] == "5" + + +def test_rows_filter_lte(httpx_mock): + result = _invoke(httpx_mock, ["-f", "age", "lte", "5"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["age__lte"] == "5" + + +def test_rows_filter_not(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "not", "Cleo"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__not"] == "Cleo" + + +def test_rows_filter_contains(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "contains", "leo"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__contains"] == "leo" + + +def test_rows_filter_like(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "like", "%leo%"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__like"] == "%leo%" + + +def test_rows_filter_startswith(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "startswith", "Cl"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__startswith"] == "Cl" + + +def test_rows_filter_endswith(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "endswith", "eo"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__endswith"] == "eo" + + +def test_rows_filter_glob(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "glob", "C*"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__glob"] == "C*" + + +def test_rows_filter_isnull(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "isnull", "1"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__isnull"] == "1" + + +def test_rows_filter_notnull(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "notnull", "1"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__notnull"] == "1" + + +def test_rows_multiple_filters(httpx_mock): + result = _invoke(httpx_mock, ["-f", "name", "eq", "Cleo", "-f", "age", "gte", "3"]) + assert result.exit_code == 0 + params = list(httpx_mock.get_request().url.params.multi_items()) + param_dict = {k: v for k, v in params} + assert param_dict["name__exact"] == "Cleo" + assert param_dict["age__gte"] == "3" + + +def test_rows_custom_filter_op_passthrough(httpx_mock): + """Unknown ops are passed through to Datasette, supporting plugin-added filters.""" + result = _invoke(httpx_mock, ["-f", "name", "custom_plugin_op", "x"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["name__custom_plugin_op"] == "x" + + +# -- sorting -- + + +def test_rows_sort(httpx_mock): + result = _invoke(httpx_mock, ["--sort", "age"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_sort"] == "age" + + +def test_rows_sort_desc(httpx_mock): + result = _invoke(httpx_mock, ["--sort-desc", "age"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_sort_desc"] == "age" + + +# -- column selection -- + + +def test_rows_col(httpx_mock): + result = _invoke(httpx_mock, ["--col", "name", "--col", "age"]) + assert result.exit_code == 0 + params = list(httpx_mock.get_request().url.params.multi_items()) + col_params = [v for k, v in params if k == "_col"] + assert col_params == ["name", "age"] + + +def test_rows_nocol(httpx_mock): + result = _invoke(httpx_mock, ["--nocol", "id"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_nocol"] == "id" + + +# -- search -- + + +def test_rows_search(httpx_mock): + result = _invoke(httpx_mock, ["--search", "pancakes"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_search"] == "pancakes" + + +# -- size -- + + +def test_rows_size(httpx_mock): + result = _invoke(httpx_mock, ["--size", "10"]) + assert result.exit_code == 0 + assert dict(httpx_mock.get_request().url.params)["_size"] == "10" + + +# -- limit -- + + +def test_rows_limit(httpx_mock): + response = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}, {"id": 3}], + "columns": ["id"], + "next": None, + } + result = _invoke(httpx_mock, ["--limit", "2"], response=response) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 + + +# -- pagination with --all -- + + +def test_rows_all_pagination(httpx_mock): + page1 = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}], + "columns": ["id"], + "next": "2", + "next_url": "https://example.com/fixtures/dogs.json?_next=2&_shape=objects", + } + page2 = { + "ok": True, + "rows": [{"id": 3}], + "columns": ["id"], + "next": None, + "next_url": None, + } + httpx_mock.add_response(json=page1, status_code=200) + httpx_mock.add_response(json=page2, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "fixtures", "dogs", "-i", "https://example.com", "--all"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 3 + assert [r["id"] for r in data] == [1, 2, 3] + + +def test_rows_all_with_limit(httpx_mock): + page1 = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}], + "columns": ["id"], + "next": "2", + "next_url": "https://example.com/fixtures/dogs.json?_next=2&_shape=objects", + } + page2 = { + "ok": True, + "rows": [{"id": 3}, {"id": 4}], + "columns": ["id"], + "next": None, + } + httpx_mock.add_response(json=page1, status_code=200) + httpx_mock.add_response(json=page2, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "rows", + "fixtures", + "dogs", + "-i", + "https://example.com", + "--all", + "--limit", + "3", + ], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 3 + + +def test_rows_no_all_ignores_next(httpx_mock): + """Without --all, pagination is not followed even if next is present.""" + response = { + "ok": True, + "rows": [{"id": 1}, {"id": 2}], + "columns": ["id"], + "next": "2", + "next_url": "https://example.com/fixtures/dogs.json?_next=2&_shape=objects", + } + result = _invoke(httpx_mock, response=response) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 # only first page + + +# -- error handling -- + + +def test_rows_api_error(httpx_mock): + httpx_mock.add_response( + json={"ok": False, "error": "Table not found: dogs"}, + status_code=404, + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "fixtures", "dogs", "-i", "https://example.com"], + ) + assert result.exit_code == 1 + assert "Table not found" in result.output + + +# -- uses default instance -- + + +def test_rows_default_instance(httpx_mock, mocker, tmpdir): + mocker.patch("dclient.cli.get_config_dir", return_value=pathlib.Path(tmpdir)) + config_file = pathlib.Path(tmpdir) / "config.json" + config_file.write_text( + json.dumps( + { + "default_instance": "prod", + "instances": { + "prod": { + "url": "https://prod.example.com", + "default_database": "main", + } + }, + } + ) + ) + httpx_mock.add_response(json=TABLE_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke(cli, ["rows", "data", "dogs"]) + assert result.exit_code == 0 + request = httpx_mock.get_request() + assert request.url.host == "prod.example.com" + assert request.url.path == "/data/dogs.json"