diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index e423b8fa..5598dc12 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -3,7 +3,8 @@ name: Deploy latest.datasette.io on: push: branches: - - main + - main + - 1.0-dev permissions: contents: read @@ -68,6 +69,8 @@ jobs: gcloud config set project datasette-222320 export SUFFIX="-${GITHUB_REF#refs/heads/}" export SUFFIX=${SUFFIX#-main} + # Replace 1.0 with one-dot-zero in SUFFIX + export SUFFIX=${SUFFIX//1.0/one-dot-zero} datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ -m fixtures.json \ --plugins-dir=plugins \ diff --git a/datasette/app.py b/datasette/app.py index 246269f3..7aa2ac4b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -27,19 +27,21 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .views.base import ureg -from .views.database import DatabaseDownload, DatabaseView +from .views.database import DatabaseDownload, DatabaseView, TableCreateView from .views.index import IndexView from .views.special import ( JsonDataView, PatternPortfolioView, AuthTokenView, + ApiExplorerView, + CreateTokenView, LogoutView, AllowDebugView, PermissionsDebugView, MessagesDebugView, ) -from .views.table import TableView -from .views.row import RowView +from .views.table import TableView, TableInsertView, TableDropView +from .views.row import RowView, RowDeleteView, RowUpdateView from .renderer import json_renderer from .url_builder import Urls from .database import Database, QueryInterrupted @@ -60,13 +62,19 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, + tilde_decode, to_css_class, + urlsafe_components, + row_sql_params_pks, ) from .utils.asgi import ( AsgiLifespan, Base400, Forbidden, NotFound, + DatabaseNotFound, + TableNotFound, + RowNotFound, Request, Response, asgi_static, @@ -98,6 +106,11 @@ SETTINGS = ( 1000, "Maximum rows that can be returned from a table or custom query", ), + Setting( + "max_insert_rows", + 100, + "Maximum rows that can be inserted at a time using the bulk insert API", + ), Setting( "num_sql_threads", 3, @@ -123,6 +136,16 @@ SETTINGS = ( True, "Allow users to download the original SQLite database files", ), + Setting( + "allow_signed_tokens", + True, + "Allow users to create and use signed API tokens", + ), + Setting( + "max_signed_tokens_ttl", + 0, + "Maximum allowed expiry time for signed API tokens", + ), Setting("suggest_facets", True, "Calculate and display suggested facets"), Setting( "default_cache_ttl", @@ -181,6 +204,12 @@ async def favicon(request, send): ) +ResolvedTable = collections.namedtuple("ResolvedTable", ("db", "table", "is_view")) +ResolvedRow = collections.namedtuple( + "ResolvedRow", ("db", "table", "sql", "params", "pks", "pk_values", "row") +) + + class Datasette: # Message constants: INFO = 1 @@ -1083,6 +1112,7 @@ class Datasette: ), "base_url": self.setting("base_url"), "csrftoken": request.scope["csrftoken"] if request else lambda: "", + "datasette_version": __version__, }, **extra_template_vars, } @@ -1215,6 +1245,14 @@ class Datasette: AuthTokenView.as_view(self), r"/-/auth-token$", ) + add_route( + CreateTokenView.as_view(self), + r"/-/create-token$", + ) + add_route( + ApiExplorerView.as_view(self), + r"/-/api$", + ) add_route( LogoutView.as_view(self), r"/-/logout$", @@ -1239,6 +1277,7 @@ class Datasette: add_route( DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" ) + add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( TableView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", @@ -1247,12 +1286,63 @@ class Datasette: RowView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^/]+?)/(?P[^/]+?)(\.(?P\w+))?$", ) + add_route( + TableInsertView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/insert$", + ) + add_route( + TableDropView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", + ) + add_route( + RowDeleteView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/delete$", + ) + add_route( + RowUpdateView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/update$", + ) return [ # Compile any strings to regular expressions ((re.compile(pattern) if isinstance(pattern, str) else pattern), view) for pattern, view in routes ] + async def resolve_database(self, request): + database_route = tilde_decode(request.url_vars["database"]) + try: + return self.get_database(route=database_route) + except KeyError: + raise DatabaseNotFound( + "Database not found: {}".format(database_route), database_route + ) + + async def resolve_table(self, request): + db = await self.resolve_database(request) + table_name = tilde_decode(request.url_vars["table"]) + # Table must exist + is_view = False + table_exists = await db.table_exists(table_name) + if not table_exists: + is_view = await db.view_exists(table_name) + if not (table_exists or is_view): + raise TableNotFound( + "Table not found: {}".format(table_name), db.name, table_name + ) + return ResolvedTable(db, table_name, is_view) + + async def resolve_row(self, request): + db, table_name, _ = await self.resolve_table(request) + pk_values = urlsafe_components(request.url_vars["pks"]) + sql, params, pks = await row_sql_params_pks(db, table_name, pk_values) + results = await db.execute(sql, params, truncate=True) + row = results.first() + if row is None: + raise RowNotFound( + "Row not found: {}".format(pk_values), db.name, table_name, pk_values + ) + return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) + def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() diff --git a/datasette/database.py b/datasette/database.py index dfca179c..d8043c24 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -338,6 +338,12 @@ class Database: ) return bool(results.rows) + async def view_exists(self, table): + results = await self.execute( + "select 1 from sqlite_master where type='view' and name=?", params=(table,) + ) + return bool(results.rows) + async def table_names(self): results = await self.execute( "select name from sqlite_master where type='table'" diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index b58d8d1b..2aaf72d6 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -1,11 +1,23 @@ from datasette import hookimpl from datasette.utils import actor_matches_allow +import click +import itsdangerous +import json +import time -@hookimpl(tryfirst=True) -def permission_allowed(datasette, actor, action, resource): +@hookimpl(tryfirst=True, specname="permission_allowed") +def permission_allowed_default(datasette, actor, action, resource): async def inner(): - if action in ("permissions-debug", "debug-menu"): + if action in ( + "permissions-debug", + "debug-menu", + "insert-row", + "create-table", + "drop-table", + "delete-row", + "update-row", + ): if actor and actor.get("id") == "root": return True elif action == "view-instance": @@ -45,3 +57,132 @@ def permission_allowed(datasette, actor, action, resource): return actor_matches_allow(actor, database_allow_sql) return inner + + +@hookimpl(specname="permission_allowed") +def permission_allowed_actor_restrictions(actor, action, resource): + if actor is None: + return None + if "_r" not in actor: + # No restrictions, so we have no opinion + return None + _r = actor.get("_r") + action_initials = "".join([word[0] for word in action.split("-")]) + # If _r is defined then we use those to further restrict the actor + # Crucially, we only use this to say NO (return False) - we never + # use it to return YES (True) because that might over-ride other + # restrictions placed on this actor + all_allowed = _r.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if action_initials in all_allowed: + return None + # How about for the current database? + if action in ("view-database", "view-database-download", "execute-sql"): + database_allowed = _r.get("d", {}).get(resource) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if action_initials in database_allowed: + return None + # Or the current table? That's any time the resource is (database, table) + if not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = _r.get("t", {}).get(database, {}).get(table) + # TODO: What should this do for canned queries? + if table_allowed is not None: + assert isinstance(table_allowed, list) + if action_initials in table_allowed: + return None + # This action is not specifically allowed, so reject it + return False + + +@hookimpl +def actor_from_request(datasette, request): + prefix = "dstok_" + if not datasette.setting("allow_signed_tokens"): + return None + max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") + authorization = request.headers.get("authorization") + if not authorization: + return None + if not authorization.startswith("Bearer "): + return None + token = authorization[len("Bearer ") :] + if not token.startswith(prefix): + return None + token = token[len(prefix) :] + try: + decoded = datasette.unsign(token, namespace="token") + except itsdangerous.BadSignature: + return None + if "t" not in decoded: + # Missing timestamp + return None + created = decoded["t"] + if not isinstance(created, int): + # Invalid timestamp + return None + duration = decoded.get("d") + if duration is not None and not isinstance(duration, int): + # Invalid duration + return None + if (duration is None and max_signed_tokens_ttl) or ( + duration is not None + and max_signed_tokens_ttl + and duration > max_signed_tokens_ttl + ): + duration = max_signed_tokens_ttl + if duration: + if time.time() - created > duration: + # Expired + return None + actor = {"id": decoded["a"], "token": "dstok"} + if duration: + actor["token_expires"] = created + duration + return actor + + +@hookimpl +def register_commands(cli): + from datasette.app import Datasette + + @cli.command() + @click.argument("id") + @click.option( + "--secret", + help="Secret used for signing the API tokens", + envvar="DATASETTE_SECRET", + required=True, + ) + @click.option( + "-e", + "--expires-after", + help="Token should expire after this many seconds", + type=int, + ) + @click.option( + "--debug", + help="Show decoded token", + is_flag=True, + ) + def create_token(id, secret, expires_after, debug): + "Create a signed API token for the specified actor ID" + ds = Datasette(secret=secret) + bits = {"a": id, "token": "dstok", "t": int(time.time())} + if expires_after: + bits["d"] = expires_after + token = ds.sign(bits, namespace="token") + click.echo("dstok_{}".format(token)) + if debug: + click.echo("\nDecoded:\n") + click.echo(json.dumps(ds.unsign(token, namespace="token"), indent=2)) + + +@hookimpl +def skip_csrf(scope): + # Skip CSRF check for requests with content-type: application/json + if scope["type"] == "http": + headers = scope.get("headers") or {} + if dict(headers).get(b"content-type") == b"application/json": + return True diff --git a/datasette/permissions.py b/datasette/permissions.py new file mode 100644 index 00000000..91c9e774 --- /dev/null +++ b/datasette/permissions.py @@ -0,0 +1,19 @@ +import collections + +Permission = collections.namedtuple( + "Permission", ("name", "abbr", "takes_database", "takes_table", "default") +) + +PERMISSIONS = ( + Permission("view-instance", "vi", False, False, True), + Permission("view-database", "vd", True, False, True), + Permission("view-database-download", "vdd", True, False, True), + Permission("view-table", "vt", True, True, True), + Permission("view-query", "vq", True, True, True), + Permission("insert-row", "ir", True, True, False), + Permission("delete-row", "dr", True, True, False), + Permission("drop-table", "dt", True, True, False), + Permission("execute-sql", "es", True, False, True), + Permission("permissions-debug", "pd", False, False, False), + Permission("debug-menu", "dm", False, False, False), +) diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..d83b8186 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,56 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === "object" && typeof module !== "undefined" + ? (module.exports = factory()) + : typeof define === "function" && define.amd + ? define(factory) + : (global.jsonFormatHighlight = factory()); +})(this, function () { + "use strict"; + + var defaultColors = { + keyColor: "dimgray", + numberColor: "lightskyblue", + stringColor: "lightcoral", + trueColor: "lightseagreen", + falseColor: "#f66578", + nullColor: "cornflowerblue", + }; + + function index(json, colorOptions) { + if (colorOptions === void 0) colorOptions = {}; + + if (!json) { + return; + } + if (typeof json !== "string") { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, "&").replace(//g, ">"); + return json.replace( + /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, + function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) + ? colors.trueColor + : /false/.test(match) + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; + } + return '' + match + ""; + } + ); + } + + return index; +}); diff --git a/datasette/templates/_close_open_menus.html b/datasette/templates/_close_open_menus.html index 65eebddf..3302d77d 100644 --- a/datasette/templates/_close_open_menus.html +++ b/datasette/templates/_close_open_menus.html @@ -9,7 +9,7 @@ document.body.addEventListener('click', (ev) => { if (target && target.tagName == 'DETAILS') { detailsClickedWithin = target; } - Array.from(document.getElementsByTagName('details')).filter( + Array.from(document.querySelectorAll('details.details-menu')).filter( (details) => details.open && details != detailsClickedWithin ).forEach(details => details.open = false); }); diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 0f1b30f0..04181531 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -35,7 +35,7 @@ p.message-warning {

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

-
+

diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html new file mode 100644 index 00000000..ea95c023 --- /dev/null +++ b/datasette/templates/api_explorer.html @@ -0,0 +1,208 @@ +{% extends "base.html" %} + +{% block title %}API Explorer{% endblock %} + +{% block extra_head %} + +{% endblock %} + +{% block content %} + +

API Explorer

+ +

Use this tool to try out the + {% if datasette_version %} + Datasette API. + {% else %} + Datasette API. + {% endif %} +

+
+ GET + +
+ + + +
+ +
+
+ POST +
+
+ + +
+
+ + +
+

+ +
+ + + + + +{% if example_links %} +

API endpoints

+
    + {% for database in example_links %} +
  • Database: {{ database.name }}
    • +
      + {% for link in database.links %} +
    • {{ link.path }} - {{ link.label }}
      • + {% endfor %} + {% for table in database.tables %} +
    • {{ table.name }} +
        + {% for link in table.links %} +
      • {{ link.path }} - {{ link.label }}
        • + {% endfor %} +
      +
      • + {% endfor %} +
    + {% endfor %} +
+{% endif %} + +{% endblock %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 87c939ac..4b763398 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -19,7 +19,7 @@
/-/insert + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "row": { + "column1": "value1", + "column2": "value2" + } + } + +If successful, this will return a ``201`` status code and the newly inserted row, for example: + +.. code-block:: json + + { + "rows": [ + { + "id": 1, + "column1": "value1", + "column2": "value2" + } + ] + } + +To insert multiple rows at a time, use the same API method but send a list of dictionaries as the ``"rows"`` key: + +:: + + POST //
/-/insert + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "rows": [ + { + "column1": "value1", + "column2": "value2" + }, + { + "column1": "value3", + "column2": "value4" + } + ] + } + +If successful, this will return a ``201`` status code and an empty ``{}`` response body. + +To return the newly inserted rows, add the ``"return": true`` key to the request body: + +.. code-block:: json + + { + "rows": [ + { + "column1": "value1", + "column2": "value2" + }, + { + "column1": "value3", + "column2": "value4" + } + ], + "return": true + } + +This will return the same ``"rows"`` key as the single row example above. There is a small performance penalty for using this option. + +.. _RowUpdateView: + +Updating a row +~~~~~~~~~~~~~~ + +To update a row, make a ``POST`` to ``//
//-/update``. This requires the :ref:`permissions_update_row` permission. + +:: + + POST //
//-/update + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "update": { + "text_column": "New text string", + "integer_column": 3, + "float_column": 3.14 + } + } + +```` here is the :ref:`tilde-encoded ` primary key value of the row to delete - or a comma-separated list of primary key values if the table has a composite primary key. + +You only need to pass the columns you want to update. Any other columns will be left unchanged. + +If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. + +Add ``"return": true`` to the request body to return the updated row: + +.. code-block:: json + + { + "update": { + "title": "New title" + }, + "return": true + } + +The returned JSON will look like this: + +.. code-block:: json + + { + "ok": true, + "row": { + "id": 1, + "title": "New title", + "other_column": "Will be present here too" + } + } + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + +.. _RowDeleteView: + +Deleting a row +~~~~~~~~~~~~~~ + +To delete a row, make a ``POST`` to ``//
//-/delete``. This requires the :ref:`permissions_delete_row` permission. + +:: + + POST //
//-/delete + Content-Type: application/json + Authorization: Bearer dstok_ + +```` here is the :ref:`tilde-encoded ` primary key value of the row to delete - or a comma-separated list of primary key values if the table has a composite primary key. + +If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + +.. _TableCreateView: + +Creating a table +~~~~~~~~~~~~~~~~ + +To create a table, make a ``POST`` to ``//-/create``. This requires the :ref:`permissions_create_table` permission. + +:: + + POST //-/create + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "table": "name_of_new_table", + "columns": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "title", + "type": "text" + } + ], + "pk": "id" + } + +The JSON here describes the table that will be created: + +* ``table`` is the name of the table to create. This field is required. +* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. + + - ``name`` is the name of the column. This is required. + - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + +* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. + + If the primary key is an integer column, it will be configured to automatically increment for each new record. + + If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an integer ID column for you. + +* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. + +If the table is successfully created this will return a ``201`` status code and the following response: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "name_of_new_table", + "table_url": "http://127.0.0.1:8001/data/name_of_new_table", + "table_api_url": "http://127.0.0.1:8001/data/name_of_new_table.json", + "schema": "CREATE TABLE [name_of_new_table] (\n [id] INTEGER PRIMARY KEY,\n [title] TEXT\n)" + } + +.. _TableCreateView_example: + +Creating a table from example data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Instead of specifying ``columns`` directly you can instead pass a single example row or a list of rows. Datasette will create a table with a schema that matches those rows and insert them for you: + +:: + + POST //-/create + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "table": "creatures", + "rows": [ + { + "id": 1, + "name": "Tarantula" + }, + { + "id": 2, + "name": "Kākāpō" + } + ], + "pk": "id" + } + +The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "creatures", + "table_url": "http://127.0.0.1:8001/data/creatures", + "table_api_url": "http://127.0.0.1:8001/data/creatures.json", + "schema": "CREATE TABLE [creatures] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + "row_count": 2 + } + +.. _TableDropView: + +Dropping tables +~~~~~~~~~~~~~~~ + +To drop a table, make a ``POST`` to ``//
/-/drop``. This requires the :ref:`permissions_drop_table` permission. + +:: + + POST //
/-/drop + Content-Type: application/json + Authorization: Bearer dstok_ + +Without a POST body this will return a status ``200`` with a note about how many rows will be deleted: + +.. code-block:: json + + { + "ok": true, + "database": "", + "table": "
", + "row_count": 5, + "message": "Pass \"confirm\": true to confirm" + } + +If you pass the following POST body: + +.. code-block:: json + + { + "confirm": true + } + +Then the table will be dropped and a status ``200`` response of ``{"ok": true}`` will be returned. + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/docs/plugins.rst b/docs/plugins.rst index 29078054..71eaa935 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -151,7 +151,10 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "templates": false, "version": null, "hooks": [ - "permission_allowed" + "actor_from_request", + "permission_allowed", + "register_commands", + "skip_csrf" ] }, { diff --git a/docs/settings.rst b/docs/settings.rst index a6d50543..b86b18bd 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -96,6 +96,17 @@ You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_returned_rows 2000 +.. _setting_max_insert_rows: + +max_insert_rows +~~~~~~~~~~~~~~~ + +Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100. + +You can increase or decrease this limit like so:: + + datasette mydatabase.db --setting max_insert_rows 1000 + .. _setting_num_sql_threads: num_sql_threads @@ -169,6 +180,34 @@ Should users be able to download the original SQLite database using a link on th datasette mydatabase.db --setting allow_download off +.. _setting_allow_signed_tokens: + +allow_signed_tokens +~~~~~~~~~~~~~~~~~~~ + +Should users be able to create signed API tokens to access Datasette? + +This is turned on by default. Use the following to turn it off:: + + datasette mydatabase.db --setting allow_signed_tokens off + +Turning this setting off will disable the ``/-/create-token`` page, :ref:`described here `. It will also cause any incoming ``Authorization: Bearer dstok_...`` API tokens to be ignored. + +.. _setting_max_signed_tokens_ttl: + +max_signed_tokens_ttl +~~~~~~~~~~~~~~~~~~~~~ + +Maximum allowed expiry time for signed API tokens created by users. + +Defaults to ``0`` which means no limit - tokens can be created that will never expire. + +Set this to a value in seconds to limit the maximum expiry time. For example, to set that limit to 24 hours you would use:: + + datasette mydatabase.db --setting max_signed_tokens_ttl 86400 + +This setting is enforced when incoming tokens are processed. + .. _setting_default_cache_ttl: default_cache_ttl diff --git a/setup.py b/setup.py index 625557ae..99e2a4ad 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", + "sqlite-utils>=3.30", ], entry_points=""" [console_scripts] diff --git a/tests/fixtures.py b/tests/fixtures.py index 744400cb..ba5f065e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -129,10 +129,14 @@ def make_app_client( for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) + # Close the connection to avoid "too many open files" errors + conn.close() if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) - sqlite3.connect(extra_filepath).executescript(extra_sql) + c2 = sqlite3.connect(extra_filepath) + c2.executescript(extra_sql) + c2.close() # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) diff --git a/tests/test_api.py b/tests/test_api.py index 4027a7a5..de0223e2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -808,8 +808,11 @@ def test_settings_json(app_client): "facet_suggest_time_limit_ms": 50, "facet_time_limit_ms": 200, "max_returned_rows": 100, + "max_insert_rows": 100, "sql_time_limit_ms": 200, "allow_download": True, + "allow_signed_tokens": True, + "max_signed_tokens_ttl": 0, "allow_facet": True, "suggest_facets": True, "default_cache_ttl": 5, diff --git a/tests/test_api_write.py b/tests/test_api_write.py new file mode 100644 index 00000000..70fc0989 --- /dev/null +++ b/tests/test_api_write.py @@ -0,0 +1,924 @@ +from datasette.app import Datasette +from datasette.utils import sqlite3 +import pytest +import time + + +@pytest.fixture +def ds_write(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + db_path_immutable = str(db_directory / "immutable.db") + db1 = sqlite3.connect(str(db_path)) + db2 = sqlite3.connect(str(db_path_immutable)) + for db in (db1, db2): + db.execute("vacuum") + db.execute( + "create table docs (id integer primary key, title text, score float, age integer)" + ) + ds = Datasette([db_path], immutables=[db_path_immutable]) + yield ds + db.close() + + +def write_token(ds, actor_id="root"): + return "dstok_{}".format( + ds.sign( + {"a": actor_id, "token": "dstok", "t": int(time.time())}, namespace="token" + ) + ) + + +@pytest.mark.asyncio +async def test_write_row(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/docs/-/insert", + json={"row": {"title": "Test", "score": 1.2, "age": 5}}, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + expected_row = {"id": 1, "title": "Test", "score": 1.2, "age": 5} + assert response.status_code == 201 + assert response.json()["rows"] == [expected_row] + rows = (await ds_write.get_database("data").execute("select * from docs")).rows + assert dict(rows[0]) == expected_row + + +@pytest.mark.asyncio +@pytest.mark.parametrize("return_rows", (True, False)) +async def test_write_rows(ds_write, return_rows): + token = write_token(ds_write) + data = { + "rows": [ + {"title": "Test {}".format(i), "score": 1.0, "age": 5} for i in range(20) + ] + } + if return_rows: + data["return"] = True + response = await ds_write.client.post( + "/data/docs/-/insert", + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 201 + actual_rows = [ + dict(r) + for r in ( + await ds_write.get_database("data").execute("select * from docs") + ).rows + ] + assert len(actual_rows) == 20 + assert actual_rows == [ + {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} + for i in range(20) + ] + assert response.json()["ok"] is True + if return_rows: + assert response.json()["rows"] == actual_rows + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,input,special_case,expected_status,expected_errors", + ( + ( + "/data2/docs/-/insert", + {}, + None, + 404, + ["Database not found: data2"], + ), + ( + "/data/docs2/-/insert", + {}, + None, + 404, + ["Table not found: docs2"], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"} for i in range(10)]}, + "bad_token", + 403, + ["Permission denied"], + ), + ( + "/data/docs/-/insert", + {}, + "invalid_json", + 400, + [ + "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ], + ), + ( + "/data/docs/-/insert", + {}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), + ( + "/data/docs/-/insert", + [], + None, + 400, + ["JSON must be a dictionary"], + ), + ( + "/data/docs/-/insert", + {"row": "blah"}, + None, + 400, + ['"row" must be a dictionary'], + ), + ( + "/data/docs/-/insert", + {"blah": "blah"}, + None, + 400, + ['JSON must have one or other of "row" or "rows"'], + ), + ( + "/data/docs/-/insert", + {"rows": "blah"}, + None, + 400, + ['"rows" must be a list'], + ), + ( + "/data/docs/-/insert", + {"rows": ["blah"]}, + None, + 400, + ['"rows" must be a list of dictionaries'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"} for i in range(101)]}, + None, + 400, + ["Too many rows, maximum allowed is 100"], + ), + ( + "/data/docs/-/insert", + {"rows": [{"id": 1, "title": "Test"}]}, + "duplicate_id", + 400, + ["UNIQUE constraint failed: docs.id"], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "ignore": True, "replace": True}, + None, + 400, + ['Cannot use "ignore" and "replace" at the same time'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "invalid_param": True}, + None, + 400, + ['Invalid parameter: "invalid_param"'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "one": True, "two": True}, + None, + 400, + ['Invalid parameter: "one", "two"'], + ), + # Validate columns of each row + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test", "bad": 1, "worse": 2} for i in range(2)]}, + None, + 400, + [ + "Row 0 has invalid columns: bad, worse", + "Row 1 has invalid columns: bad, worse", + ], + ), + ), +) +async def test_write_row_errors( + ds_write, path, input, special_case, expected_status, expected_errors +): + token = write_token(ds_write) + if special_case == "duplicate_id": + await ds_write.get_database("data").execute_write( + "insert into docs (id) values (1)" + ) + if special_case == "bad_token": + token += "bad" + kwargs = dict( + json=input, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "text/plain" + if special_case == "invalid_content_type" + else "application/json", + }, + ) + if special_case == "invalid_json": + del kwargs["json"] + kwargs["content"] = "{bad json" + response = await ds_write.client.post( + path, + **kwargs, + ) + assert response.status_code == expected_status + assert response.json()["ok"] is False + assert response.json()["errors"] == expected_errors + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "ignore,replace,expected_rows", + ( + ( + True, + False, + [ + {"id": 1, "title": "Exists", "score": None, "age": None}, + ], + ), + ( + False, + True, + [ + {"id": 1, "title": "One", "score": None, "age": None}, + ], + ), + ), +) +@pytest.mark.parametrize("should_return", (True, False)) +async def test_insert_ignore_replace( + ds_write, ignore, replace, expected_rows, should_return +): + await ds_write.get_database("data").execute_write( + "insert into docs (id, title) values (1, 'Exists')" + ) + token = write_token(ds_write) + data = {"rows": [{"id": 1, "title": "One"}]} + if ignore: + data["ignore"] = True + if replace: + data["replace"] = True + if should_return: + data["return"] = True + response = await ds_write.client.post( + "/data/docs/-/insert", + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 201 + actual_rows = [ + dict(r) + for r in ( + await ds_write.get_database("data").execute("select * from docs") + ).rows + ] + assert actual_rows == expected_rows + assert response.json()["ok"] is True + if should_return: + assert response.json()["rows"] == expected_rows + + +async def _insert_row(ds): + insert_response = await ds.client.post( + "/data/docs/-/insert", + json={"row": {"title": "Row one", "score": 1.2, "age": 5}, "return": True}, + headers={ + "Authorization": "Bearer {}".format(write_token(ds)), + "Content-Type": "application/json", + }, + ) + assert insert_response.status_code == 201 + return insert_response.json()["rows"][0]["id"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table")) +async def test_delete_row_errors(ds_write, scenario): + if scenario == "no_token": + token = "bad_token" + elif scenario == "no_perm": + token = write_token(ds_write, actor_id="not-root") + else: + token = write_token(ds_write) + + pk = await _insert_row(ds_write) + + path = "/data/{}/{}/-/delete".format( + "docs" if scenario != "bad_table" else "bad_table", pk + ) + response = await ds_write.client.post( + path, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 403 if scenario in ("no_token", "bad_token") else 404 + assert response.json()["ok"] is False + assert ( + response.json()["errors"] == ["Permission denied"] + if scenario == "no_token" + else ["Table not found: bad_table"] + ) + assert len((await ds_write.client.get("/data/docs.json?_shape=array")).json()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "table,row_for_create,pks,delete_path", + ( + ("rowid_table", {"name": "rowid row"}, None, None), + ("pk_table", {"id": 1, "name": "ID table"}, "id", "1"), + ( + "compound_pk_table", + {"type": "article", "key": "k"}, + ["type", "key"], + "article,k", + ), + ), +) +async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): + # First create the table with that example row + create_data = { + "table": table, + "row": row_for_create, + } + if pks: + if isinstance(pks, str): + create_data["pk"] = pks + else: + create_data["pks"] = pks + create_response = await ds_write.client.post( + "/data/-/create", + json=create_data, + headers={ + "Authorization": "Bearer {}".format(write_token(ds_write)), + }, + ) + assert create_response.status_code == 201, create_response.json() + # Should be a single row + assert ( + await ds_write.client.get( + "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) + ) + ).json() == [1] + # Now delete the row + if delete_path is None: + # Special case for that rowid table + delete_path = ( + await ds_write.client.get( + "/data.json?_shape=arrayfirst&sql=select+rowid+from+{}".format(table) + ) + ).json()[0] + + delete_response = await ds_write.client.post( + "/data/{}/{}/-/delete".format(table, delete_path), + headers={ + "Authorization": "Bearer {}".format(write_token(ds_write)), + }, + ) + assert delete_response.status_code == 200 + assert ( + await ds_write.client.get( + "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) + ) + ).json() == [0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table")) +async def test_update_row_check_permission(ds_write, scenario): + if scenario == "no_token": + token = "bad_token" + elif scenario == "no_perm": + token = write_token(ds_write, actor_id="not-root") + else: + token = write_token(ds_write) + + pk = await _insert_row(ds_write) + + path = "/data/{}/{}/-/delete".format( + "docs" if scenario != "bad_table" else "bad_table", pk + ) + + response = await ds_write.client.post( + path, + json={"update": {"title": "New title"}}, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 403 if scenario in ("no_token", "bad_token") else 404 + assert response.json()["ok"] is False + assert ( + response.json()["errors"] == ["Permission denied"] + if scenario == "no_token" + else ["Table not found: bad_table"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "input,expected_errors", + ( + ({"title": "New title"}, None), + ({"title": None}, None), + ({"score": 1.6}, None), + ({"age": 10}, None), + ({"title": "New title", "score": 1.6}, None), + ({"title2": "New title"}, ["no such column: title2"]), + ), +) +@pytest.mark.parametrize("use_return", (True, False)) +async def test_update_row(ds_write, input, expected_errors, use_return): + token = write_token(ds_write) + pk = await _insert_row(ds_write) + + path = "/data/docs/{}/-/update".format(pk) + + data = {"update": input} + if use_return: + data["return"] = True + + response = await ds_write.client.post( + path, + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + if expected_errors: + assert response.status_code == 400 + assert response.json()["ok"] is False + assert response.json()["errors"] == expected_errors + return + + assert response.json()["ok"] is True + if not use_return: + assert "row" not in response.json() + else: + returned_row = response.json()["row"] + assert returned_row["id"] == pk + for k, v in input.items(): + assert returned_row[k] == v + + # And fetch the row to check it's updated + response = await ds_write.client.get( + "/data/docs/{}.json?_shape=array".format(pk), + ) + assert response.status_code == 200 + row = response.json()[0] + assert row["id"] == pk + for k, v in input.items(): + assert row[k] == v + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "scenario", ("no_token", "no_perm", "bad_table", "has_perm", "immutable") +) +async def test_drop_table(ds_write, scenario): + if scenario == "no_token": + token = "bad_token" + elif scenario == "no_perm": + token = write_token(ds_write, actor_id="not-root") + else: + token = write_token(ds_write) + should_work = scenario == "has_perm" + await ds_write.get_database("data").execute_write( + "insert into docs (id, title) values (1, 'Row 1')" + ) + path = "/{database}/{table}/-/drop".format( + database="immutable" if scenario == "immutable" else "data", + table="docs" if scenario != "bad_table" else "bad_table", + ) + response = await ds_write.client.post( + path, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + if not should_work: + assert ( + response.status_code == 403 + if scenario in ("no_token", "bad_token") + else 404 + ) + assert response.json()["ok"] is False + expected_error = "Permission denied" + if scenario == "bad_table": + expected_error = "Table not found: bad_table" + elif scenario == "immutable": + expected_error = "Database is immutable" + assert response.json()["errors"] == [expected_error] + assert (await ds_write.client.get("/data/docs")).status_code == 200 + else: + # It should show a confirmation page + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "docs", + "row_count": 1, + "message": 'Pass "confirm": true to confirm', + } + assert (await ds_write.client.get("/data/docs")).status_code == 200 + # Now send confirm: true + response2 = await ds_write.client.post( + path, + json={"confirm": True}, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response2.json() == {"ok": True} + assert (await ds_write.client.get("/data/docs")).status_code == 404 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "input,expected_status,expected_response", + ( + # Permission error with a bad token + ( + {"table": "bad", "row": {"id": 1}}, + 403, + {"ok": False, "errors": ["Permission denied"]}, + ), + # Successful creation with columns: + ( + { + "table": "one", + "columns": [ + { + "name": "id", + "type": "integer", + }, + { + "name": "title", + "type": "text", + }, + { + "name": "score", + "type": "integer", + }, + { + "name": "weight", + "type": "float", + }, + { + "name": "thumbnail", + "type": "blob", + }, + ], + "pk": "id", + }, + 201, + { + "ok": True, + "database": "data", + "table": "one", + "table_url": "http://localhost/data/one", + "table_api_url": "http://localhost/data/one.json", + "schema": ( + "CREATE TABLE [one] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [score] INTEGER,\n" + " [weight] FLOAT,\n" + " [thumbnail] BLOB\n" + ")" + ), + }, + ), + # Successful creation with rows: + ( + { + "table": "two", + "rows": [ + { + "id": 1, + "title": "Row 1", + "score": 1.5, + }, + { + "id": 2, + "title": "Row 2", + "score": 1.5, + }, + ], + "pk": "id", + }, + 201, + { + "ok": True, + "database": "data", + "table": "two", + "table_url": "http://localhost/data/two", + "table_api_url": "http://localhost/data/two.json", + "schema": ( + "CREATE TABLE [two] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [score] FLOAT\n" + ")" + ), + "row_count": 2, + }, + ), + # Successful creation with row: + ( + { + "table": "three", + "row": { + "id": 1, + "title": "Row 1", + "score": 1.5, + }, + "pk": "id", + }, + 201, + { + "ok": True, + "database": "data", + "table": "three", + "table_url": "http://localhost/data/three", + "table_api_url": "http://localhost/data/three.json", + "schema": ( + "CREATE TABLE [three] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [score] FLOAT\n" + ")" + ), + "row_count": 1, + }, + ), + # Create with row and no primary key + ( + { + "table": "four", + "row": { + "name": "Row 1", + }, + }, + 201, + { + "ok": True, + "database": "data", + "table": "four", + "table_url": "http://localhost/data/four", + "table_api_url": "http://localhost/data/four.json", + "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), + "row_count": 1, + }, + ), + # Create table with compound primary key + ( + { + "table": "five", + "row": {"type": "article", "key": 123, "title": "Article 1"}, + "pks": ["type", "key"], + }, + 201, + { + "ok": True, + "database": "data", + "table": "five", + "table_url": "http://localhost/data/five", + "table_api_url": "http://localhost/data/five.json", + "schema": ( + "CREATE TABLE [five] (\n [type] TEXT,\n [key] INTEGER,\n" + " [title] TEXT,\n PRIMARY KEY ([type], [key])\n)" + ), + "row_count": 1, + }, + ), + # Error: Table is required + ( + { + "row": {"id": 1}, + }, + 400, + { + "ok": False, + "errors": ["Table is required"], + }, + ), + # Error: Invalid table name + ( + { + "table": "sqlite_bad_name", + "row": {"id": 1}, + }, + 400, + { + "ok": False, + "errors": ["Invalid table name"], + }, + ), + # Error: JSON must be an object + ( + [], + 400, + { + "ok": False, + "errors": ["JSON must be an object"], + }, + ), + # Error: Cannot specify columns with rows or row + ( + { + "table": "bad", + "columns": [{"name": "id", "type": "integer"}], + "rows": [{"id": 1}], + }, + 400, + { + "ok": False, + "errors": ["Cannot specify columns with rows or row"], + }, + ), + # Error: columns, rows or row is required + ( + { + "table": "bad", + }, + 400, + { + "ok": False, + "errors": ["columns, rows or row is required"], + }, + ), + # Error: columns must be a list + ( + { + "table": "bad", + "columns": {"name": "id", "type": "integer"}, + }, + 400, + { + "ok": False, + "errors": ["columns must be a list"], + }, + ), + # Error: columns must be a list of objects + ( + { + "table": "bad", + "columns": ["id"], + }, + 400, + { + "ok": False, + "errors": ["columns must be a list of objects"], + }, + ), + # Error: Column name is required + ( + { + "table": "bad", + "columns": [{"type": "integer"}], + }, + 400, + { + "ok": False, + "errors": ["Column name is required"], + }, + ), + # Error: Unsupported column type + ( + { + "table": "bad", + "columns": [{"name": "id", "type": "bad"}], + }, + 400, + { + "ok": False, + "errors": ["Unsupported column type: bad"], + }, + ), + # Error: Duplicate column name + ( + { + "table": "bad", + "columns": [ + {"name": "id", "type": "integer"}, + {"name": "id", "type": "integer"}, + ], + }, + 400, + { + "ok": False, + "errors": ["Duplicate column name: id"], + }, + ), + # Error: rows must be a list + ( + { + "table": "bad", + "rows": {"id": 1}, + }, + 400, + { + "ok": False, + "errors": ["rows must be a list"], + }, + ), + # Error: rows must be a list of objects + ( + { + "table": "bad", + "rows": ["id"], + }, + 400, + { + "ok": False, + "errors": ["rows must be a list of objects"], + }, + ), + # Error: pk must be a string + ( + { + "table": "bad", + "row": {"id": 1}, + "pk": 1, + }, + 400, + { + "ok": False, + "errors": ["pk must be a string"], + }, + ), + # Error: Cannot specify both pk and pks + ( + { + "table": "bad", + "row": {"id": 1, "name": "Row 1"}, + "pk": "id", + "pks": ["id", "name"], + }, + 400, + { + "ok": False, + "errors": ["Cannot specify both pk and pks"], + }, + ), + # Error: pks must be a list + ( + { + "table": "bad", + "row": {"id": 1, "name": "Row 1"}, + "pks": "id", + }, + 400, + { + "ok": False, + "errors": ["pks must be a list"], + }, + ), + # Error: pks must be a list of strings + ( + {"table": "bad", "row": {"id": 1, "name": "Row 1"}, "pks": [1, 2]}, + 400, + {"ok": False, "errors": ["pks must be a list of strings"]}, + ), + ), +) +async def test_create_table(ds_write, input, expected_status, expected_response): + # Special case for expected status of 403 + if expected_status == 403: + token = "bad_token" + else: + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json=input, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == expected_status + data = response.json() + assert data == expected_response diff --git a/tests/test_auth.py b/tests/test_auth.py index 4ef35a76..fa1b2e46 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,5 +1,7 @@ from .fixtures import app_client +from click.testing import CliRunner from datasette.utils import baseconv +from datasette.cli import cli import pytest import time @@ -110,3 +112,180 @@ def test_no_logout_button_in_navigation_if_no_ds_actor_cookie(app_client, path): response = app_client.get(path + "?_bot=1") assert "bot" in response.text assert '
' not in response.text + + +@pytest.mark.parametrize( + "post_data,errors,expected_duration", + ( + ({"expire_type": ""}, [], None), + ({"expire_type": "x"}, ["Invalid expire duration"], None), + ({"expire_type": "minutes"}, ["Invalid expire duration"], None), + ( + {"expire_type": "minutes", "expire_duration": "x"}, + ["Invalid expire duration"], + None, + ), + ( + {"expire_type": "minutes", "expire_duration": "-1"}, + ["Invalid expire duration"], + None, + ), + ( + {"expire_type": "minutes", "expire_duration": "0"}, + ["Invalid expire duration"], + None, + ), + ( + {"expire_type": "minutes", "expire_duration": "10"}, + [], + 600, + ), + ( + {"expire_type": "hours", "expire_duration": "10"}, + [], + 10 * 60 * 60, + ), + ( + {"expire_type": "days", "expire_duration": "3"}, + [], + 60 * 60 * 24 * 3, + ), + ), +) +def test_auth_create_token(app_client, post_data, errors, expected_duration): + assert app_client.get("/-/create-token").status == 403 + ds_actor = app_client.actor_cookie({"id": "test"}) + response = app_client.get("/-/create-token", cookies={"ds_actor": ds_actor}) + assert response.status == 200 + assert ">Create an API token<" in response.text + # Now try actually creating one + response2 = app_client.post( + "/-/create-token", + post_data, + csrftoken_from=True, + cookies={"ds_actor": ds_actor}, + ) + assert response2.status == 200 + if errors: + for error in errors: + assert '

{}

'.format(error) in response2.text + else: + # Extract token from page + token = response2.text.split('value="dstok_')[1].split('"')[0] + details = app_client.ds.unsign(token, "token") + assert details.keys() == {"a", "t", "d"} or details.keys() == {"a", "t"} + assert details["a"] == "test" + if expected_duration is None: + assert "d" not in details + else: + assert details["d"] == expected_duration + # And test that token + response3 = app_client.get( + "/-/actor.json", + headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, + ) + assert response3.status == 200 + assert response3.json["actor"]["id"] == "test" + + +def test_auth_create_token_not_allowed_for_tokens(app_client): + ds_tok = app_client.ds.sign({"a": "test", "token": "dstok"}, "token") + response = app_client.get( + "/-/create-token", + headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, + ) + assert response.status == 403 + + +def test_auth_create_token_not_allowed_if_allow_signed_tokens_off(app_client): + app_client.ds._settings["allow_signed_tokens"] = False + try: + ds_actor = app_client.actor_cookie({"id": "test"}) + response = app_client.get("/-/create-token", cookies={"ds_actor": ds_actor}) + assert response.status == 403 + finally: + app_client.ds._settings["allow_signed_tokens"] = True + + +@pytest.mark.parametrize( + "scenario,should_work", + ( + ("allow_signed_tokens_off", False), + ("no_token", False), + ("no_timestamp", False), + ("invalid_token", False), + ("expired_token", False), + ("valid_unlimited_token", True), + ("valid_expiring_token", True), + ), +) +def test_auth_with_dstok_token(app_client, scenario, should_work): + token = None + _time = int(time.time()) + if scenario in ("valid_unlimited_token", "allow_signed_tokens_off"): + token = app_client.ds.sign({"a": "test", "t": _time}, "token") + elif scenario == "valid_expiring_token": + token = app_client.ds.sign({"a": "test", "t": _time - 50, "d": 1000}, "token") + elif scenario == "expired_token": + token = app_client.ds.sign({"a": "test", "t": _time - 2000, "d": 1000}, "token") + elif scenario == "no_timestamp": + token = app_client.ds.sign({"a": "test"}, "token") + elif scenario == "invalid_token": + token = "invalid" + if token: + token = "dstok_{}".format(token) + if scenario == "allow_signed_tokens_off": + app_client.ds._settings["allow_signed_tokens"] = False + headers = {} + if token: + headers["Authorization"] = "Bearer {}".format(token) + response = app_client.get("/-/actor.json", headers=headers) + try: + if should_work: + assert response.json.keys() == {"actor"} + actor = response.json["actor"] + expected_keys = {"id", "token"} + if scenario != "valid_unlimited_token": + expected_keys.add("token_expires") + assert actor.keys() == expected_keys + assert actor["id"] == "test" + assert actor["token"] == "dstok" + if scenario != "valid_unlimited_token": + assert isinstance(actor["token_expires"], int) + else: + assert response.json == {"actor": None} + finally: + app_client.ds._settings["allow_signed_tokens"] = True + + +@pytest.mark.parametrize("expires", (None, 1000, -1000)) +def test_cli_create_token(app_client, expires): + secret = app_client.ds._secret + runner = CliRunner(mix_stderr=False) + args = ["create-token", "--secret", secret, "test"] + if expires: + args += ["--expires-after", str(expires)] + result = runner.invoke(cli, args) + assert result.exit_code == 0 + token = result.output.strip() + assert token.startswith("dstok_") + details = app_client.ds.unsign(token[len("dstok_") :], "token") + expected_keys = {"a", "token", "t"} + if expires: + expected_keys.add("d") + assert details.keys() == expected_keys + assert details["a"] == "test" + response = app_client.get( + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + ) + if expires is None or expires > 0: + expected_actor = { + "id": "test", + "token": "dstok", + } + if expires and expires > 0: + expected_actor["token_expires"] = details["t"] + expires + assert response.json == {"actor": expected_actor} + else: + expected_actor = None + assert response.json == {"actor": expected_actor} diff --git a/tests/test_docs.py b/tests/test_docs.py index cd5a6c13..e9b813fe 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -62,7 +62,7 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView")) + view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) return view_labels diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 4e33beed..647ae7bd 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -78,6 +78,19 @@ async def test_table_exists(db, tables, exists): assert exists == actual +@pytest.mark.parametrize( + "view,expected", + ( + ("not_a_view", False), + ("paginated_view", True), + ), +) +@pytest.mark.asyncio +async def test_view_exists(db, view, expected): + actual = await db.view_exists(view) + assert actual == expected + + @pytest.mark.parametrize( "table,expected", ( @@ -400,6 +413,17 @@ async def test_table_names(db): ] +@pytest.mark.asyncio +async def test_view_names(db): + view_names = await db.view_names() + assert view_names == [ + "paginated_view", + "simple_view", + "searchable_view", + "searchable_view_configured_by_metadata", + ] + + @pytest.mark.asyncio async def test_execute_write_block_true(db): await db.execute_write( diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0364707a..4eb18cee 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,7 +1,9 @@ +from datasette.app import Datasette from .fixtures import app_client, assert_permissions_checked, make_app_client from bs4 import BeautifulSoup as Soup import copy import json +import pytest_asyncio import pytest import re import urllib @@ -21,6 +23,18 @@ def padlock_client(): yield client +@pytest_asyncio.fixture +async def perms_ds(): + ds = Datasette() + await ds.invoke_startup() + one = ds.add_memory_database("perms_ds_one") + two = ds.add_memory_database("perms_ds_two") + await one.execute_write("create table if not exists t1 (id integer primary key)") + await one.execute_write("create table if not exists t2 (id integer primary key)") + await two.execute_write("create table if not exists t1 (id integer primary key)") + return ds + + @pytest.mark.parametrize( "allow,expected_anon,expected_auth", [ @@ -260,6 +274,7 @@ def test_execute_sql(metadata): schema_json = schema_re.search(response_text).group(1) schema = json.loads(schema_json) assert set(schema["attraction_characteristic"]) == {"name", "pk"} + assert schema["paginated_view"] == [] assert form_fragment in response_text query_response = client.get("/fixtures?sql=select+1", cookies=cookies) assert query_response.status == 200 @@ -540,3 +555,88 @@ def test_padlocks_on_database_page(cascade_app_client): assert ">simple_view" in response.text finally: cascade_app_client.ds._metadata_local = previous_metadata + + +DEF = "USE_DEFAULT" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "actor,permission,resource_1,resource_2,expected_result", + ( + # Without restrictions the defaults apply + ({"id": "t"}, "view-instance", None, None, DEF), + ({"id": "t"}, "view-database", "one", None, DEF), + ({"id": "t"}, "view-table", "one", "t1", DEF), + # If there is an _r block, everything gets denied unless explicitly allowed + ({"id": "t", "_r": {}}, "view-instance", None, None, False), + ({"id": "t", "_r": {}}, "view-database", "one", None, False), + ({"id": "t", "_r": {}}, "view-table", "one", "t1", False), + # Explicit allowing works at the "a" for all level: + ({"id": "t", "_r": {"a": ["vi"]}}, "view-instance", None, None, DEF), + ({"id": "t", "_r": {"a": ["vd"]}}, "view-database", "one", None, DEF), + ({"id": "t", "_r": {"a": ["vt"]}}, "view-table", "one", "t1", DEF), + # But not if it's the wrong permission + ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, False), + ({"id": "t", "_r": {"a": ["vi"]}}, "view-database", "one", None, False), + ({"id": "t", "_r": {"a": ["vd"]}}, "view-table", "one", "t1", False), + # Works at the "d" for database level: + ({"id": "t", "_r": {"d": {"one": ["vd"]}}}, "view-database", "one", None, DEF), + ( + {"id": "t", "_r": {"d": {"one": ["vdd"]}}}, + "view-database-download", + "one", + None, + DEF, + ), + ({"id": "t", "_r": {"d": {"one": ["es"]}}}, "execute-sql", "one", None, DEF), + # Works at the "t" for table level: + ( + {"id": "t", "_r": {"t": {"one": {"t1": ["vt"]}}}}, + "view-table", + "one", + "t1", + DEF, + ), + ( + {"id": "t", "_r": {"t": {"one": {"t1": ["vt"]}}}}, + "view-table", + "one", + "t2", + False, + ), + ), +) +async def test_actor_restricted_permissions( + perms_ds, actor, permission, resource_1, resource_2, expected_result +): + cookies = {"ds_actor": perms_ds.sign({"a": {"id": "root"}}, "actor")} + csrftoken = (await perms_ds.client.get("/-/permissions", cookies=cookies)).cookies[ + "ds_csrftoken" + ] + cookies["ds_csrftoken"] = csrftoken + response = await perms_ds.client.post( + "/-/permissions", + data={ + "actor": json.dumps(actor), + "permission": permission, + "resource_1": resource_1, + "resource_2": resource_2, + "csrftoken": csrftoken, + }, + cookies=cookies, + ) + expected_resource = [] + if resource_1: + expected_resource.append(resource_1) + if resource_2: + expected_resource.append(resource_2) + if len(expected_resource) == 1: + expected_resource = expected_resource[0] + expected = { + "actor": actor, + "permission": permission, + "resource": expected_resource, + "result": expected_result, + } + assert response.json() == expected diff --git a/tests/test_plugins.py b/tests/test_plugins.py index e0a7bc76..de3fde8e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -971,6 +971,7 @@ def test_hook_register_commands(): "plugins", "publish", "uninstall", + "create-token", } # Now install a plugin @@ -1001,6 +1002,7 @@ def test_hook_register_commands(): "uninstall", "verify", "unverify", + "create-token", } pm.unregister(name="verify") importlib.reload(cli)