From af4cb39d8f67fbb48cb35365a309eb96d06911af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 May 2018 10:40:33 -0700 Subject: [PATCH 1/4] ?_shape=array experimental feature --- datasette/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 0b73d0e1..30f0d7a1 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -254,7 +254,7 @@ class BaseView(RenderMixin): ) # Deal with the _shape option shape = request.args.get('_shape', 'lists') - if shape in ('objects', 'object'): + if shape in ('objects', 'object', 'array'): columns = data.get('columns') rows = data.get('rows') if rows and columns: @@ -283,6 +283,8 @@ class BaseView(RenderMixin): 'database': name, 'database_hash': hash, } + if shape == 'array': + data = data['rows'] headers = {} if self.ds.cors: From 379caddccbe12b79e876fc03dbec20258beccc54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 May 2018 17:08:16 -0700 Subject: [PATCH 2/4] New ?_shape=array option + tweaks to _shape, closes #245 * Default is now ?_shape=arrays (renamed from lists) * New ?_shape=array returns an array of objects as the root object * Changed ?_shape=object to return the object as the root * Updated docs --- datasette/app.py | 17 +++++++++++++---- docs/json_api.rst | 32 +++++++++++++++++++++++++++----- tests/test_api.py | 38 ++++++++++++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 30f0d7a1..ec1c8cbd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -253,7 +253,7 @@ class BaseView(RenderMixin): forward_querystring=False ) # Deal with the _shape option - shape = request.args.get('_shape', 'lists') + shape = request.args.get('_shape', 'arrays') if shape in ('objects', 'object', 'array'): columns = data.get('columns') rows = data.get('rows') @@ -275,7 +275,7 @@ class BaseView(RenderMixin): for row in data['rows']: pk_string = path_from_row_pks(row, pks, not pks) object_rows[pk_string] = row - data['rows'] = object_rows + data = object_rows if error: data = { 'ok': False, @@ -283,9 +283,18 @@ class BaseView(RenderMixin): 'database': name, 'database_hash': hash, } - if shape == 'array': + elif shape == 'array': data = data['rows'] - + elif shape == 'arrays': + pass + else: + status_code = 400 + data = { + 'ok': False, + 'error': 'Invalid _shape: {}'.format(shape), + 'status': 400, + 'title': None, + } headers = {} if self.ds.cors: headers['Access-Control-Allow-Origin'] = '*' diff --git a/docs/json_api.rst b/docs/json_api.rst index e750dbbf..af212ea5 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -59,13 +59,35 @@ The ``_shape`` parameter can be used to access alternative formats for the ``rows`` key which may be more convenient for your application. There are three options: -* ``?_shape=lists`` - the default option, shown above -* ``?_shape=objects`` - a list of JSON key/value objects -* ``?_shape=object`` - a JSON object keyed using the primary keys of the rows +* ``?_shape=arrays`` - ``"rows"`` is the default option, shown above +* ``?_shape=objects`` - ``"rows"`` is a list of JSON key/value objects +* ``?_shape=array`` - the entire response is an array of objects +* ``?_shape=object`` - the entire response is a JSON object keyed using the primary keys of the rows ``objects`` looks like this:: - "rows": [ + { + "database": "sf-trees", + ... + "rows": [ + { + "id": 1, + "value": "Myoporum laetum :: Myoporum" + }, + { + "id": 2, + "value": "Metrosideros excelsa :: New Zealand Xmas Tree" + }, + { + "id": 3, + "value": "Pinus radiata :: Monterey Pine" + } + ] + } + +``array`` looks like this:: + + [ { "id": 1, "value": "Myoporum laetum :: Myoporum" @@ -82,7 +104,7 @@ options: ``object`` looks like this:: - "rows": { + { "1": { "id": 1, "value": "Myoporum laetum :: Myoporum" diff --git a/tests/test_api.py b/tests/test_api.py index 40c23d00..6d5fa40a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -330,9 +330,9 @@ def test_jsono_redirects_to_shape_objects(app_client): assert response.headers['Location'].endswith('?_shape=objects') -def test_table_shape_lists(app_client): +def test_table_shape_arrays(app_client): response = app_client.get( - '/test_tables/simple_primary_key.json?_shape=lists', + '/test_tables/simple_primary_key.json?_shape=arrays', gather_request=False ) assert [ @@ -359,6 +359,36 @@ def test_table_shape_objects(app_client): }] == response.json['rows'] +def test_table_shape_array(app_client): + response = app_client.get( + '/test_tables/simple_primary_key.json?_shape=array', + gather_request=False + ) + assert [{ + 'id': '1', + 'content': 'hello', + }, { + 'id': '2', + 'content': 'world', + }, { + 'id': '3', + 'content': '', + }] == response.json + + +def test_table_shape_invalid(app_client): + response = app_client.get( + '/test_tables/simple_primary_key.json?_shape=invalid', + gather_request=False + ) + assert { + 'ok': False, + 'error': 'Invalid _shape: invalid', + 'status': 400, + 'title': None, + } == response.json + + def test_table_shape_object(app_client): response = app_client.get( '/test_tables/simple_primary_key.json?_shape=object', @@ -377,7 +407,7 @@ def test_table_shape_object(app_client): 'id': '3', 'content': '', } - } == response.json['rows'] + } == response.json def test_table_shape_object_compound_primary_Key(app_client): @@ -391,7 +421,7 @@ def test_table_shape_object_compound_primary_Key(app_client): 'pk2': 'b', 'content': 'c', } - } == response.json['rows'] + } == response.json def test_table_with_slashes_in_name(app_client): From 8081ec7bb48f71c396a5d6bae6bb2cd397c4dc00 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 May 2018 17:11:46 -0700 Subject: [PATCH 3/4] Renamed ?_sql_time_limit_ms= to ?_timelimit, closes #242 --- datasette/app.py | 8 ++++---- docs/json_api.rst | 2 +- docs/sql_queries.rst | 4 ++-- tests/test_api.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ec1c8cbd..75be58bc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -358,8 +358,8 @@ class BaseView(RenderMixin): params[named_parameter] = '' extra_args = {} - if params.get('_sql_time_limit_ms'): - extra_args['custom_time_limit'] = int(params['_sql_time_limit_ms']) + if params.get('_timelimit'): + extra_args['custom_time_limit'] = int(params['_timelimit']) rows, truncated, description = await self.execute( name, sql, params, truncate=True, **extra_args ) @@ -894,8 +894,8 @@ class TableView(RowTableShared): offset=offset, ) - if request.raw_args.get('_sql_time_limit_ms'): - extra_args['custom_time_limit'] = int(request.raw_args['_sql_time_limit_ms']) + if request.raw_args.get('_timelimit'): + extra_args['custom_time_limit'] = int(request.raw_args['_timelimit']) rows, truncated, description = await self.execute( name, sql, params, truncate=True, **extra_args diff --git a/docs/json_api.rst b/docs/json_api.rst index af212ea5..aa9a914f 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -154,7 +154,7 @@ The Datasette table view takes a number of special querystring arguments: You can pass multiple ``_group_count`` columns to return counts against unique combinations of those columns. -``?_sql_time_limit_ms=MS`` +``?_timelimit=MS`` Sets a custom time limit for the query in ms. You can use this for optimistic queries where you would like Datasette to give up if the query takes too long, for example if you want to implement autocomplete search but only if diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 07276d94..e02fad3d 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -62,9 +62,9 @@ If this time limit is too short for you, you can customize it using the datasette mydatabase.db --sql_time_limit_ms=3500 You can optionally set a lower time limit for an individual query using the -``_sql_time_limit_ms`` query string argument:: +``_timelimit`` query string argument:: - /my-database/my-table?qSpecies=44&_sql_time_limit_ms=100 + /my-database/my-table?qSpecies=44&_timelimit=100 This would set the time limit to 100ms for that specific query. This feature is useful if you are working with databases of unknown size and complexity - diff --git a/tests/test_api.py b/tests/test_api.py index 6d5fa40a..08763b0f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -269,7 +269,7 @@ def test_custom_sql_time_limit(app_client): ) assert 200 == response.status response = app_client.get( - '/test_tables.json?sql=select+sleep(0.01)&_sql_time_limit_ms=5', + '/test_tables.json?sql=select+sleep(0.01)&_timelimit=5', gather_request=False ) assert 400 == response.status From 3807d93b98573e142858c5871b8b4aadda71d28f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 May 2018 17:17:39 -0700 Subject: [PATCH 4/4] Bump up time limit for test_paginate_tables_and_views It was intermittently failing in Travis CI on Python 3.5: https://travis-ci.org/simonw/datasette/jobs/373713476 --- tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 08763b0f..81a95c36 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -453,11 +453,11 @@ def test_table_with_reserved_word_name(app_client): ('/test_tables/paginated_view.json?_size=25', 201, 9), ('/test_tables/123_starts_with_digits.json', 0, 1), ]) -def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages): +def test_paginate_tables_and_views(app_client_longer_time_limit, path, expected_rows, expected_pages): fetched = [] count = 0 while path: - response = app_client.get(path, gather_request=False) + response = app_client_longer_time_limit.get(path, gather_request=False) count += 1 fetched.extend(response.json['rows']) path = response.json['next_url']