Compare commits

...

4 commits

Author SHA1 Message Date
Simon Willison
3807d93b98
Bump up time limit for test_paginate_tables_and_views
It was intermittently failing in Travis CI on Python 3.5:

https://travis-ci.org/simonw/datasette/jobs/373713476
2018-05-01 17:17:39 -07:00
Simon Willison
8081ec7bb4
Renamed ?_sql_time_limit_ms= to ?_timelimit, closes #242 2018-05-01 17:11:46 -07:00
Simon Willison
379caddccb
New ?_shape=array option + tweaks to _shape, closes #245
* Default is now ?_shape=arrays (renamed from lists)
* New ?_shape=array returns an array of objects as the root object
* Changed ?_shape=object to return the object as the root
* Updated docs
2018-05-01 17:08:16 -07:00
Simon Willison
af4cb39d8f
?_shape=array experimental feature 2018-05-01 10:40:33 -07:00
4 changed files with 86 additions and 23 deletions

View file

@ -253,8 +253,8 @@ class BaseView(RenderMixin):
forward_querystring=False
)
# Deal with the _shape option
shape = request.args.get('_shape', 'lists')
if shape in ('objects', 'object'):
shape = request.args.get('_shape', 'arrays')
if shape in ('objects', 'object', 'array'):
columns = data.get('columns')
rows = data.get('rows')
if rows and columns:
@ -275,7 +275,7 @@ class BaseView(RenderMixin):
for row in data['rows']:
pk_string = path_from_row_pks(row, pks, not pks)
object_rows[pk_string] = row
data['rows'] = object_rows
data = object_rows
if error:
data = {
'ok': False,
@ -283,7 +283,18 @@ class BaseView(RenderMixin):
'database': name,
'database_hash': hash,
}
elif shape == 'array':
data = data['rows']
elif shape == 'arrays':
pass
else:
status_code = 400
data = {
'ok': False,
'error': 'Invalid _shape: {}'.format(shape),
'status': 400,
'title': None,
}
headers = {}
if self.ds.cors:
headers['Access-Control-Allow-Origin'] = '*'
@ -347,8 +358,8 @@ class BaseView(RenderMixin):
params[named_parameter] = ''
extra_args = {}
if params.get('_sql_time_limit_ms'):
extra_args['custom_time_limit'] = int(params['_sql_time_limit_ms'])
if params.get('_timelimit'):
extra_args['custom_time_limit'] = int(params['_timelimit'])
rows, truncated, description = await self.execute(
name, sql, params, truncate=True, **extra_args
)
@ -883,8 +894,8 @@ class TableView(RowTableShared):
offset=offset,
)
if request.raw_args.get('_sql_time_limit_ms'):
extra_args['custom_time_limit'] = int(request.raw_args['_sql_time_limit_ms'])
if request.raw_args.get('_timelimit'):
extra_args['custom_time_limit'] = int(request.raw_args['_timelimit'])
rows, truncated, description = await self.execute(
name, sql, params, truncate=True, **extra_args

View file

@ -59,12 +59,16 @@ The ``_shape`` parameter can be used to access alternative formats for the
``rows`` key which may be more convenient for your application. There are three
options:
* ``?_shape=lists`` - the default option, shown above
* ``?_shape=objects`` - a list of JSON key/value objects
* ``?_shape=object`` - a JSON object keyed using the primary keys of the rows
* ``?_shape=arrays`` - ``"rows"`` is the default option, shown above
* ``?_shape=objects`` - ``"rows"`` is a list of JSON key/value objects
* ``?_shape=array`` - the entire response is an array of objects
* ``?_shape=object`` - the entire response is a JSON object keyed using the primary keys of the rows
``objects`` looks like this::
{
"database": "sf-trees",
...
"rows": [
{
"id": 1,
@ -79,10 +83,28 @@ options:
"value": "Pinus radiata :: Monterey Pine"
}
]
}
``array`` looks like this::
[
{
"id": 1,
"value": "Myoporum laetum :: Myoporum"
},
{
"id": 2,
"value": "Metrosideros excelsa :: New Zealand Xmas Tree"
},
{
"id": 3,
"value": "Pinus radiata :: Monterey Pine"
}
]
``object`` looks like this::
"rows": {
{
"1": {
"id": 1,
"value": "Myoporum laetum :: Myoporum"
@ -132,7 +154,7 @@ The Datasette table view takes a number of special querystring arguments:
You can pass multiple ``_group_count`` columns to return counts against
unique combinations of those columns.
``?_sql_time_limit_ms=MS``
``?_timelimit=MS``
Sets a custom time limit for the query in ms. You can use this for optimistic
queries where you would like Datasette to give up if the query takes too
long, for example if you want to implement autocomplete search but only if

View file

@ -62,9 +62,9 @@ If this time limit is too short for you, you can customize it using the
datasette mydatabase.db --sql_time_limit_ms=3500
You can optionally set a lower time limit for an individual query using the
``_sql_time_limit_ms`` query string argument::
``_timelimit`` query string argument::
/my-database/my-table?qSpecies=44&_sql_time_limit_ms=100
/my-database/my-table?qSpecies=44&_timelimit=100
This would set the time limit to 100ms for that specific query. This feature
is useful if you are working with databases of unknown size and complexity -

View file

@ -269,7 +269,7 @@ def test_custom_sql_time_limit(app_client):
)
assert 200 == response.status
response = app_client.get(
'/test_tables.json?sql=select+sleep(0.01)&_sql_time_limit_ms=5',
'/test_tables.json?sql=select+sleep(0.01)&_timelimit=5',
gather_request=False
)
assert 400 == response.status
@ -330,9 +330,9 @@ def test_jsono_redirects_to_shape_objects(app_client):
assert response.headers['Location'].endswith('?_shape=objects')
def test_table_shape_lists(app_client):
def test_table_shape_arrays(app_client):
response = app_client.get(
'/test_tables/simple_primary_key.json?_shape=lists',
'/test_tables/simple_primary_key.json?_shape=arrays',
gather_request=False
)
assert [
@ -359,6 +359,36 @@ def test_table_shape_objects(app_client):
}] == response.json['rows']
def test_table_shape_array(app_client):
response = app_client.get(
'/test_tables/simple_primary_key.json?_shape=array',
gather_request=False
)
assert [{
'id': '1',
'content': 'hello',
}, {
'id': '2',
'content': 'world',
}, {
'id': '3',
'content': '',
}] == response.json
def test_table_shape_invalid(app_client):
response = app_client.get(
'/test_tables/simple_primary_key.json?_shape=invalid',
gather_request=False
)
assert {
'ok': False,
'error': 'Invalid _shape: invalid',
'status': 400,
'title': None,
} == response.json
def test_table_shape_object(app_client):
response = app_client.get(
'/test_tables/simple_primary_key.json?_shape=object',
@ -377,7 +407,7 @@ def test_table_shape_object(app_client):
'id': '3',
'content': '',
}
} == response.json['rows']
} == response.json
def test_table_shape_object_compound_primary_Key(app_client):
@ -391,7 +421,7 @@ def test_table_shape_object_compound_primary_Key(app_client):
'pk2': 'b',
'content': 'c',
}
} == response.json['rows']
} == response.json
def test_table_with_slashes_in_name(app_client):
@ -423,11 +453,11 @@ def test_table_with_reserved_word_name(app_client):
('/test_tables/paginated_view.json?_size=25', 201, 9),
('/test_tables/123_starts_with_digits.json', 0, 1),
])
def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages):
def test_paginate_tables_and_views(app_client_longer_time_limit, path, expected_rows, expected_pages):
fetched = []
count = 0
while path:
response = app_client.get(path, gather_request=False)
response = app_client_longer_time_limit.get(path, gather_request=False)
count += 1
fetched.extend(response.json['rows'])
path = response.json['next_url']