New ?_shape=objects/object/lists param for JSON API (#192)

New _shape= parameter replacing old .jsono extension

Now instead of this:

	/database/table.jsono

We use the _shape parameter like this:

	/database/table.json?_shape=objects

Also introduced a new _shape called 'object' which looks like this:

	/database/table.json?_shape=object

Returning an object for the rows key:

	...
	"rows": {
		"pk1": {
			...
		},
		"pk2": {
			...
		}
	}

Refs #122

README.md

@@ -73,7 +73,7 @@ http://localhost:8001/History/downloads.json will return that data as JSON:
     }
 
 
-http://localhost:8001/History/downloads.jsono will return that data as JSON in a more convenient but less efficient format:
+http://localhost:8001/History/downloads.json?_shape=objects will return that data as JSON in a more convenient but less efficient format:
 
     {
         ...

datasette/app.py

@@ -221,8 +221,20 @@ class BaseView(RenderMixin):
             if value:
                 data[key] = value
         if as_json:
-            # Special case for .jsono extension
+            # Special case for .jsono extension - redirect to _shape=objects
             if as_json == '.jsono':
+                return self.redirect(
+                    request,
+                    path_with_added_args(
+                        request,
+                        {'_shape': 'objects'},
+                        path=request.path.rsplit('.jsono', 1)[0] + '.json'
+                    ),
+                    forward_querystring=False
+                )
+            # Deal with the _shape option
+            shape = request.args.get('_shape', 'lists')
+            if shape in ('objects', 'object'):
                 columns = data.get('columns')
                 rows = data.get('rows')
                 if rows and columns:
@@ -230,6 +242,28 @@ class BaseView(RenderMixin):
                         dict(zip(columns, row))
                         for row in rows
                     ]
+                if shape == 'object':
+                    error = None
+                    if 'primary_keys' not in data:
+                        error = '_shape=object is only available on tables'
+                    else:
+                        pks = data['primary_keys']
+                        if not pks:
+                            error = '_shape=object not available for tables with no primary keys'
+                        else:
+                            object_rows = {}
+                            for row in data['rows']:
+                                pk_string = path_from_row_pks(row, pks, not pks)
+                                object_rows[pk_string] = row
+                            data['rows'] = object_rows
+                    if error:
+                        data = {
+                            'ok': False,
+                            'error': error,
+                            'database': name,
+                            'database_hash': hash,
+                        }
+
             headers = {}
             if self.ds.cors:
                 headers['Access-Control-Allow-Origin'] = '*'
@@ -278,6 +312,8 @@ class BaseView(RenderMixin):
         params = request.raw_args
         if 'sql' in params:
             params.pop('sql')
+        if '_shape' in params:
+            params.pop('_shape')
         # Extract any :named parameters
         named_parameters = self.re_named_parameter.findall(sql)
         named_parameter_values = {

datasette/templates/query.html

@@ -40,7 +40,7 @@
 </form>
 
 {% if rows %}
-<p>This data as <a href="{{ url_json }}">.json</a>, <a href="{{ url_jsono }}">.jsono</a></p>
+<p>This data as <a href="{{ url_json }}">.json</a></p>
 <table>
     <thead>
         <tr>

datasette/templates/row.html

@@ -22,7 +22,7 @@
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
-<p>This data as <a href="{{ url_json }}">.json</a>, <a href="{{ url_jsono }}">.jsono</a></p>
+<p>This data as <a href="{{ url_json }}">.json</a></p>
 
 {% include custom_rows_and_columns_templates %}
 

datasette/templates/table.html

@@ -74,7 +74,7 @@
     <p><a class="not-underlined" title="{{ query.sql }}" href="/{{ database }}-{{ database_hash }}?{{ {'sql': query.sql}|urlencode|safe }}{% if query.params %}&amp;{{ query.params|urlencode|safe }}{% endif %}">&#x270e; <span class="underlined">View and edit SQL</span></a></p>
 {% endif %}
 
-<p>This data as <a href="{{ url_json }}">.json</a>, <a href="{{ url_jsono }}">.jsono</a></p>
+<p>This data as <a href="{{ url_json }}">.json</a></p>
 
 {% include custom_rows_and_columns_templates %}
 

datasette/utils.py

@@ -134,7 +134,8 @@ def validate_sql_select(sql):
             raise InvalidSql(msg)
 
 
-def path_with_added_args(request, args):
+def path_with_added_args(request, args, path=None):
+    path = path or request.path
     if isinstance(args, dict):
         args = args.items()
     arg_keys = set(a[0] for a in args)
@@ -151,7 +152,7 @@ def path_with_added_args(request, args):
     query_string = urllib.parse.urlencode(sorted(current))
     if query_string:
         query_string = '?{}'.format(query_string)
-    return request.path + query_string
+    return path + query_string
 
 
 def path_with_ext(request, ext):

docs/getting_started.rst

@@ -62,7 +62,7 @@ JSON:
         ]
     }
 
-http://localhost:8001/History/downloads.jsono will return that data as
+http://localhost:8001/History/downloads.json?_shape=objects will return that data as
 JSON in a more convenient but less efficient format:
 
 ::

docs/json_api.rst

@@ -0,0 +1,105 @@
+The Datasette JSON API
+======================
+
+Datasette provides a JSON API for your SQLite databases. Anything you can do
+through the Datasette user interface can also be accessed as JSON via the API.
+
+To access the API for a page, either click on the ``.json`` link on that page or
+edit the URL and add a ``.json`` extension to it.
+
+If you started Datasette with the ``--cors`` option, each JSON endpoint will be
+served with the following additional HTTP header::
+
+    Access-Control-Allow-Origin: *
+
+This means JavaScript running on any domain will be able to make cross-origin
+requests to fetch the data.
+
+If you start Datasette without the ``--cors`` option only JavaScript running on
+the same domain as Datasette will be able to access the API.
+
+Different shapes
+----------------
+
+The default JSON representation of data from a SQLite table or custom query
+looks like this::
+
+    {
+        "database": "sf-trees",
+        "table": "qSpecies",
+        "columns": [
+            "id",
+            "value"
+        ],
+        "rows": [
+            [
+                1,
+                "Myoporum laetum :: Myoporum"
+            ],
+            [
+                2,
+                "Metrosideros excelsa :: New Zealand Xmas Tree"
+            ],
+            [
+                3,
+                "Pinus radiata :: Monterey Pine"
+            ]
+        ],
+        "truncated": false,
+        "next": "100",
+        "next_url": "http://127.0.0.1:8001/sf-trees-02c8ef1/qSpecies.json?_next=100",
+        "query_ms": 1.9571781158447266
+    }
+
+The ``columns`` key lists the columns that are being returned, and the ``rows``
+key then returns a list of lists, each one representing a row. The order of the
+values in each row corresponds to the columns.
+
+The ``_shape`` parameter can be used to access alternative formats for the
+``rows`` key which may be more convenient for your application. There are three
+options:
+
+* ``?_shape=lists`` - the default option, shown above
+* ``?_shape=objects`` - a list of JSON key/value objects
+* ``?_shape=object`` - a JSON object keyed using the primary keys of the rows
+
+``objects`` looks like this::
+
+    "rows": [
+        {
+            "id": 1,
+            "value": "Myoporum laetum :: Myoporum"
+        },
+        {
+            "id": 2,
+            "value": "Metrosideros excelsa :: New Zealand Xmas Tree"
+        },
+        {
+            "id": 3,
+            "value": "Pinus radiata :: Monterey Pine"
+        }
+    ]
+
+``object`` looks like this::
+
+    "rows": {
+        "1": {
+            "id": 1,
+            "value": "Myoporum laetum :: Myoporum"
+        },
+        "2": {
+            "id": 2,
+            "value": "Metrosideros excelsa :: New Zealand Xmas Tree"
+        },
+        "3": {
+            "id": 3,
+            "value": "Pinus radiata :: Monterey Pine"
+        }
+    ]
+
+The ``object`` shape is only available for queries against tables - custom SQL
+queries and views do not have an obvious primary key so cannot be returned using
+this format.
+
+The ``object`` keys are always strings. If your table has a compound primary
+key, the ``object`` keys will be a comma-separated string.

tests/test_api.py

@@ -118,7 +118,7 @@ def test_database_page(app_client):
 
 def test_custom_sql(app_client):
     response = app_client.get(
-        '/test_tables.jsono?sql=select+content+from+simple_primary_key',
+        '/test_tables.json?sql=select+content+from+simple_primary_key&_shape=objects',
         gather_request=False
     )
     data = response.json
@@ -138,7 +138,7 @@ def test_custom_sql(app_client):
 
 def test_sql_time_limit(app_client):
     response = app_client.get(
-        '/test_tables.jsono?sql=select+sleep(0.5)',
+        '/test_tables.json?sql=select+sleep(0.5)',
         gather_request=False
     )
     assert 400 == response.status
@@ -147,12 +147,12 @@ def test_sql_time_limit(app_client):
 
 def test_custom_sql_time_limit(app_client):
     response = app_client.get(
-        '/test_tables.jsono?sql=select+sleep(0.01)',
+        '/test_tables.json?sql=select+sleep(0.01)',
         gather_request=False
     )
     assert 200 == response.status
     response = app_client.get(
-        '/test_tables.jsono?sql=select+sleep(0.01)&_sql_time_limit_ms=5',
+        '/test_tables.json?sql=select+sleep(0.01)&_sql_time_limit_ms=5',
         gather_request=False
     )
     assert 400 == response.status
@@ -170,7 +170,7 @@ def test_invalid_custom_sql(app_client):
 
 
 def test_table_json(app_client):
-    response = app_client.get('/test_tables/simple_primary_key.jsono', gather_request=False)
+    response = app_client.get('/test_tables/simple_primary_key.json?_shape=objects', gather_request=False)
     assert response.status == 200
     data = response.json
     assert data['query']['sql'] == 'select * from simple_primary_key order by pk limit 51'
@@ -187,8 +187,87 @@ def test_table_json(app_client):
     }]
 
 
+def test_jsono_redirects_to_shape_objects(app_client):
+    response_1 = app_client.get(
+        '/test_tables/simple_primary_key.jsono',
+        allow_redirects=False,
+        gather_request=False
+    )
+    response = app_client.get(
+        response_1.headers['Location'],
+        allow_redirects=False,
+        gather_request=False
+    )
+    assert response.status == 302
+    assert response.headers['Location'].endswith('?_shape=objects')
+
+
+def test_table_shape_lists(app_client):
+    response = app_client.get(
+        '/test_tables/simple_primary_key.json?_shape=lists',
+        gather_request=False
+    )
+    assert [
+        ['1', 'hello'],
+        ['2', 'world'],
+        ['3', ''],
+    ] == response.json['rows']
+
+
+def test_table_shape_objects(app_client):
+    response = app_client.get(
+        '/test_tables/simple_primary_key.json?_shape=objects',
+        gather_request=False
+    )
+    assert [{
+        'pk': '1',
+        'content': 'hello',
+    }, {
+        'pk': '2',
+        'content': 'world',
+    }, {
+        'pk': '3',
+        'content': '',
+    }] == response.json['rows']
+
+
+def test_table_shape_object(app_client):
+    response = app_client.get(
+        '/test_tables/simple_primary_key.json?_shape=object',
+        gather_request=False
+    )
+    assert {
+        '1': {
+            'pk': '1',
+            'content': 'hello',
+        },
+        '2': {
+            'pk': '2',
+            'content': 'world',
+        },
+        '3': {
+            'pk': '3',
+            'content': '',
+        }
+    } == response.json['rows']
+
+
+def test_table_shape_object_compound_primary_Key(app_client):
+    response = app_client.get(
+        '/test_tables/compound_primary_key.json?_shape=object',
+        gather_request=False
+    )
+    assert {
+        'a,b': {
+            'pk1': 'a',
+            'pk2': 'b',
+            'content': 'c',
+        }
+    } == response.json['rows']
+
+
 def test_table_with_slashes_in_name(app_client):
-    response = app_client.get('/test_tables/table%2Fwith%2Fslashes.csv.jsono', gather_request=False)
+    response = app_client.get('/test_tables/table%2Fwith%2Fslashes.csv.json?_shape=objects', gather_request=False)
     assert response.status == 200
     data = response.json
     assert data['rows'] == [{
@@ -198,7 +277,7 @@ def test_table_with_slashes_in_name(app_client):
 
 
 def test_table_with_reserved_word_name(app_client):
-    response = app_client.get('/test_tables/select.jsono', gather_request=False)
+    response = app_client.get('/test_tables/select.json?_shape=objects', gather_request=False)
     assert response.status == 200
     data = response.json
     assert data['rows'] == [{
@@ -210,9 +289,9 @@ def test_table_with_reserved_word_name(app_client):
 
 
 @pytest.mark.parametrize('path,expected_rows,expected_pages', [
-    ('/test_tables/no_primary_key.jsono', 201, 5),
+    ('/test_tables/no_primary_key.json', 201, 5),
-    ('/test_tables/paginated_view.jsono', 201, 5),
+    ('/test_tables/paginated_view.json', 201, 5),
-    ('/test_tables/123_starts_with_digits.jsono', 0, 1),
+    ('/test_tables/123_starts_with_digits.json', 0, 1),
 ])
 def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages):
     fetched = []
@@ -232,7 +311,7 @@ def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pag
 
 def test_paginate_compound_keys(app_client):
     fetched = []
-    path = '/test_tables/compound_three_primary_keys.jsono'
+    path = '/test_tables/compound_three_primary_keys.json?_shape=objects'
     page = 0
     while path:
         page += 1
@@ -250,7 +329,7 @@ def test_paginate_compound_keys(app_client):
 
 def test_paginate_compound_keys_with_extra_filters(app_client):
     fetched = []
-    path = '/test_tables/compound_three_primary_keys.jsono?content__contains=d'
+    path = '/test_tables/compound_three_primary_keys.json?content__contains=d&_shape=objects'
     page = 0
     while path:
         page += 1
@@ -289,7 +368,7 @@ def test_table_filter_queries(app_client, path, expected_rows):
 
 def test_max_returned_rows(app_client):
     response = app_client.get(
-        '/test_tables.jsono?sql=select+content+from+no_primary_key',
+        '/test_tables.json?sql=select+content+from+no_primary_key',
         gather_request=False
     )
     data = response.json
@@ -302,7 +381,7 @@ def test_max_returned_rows(app_client):
 
 
 def test_view(app_client):
-    response = app_client.get('/test_tables/simple_view.jsono', gather_request=False)
+    response = app_client.get('/test_tables/simple_view.json?_shape=objects', gather_request=False)
     assert response.status == 200
     data = response.json
     assert data['rows'] == [{
@@ -318,7 +397,7 @@ def test_view(app_client):
 
 
 def test_row(app_client):
-    response = app_client.get('/test_tables/simple_primary_key/1.jsono', gather_request=False)
+    response = app_client.get('/test_tables/simple_primary_key/1.json?_shape=objects', gather_request=False)
     assert response.status == 200
     assert [{'pk': '1', 'content': 'hello'}] == response.json['rows']