Streaming works locally, needs cleanup + _dl= option

datasette/views/base.py

@@ -149,7 +149,35 @@ class BaseView(RenderMixin):
 
         return await self.view_get(request, name, hash, **kwargs)
 
+    async def as_csv_stream(self, request, name, hash, **kwargs):
+        assert not request.args.get("_next")  # TODO: real error
+        kwargs['_size'] = 'max'
+
+        async def stream_fn(r):
+            first = True
+            next = None
+            writer = csv.writer(r)
+            while first or next:
+                if next:
+                    kwargs['_next'] = next
+                data, extra_template_data, templates = await self.data(
+                    request, name, hash, **kwargs
+                )
+                if first:
+                    writer.writerow(data["columns"])
+                    first = False
+                next = data["next"]
+                for row in data["rows"]:
+                    writer.writerow(row)
+
+        return response.stream(
+            stream_fn,
+            content_type="text/plain; charset=utf-8"
+        )
+
     async def as_csv(self, request, name, hash, **kwargs):
+        if request.args.get("_stream"):
+            return await self.as_csv_stream(request, name, hash, **kwargs)
         try:
             response_or_template_contexts = await self.data(
                 request, name, hash, **kwargs

datasette/views/table.py

@@ -220,7 +220,7 @@ class RowTableShared(BaseView):
 
 class TableView(RowTableShared):
 
-    async def data(self, request, name, hash, table, default_labels=False):
+    async def data(self, request, name, hash, table, default_labels=False,  _next=None, _size=None):
         canned_query = self.ds.get_canned_query(name, table)
         if canned_query is not None:
             return await self.custom_sql(
@@ -375,7 +375,7 @@ class TableView(RowTableShared):
 
         count_sql = "select count(*) {}".format(from_sql)
 
-        _next = special_args.get("_next")
+        _next = _next or special_args.get("_next")
         offset = ""
         if _next:
             if is_view:
@@ -462,7 +462,7 @@ class TableView(RowTableShared):
 
         extra_args = {}
         # Handle ?_size=500
-        page_size = request.raw_args.get("_size")
+        page_size = _size or request.raw_args.get("_size")
         if page_size:
             if page_size == "max":
                 page_size = self.max_returned_rows
@@ -512,6 +512,8 @@ class TableView(RowTableShared):
         facet_results = {}
         facets_timed_out = []
         for column in facets:
+            if _next:
+                continue
             facet_sql = """
                 select {col} as value, count(*) as count
                 {from_sql} {and_or_where} {col} is not null
@@ -665,6 +667,8 @@ class TableView(RowTableShared):
                 for facet_column in columns:
                     if facet_column in facets:
                         continue
+                    if _next:
+                        continue
                     if not self.ds.config["suggest_facets"]:
                         continue
                     suggested_facet_sql = '''