Get publish cloudrun working with latest Cloud Run (#2581)

Refs:
- #2511

Filter out bad services, refs:
- https://github.com/simonw/datasette/pull/2581#issuecomment-3492243400

.github/workflows/deploy-latest.yml

@@ -102,12 +102,13 @@ jobs:
     #       jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \
     #       > metadata.json
     #     cat metadata.json
-    - name: Set up Cloud Run
+    - id: auth
-      uses: google-github-actions/setup-gcloud@v0
+      name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        version: '318.0.0'
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
-        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+    - name: Set up Cloud SDK
-        service_account_key: ${{ secrets.GCP_SA_KEY }}
+      uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}

.github/workflows/publish.yml

@@ -73,12 +73,13 @@ jobs:
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - name: Set up Cloud Run
+    - id: auth
-      uses: google-github-actions/setup-gcloud@v0
+      name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        version: '318.0.0'
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
-        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+    - name: Set up Cloud SDK
-        service_account_key: ${{ secrets.GCP_SA_KEY }}
+      uses: google-github-actions/setup-gcloud@v3
     - name: Deploy stable-docs.datasette.io to Cloud Run
       run: |-
         gcloud config set run/region us-central1

datasette/publish/cloudrun.py

@@ -3,7 +3,7 @@ import click
 import json
 import os
 import re
-from subprocess import check_call, check_output
+from subprocess import CalledProcessError, check_call, check_output
 
 from .common import (
     add_common_publish_arguments_and_options,
@@ -23,7 +23,9 @@ def publish_subcommand(publish):
         help="Application name to use when building",
     )
     @click.option(
-        "--service", default="", help="Cloud Run service to deploy (or over-write)"
+        "--service",
+        default="",
+        help="Cloud Run service to deploy (or over-write)",
     )
     @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
     @click.option(
@@ -55,13 +57,32 @@ def publish_subcommand(publish):
     @click.option(
         "--max-instances",
         type=int,
-        help="Maximum Cloud Run instances",
+        default=1,
+        show_default=True,
+        help="Maximum Cloud Run instances (use 0 to remove the limit)",
     )
     @click.option(
         "--min-instances",
         type=int,
         help="Minimum Cloud Run instances",
     )
+    @click.option(
+        "--artifact-repository",
+        default="datasette",
+        show_default=True,
+        help="Artifact Registry repository to store the image",
+    )
+    @click.option(
+        "--artifact-region",
+        default="us",
+        show_default=True,
+        help="Artifact Registry location (region or multi-region)",
+    )
+    @click.option(
+        "--artifact-project",
+        default=None,
+        help="Project ID for Artifact Registry (defaults to the active project)",
+    )
     def cloudrun(
         files,
         metadata,
@@ -91,6 +112,9 @@ def publish_subcommand(publish):
         apt_get_extras,
         max_instances,
         min_instances,
+        artifact_repository,
+        artifact_region,
+        artifact_project,
     ):
         "Publish databases to Datasette running on Cloud Run"
         fail_if_publish_binary_not_installed(
@@ -100,6 +124,21 @@ def publish_subcommand(publish):
             "gcloud config get-value project", shell=True, universal_newlines=True
         ).strip()
 
+        artifact_project = artifact_project or project
+
+        # Ensure Artifact Registry exists for the target image
+        _ensure_artifact_registry(
+            artifact_project=artifact_project,
+            artifact_region=artifact_region,
+            artifact_repository=artifact_repository,
+        )
+
+        artifact_host = (
+            artifact_region
+            if artifact_region.endswith("-docker.pkg.dev")
+            else f"{artifact_region}-docker.pkg.dev"
+        )
+
         if not service:
             # Show the user their current services, then prompt for one
             click.echo("Please provide a service name for this deployment\n")
@@ -117,6 +156,11 @@ def publish_subcommand(publish):
                 click.echo("")
             service = click.prompt("Service name", type=str)
 
+        image_id = (
+            f"{artifact_host}/{artifact_project}/"
+            f"{artifact_repository}/datasette-{service}"
+        )
+
         extra_metadata = {
             "title": title,
             "license": license,
@@ -173,7 +217,6 @@ def publish_subcommand(publish):
                     print(fp.read())
                 print("\n====================\n")
 
-            image_id = f"gcr.io/{project}/datasette-{service}"
             check_call(
                 "gcloud builds submit --tag {}{}".format(
                     image_id, " --timeout {}".format(timeout) if timeout else ""
@@ -187,7 +230,7 @@ def publish_subcommand(publish):
             ("--max-instances", max_instances),
             ("--min-instances", min_instances),
         ):
-            if value:
+            if value is not None:
                 extra_deploy_options.append("{} {}".format(option, value))
         check_call(
             "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format(
@@ -199,6 +242,52 @@ def publish_subcommand(publish):
         )
 
 
+def _ensure_artifact_registry(artifact_project, artifact_region, artifact_repository):
+    """Ensure Artifact Registry API is enabled and the repository exists."""
+
+    enable_cmd = (
+        "gcloud services enable artifactregistry.googleapis.com "
+        f"--project {artifact_project} --quiet"
+    )
+    try:
+        check_call(enable_cmd, shell=True)
+    except CalledProcessError as exc:
+        raise click.ClickException(
+            "Failed to enable artifactregistry.googleapis.com. "
+            "Please ensure you have permissions to manage services."
+        ) from exc
+
+    describe_cmd = (
+        "gcloud artifacts repositories describe {repo} --project {project} "
+        "--location {location} --quiet"
+    ).format(
+        repo=artifact_repository,
+        project=artifact_project,
+        location=artifact_region,
+    )
+    try:
+        check_call(describe_cmd, shell=True)
+        return
+    except CalledProcessError:
+        create_cmd = (
+            "gcloud artifacts repositories create {repo} --repository-format=docker "
+            '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet'
+        ).format(
+            repo=artifact_repository,
+            location=artifact_region,
+            project=artifact_project,
+        )
+        try:
+            check_call(create_cmd, shell=True)
+            click.echo(f"Created Artifact Registry repository '{artifact_repository}'")
+        except CalledProcessError as exc:
+            raise click.ClickException(
+                "Failed to create Artifact Registry repository. "
+                "Use --artifact-repository/--artifact-region to point to an existing repo "
+                "or create one manually."
+            ) from exc
+
+
 def get_existing_services():
     services = json.loads(
         check_output(
@@ -214,6 +303,7 @@ def get_existing_services():
             "url": service["status"]["address"]["url"],
         }
         for service in services
+        if "url" in service["status"]
     ]
 
 

docs/cli-reference.rst

@@ -489,8 +489,15 @@ See :ref:`publish_cloud_run`.
       --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
       --timeout INTEGER               Build timeout in seconds
       --apt-get-install TEXT          Additional packages to apt-get install
-      --max-instances INTEGER         Maximum Cloud Run instances
+      --max-instances INTEGER         Maximum Cloud Run instances (use 0 to remove
+                                      the limit)  [default: 1]
       --min-instances INTEGER         Minimum Cloud Run instances
+      --artifact-repository TEXT      Artifact Registry repository to store the
+                                      image  [default: datasette]
+      --artifact-region TEXT          Artifact Registry location (region or multi-
+                                      region)  [default: us]
+      --artifact-project TEXT         Project ID for Artifact Registry (defaults to
+                                      the active project)
       --help                          Show this message and exit.
 
 

tests/test_publish_cloudrun.py

@@ -57,12 +57,20 @@ def test_publish_cloudrun_prompts_for_service(
         "Service name: input-service"
     ) == result.output.strip()
     assert 0 == result.exit_code
-    tag = "gcr.io/myproject/datasette-input-service"
+    tag = "us-docker.pkg.dev/myproject/datasette/datasette-input-service"
     mock_call.assert_has_calls(
         [
+            mock.call(
+                "gcloud services enable artifactregistry.googleapis.com --project myproject --quiet",
+                shell=True,
+            ),
+            mock.call(
+                "gcloud artifacts repositories describe datasette --project myproject --location us --quiet",
+                shell=True,
+            ),
             mock.call(f"gcloud builds submit --tag {tag}", shell=True),
             mock.call(
-                "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service".format(
+                "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format(
                     tag
                 ),
                 shell=True,
@@ -86,12 +94,20 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory):
         cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"]
     )
     assert 0 == result.exit_code
-    tag = f"gcr.io/{mock_output.return_value}/datasette-test"
+    tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test"
     mock_call.assert_has_calls(
         [
+            mock.call(
+                f"gcloud services enable artifactregistry.googleapis.com --project {mock_output.return_value} --quiet",
+                shell=True,
+            ),
+            mock.call(
+                f"gcloud artifacts repositories describe datasette --project {mock_output.return_value} --location us --quiet",
+                shell=True,
+            ),
             mock.call(f"gcloud builds submit --tag {tag}", shell=True),
             mock.call(
-                "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test".format(
+                "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format(
                     tag
                 ),
                 shell=True,
@@ -167,7 +183,7 @@ def test_publish_cloudrun_memory_cpu(
         assert 2 == result.exit_code
         return
     assert 0 == result.exit_code
-    tag = f"gcr.io/{mock_output.return_value}/datasette-test"
+    tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test"
     expected_call = (
         "gcloud run deploy --allow-unauthenticated --platform=managed"
         " --image {} test".format(tag)
@@ -179,8 +195,18 @@ def test_publish_cloudrun_memory_cpu(
         expected_call += " --cpu {}".format(cpu)
     if timeout:
         expected_build_call += f" --timeout {timeout}"
+    # max_instances defaults to 1
+    expected_call += " --max-instances 1"
     mock_call.assert_has_calls(
         [
+            mock.call(
+                f"gcloud services enable artifactregistry.googleapis.com --project {mock_output.return_value} --quiet",
+                shell=True,
+            ),
+            mock.call(
+                f"gcloud artifacts repositories describe datasette --project {mock_output.return_value} --location us --quiet",
+                shell=True,
+            ),
             mock.call(expected_build_call, shell=True),
             mock.call(
                 expected_call,