diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 6907b438..8ffdbfd5 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -102,12 +102,13 @@ jobs: # jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ # > metadata.json # cat metadata.json - - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@v0 + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 with: - version: '318.0.0' - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Deploy to Cloud Run env: LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 14bfaded..e94d0bdd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,12 +73,13 @@ jobs: DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@v0 + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 with: - version: '318.0.0' - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Deploy stable-docs.datasette.io to Cloud Run run: |- gcloud config set run/region us-central1 diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 760ff0d1..63d22fe8 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -3,7 +3,7 @@ import click import json import os import re -from subprocess import check_call, check_output +from subprocess import CalledProcessError, check_call, check_output from .common import ( add_common_publish_arguments_and_options, @@ -23,7 +23,9 @@ def publish_subcommand(publish): help="Application name to use when building", ) @click.option( - "--service", default="", help="Cloud Run service to deploy (or over-write)" + "--service", + default="", + help="Cloud Run service to deploy (or over-write)", ) @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") @click.option( @@ -55,13 +57,32 @@ def publish_subcommand(publish): @click.option( "--max-instances", type=int, - help="Maximum Cloud Run instances", + default=1, + show_default=True, + help="Maximum Cloud Run instances (use 0 to remove the limit)", ) @click.option( "--min-instances", type=int, help="Minimum Cloud Run instances", ) + @click.option( + "--artifact-repository", + default="datasette", + show_default=True, + help="Artifact Registry repository to store the image", + ) + @click.option( + "--artifact-region", + default="us", + show_default=True, + help="Artifact Registry location (region or multi-region)", + ) + @click.option( + "--artifact-project", + default=None, + help="Project ID for Artifact Registry (defaults to the active project)", + ) def cloudrun( files, metadata, @@ -91,6 +112,9 @@ def publish_subcommand(publish): apt_get_extras, max_instances, min_instances, + artifact_repository, + artifact_region, + artifact_project, ): "Publish databases to Datasette running on Cloud Run" fail_if_publish_binary_not_installed( @@ -100,6 +124,21 @@ def publish_subcommand(publish): "gcloud config get-value project", shell=True, universal_newlines=True ).strip() + artifact_project = artifact_project or project + + # Ensure Artifact Registry exists for the target image + _ensure_artifact_registry( + artifact_project=artifact_project, + artifact_region=artifact_region, + artifact_repository=artifact_repository, + ) + + artifact_host = ( + artifact_region + if artifact_region.endswith("-docker.pkg.dev") + else f"{artifact_region}-docker.pkg.dev" + ) + if not service: # Show the user their current services, then prompt for one click.echo("Please provide a service name for this deployment\n") @@ -117,6 +156,11 @@ def publish_subcommand(publish): click.echo("") service = click.prompt("Service name", type=str) + image_id = ( + f"{artifact_host}/{artifact_project}/" + f"{artifact_repository}/datasette-{service}" + ) + extra_metadata = { "title": title, "license": license, @@ -173,7 +217,6 @@ def publish_subcommand(publish): print(fp.read()) print("\n====================\n") - image_id = f"gcr.io/{project}/datasette-{service}" check_call( "gcloud builds submit --tag {}{}".format( image_id, " --timeout {}".format(timeout) if timeout else "" @@ -187,7 +230,7 @@ def publish_subcommand(publish): ("--max-instances", max_instances), ("--min-instances", min_instances), ): - if value: + if value is not None: extra_deploy_options.append("{} {}".format(option, value)) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( @@ -199,6 +242,52 @@ def publish_subcommand(publish): ) +def _ensure_artifact_registry(artifact_project, artifact_region, artifact_repository): + """Ensure Artifact Registry API is enabled and the repository exists.""" + + enable_cmd = ( + "gcloud services enable artifactregistry.googleapis.com " + f"--project {artifact_project} --quiet" + ) + try: + check_call(enable_cmd, shell=True) + except CalledProcessError as exc: + raise click.ClickException( + "Failed to enable artifactregistry.googleapis.com. " + "Please ensure you have permissions to manage services." + ) from exc + + describe_cmd = ( + "gcloud artifacts repositories describe {repo} --project {project} " + "--location {location} --quiet" + ).format( + repo=artifact_repository, + project=artifact_project, + location=artifact_region, + ) + try: + check_call(describe_cmd, shell=True) + return + except CalledProcessError: + create_cmd = ( + "gcloud artifacts repositories create {repo} --repository-format=docker " + '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' + ).format( + repo=artifact_repository, + location=artifact_region, + project=artifact_project, + ) + try: + check_call(create_cmd, shell=True) + click.echo(f"Created Artifact Registry repository '{artifact_repository}'") + except CalledProcessError as exc: + raise click.ClickException( + "Failed to create Artifact Registry repository. " + "Use --artifact-repository/--artifact-region to point to an existing repo " + "or create one manually." + ) from exc + + def get_existing_services(): services = json.loads( check_output( @@ -214,6 +303,7 @@ def get_existing_services(): "url": service["status"]["address"]["url"], } for service in services + if "url" in service["status"] ] diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 0e224916..f002d05a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -489,8 +489,15 @@ See :ref:`publish_cloud_run`. --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run --timeout INTEGER Build timeout in seconds --apt-get-install TEXT Additional packages to apt-get install - --max-instances INTEGER Maximum Cloud Run instances + --max-instances INTEGER Maximum Cloud Run instances (use 0 to remove + the limit) [default: 1] --min-instances INTEGER Minimum Cloud Run instances + --artifact-repository TEXT Artifact Registry repository to store the + image [default: datasette] + --artifact-region TEXT Artifact Registry location (region or multi- + region) [default: us] + --artifact-project TEXT Project ID for Artifact Registry (defaults to + the active project) --help Show this message and exit. diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 818fa2d3..f53e5059 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -57,12 +57,20 @@ def test_publish_cloudrun_prompts_for_service( "Service name: input-service" ) == result.output.strip() assert 0 == result.exit_code - tag = "gcr.io/myproject/datasette-input-service" + tag = "us-docker.pkg.dev/myproject/datasette/datasette-input-service" mock_call.assert_has_calls( [ + mock.call( + "gcloud services enable artifactregistry.googleapis.com --project myproject --quiet", + shell=True, + ), + mock.call( + "gcloud artifacts repositories describe datasette --project myproject --location us --quiet", + shell=True, + ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service".format( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format( tag ), shell=True, @@ -86,12 +94,20 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] ) assert 0 == result.exit_code - tag = f"gcr.io/{mock_output.return_value}/datasette-test" + tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test" mock_call.assert_has_calls( [ + mock.call( + f"gcloud services enable artifactregistry.googleapis.com --project {mock_output.return_value} --quiet", + shell=True, + ), + mock.call( + f"gcloud artifacts repositories describe datasette --project {mock_output.return_value} --location us --quiet", + shell=True, + ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test".format( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format( tag ), shell=True, @@ -167,7 +183,7 @@ def test_publish_cloudrun_memory_cpu( assert 2 == result.exit_code return assert 0 == result.exit_code - tag = f"gcr.io/{mock_output.return_value}/datasette-test" + tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test" expected_call = ( "gcloud run deploy --allow-unauthenticated --platform=managed" " --image {} test".format(tag) @@ -179,8 +195,18 @@ def test_publish_cloudrun_memory_cpu( expected_call += " --cpu {}".format(cpu) if timeout: expected_build_call += f" --timeout {timeout}" + # max_instances defaults to 1 + expected_call += " --max-instances 1" mock_call.assert_has_calls( [ + mock.call( + f"gcloud services enable artifactregistry.googleapis.com --project {mock_output.return_value} --quiet", + shell=True, + ), + mock.call( + f"gcloud artifacts repositories describe datasette --project {mock_output.return_value} --location us --quiet", + shell=True, + ), mock.call(expected_build_call, shell=True), mock.call( expected_call,