From 532e2938d0f82687ceda629ba50eaeae2a3a4102 Mon Sep 17 00:00:00 2001 From: lukedeng Date: Thu, 24 Oct 2024 22:06:48 +0000 Subject: [PATCH] Release 4.2.0 (2024-10-19) ### Features * [BYOB] Add udf_execution metrics for generateBid * add proto field to enable buyer debugging for eligible request * adding aws inference dashboard * Enable B&A experiments in GCP by splitting traffic * enable buyer debugging for eligible request * Implement Data Version Header for KV Server Clients * Implement v2 bidding support - add branch selection * Implement v2 bidding support - add kv_buyer_signals_adapter * Implement v2 bidding support - update kv dependency * output packer logs to stderr on AMI creation failure * Pass data version to GenerateBidsRawRequest * Support model deletion in model store * Update Kv client to match the latest contract ### Bug Fixes * Accumulate js_execution.errors_count metric * Copy common signals instead of move in BuildProtectedAudienceBidRequest * Fix changelog generation * GCP demo terraform outputs url * Prevent a potential read-after-delete bug in the inference benchmark * start_bidding runs inference default js file * typo in bidding_service_integration_test and buyer_code_wrapper Bug: 374379327 GitOrigin-RevId: fe009c936554bbaa97d98a8823c1e9766de78b86 Change-Id: I5dc590ef23daa175d2edb5ac156916165871f963 --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 34 +- WORKSPACE | 8 +- api/bidding_auction_servers.proto | 24 + builders/.pre-commit-config.yaml | 2 +- builders/CHANGELOG.md | 30 + builders/images/build-amazonlinux2/Dockerfile | 2 +- .../images/build-amazonlinux2023/Dockerfile | 2 +- .../images/build-amazonlinux2023/install_apps | 16 +- builders/images/build-debian/Dockerfile | 5 +- builders/images/build-debian/install_apps | 5 +- builders/images/install_golang_apps | 7 + builders/images/presubmit/install_apps | 4 +- builders/tests/data/hashes/build-amazonlinux2 | 2 +- .../tests/data/hashes/build-amazonlinux2023 | 2 +- builders/tests/data/hashes/build-debian | 2 +- builders/tests/data/hashes/presubmit | 2 +- builders/version.txt | 2 +- .../terraform/environment/demo/buyer/buyer.tf | 8 +- .../aws/terraform/modules/buyer/service.tf | 6 + .../dashboards/buyer_dashboard/main.tf | 8 +- .../dashboards/inference_dashboard/main.tf | 531 ++++++++ .../inference_dashboard/variables.tf | 23 + .../dashboards/seller_dashboard/main.tf | 8 +- .../terraform/environment/demo/buyer/buyer.tf | 146 ++- .../environment/demo/buyer/buyer_outputs.tf | 2 +- .../environment/demo/seller/seller.tf | 142 ++- .../environment/demo/seller/seller_outputs.tf | 2 +- .../gcp/terraform/modules/buyer/outputs.tf | 21 + .../gcp/terraform/modules/buyer/service.tf | 52 +- .../terraform/modules/buyer/service_vars.tf | 12 - .../gcp/terraform/modules/seller/outputs.tf | 21 + .../gcp/terraform/modules/seller/service.tf | 47 +- .../terraform/modules/seller/service_vars.tf | 13 - .../dashboards/buyer_dashboard/main.tf | 64 +- .../dashboards/inference_dashboard/main.tf | 83 +- .../dashboards/seller_dashboard/main.tf | 82 +- .../services/frontend_load_balancing/main.tf | 69 + .../frontend_load_balancing/outputs.tf | 20 + .../frontend_load_balancing/variables.tf | 66 + .../terraform/services/load_balancing/main.tf | 39 - .../services/load_balancing/outputs.tf | 4 + .../services/load_balancing/variables.tf | 21 - production/packaging/aws/build_and_test | 31 +- .../ami/debug/otel_collector_config.yaml | 61 +- .../packaging/aws/common/ami/image.pkr.hcl | 4 +- production/packaging/aws/common/ami/setup.sh | 16 +- services/auction_service/BUILD | 3 + services/auction_service/auction_main.cc | 5 +- .../auction_service_integration_test.cc | 2 +- .../auction_service_integration_test_util.cc | 6 +- .../auction_service_integration_test_util.h | 10 +- ...tion_service_reporting_integration_test.cc | 63 +- .../buyer_reporting_test_constants.h | 16 +- .../auction_service/data/runtime_config.h | 3 + .../private_aggregation_manager.cc | 2 +- .../reporting/buyer/buyer_reporting_helper.cc | 50 +- .../reporting/buyer/buyer_reporting_helper.h | 6 + .../buyer/buyer_reporting_helper_test.cc | 17 + .../buyer/pa_buyer_reporting_manager.cc | 4 +- .../buyer/pas_buyer_reporting_manager.cc | 5 +- .../reporting/reporting_helper.cc | 4 +- .../reporting/reporting_helper.h | 3 + .../reporting_helper_test_constants.h | 2 + .../reporting/reporting_test_util.cc | 10 +- services/auction_service/score_ads_reactor.cc | 640 +++++++--- services/auction_service/score_ads_reactor.h | 117 +- .../auction_service/score_ads_reactor_test.cc | 1105 ++++++++++++++++- .../score_ads_reactor_test_util.cc | 34 +- .../score_ads_reactor_test_util.h | 8 + .../udf_fetcher/buyer_reporting_fetcher.cc | 2 +- .../buyer_reporting_udf_fetch_manager.cc | 2 +- services/auction_service/utils/proto_utils.cc | 22 +- services/auction_service/utils/proto_utils.h | 7 +- .../auction_service/utils/proto_utils_test.cc | 14 +- services/bidding_service/BUILD | 2 +- .../base_generate_bids_reactor.h | 9 +- services/bidding_service/bidding_main.cc | 13 +- .../bidding_service_integration_test.cc | 81 +- .../code_wrapper/buyer_code_wrapper.h | 54 +- .../buyer_code_wrapper_test_constants.h | 116 +- services/bidding_service/constants.h | 10 +- .../bidding_service/data/runtime_config.h | 4 + .../egress_features/boolean_feature.cc | 2 +- .../egress_features/histogram_feature.cc | 2 +- .../egress_features/signed_int_feature.cc | 2 +- .../egress_features/unsigned_int_feature.cc | 2 +- .../generate_bids_binary_reactor.cc | 26 +- .../generate_bids_binary_reactor.h | 4 + .../generate_bids_binary_reactor_test.cc | 34 + .../bidding_service/generate_bids_reactor.cc | 80 +- .../bidding_service/generate_bids_reactor.h | 1 + .../generate_bids_reactor_test.cc | 56 +- .../generate_bids_reactor_test_utils.cc | 6 +- .../inference/periodic_model_fetcher.cc | 5 +- ...ected_app_signals_generate_bids_reactor.cc | 122 +- ...tected_app_signals_generate_bids_reactor.h | 4 +- ..._app_signals_generate_bids_reactor_test.cc | 98 +- services/buyer_frontend_service/BUILD | 40 +- .../buyer_frontend_main.cc | 5 + .../buyer_frontend_service.cc | 2 + .../data/bidding_signals.h | 1 + .../data/get_bids_config.h | 4 + .../get_bids_unary_reactor.cc | 156 ++- .../get_bids_unary_reactor.h | 22 +- .../kv_buyer_signals_adapter.cc | 188 +++ .../kv_buyer_signals_adapter.h | 60 + .../kv_buyer_signals_adapter_test.cc | 436 +++++++ .../buyer_frontend_service/providers/BUILD | 16 +- .../http_bidding_signals_async_provider.cc | 7 +- ...ttp_bidding_signals_async_provider_test.cc | 15 +- services/buyer_frontend_service/util/BUILD | 2 +- .../util/buyer_frontend_test_utils.cc | 4 +- .../util/proto_factory.cc | 4 +- .../util/proto_factory.h | 3 +- .../util/proto_factory_test.cc | 52 +- .../config/trusted_server_config_client.h | 6 +- .../common/clients/http/http_fetcher_async.h | 12 + .../http/multi_curl_http_fetcher_async.cc | 43 +- .../http/multi_curl_http_fetcher_async.h | 13 + .../buyer_key_value_async_http_client.cc | 34 +- .../buyer/buyer_key_value_async_http_client.h | 4 + .../buyer_key_value_async_http_client_test.cc | 213 ++-- services/common/clients/kv_server/BUILD | 2 +- .../clients/kv_server/kv_async_client.cc | 79 +- .../clients/kv_server/kv_async_client.h | 1 - services/common/code_dispatch/BUILD | 1 - .../code_dispatch/code_dispatch_reactor.h | 10 +- .../common/constants/common_service_flags.cc | 5 + .../common/constants/common_service_flags.h | 6 +- .../data_fetch/periodic_bucket_fetcher.cc | 2 +- .../common/data_fetch/periodic_url_fetcher.cc | 2 +- services/common/loggers/BUILD | 2 + services/common/loggers/request_log_context.h | 13 +- services/common/metric/server_definition.h | 53 +- .../private_aggregation_post_auction_util.cc | 4 +- services/common/test/BUILD | 1 + services/common/test/mocks.h | 4 + services/common/test/random.cc | 10 +- services/common/test/random.h | 3 +- services/common/util/BUILD | 30 - services/common/util/async_task_tracker.cc | 2 +- services/common/util/binary_http_utils.h | 96 -- .../common/util/binary_http_utils_test.cc | 85 -- services/common/util/hpke_utils.cc | 2 +- services/common/util/json_util.h | 28 +- services/common/util/json_util_test.cc | 32 + services/common/util/read_system.cc | 171 ++- services/common/util/read_system.h | 41 +- services/common/util/read_system_test.cc | 20 +- services/common/util/reporting_util.cc | 19 +- services/common/util/reporting_util.h | 8 + services/common/util/tcmalloc_utils.h | 4 +- services/inference_sidecar/README.md | 16 +- .../inference_sidecar/common/benchmark/BUILD | 1 + .../common/benchmark/module_benchmark.cc | 15 +- .../common/model/model_store.h | 60 +- .../common/model/model_store_test.cc | 122 +- .../common/sandbox/sandbox_executor.cc | 15 + .../common/sandbox/sandbox_executor.h | 3 + .../modules/pytorch_v2_1_1/pytorch.cc | 3 +- .../modules/tensorflow_v2_14_0/tensorflow.cc | 85 +- .../modules/tensorflow_v2_14_0/tensorflow.h | 3 + .../select_ad_reactor.cc | 22 +- .../select_ad_reactor.h | 9 +- .../select_ad_reactor_app.cc | 14 +- .../select_ad_reactor_app.h | 4 +- .../select_ad_reactor_test.cc | 123 +- .../select_ad_reactor_web.cc | 9 +- .../select_ad_reactor_web.h | 5 +- .../select_auction_result_reactor.cc | 4 +- .../select_auction_result_reactor.h | 3 +- .../seller_frontend_service.cc | 19 +- .../seller_frontend_service.h | 13 +- .../util/encryption_util.cc | 2 +- .../util/proto_mapping_util.cc | 6 +- .../util/select_ad_reactor_test_utils.cc | 16 +- .../util/select_ad_reactor_test_utils.h | 11 +- .../seller_frontend_service/util/web_utils.cc | 8 +- .../util/web_utils_test.cc | 8 +- tools/debug/start_auction | 4 + tools/debug/start_bfe | 2 + tools/debug/start_bidding | 11 +- tools/debug/start_bidding_byob | 8 +- tools/debug/start_sfe | 4 + tools/load_testing/README.md | 4 +- .../payload_generator/payload_packaging.cc | 4 +- .../payload_packaging_test.cc | 4 +- version.txt | 2 +- 189 files changed, 5676 insertions(+), 1692 deletions(-) create mode 100644 production/deploy/aws/terraform/services/dashboards/inference_dashboard/main.tf create mode 100644 production/deploy/aws/terraform/services/dashboards/inference_dashboard/variables.tf create mode 100644 production/deploy/gcp/terraform/modules/buyer/outputs.tf create mode 100644 production/deploy/gcp/terraform/modules/seller/outputs.tf create mode 100644 production/deploy/gcp/terraform/services/frontend_load_balancing/main.tf create mode 100644 production/deploy/gcp/terraform/services/frontend_load_balancing/outputs.tf create mode 100644 production/deploy/gcp/terraform/services/frontend_load_balancing/variables.tf create mode 100644 services/buyer_frontend_service/kv_buyer_signals_adapter.cc create mode 100644 services/buyer_frontend_service/kv_buyer_signals_adapter.h create mode 100644 services/buyer_frontend_service/kv_buyer_signals_adapter_test.cc delete mode 100644 services/common/util/binary_http_utils.h delete mode 100644 services/common/util/binary_http_utils_test.cc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0e755d2..9b989983 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,7 +145,7 @@ repos: )$ - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.13.0 + rev: v0.7.1 hooks: - id: markdownlint-cli2 name: lint markdown diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8fe0e6..2f6497b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,38 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. -## 4.1.0 (2024-10-10) +## 4.2.0 (2024-10-19) -### ⚠ BREAKING CHANGES +### Features + +* [BYOB] Add udf_execution metrics for generateBid +* add proto field to enable buyer debugging for eligible request +* adding aws inference dashboard +* Enable B&A experiments in GCP by splitting traffic +* enable buyer debugging for eligible request +* Implement Data Version Header for KV Server Clients +* Implement v2 bidding support - add branch selection +* Implement v2 bidding support - add kv_buyer_signals_adapter +* Implement v2 bidding support - update kv dependency +* output packer logs to stderr on AMI creation failure +* Pass data version to GenerateBidsRawRequest +* Support model deletion in model store +* Update Kv client to match the latest contract + + +### Bug Fixes + +* Accumulate js_execution.errors_count metric +* Copy common signals instead of move in BuildProtectedAudienceBidRequest +* Fix changelog generation +* GCP demo terraform outputs url +* Prevent a potential read-after-delete bug in the inference benchmark +* start_bidding runs inference default js file +* typo in bidding_service_integration_test and buyer_code_wrapper + +## 4.1.0 (2024-10-10) -* All sellers and buyers must update the js_num_workers parameter to udf_num_workers in terraform -* Sellers must wait for all the integrated buyers to update to 4.1.0 before updating to accomodate serialization changes in the GetBidsRequest. ### Features @@ -39,6 +64,7 @@ All notable changes to this project will be documented in this file. See [commit * SFE sends updateIfOlderThanMs data to client * update start_bidding with minimal defaults and local file flag + ### Bug Fixes * BYOB generateBid should only parse first bid from binary response diff --git a/WORKSPACE b/WORKSPACE index 6f97c1f3..7949ec9c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -87,11 +87,11 @@ http_archive( http_archive( name = "service_value_key_fledge_privacysandbox", - # commit d186bcf8d643888fac81784f9c91ffb2fb48791c 2024-04-19 - sha256 = "53521403ffbe18a306417b8e7b41c2a13127ba0892ad921229861a451ad2045e", - strip_prefix = "protected-auction-key-value-service-d186bcf8d643888fac81784f9c91ffb2fb48791c", + # commit bdadee7c80dd84197f9253d4fd92c310f457be00 2024-09-26 + sha256 = "6f91715f5ac946b2c5a9c4536f8e7deebf74b73e5b75e93163fd0e6276731ac1", + strip_prefix = "protected-auction-key-value-service-bdadee7c80dd84197f9253d4fd92c310f457be00", urls = [ - "https://github.com/privacysandbox/protected-auction-key-value-service/archive/d186bcf8d643888fac81784f9c91ffb2fb48791c.zip", + "https://github.com/privacysandbox/protected-auction-key-value-service/archive/bdadee7c80dd84197f9253d4fd92c310f457be00.zip", ], ) diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto index b24a1186..415ced6b 100644 --- a/api/bidding_auction_servers.proto +++ b/api/bidding_auction_servers.proto @@ -74,6 +74,11 @@ message ProtectedAudienceInput { // Clients can indicate whether K-Anonymity is enforced for the traffic. // By default, this would be false, i.e. K-Anonymity is not enforced. bool enforce_kanon = 9; + + // Clients can indicate whether this request can be used for debug. + // By default, this would be false, i.e. cannot be used for debug. + // Currently Android request with ad-id is eligible. + bool prod_debug = 10; } // ProtectedAuctionInput is generated and encrypted by the client, @@ -128,6 +133,11 @@ message ProtectedAuctionInput { // Clients can indicate whether K-Anonymity is enforced for the traffic. // By default, this would be false, i.e. K-Anonymity is not enforced. bool enforce_kanon = 9; + + // Clients can indicate whether this request can be used for debug. + // By default, this would be false, i.e. cannot be used for debug. + // Currently Android request with ad-id is eligible. + bool prod_debug = 10; } // Grouping of data pertaining to protected app signals. @@ -976,6 +986,10 @@ message GetBidsRequest { // Clients can indicate whether K-Anonymity is enforced for the traffic. // By default, this would be false, i.e. K-Anonymity is not enforced. bool enforce_kanon = 16; + + // A boolean value which indicates whether this request can be used for debugging. + // current eligible request: Android request with ad-id + bool is_debug_eligible = 17; } // Encrypted GetBidsRawRequest. @@ -1226,6 +1240,13 @@ message GenerateBidsRequest { // Clients can indicate whether K-Anonymity is enforced for the traffic. // By default, this would be false, i.e. K-Anonymity is not enforced. bool enforce_kanon = 12; + + // Optional. Value of the data version header from the buyer kv server. + // This is passed through to the buyer reporting function. + uint32 data_version = 13; + + // If BFE is sampling this request for debug. + bool is_sampled_for_debug = 14; } // Encrypted GenerateBidsRawRequest. @@ -1327,6 +1348,9 @@ message GenerateProtectedAppSignalsBidsRequest { // Clients can indicate whether K-Anonymity is enforced for the traffic. // By default, this would be false, i.e. K-Anonymity is not enforced. bool enforce_kanon = 12; + + // If BFE is sampling this request for debug. + bool is_sampled_for_debug = 13; } // Encrypted GenerateProtectedAppSignalsBidsRawRequest. diff --git a/builders/.pre-commit-config.yaml b/builders/.pre-commit-config.yaml index 3f853aab..67c5de5b 100644 --- a/builders/.pre-commit-config.yaml +++ b/builders/.pre-commit-config.yaml @@ -109,7 +109,7 @@ repos: - markdown - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.13.0 + rev: v0.7.1 hooks: - id: markdownlint-cli2 name: lint markdown diff --git a/builders/CHANGELOG.md b/builders/CHANGELOG.md index 3d0ac973..164f0996 100644 --- a/builders/CHANGELOG.md +++ b/builders/CHANGELOG.md @@ -2,6 +2,36 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 0.71.0 (2024-10-18) + + +### Dependencies + +* **deps:** Pin AmazonLinux2023 Nitro CLI versions +* **deps:** Update AmazonLinux base images to 2024-10-01 +* **deps:** Upgrade pre-commit and pylint + +## 0.70.0 (2024-10-10) + + +### Features + +* add awk, crane and jq to build-* images for the oci_pull credential helper + +## 0.69.1 (2024-09-19) + +### Bug Fixes + +* Use digest to ensure deterministic environment for build system + + +## 0.69.0 (2024-09-15) + + +### Features + +* Pin build-debian ubuntu base image from 20.04 to focal-20240530 + ## 0.68.1 (2024-08-21) ### Bug Fixes diff --git a/builders/images/build-amazonlinux2/Dockerfile b/builders/images/build-amazonlinux2/Dockerfile index 6ccce805..057b02ed 100644 --- a/builders/images/build-amazonlinux2/Dockerfile +++ b/builders/images/build-amazonlinux2/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM amazonlinux:2.0.20240412.0 +FROM amazonlinux:2.0.20241001.0 COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/ COPY get_workspace_mount /usr/local/bin diff --git a/builders/images/build-amazonlinux2023/Dockerfile b/builders/images/build-amazonlinux2023/Dockerfile index c7d3509b..dc5d5fbf 100644 --- a/builders/images/build-amazonlinux2023/Dockerfile +++ b/builders/images/build-amazonlinux2023/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM amazonlinux:2023.5.20240722.0 +FROM amazonlinux:2023.5.20241001.1 COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/ COPY get_workspace_mount /usr/local/bin diff --git a/builders/images/build-amazonlinux2023/install_apps b/builders/images/build-amazonlinux2023/install_apps index 16378fa5..1987184d 100755 --- a/builders/images/build-amazonlinux2023/install_apps +++ b/builders/images/build-amazonlinux2023/install_apps @@ -46,9 +46,19 @@ function install_python() { } function install_nitro() { + local -r arch="$1" + local -r -A extensions=( + [amd64]="x86_64" + [arm64]="aarch64" + ) + local -r ext="${extensions[${arch}]}" + if [[ -z ${ext} ]]; then + printf "Unrecognized or unsupported architecture for nitro: %s\n" "${arch}" &>/dev/stderr + return 1 + fi dnf install -y \ - "aws-nitro-enclaves-cli-1.3.*" \ - "aws-nitro-enclaves-cli-devel-1.3.*" + "aws-nitro-enclaves-cli-1.3.1-0.amzn2023.${ext}" \ + "aws-nitro-enclaves-cli-devel-1.3.1-0.amzn2023.${ext}" } function install_gcc() { @@ -91,7 +101,7 @@ fi install_python dnf_update -install_nitro +install_nitro "${BUILD_ARCH}" install_misc install_clang install_golang "${BUILD_ARCH}" diff --git a/builders/images/build-debian/Dockerfile b/builders/images/build-debian/Dockerfile index 735d9b66..f555d5d7 100644 --- a/builders/images/build-debian/Dockerfile +++ b/builders/images/build-debian/Dockerfile @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG BASE_IMAGE=ubuntu:20.04 +# Use fixed manifest digest to ensure reproducible. +# https://hub.docker.com/layers/library/ubuntu/focal-20240530/images/sha256-85c08a37b74bc18a7b3f8cf89aabdfac51c525cdbc193a753f7907965e310ec2 +# ubuntu v20.04, focal-20240530 +ARG BASE_IMAGE=ubuntu@sha256:fa17826afb526a9fc7250e0fbcbfd18d03fe7a54849472f86879d8bf562c629e # ignore this hadolint error as BASE_IMAGE contains an image tag # hadolint ignore=DL3006 diff --git a/builders/images/build-debian/install_apps b/builders/images/build-debian/install_apps index fba47ca1..e74c3795 100755 --- a/builders/images/build-debian/install_apps +++ b/builders/images/build-debian/install_apps @@ -68,13 +68,15 @@ function install_misc() { bsdmainutils \ ca-certificates \ chrpath="0.16-*" \ - libcurl4="7.68.*" \ curl="7.68.*" \ file="1:5.*" \ + gawk="1:5.*" \ gettext="0.19.*" \ git="1:2.25.*" \ gnupg="2.2.*" \ google-perftools="2.*" \ + jq="1.6-*" \ + libcurl4="7.68.*" \ locales="2.31-*" \ lsb-release="11.1.*" \ openssh-client="1:8.2*" \ @@ -85,6 +87,7 @@ function install_misc() { wget="1.20.*" \ xz-utils="5.2.*" \ zip="3.0-*" + if [[ -n ${INSTALL_LOCALE} ]]; then printf "\nSetting locale to: %s\n" "${INSTALL_LOCALE}" locale-gen "${INSTALL_LOCALE}" diff --git a/builders/images/install_golang_apps b/builders/images/install_golang_apps index a6bb3cfd..3b5375ae 100755 --- a/builders/images/install_golang_apps +++ b/builders/images/install_golang_apps @@ -40,10 +40,17 @@ function install_bazelisk() { rm -rf /bazel_root/* } +function install_crane() { + go install github.com/google/go-containerregistry/cmd/crane@v0.20.2 + CRANE="$(go env GOPATH)"/bin/crane + ln -s "${CRANE}" /usr/bin/crane +} + function cleanup() { cd / go clean -cache -modcache } install_bazelisk +install_crane cleanup diff --git a/builders/images/presubmit/install_apps b/builders/images/presubmit/install_apps index 3986ec65..501d4459 100755 --- a/builders/images/presubmit/install_apps +++ b/builders/images/presubmit/install_apps @@ -82,8 +82,8 @@ function install_docker() { function install_precommit() { /usr/bin/python3.12 -m venv "${PRE_COMMIT_VENV_DIR}" "${PRE_COMMIT_VENV_DIR}"/bin/pip install \ - pre-commit~=3.7 \ - pylint~=3.1.0 + pre-commit~=4.0 \ + pylint~=3.1 "${PRE_COMMIT_TOOL}" --version # initialize pre-commit cache, which needs a git repo (a temporary will suffice) diff --git a/builders/tests/data/hashes/build-amazonlinux2 b/builders/tests/data/hashes/build-amazonlinux2 index 71f56dfb..49a5d7d1 100644 --- a/builders/tests/data/hashes/build-amazonlinux2 +++ b/builders/tests/data/hashes/build-amazonlinux2 @@ -1 +1 @@ -57ca4f2381a0fc193b0476663171c4d339b6ef66c0d1f1c24bb3f48d368b38ab +566ebf3a2f4bc0a9675b51ee68de67bf0ea6d90c1fbbb9dcd9096002a24ed365 diff --git a/builders/tests/data/hashes/build-amazonlinux2023 b/builders/tests/data/hashes/build-amazonlinux2023 index 7d1e615d..8b49a427 100644 --- a/builders/tests/data/hashes/build-amazonlinux2023 +++ b/builders/tests/data/hashes/build-amazonlinux2023 @@ -1 +1 @@ -59a82d2db8173784b0b49959c9f82ead6c2e6da78a6be21cdc78520aa43741e3 +9cde9b8ca775243a0093c486e229f3fa885bead7e12afc3bdd1408a919076e7e diff --git a/builders/tests/data/hashes/build-debian b/builders/tests/data/hashes/build-debian index 57095aed..35fa781d 100644 --- a/builders/tests/data/hashes/build-debian +++ b/builders/tests/data/hashes/build-debian @@ -1 +1 @@ -c194dafd287978093f8fe6e16e981fb22028e37345e20a4d7ca84caa43f0d4c0 +877727ca542620501a2822cc32424fe5387cc37f09012a08a8cfdc86881c39a5 diff --git a/builders/tests/data/hashes/presubmit b/builders/tests/data/hashes/presubmit index a3e4b6ff..03a9f171 100644 --- a/builders/tests/data/hashes/presubmit +++ b/builders/tests/data/hashes/presubmit @@ -1 +1 @@ -560a5a1726e7b6fd2a507f72f2563eb3938d153a7d4b4aada575a7fe772873b0 +59738b43c28ecd1582112ce61fdbb697aa4952636b99955fdecdcc624bd245fe diff --git a/builders/version.txt b/builders/version.txt index 65529192..f91d144a 100644 --- a/builders/version.txt +++ b/builders/version.txt @@ -1 +1 @@ -0.68.1 \ No newline at end of file +0.71.0 \ No newline at end of file diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf index 30b11510..f52ef332 100644 --- a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf @@ -108,6 +108,7 @@ module "buyer" { TELEMETRY_CONFIG = "" # Example: "mode: EXPERIMENT" ENABLE_OTEL_BASED_LOGGING = "" # Example: "true" CONSENTED_DEBUG_TOKEN = "" # Example: "123456" + DEBUG_SAMPLE_RATE_MICRO = "0" TEST_MODE = "" # Example: "false" BUYER_CODE_FETCH_CONFIG = "" # Example: @@ -165,9 +166,10 @@ module "buyer" { MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" - INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar_" - INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" - INFERENCE_MODEL_BUCKET_PATHS = "" # Example: "," + INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar_" + INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" + INFERENCE_MODEL_CONFIG_PATH = "" # Example: "model_config.json" + INFERENCE_MODEL_FETCH_PERIOD_MS = "" # Example: "60000" # TCMalloc related config parameters. # See: https://github.com/google/tcmalloc/blob/master/docs/tuning.md diff --git a/production/deploy/aws/terraform/modules/buyer/service.tf b/production/deploy/aws/terraform/modules/buyer/service.tf index 336e6ce0..915b153c 100644 --- a/production/deploy/aws/terraform/modules/buyer/service.tf +++ b/production/deploy/aws/terraform/modules/buyer/service.tf @@ -109,6 +109,12 @@ module "buyer_dashboard" { region = var.region } +module "inference_dashboard" { + source = "../../services/dashboards/inference_dashboard" + environment = var.environment + region = var.region +} + module "buyer_app_mesh" { # Only create if using service mesh count = var.use_service_mesh ? 1 : 0 diff --git a/production/deploy/aws/terraform/services/dashboards/buyer_dashboard/main.tf b/production/deploy/aws/terraform/services/dashboards/buyer_dashboard/main.tf index 3650083b..cd1eb5a8 100644 --- a/production/deploy/aws/terraform/services/dashboards/buyer_dashboard/main.tf +++ b/production/deploy/aws/terraform/services/dashboards/buyer_dashboard/main.tf @@ -582,7 +582,7 @@ resource "aws_cloudwatch_dashboard" "environment_dashboard" { "type": "metric", "properties": { "metrics": [ - [ { "expression": "SEARCH(' service.name=\"bidding\" deployment.environment=\"${var.environment}\" Noise=(\"Raw\" OR \"Noised\") MetricName=\"js_execution.duration_ms\" ', 'Average', 60)", "id": "e1", "label": "$${PROP('Dim.service.name')} $${PROP('Dim.deployment.environment')} $${PROP('Dim.Noise')} $${PROP('Dim.service.instance.id')}" } ] + [ { "expression": "SEARCH(' service.name=\"bidding\" deployment.environment=\"${var.environment}\" Noise=(\"Raw\" OR \"Noised\") MetricName=\"udf_execution.duration_ms\" ', 'Average', 60)", "id": "e1", "label": "$${PROP('Dim.service.name')} $${PROP('Dim.deployment.environment')} $${PROP('Dim.Noise')} $${PROP('Dim.service.instance.id')}" } ] ], "timezone": "UTC", "region": "${var.region}", @@ -594,7 +594,7 @@ resource "aws_cloudwatch_dashboard" "environment_dashboard" { "showUnits": false } }, - "title": "js_execution.duration_ms [MEAN]" + "title": "udf_execution.duration_ms [MEAN]" } }, { @@ -605,7 +605,7 @@ resource "aws_cloudwatch_dashboard" "environment_dashboard" { "type": "metric", "properties": { "metrics": [ - [ { "expression": "SEARCH(' service.name=\"bidding\" deployment.environment=\"${var.environment}\" Noise=(\"Raw\" OR \"Noised\") MetricName=\"js_execution.errors_count\" ', 'Average', 60)", "id": "e1", "label": "$${PROP('Dim.service.name')} $${PROP('Dim.deployment.environment')} $${PROP('Dim.Noise')} $${PROP('Dim.service.instance.id')}" } ] + [ { "expression": "SEARCH(' service.name=\"bidding\" deployment.environment=\"${var.environment}\" Noise=(\"Raw\" OR \"Noised\") MetricName=\"udf_execution.errors_count\" ', 'Average', 60)", "id": "e1", "label": "$${PROP('Dim.service.name')} $${PROP('Dim.deployment.environment')} $${PROP('Dim.Noise')} $${PROP('Dim.service.instance.id')}" } ] ], "timezone": "UTC", "region": "${var.region}", @@ -617,7 +617,7 @@ resource "aws_cloudwatch_dashboard" "environment_dashboard" { "showUnits": false } }, - "title": "js_execution.errors_count [MEAN]" + "title": "udf_execution.errors_count [MEAN]" } }, { diff --git a/production/deploy/aws/terraform/services/dashboards/inference_dashboard/main.tf b/production/deploy/aws/terraform/services/dashboards/inference_dashboard/main.tf new file mode 100644 index 00000000..46adfe3f --- /dev/null +++ b/production/deploy/aws/terraform/services/dashboards/inference_dashboard/main.tf @@ -0,0 +1,531 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "aws_cloudwatch_dashboard" "environment_dashboard" { + dashboard_name = "${var.environment}-inference-metrics" + + # https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Dashboard-Body-Structure.html + dashboard_body = < 0. + region_config = local.default_region_config + runtime_flag_override = {} + } + + # optional: experiment arm 1 + "${local.environment}-1" = { + image_tag = "" # Image built and uploaded by production/packaging/build_and_test_all_in_docker + traffic_weight = 0 # traffic_weight for this arm, between 0~1000. Arm with 0 weight will be skipped. + region_config = local.default_region_config # region config can be different for each arm + runtime_flag_override = { + # Example: MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "12345" + } + } + + # optional: more experiment arm + # "${local.environment}-2" = ... + } } provider "google" { @@ -39,13 +97,14 @@ module "secrets" { } module "buyer" { + for_each = { for key, value in local.buyer_traffic_splits : key => value if value.traffic_weight > 0 } source = "../../../modules/buyer" - environment = local.environment + environment = each.key gcp_project_id = local.gcp_project_id - bidding_image = "${local.image_repo}/bidding_service:${local.environment}" # Image built and uploaded by production/packaging/build_and_test_all_in_docker - buyer_frontend_image = "${local.image_repo}/buyer_frontend_service:${local.environment}" # Image built and uploaded by production/packaging/build_and_test_all_in_docker + bidding_image = "${local.image_repo}/bidding_service:${each.value.image_tag}" + buyer_frontend_image = "${local.image_repo}/buyer_frontend_service:${each.value.image_tag}" - runtime_flags = { + runtime_flags = merge({ BIDDING_PORT = "50051" # Do not change unless you are modifying the default GCP architecture. BUYER_FRONTEND_PORT = "50051" # Do not change unless you are modifying the default GCP architecture. BUYER_FRONTEND_HEALTHCHECK_PORT = "50050" # Do not change unless you are modifying the default GCP architecture. @@ -108,9 +167,10 @@ module "buyer" { JS_WORKER_QUEUE_LEN = "" # Example: "200". ROMA_TIMEOUT_MS = "" # Example: "10000" TELEMETRY_CONFIG = "" # Example: "mode: EXPERIMENT" - COLLECTOR_ENDPOINT = "" # Example: "collector-buyer-1-${local.environment}.bfe-gcp.com:4317" + COLLECTOR_ENDPOINT = "" # Example: "collector-buyer-1-${each.key}.bfe-gcp.com:4317" ENABLE_OTEL_BASED_LOGGING = "" # Example: "false" CONSENTED_DEBUG_TOKEN = "" # Example: "" + DEBUG_SAMPLE_RATE_MICRO = "0" # Coordinator-based attestation flags. # These flags are production-ready and you do not need to change them. @@ -137,7 +197,8 @@ module "buyer" { INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar_" INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" - INFERENCE_MODEL_BUCKET_PATHS = "" # Example: "," + INFERENCE_MODEL_CONFIG_PATH = "" # Example: "model_config.json" + INFERENCE_MODEL_FETCH_PERIOD_MS = "" # Example: "60000" INFERENCE_SIDECAR_RUNTIME_CONFIG = "" # Example: # "{ # "num_interop_threads": 4, @@ -151,17 +212,11 @@ module "buyer" { BIDDING_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "10737418240" # Example: 10737418240 BFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND = "4096" BFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "10737418240" - } + }, each.value.runtime_flag_override) - # Please create a Google Cloud domain name, dns zone, and SSL certificate. - # See demo/project_setup_utils/domain_setup/README.md for more details. - # If you specify a certificate_map_id, you do not need to specify an ssl_certificate_id. - frontend_domain_name = "" # Example: bfe-gcp.com - frontend_dns_zone = "" # Example: "bfe-gcp-com" - frontend_domain_ssl_certificate_id = "" # Example: "projects/${local.gcp_project_id}/global/sslCertificates/bfe-${local.environment}" - frontend_certificate_map_id = "" # Example: "//certificatemanager.googleapis.com/projects/test/locations/global/certificateMaps/wildcard-cert-map" - - operator = "" # Example: "buyer-1" + frontend_domain_name = local.buyer_domain_name + frontend_dns_zone = local.frontend_dns_zone + operator = local.buyer_operator service_account_email = "" # Example: "terraform-sa@{local.gcp_project_id}.iam.gserviceaccount.com" vm_startup_delay_seconds = 200 # Example: 200 cpu_utilization_percent = 0.6 # Example: 0.6 @@ -174,34 +229,37 @@ module "buyer" { gcs_hmac_key = module.secrets.gcs_hmac_key gcs_hmac_secret = module.secrets.gcs_hmac_secret gcs_bucket = "" # Example: ${name of a gcs bucket} - gcs_bucket_prefix = "" # Example: "consented-eventmessage-${local.environment}" - file_prefix = "" # Example: "operator-name" + gcs_bucket_prefix = "" # Example: "consented-eventmessage-${each.key}" + file_prefix = "" # Example: local.buyer_operator }) - region_config = { - # Example config provided for us-central1 and you may add your own regions. - "us-central1" = { - collector = { - machine_type = "e2-micro" - min_replicas = 1 - max_replicas = 1 - zones = null # Null signifies no zone preference. - max_rate_per_instance = null # Null signifies no max. - } - backend = { - machine_type = "n2d-standard-64" - min_replicas = 1 - max_replicas = 5 - zones = null # Null signifies no zone preference. - max_rate_per_instance = null # Null signifies no max. - } - frontend = { - machine_type = "n2d-standard-64" - min_replicas = 1 - max_replicas = 2 - zones = null # Null signifies no zone preference. - max_rate_per_instance = null # Null signifies no max. - } - } - } + region_config = each.value.region_config enable_tee_container_log_redirect = false } + +module "buyer_frontend_load_balancing" { + source = "../../services/frontend_load_balancing" + environment = local.environment + operator = local.buyer_operator + frontend_ip_address = module.buyer[local.environment].frontend_address + frontend_domain_name = local.buyer_domain_name + frontend_dns_zone = local.frontend_dns_zone + + frontend_domain_ssl_certificate_id = local.frontend_domain_ssl_certificate_id + frontend_certificate_map_id = local.frontend_certificate_map_id + frontend_service_name = "bfe" + google_compute_backend_service_ids = { + for buyer_key, buyer in module.buyer : + buyer_key => buyer.google_compute_backend_service_id + } + traffic_weights = { for key, value in local.buyer_traffic_splits : key => value.traffic_weight } +} + +module "buyer_dashboard" { + source = "../../services/dashboards/buyer_dashboard" + environment = join("|", [for k, v in local.buyer_traffic_splits : k if v.traffic_weight > 0]) +} + +module "inference_dashboard" { + source = "../../services/dashboards/inference_dashboard" + environment = join("|", [for k, v in local.buyer_traffic_splits : k if v.traffic_weight > 0]) +} diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer_outputs.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer_outputs.tf index eb6b8a13..c66b5475 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/buyer_outputs.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/buyer_outputs.tf @@ -13,6 +13,6 @@ # limitations under the License. output "buyer_frontend_url" { - value = "https://${module.buyer.operator}-${module.buyer.environment}.${module.buyer.frontend_domain_name}" + value = module.buyer_frontend_load_balancing.frontend_url description = "This is the globally load-balanced entrypoint to the buyer frontend service, to be used by seller front ends." } diff --git a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf index 61d84a5c..f67d088a 100644 --- a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf @@ -13,10 +13,67 @@ # limitations under the License. locals { - gcp_project_id = "" # Example: "your-gcp-project-123" - environment = "" # # Must be <= 3 characters. Example: "abc" - image_repo = "" # Example: "us-docker.pkg.dev/your-gcp-project-123/services" + gcp_project_id = "" # Example: "your-gcp-project-123" + environment = "" # Must be <= 3 characters. Example: "abc" + image_repo = "" # Example: "us-docker.pkg.dev/your-gcp-project-123/services" + seller_operator = "" # Example: "seller-1" + default_region_config = { + # Example config provided for us-central1 and you may add your own regions. + "us-central1" = { + collector = { + machine_type = "e2-micro" + min_replicas = 1 + max_replicas = 1 + zones = null # Null signifies no zone preference. + max_rate_per_instance = null # Null signifies no max. + } + backend = { + machine_type = "n2d-standard-64" + min_replicas = 1 + max_replicas = 5 + zones = null # Null signifies no zone preference. + max_rate_per_instance = null # Null signifies no max. + } + frontend = { + machine_type = "n2d-standard-64" + min_replicas = 1 + max_replicas = 2 + zones = null # Null signifies no zone preference. + max_rate_per_instance = null # Null signifies no max. + } + } + } + # Please create a Google Cloud domain name, dns zone, and SSL certificate. + # See demo/project_setup_utils/domain_setup/README.md for more details. + # If you specify a certificate_map_id, you do not need to specify an ssl_certificate_id. + frontend_domain_ssl_certificate_id = "" # Example: "projects/${local.gcp_project_id}/global/sslCertificates/sfe-${local.environment}" + frontend_certificate_map_id = "" # Example: "//certificatemanager.googleapis.com/projects/test/locations/global/certificateMaps/wildcard-cert-map" + seller_domain_name = "" # Example: sfe-gcp.com + frontend_dns_zone = "" # Example: "sfe-gcp-com" + + seller_traffic_splits = { + # default + "${local.environment}" = { + image_tag = "" # Image built and uploaded by production/packaging/build_and_test_all_in_docker + traffic_weight = 1000 # traffic_weight for this arm, between 0~1000. default's weight must > 0. + region_config = local.default_region_config + runtime_flag_override = {} + } + + # optional: experiment arm 1 + "${local.environment}-1" = { + image_tag = "" # Image built and uploaded by production/packaging/build_and_test_all_in_docker + traffic_weight = 0 # traffic_weight for this arm, between 0~1000. Arm with 0 weight will be skipped. + region_config = local.default_region_config # region config can be different for each arm + runtime_flag_override = { + # Example: MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "12345" + } + } + + # optional: more experiment arm + # "${local.environment}-2" = ... + } } provider "google" { @@ -40,14 +97,15 @@ module "secrets" { } module "seller" { + for_each = { for key, value in local.seller_traffic_splits : key => value if value.traffic_weight > 0 } source = "../../../modules/seller" - environment = local.environment + environment = each.key gcp_project_id = local.gcp_project_id - auction_image = "${local.image_repo}/auction_service:${local.environment}" # Image built and uploaded by production/packaging/build_and_test_all_in_docker - seller_frontend_image = "${local.image_repo}/seller_frontend_service:${local.environment}" # Image built and uploaded by production/packaging/build_and_test_all_in_docker + auction_image = "${local.image_repo}/auction_service:${each.value.image_tag}" + seller_frontend_image = "${local.image_repo}/seller_frontend_service:${each.value.image_tag}" envoy_port = 51052 # Do not change. Must match production/packaging/gcp/seller_frontend_service/bin/envoy.yaml - runtime_flags = { + runtime_flags = merge({ SELLER_FRONTEND_PORT = "50051" # Do not change unless you are modifying the default GCP architecture. SELLER_FRONTEND_HEALTHCHECK_PORT = "50050" # Do not change unless you are modifying the default GCP architecture. AUCTION_PORT = "50051" # Do not change unless you are modifying the default GCP architecture. @@ -92,7 +150,7 @@ module "seller" { JS_WORKER_QUEUE_LEN = "" # Example: "200". ROMA_TIMEOUT_MS = "" # Example: "10000" TELEMETRY_CONFIG = "" # Example: "mode: EXPERIMENT" - COLLECTOR_ENDPOINT = "" # Example: "collector-seller-1-${local.environment}.sfe-gcp.com:4317" + COLLECTOR_ENDPOINT = "" # Example: "collector-seller-1-${each.key}.sfe-gcp.com:4317" ENABLE_OTEL_BASED_LOGGING = "" # Example: "false" CONSENTED_DEBUG_TOKEN = "" # Example: "" ENABLE_REPORT_WIN_INPUT_NOISING = "" # Example: "true" @@ -132,17 +190,11 @@ module "seller" { AUCTION_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "10737418240" SFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND = "4096" SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "10737418240" - } + }, each.value.runtime_flag_override) - # Please create a Google Cloud domain name, dns zone, and SSL certificate. - # See demo/project_setup_utils/domain_setup/README.md for more details. - # If you specify a certificate_map_id, you do not need to specify an ssl_certificate_id. - frontend_domain_name = "" # Example: sfe-gcp.com - frontend_dns_zone = "" # Example: "sfe-gcp-com" - frontend_domain_ssl_certificate_id = "" # Example: "projects/${local.gcp_project_id}/global/sslCertificates/sfe-${local.environment}" - frontend_certificate_map_id = "" # Example: "//certificatemanager.googleapis.com/projects/test/locations/global/certificateMaps/wildcard-cert-map" - - operator = "" # Example: "seller-1" + frontend_domain_name = local.seller_domain_name + frontend_dns_zone = local.frontend_dns_zone + operator = local.seller_operator service_account_email = "" # Example: "terraform-sa@{local.gcp_project_id}.iam.gserviceaccount.com" vm_startup_delay_seconds = 200 # Example: 200 cpu_utilization_percent = 0.6 # Example: 0.6 @@ -155,34 +207,32 @@ module "seller" { gcs_hmac_key = module.secrets.gcs_hmac_key gcs_hmac_secret = module.secrets.gcs_hmac_secret gcs_bucket = "" # Example: ${name of a gcs bucket} - gcs_bucket_prefix = "" # Example: "consented-eventmessage-${local.environment}" - file_prefix = "" # Example: "operator-name" + gcs_bucket_prefix = "" # Example: "consented-eventmessage-${each.key}" + file_prefix = "" # Example: local.seller_operator }) - region_config = { - # Example config provided for us-central1 and you may add your own regions. - "us-central1" = { - collector = { - machine_type = "e2-micro" - min_replicas = 1 - max_replicas = 1 - zones = null # Null signifies no zone preference. - max_rate_per_instance = null # Null signifies no max. - } - backend = { - machine_type = "n2d-standard-64" - min_replicas = 1 - max_replicas = 5 - zones = null # Null signifies no zone preference. - max_rate_per_instance = null # Null signifies no max. - } - frontend = { - machine_type = "n2d-standard-64" - min_replicas = 1 - max_replicas = 2 - zones = null # Null signifies no zone preference. - max_rate_per_instance = null # Null signifies no max. - } - } - } + region_config = each.value.region_config enable_tee_container_log_redirect = false } + +module "seller_frontend_load_balancing" { + source = "../../services/frontend_load_balancing" + environment = local.environment + operator = local.seller_operator + frontend_ip_address = module.seller[local.environment].frontend_address + frontend_domain_name = local.seller_domain_name + frontend_dns_zone = local.frontend_dns_zone + + frontend_domain_ssl_certificate_id = local.frontend_domain_ssl_certificate_id + frontend_certificate_map_id = local.frontend_certificate_map_id + frontend_service_name = "sfe" + google_compute_backend_service_ids = { + for seller_key, seller in module.seller : + seller_key => seller.google_compute_backend_service_id + } + traffic_weights = { for key, value in local.seller_traffic_splits : key => value.traffic_weight } +} + +module "seller_dashboard" { + source = "../../services/dashboards/seller_dashboard" + environment = join("|", [for k, v in local.seller_traffic_splits : k if v.traffic_weight > 0]) +} diff --git a/production/deploy/gcp/terraform/environment/demo/seller/seller_outputs.tf b/production/deploy/gcp/terraform/environment/demo/seller/seller_outputs.tf index 7dbe4deb..8262dd5b 100644 --- a/production/deploy/gcp/terraform/environment/demo/seller/seller_outputs.tf +++ b/production/deploy/gcp/terraform/environment/demo/seller/seller_outputs.tf @@ -13,6 +13,6 @@ # limitations under the License. output "seller_frontend_url" { - value = "https://${module.seller.operator}-${module.seller.environment}.${module.seller.frontend_domain_name}" + value = module.seller_frontend_load_balancing.frontend_url description = "This is the globally load-balanced entrypoint to the seller frontend service, to be used by seller ad services." } diff --git a/production/deploy/gcp/terraform/modules/buyer/outputs.tf b/production/deploy/gcp/terraform/modules/buyer/outputs.tf new file mode 100644 index 00000000..28a3ec8d --- /dev/null +++ b/production/deploy/gcp/terraform/modules/buyer/outputs.tf @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "google_compute_backend_service_id" { + value = module.load_balancing.google_compute_backend_service_id +} + +output "frontend_address" { + value = module.networking.frontend_address +} diff --git a/production/deploy/gcp/terraform/modules/buyer/service.tf b/production/deploy/gcp/terraform/modules/buyer/service.tf index a0e85b72..598f7237 100644 --- a/production/deploy/gcp/terraform/modules/buyer/service.tf +++ b/production/deploy/gcp/terraform/modules/buyer/service.tf @@ -66,39 +66,25 @@ module "autoscaling" { } module "load_balancing" { - source = "../../services/load_balancing" - environment = var.environment - operator = var.operator - gcp_project_id = var.gcp_project_id - subnets = module.networking.subnets - mesh = module.networking.mesh - frontend_ip_address = module.networking.frontend_address - frontend_domain_name = var.frontend_domain_name - frontend_dns_zone = var.frontend_dns_zone - frontend_domain_ssl_certificate_id = var.frontend_domain_ssl_certificate_id - frontend_certificate_map_id = var.frontend_certificate_map_id - frontend_instance_group_managers = module.autoscaling.frontend_instance_group_managers - frontend_service_name = "bfe" - frontend_service_port = tonumber(var.runtime_flags["BUYER_FRONTEND_PORT"]) - frontend_service_healthcheck_port = tonumber(var.runtime_flags["BUYER_FRONTEND_HEALTHCHECK_PORT"]) - backend_instance_group_managers = module.autoscaling.backend_instance_group_managers - backend_service_name = "bidding" - backend_address = var.runtime_flags["BIDDING_SERVER_ADDR"] - backend_service_port = tonumber(var.runtime_flags["BIDDING_PORT"]) - collector_instance_group_managers = module.autoscaling.collector_instance_group_managers - collector_service_name = "collector" - collector_service_port = var.collector_service_port - region_config = var.region_config -} - -module "buyer_dashboard" { - source = "../../services/dashboards/buyer_dashboard" - environment = var.environment -} - -module "inference_dashboard" { - source = "../../services/dashboards/inference_dashboard" - environment = var.environment + source = "../../services/load_balancing" + environment = var.environment + operator = var.operator + gcp_project_id = var.gcp_project_id + subnets = module.networking.subnets + mesh = module.networking.mesh + frontend_domain_name = var.frontend_domain_name + frontend_dns_zone = var.frontend_dns_zone + frontend_instance_group_managers = module.autoscaling.frontend_instance_group_managers + frontend_service_name = "bfe" + frontend_service_healthcheck_port = tonumber(var.runtime_flags["BUYER_FRONTEND_HEALTHCHECK_PORT"]) + backend_instance_group_managers = module.autoscaling.backend_instance_group_managers + backend_service_name = "bidding" + backend_address = var.runtime_flags["BIDDING_SERVER_ADDR"] + backend_service_port = tonumber(var.runtime_flags["BIDDING_PORT"]) + collector_instance_group_managers = module.autoscaling.collector_instance_group_managers + collector_service_name = "collector" + collector_service_port = var.collector_service_port + region_config = var.region_config } resource "google_secret_manager_secret" "runtime_flag_secrets" { diff --git a/production/deploy/gcp/terraform/modules/buyer/service_vars.tf b/production/deploy/gcp/terraform/modules/buyer/service_vars.tf index d1eb3dc0..a923ed70 100644 --- a/production/deploy/gcp/terraform/modules/buyer/service_vars.tf +++ b/production/deploy/gcp/terraform/modules/buyer/service_vars.tf @@ -86,18 +86,6 @@ variable "frontend_dns_zone" { type = string } -variable "frontend_domain_ssl_certificate_id" { - description = "A GCP ssl certificate id. Example: projects/test-project/global/sslCertificates/dev. Used to terminate client-to-external-LB connections. Unused if frontend_certificate_map_id is specified." - type = string - default = "" -} - -variable "frontend_certificate_map_id" { - description = "A certificate manager certificate map resource id. Example: projects/test-project/locations/global/certificateMaps/wildcard-cert-map. Takes precedence over frontend_domain_ssl_certificate_id." - type = string - default = "" -} - variable "vm_startup_delay_seconds" { description = "The time it takes to get a service up and responding to heartbeats (in seconds)." type = number diff --git a/production/deploy/gcp/terraform/modules/seller/outputs.tf b/production/deploy/gcp/terraform/modules/seller/outputs.tf new file mode 100644 index 00000000..28a3ec8d --- /dev/null +++ b/production/deploy/gcp/terraform/modules/seller/outputs.tf @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "google_compute_backend_service_id" { + value = module.load_balancing.google_compute_backend_service_id +} + +output "frontend_address" { + value = module.networking.frontend_address +} diff --git a/production/deploy/gcp/terraform/modules/seller/service.tf b/production/deploy/gcp/terraform/modules/seller/service.tf index 15335add..1e3900a0 100644 --- a/production/deploy/gcp/terraform/modules/seller/service.tf +++ b/production/deploy/gcp/terraform/modules/seller/service.tf @@ -64,34 +64,25 @@ module "autoscaling" { } module "load_balancing" { - source = "../../services/load_balancing" - environment = var.environment - operator = var.operator - gcp_project_id = var.gcp_project_id - subnets = module.networking.subnets - mesh = module.networking.mesh - frontend_ip_address = module.networking.frontend_address - frontend_domain_name = var.frontend_domain_name - frontend_dns_zone = var.frontend_dns_zone - frontend_domain_ssl_certificate_id = var.frontend_domain_ssl_certificate_id - frontend_certificate_map_id = var.frontend_certificate_map_id - frontend_instance_group_managers = module.autoscaling.frontend_instance_group_managers - frontend_service_name = "sfe" - frontend_service_port = var.envoy_port - frontend_service_healthcheck_port = tonumber(var.runtime_flags["SELLER_FRONTEND_HEALTHCHECK_PORT"]) - backend_instance_group_managers = module.autoscaling.backend_instance_group_managers - backend_address = var.runtime_flags["AUCTION_SERVER_HOST"] - backend_service_name = "auction" - backend_service_port = tonumber(var.runtime_flags["AUCTION_PORT"]) - collector_instance_group_managers = module.autoscaling.collector_instance_group_managers - collector_service_name = "collector" - collector_service_port = var.collector_service_port - region_config = var.region_config -} - -module "seller_dashboard" { - source = "../../services/dashboards/seller_dashboard" - environment = var.environment + source = "../../services/load_balancing" + environment = var.environment + operator = var.operator + gcp_project_id = var.gcp_project_id + subnets = module.networking.subnets + mesh = module.networking.mesh + frontend_domain_name = var.frontend_domain_name + frontend_dns_zone = var.frontend_dns_zone + frontend_instance_group_managers = module.autoscaling.frontend_instance_group_managers + frontend_service_name = "sfe" + frontend_service_healthcheck_port = tonumber(var.runtime_flags["SELLER_FRONTEND_HEALTHCHECK_PORT"]) + backend_instance_group_managers = module.autoscaling.backend_instance_group_managers + backend_address = var.runtime_flags["AUCTION_SERVER_HOST"] + backend_service_name = "auction" + backend_service_port = tonumber(var.runtime_flags["AUCTION_PORT"]) + collector_instance_group_managers = module.autoscaling.collector_instance_group_managers + collector_service_name = "collector" + collector_service_port = var.collector_service_port + region_config = var.region_config } resource "google_secret_manager_secret" "runtime_flag_secrets" { diff --git a/production/deploy/gcp/terraform/modules/seller/service_vars.tf b/production/deploy/gcp/terraform/modules/seller/service_vars.tf index 86315a4f..709fa8d1 100644 --- a/production/deploy/gcp/terraform/modules/seller/service_vars.tf +++ b/production/deploy/gcp/terraform/modules/seller/service_vars.tf @@ -85,19 +85,6 @@ variable "frontend_dns_zone" { type = string } -variable "frontend_domain_ssl_certificate_id" { - description = "A GCP ssl certificate id. Example: projects/test-project/global/sslCertificates/dev. Used to terminate client-to-external-LB connections. Unused if frontend_certificate_map_id is specified." - type = string - default = "" -} - -variable "frontend_certificate_map_id" { - description = "A certificate manager certificate map resource id. Example: projects/test-project/locations/global/certificateMaps/wildcard-cert-map. Takes precedence over frontend_domain_ssl_certificate_id." - type = string - default = "" -} - - variable "vm_startup_delay_seconds" { description = "The time it takes to get a service up and responding to heartbeats (in seconds)." type = number diff --git a/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf index 183561d1..b5bb321a 100644 --- a/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf @@ -36,7 +36,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "60s", "crossSeriesReducer": "REDUCE_MEAN", @@ -88,7 +88,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"!=\"total cpu cores\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"!=\"total cpu cores\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -128,7 +128,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -167,7 +167,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"main process\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"main process\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -208,7 +208,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"MemAvailable:\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"MemAvailable:\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -247,7 +247,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/request.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/request.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -287,7 +287,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/response.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/response.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -303,7 +303,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { { "height": 19, "widget": { - "title": "js_execution.duration_ms [95TH PERCENTILE]", + "title": "udf_execution.duration_ms [95TH PERCENTILE]", "xyChart": { "chartOptions": {}, "dataSets": [ @@ -326,7 +326,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/js_execution.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/udf_execution.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -356,7 +356,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -387,7 +387,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { { "height": 19, "widget": { - "title": "js_execution.errors_count [MEAN]", + "title": "udf_execution.errors_count [MEAN]", "xyChart": { "chartOptions": {}, "dataSets": [ @@ -401,7 +401,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/js_execution.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/udf_execution.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -445,7 +445,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.bids_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.bids_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -488,7 +488,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.zero_bid_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.zero_bid_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -541,7 +541,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -572,7 +572,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/initiated_request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -625,7 +625,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.zero_bid_percent\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.zero_bid_percent\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -664,7 +664,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -703,7 +703,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -742,7 +742,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\" metric.label.\"service_name\"=\"bfe\"" + "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\") metric.label.\"service_name\"=\"bfe\"" } } } @@ -782,7 +782,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_response.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\" metric.label.\"service_name\"=\"bfe\"" + "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_response.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\") metric.label.\"service_name\"=\"bfe\"" } } } @@ -813,7 +813,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/bfe.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/bfe.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -857,7 +857,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/bidding.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/bidding.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -902,7 +902,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -946,7 +946,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/request.failed_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/request.failed_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "60s", "crossSeriesReducer": "REDUCE_MEAN", @@ -999,7 +999,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_response.to_bidding.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_response.to_bidding.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1039,7 +1039,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.thread.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.thread.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1080,7 +1080,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.failure_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.failure_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1121,7 +1121,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_parsed_on_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_parsed_on_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1161,7 +1161,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_cached_after_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_cached_after_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1201,7 +1201,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"=\"total cpu cores\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"=\"total cpu cores\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1240,7 +1240,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bidding.inference.request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/bidding.inference.request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } diff --git a/production/deploy/gcp/terraform/services/dashboards/inference_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/inference_dashboard/main.tf index 285d6062..420adace 100644 --- a/production/deploy/gcp/terraform/services/dashboards/inference_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/inference_dashboard/main.tf @@ -46,7 +46,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"!=\"total cpu cores\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"!=\"total cpu cores\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -86,7 +86,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"main process\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"main process\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -99,6 +99,45 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "width": 24, "yPos": 57 }, + { + "height": 19, + "widget": { + "title": "system.memory.usage_kb for inference process [MEAN]", + "xyChart": { + "chartOptions": {}, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"service_name\"", + "metric.label.\"deployment_environment\"", + "metric.label.\"operator\"", + "metric.label.\"Noise\"", + "resource.label.\"task_id\"", + "metric.label.\"service_version\"" + ], + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"inference process\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + } + } + } + ], + "yAxis": { + "scale": "LINEAR" + } + } + }, + "width": 24, + "yPos": 209 + }, { "height": 19, "widget": { @@ -126,7 +165,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"MemAvailable:\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"MemAvailable:\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -167,7 +206,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.thread.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.thread.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -206,7 +245,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"=\"total cpu cores\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"=\"total cpu cores\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -237,7 +276,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/inference.request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -288,7 +327,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -327,7 +366,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.request.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -366,7 +405,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/inference.response.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.response.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -397,7 +436,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.failed_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/inference.request.failed_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -441,7 +480,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/inference.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/inference.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -485,7 +524,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/inference.request.count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -539,7 +578,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.duration_ms_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.request.duration_ms_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -570,7 +609,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.failed_count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/inference.request.failed_count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -614,7 +653,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/inference.request.batch_count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/inference.request.batch_count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -668,7 +707,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/inference.cloud_fetch.success_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.cloud_fetch.success_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -708,7 +747,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/inference.cloud_fetch.failed_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.cloud_fetch.failed_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -749,7 +788,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.recent_success_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.recent_success_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -788,7 +827,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.failed_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.failed_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -829,7 +868,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.recent_failure_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.recent_failure_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -869,7 +908,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.available_models\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/inference.model_registration.available_models\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -910,7 +949,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/bidding.inference.request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/bidding.inference.request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } diff --git a/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf index 5277a446..ab74515d 100644 --- a/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf @@ -36,7 +36,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "60s", "crossSeriesReducer": "REDUCE_MEAN", @@ -88,7 +88,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"!=\"total cpu cores\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"!=\"total cpu cores\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -128,7 +128,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/request.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -167,7 +167,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"main process\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"main process\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -208,7 +208,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"MemAvailable:\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.memory.usage_kb\" resource.type=\"generic_task\" metric.label.\"label\"=\"MemAvailable:\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -247,7 +247,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/request.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/request.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -287,7 +287,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/response.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/response.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -303,7 +303,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { { "height": 19, "widget": { - "title": "js_execution.duration_ms [95TH PERCENTILE]", + "title": "udf_execution.duration_ms [95TH PERCENTILE]", "xyChart": { "chartOptions": {}, "dataSets": [ @@ -326,7 +326,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/js_execution.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/udf_execution.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -356,7 +356,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.errors_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.errors_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -387,7 +387,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { { "height": 19, "widget": { - "title": "js_execution.errors_count [MEAN]", + "title": "udf_execution.errors_count [MEAN]", "xyChart": { "chartOptions": {}, "dataSets": [ @@ -401,7 +401,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/js_execution.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/udf_execution.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -445,7 +445,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bids_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bids_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -488,7 +488,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bid_rejected_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bid_rejected_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -542,7 +542,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -572,7 +572,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/initiated_request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -625,7 +625,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bid_rejected_percent\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bid_rejected_percent\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -664,7 +664,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -703,7 +703,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -742,7 +742,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\" metric.label.\"service_name\"=\"sfe\"" + "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\") metric.label.\"service_name\"=\"sfe\"" } } } @@ -781,7 +781,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\" metric.label.\"service_name\"=\"sfe\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_kv.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\") metric.label.\"service_name\"=\"sfe\"" } } } @@ -811,7 +811,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -855,7 +855,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/auction.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/auction.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -900,7 +900,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -945,7 +945,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -990,7 +990,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/request.failed_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/request.failed_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "60s", "crossSeriesReducer": "REDUCE_MEAN", @@ -1043,7 +1043,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_auction.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_auction.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1084,7 +1084,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.thread.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.thread.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1115,7 +1115,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.errors_count_by_buyer\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.errors_count_by_buyer\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -1160,7 +1160,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -1199,7 +1199,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "timeSeriesQueryLanguage": "fetch generic_task\n| { metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.duration_ms'\n ; metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.count' }\n| filter (metric.deployment_environment == '${var.environment}')\n| group_by\n [metric.buyer, metric.service_name, metric.deployment_environment,\n metric.operator, metric.Noise, resource.task_id, metric.service_version]\n| align rate(1m)\n| outer_join 0\n| div" + "timeSeriesQueryLanguage": "fetch generic_task\n| { metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.duration_ms'\n ; metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.count' }\n| filter (metric.deployment_environment =~ '${var.environment}')\n| group_by\n [metric.buyer, metric.service_name, metric.deployment_environment,\n metric.operator, metric.Noise, resource.task_id, metric.service_version]\n| align rate(1m)\n| outer_join 0\n| div" } } ], @@ -1228,7 +1228,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -1272,7 +1272,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_bfe.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_bfe.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -1327,7 +1327,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.failure_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.failure_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1368,7 +1368,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_parsed_on_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_parsed_on_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1409,7 +1409,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_cached_after_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.key_fetch.num_keys_cached_after_recent_fetch\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1448,7 +1448,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"=\"total cpu cores\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/system.cpu.percent\" resource.type=\"generic_task\" metric.label.\"label\"=\"total cpu cores\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1478,7 +1478,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.business_logic.request_with_winner_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "filter": "metric.type=\"workload.googleapis.com/sfe.business_logic.request_with_winner_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")", "secondaryAggregation": { "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", @@ -1531,7 +1531,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.business_logic.request_with_winner.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.business_logic.request_with_winner.duration_ms\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1571,7 +1571,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.protected_ciphertext.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.protected_ciphertext.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } @@ -1611,7 +1611,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { ], "perSeriesAligner": "ALIGN_DELTA" }, - "filter": "metric.type=\"workload.googleapis.com/sfe.auction_config.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + "filter": "metric.type=\"workload.googleapis.com/sfe.auction_config.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=monitoring.regex.full_match(\"${var.environment}\")" } } } diff --git a/production/deploy/gcp/terraform/services/frontend_load_balancing/main.tf b/production/deploy/gcp/terraform/services/frontend_load_balancing/main.tf new file mode 100644 index 00000000..d5f4cdc5 --- /dev/null +++ b/production/deploy/gcp/terraform/services/frontend_load_balancing/main.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +############################################################### +# +# EXTERNAL LB +# +# The external lb uses HTTP/2 (gRPC) with TLS. +############################################################### + +resource "google_compute_url_map" "default" { + name = "${var.operator}-${var.environment}-xlb-grpc-map" + default_route_action { + dynamic "weighted_backend_services" { + for_each = var.google_compute_backend_service_ids + content { + backend_service = weighted_backend_services.value + weight = var.traffic_weights[weighted_backend_services.key] + } + } + } +} + +resource "google_compute_target_https_proxy" "default" { + name = "${var.operator}-${var.environment}-https-lb-proxy" + url_map = google_compute_url_map.default.id + ssl_certificates = var.frontend_certificate_map_id == "" ? [var.frontend_domain_ssl_certificate_id] : null + certificate_map = var.frontend_certificate_map_id == "" ? null : var.frontend_certificate_map_id +} + +resource "google_compute_global_forwarding_rule" "xlb_https" { + name = "${var.operator}-${var.environment}-xlb-https-forwarding-rule" + provider = google-beta + + ip_protocol = "TCP" + port_range = "443" + load_balancing_scheme = "EXTERNAL_MANAGED" + target = google_compute_target_https_proxy.default.id + ip_address = var.frontend_ip_address + + labels = { + environment = var.environment + operator = var.operator + service = var.frontend_service_name + } +} + +resource "google_dns_record_set" "default" { + name = "${var.operator}-${var.environment}.${var.frontend_domain_name}." + managed_zone = var.frontend_dns_zone + type = "A" + ttl = 10 + rrdatas = [ + var.frontend_ip_address + ] +} diff --git a/production/deploy/gcp/terraform/services/frontend_load_balancing/outputs.tf b/production/deploy/gcp/terraform/services/frontend_load_balancing/outputs.tf new file mode 100644 index 00000000..dfe7033b --- /dev/null +++ b/production/deploy/gcp/terraform/services/frontend_load_balancing/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "frontend_url" { + value = "https://${trim(google_dns_record_set.default.name, ".")}" + description = "This is the globally load-balanced entrypoint to the frontend service." +} diff --git a/production/deploy/gcp/terraform/services/frontend_load_balancing/variables.tf b/production/deploy/gcp/terraform/services/frontend_load_balancing/variables.tf new file mode 100644 index 00000000..d4fcf215 --- /dev/null +++ b/production/deploy/gcp/terraform/services/frontend_load_balancing/variables.tf @@ -0,0 +1,66 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "operator" { + description = "Operator name used to identify the resource owner." + type = string +} + +variable "environment" { + description = "Assigned environment name to group related resources." + type = string +} + +variable "frontend_domain_name" { + description = "Domain name for global external LB" + type = string +} + +variable "frontend_dns_zone" { + description = "DNS zone for the frontend domain" + type = string +} + +variable "frontend_ip_address" { + description = "Frontend ip address" + type = string +} +variable "frontend_domain_ssl_certificate_id" { + description = "A GCP ssl certificate id. Example: projects/test-project/global/sslCertificates/dev. Used to terminate client-to-external-LB connections. Unused if frontend_certificate_map_id is specified." + type = string + default = "" +} + +variable "frontend_certificate_map_id" { + description = "A certificate manager certificate map resource id. Example: projects/test-project/locations/global/certificateMaps/wildcard-cert-map. Takes precedence over frontend_domain_ssl_certificate_id." + type = string + default = "" +} + +variable "frontend_service_name" { + type = string +} + + +variable "google_compute_backend_service_ids" { + description = "a map with environment as key, the value is google_compute_backend_service_id" + type = map(string) +} + +variable "traffic_weights" { + description = "a map with environment as key, the value is traffic_weight between 0~1000" + type = map(number) +} diff --git a/production/deploy/gcp/terraform/services/load_balancing/main.tf b/production/deploy/gcp/terraform/services/load_balancing/main.tf index 20e4ddcb..8279a9bd 100644 --- a/production/deploy/gcp/terraform/services/load_balancing/main.tf +++ b/production/deploy/gcp/terraform/services/load_balancing/main.tf @@ -193,45 +193,6 @@ resource "google_compute_backend_service" "default" { depends_on = [var.mesh, google_network_services_grpc_route.default] } -resource "google_compute_url_map" "default" { - name = "${var.operator}-${var.environment}-xlb-grpc-map" - default_service = google_compute_backend_service.default.id -} - - -resource "google_compute_target_https_proxy" "default" { - name = "${var.operator}-${var.environment}-https-lb-proxy" - url_map = google_compute_url_map.default.id - ssl_certificates = var.frontend_certificate_map_id == "" ? [var.frontend_domain_ssl_certificate_id] : null - certificate_map = var.frontend_certificate_map_id == "" ? null : var.frontend_certificate_map_id -} - -resource "google_compute_global_forwarding_rule" "xlb_https" { - name = "${var.operator}-${var.environment}-xlb-https-forwarding-rule" - provider = google-beta - - ip_protocol = "TCP" - port_range = "443" - load_balancing_scheme = "EXTERNAL_MANAGED" - target = google_compute_target_https_proxy.default.id - ip_address = var.frontend_ip_address - - labels = { - environment = var.environment - operator = var.operator - service = var.frontend_service_name - } -} - -resource "google_dns_record_set" "default" { - name = "${var.operator}-${var.environment}.${var.frontend_domain_name}." - managed_zone = var.frontend_dns_zone - type = "A" - ttl = 10 - rrdatas = [ - var.frontend_ip_address - ] -} resource "google_compute_health_check" "frontend" { name = "${var.operator}-${var.environment}-${var.frontend_service_name}-lb-hc" diff --git a/production/deploy/gcp/terraform/services/load_balancing/outputs.tf b/production/deploy/gcp/terraform/services/load_balancing/outputs.tf index 8190cc90..df603770 100644 --- a/production/deploy/gcp/terraform/services/load_balancing/outputs.tf +++ b/production/deploy/gcp/terraform/services/load_balancing/outputs.tf @@ -13,3 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +output "google_compute_backend_service_id" { + value = google_compute_backend_service.default.id +} diff --git a/production/deploy/gcp/terraform/services/load_balancing/variables.tf b/production/deploy/gcp/terraform/services/load_balancing/variables.tf index ce248594..4f1f0c2b 100644 --- a/production/deploy/gcp/terraform/services/load_balancing/variables.tf +++ b/production/deploy/gcp/terraform/services/load_balancing/variables.tf @@ -49,22 +49,6 @@ variable "frontend_dns_zone" { type = string } -variable "frontend_ip_address" { - description = "Frontend ip address" - type = string -} -variable "frontend_domain_ssl_certificate_id" { - description = "A GCP ssl certificate id. Example: projects/test-project/global/sslCertificates/dev. Used to terminate client-to-external-LB connections. Unused if frontend_certificate_map_id is specified." - type = string - default = "" -} - -variable "frontend_certificate_map_id" { - description = "A certificate manager certificate map resource id. Example: projects/test-project/locations/global/certificateMaps/wildcard-cert-map. Takes precedence over frontend_domain_ssl_certificate_id." - type = string - default = "" -} - variable "frontend_instance_group_managers" { description = "Frontend instance group managers." type = set(any) @@ -74,11 +58,6 @@ variable "frontend_service_name" { type = string } -variable "frontend_service_port" { - description = "The grpc port that receives traffic destined for the frontend service." - type = number -} - variable "frontend_service_healthcheck_port" { description = "The Non-TLS grpc port that receives healthcheck traffic." type = number diff --git a/production/packaging/aws/build_and_test b/production/packaging/aws/build_and_test index c6d4fe9a..d73ad460 100755 --- a/production/packaging/aws/build_and_test +++ b/production/packaging/aws/build_and_test @@ -102,7 +102,7 @@ DIST="${WORKSPACE}"/dist mkdir -p "${DIST}"/aws chmod 770 "${DIST}" "${DIST}"/aws -printf "==== build AWS artifacts using build-amazonlinux2 =====\n" +printf "==== build AWS artifacts using build-amazonlinux2023 =====\n" declare server_image="" declare image_uri="" declare image_tag="" @@ -117,7 +117,7 @@ for service in "${SERVICE_LIST[@]}"; do image_uri="bazel/production/packaging/aws/${service}" image_tag=$(mktemp --dry-run temp-XXXXXX) - builder::cbuild_al2 $" + builder::cbuild_al2023 $" set -o errexit # extract server docker image into local docker client and retag it docker load -i ${server_image} @@ -134,7 +134,7 @@ for service in "${SERVICE_LIST[@]}"; do docker image rm "${image_uri}:${image_tag}" done -builder::cbuild_al2 $" +builder::cbuild_al2023 $" trap _collect_logs EXIT function _collect_logs() { local -r -i STATUS=\$? @@ -169,9 +169,12 @@ if [[ -n ${AMI_REGIONS[0]} ]]; then printf "==== build AWS AMI (using packer) =====\n" regions="$(arr_to_string_list AMI_REGIONS)" build_version="$(git -C ${WORKSPACE} describe --tags --always || echo no-git-version)" + + packer_pids=() + for service in "${SERVICE_LIST[@]}"; do log_file="${WORKSPACE}/dist/${service}_aws_build_log.txt" - (builder::cbuild_al2 " + (builder::cbuild_al2023 " set -o errexit packer build \ -var=regions='${regions}' \ @@ -184,6 +187,24 @@ if [[ -n ${AMI_REGIONS[0]} ]]; then -var=workspace=/src/workspace \ production/packaging/aws/common/ami/image.pkr.hcl " 2>&1 >> "$log_file") & + packer_pids+=($!) # Store the packer build process ID + done + + failed=0 + i=0 + for pid in "${packer_pids[@]}"; do + wait "${pid}" || { + built_service="${SERVICE_LIST[${i}]}" + service_build_logs="${WORKSPACE}/dist/${built_service}_aws_build_log.txt" + echo "----- Log output for $built_service -----" >&2 + cat "${service_build_logs}" >&2 + echo "Error: packer build failed for process ${pid} (service: ${built_service})." >&2 + failed=1 + } + i=$((i+1)) done - wait + + if [[ $failed -eq 1 ]]; then + exit 1 + fi fi diff --git a/production/packaging/aws/common/ami/debug/otel_collector_config.yaml b/production/packaging/aws/common/ami/debug/otel_collector_config.yaml index f533ce4d..98417a4d 100644 --- a/production/packaging/aws/common/ami/debug/otel_collector_config.yaml +++ b/production/packaging/aws/common/ami/debug/otel_collector_config.yaml @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# OTEL_S3_BUCKET and OTEL_S3_PREFIX will be set as environment variables. +extensions: + health_check: receivers: otlp: @@ -20,19 +21,61 @@ receivers: grpc: endpoint: 127.0.0.1:4317 +processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s + batch/logs: + timeout: 60s + filter/drop_event: + error_mode: ignore + logs: + log_record: + - 'attributes["ps_tee_log_type"] == "event_message"' + filter/drop_non_event: + error_mode: ignore + logs: + log_record: + - 'attributes["ps_tee_log_type"] != "event_message"' + exporters: - awss3: + awsemf: + namespace: '$SERVICE' + resource_to_telemetry_conversion: + enabled: true + awscloudwatchlogs: + log_group_name: "bidding-auction" + log_stream_name: "consented-log-stream" + awss3/consent: + s3uploader: + region: $S3_REGION + s3_bucket: $S3_BUCKET + s3_prefix: $S3_PREFIX + s3_partition: minute + file_prefix: $FILE_PREFIX + marshaler: body + awss3/perfgate: s3uploader: - region: 'us-east-1' - s3_bucket: ${env:OTEL_S3_BUCKET} - s3_prefix: ${env:OTEL_S3_PREFIX} + region: 'us-west-1' + s3_bucket: 'bna-test-perfgate' + s3_prefix: 'e2e-build' s3_partition: 'minute' service: pipelines: - traces: - receivers: [otlp] - exporters: [awss3] metrics: receivers: [otlp] - exporters: [awss3] + processors: [batch/metrics] + exporters: [awsemf, awss3/perfgate] + logs/1: + receivers: [otlp] + processors: [batch/logs, filter/drop_event] + exporters: [awscloudwatchlogs] + logs/2: + receivers: [otlp] + processors: [filter/drop_non_event] + exporters: [awss3/consent] + + extensions: [health_check] diff --git a/production/packaging/aws/common/ami/image.pkr.hcl b/production/packaging/aws/common/ami/image.pkr.hcl index 9ce88be3..99fe29ff 100644 --- a/production/packaging/aws/common/ami/image.pkr.hcl +++ b/production/packaging/aws/common/ami/image.pkr.hcl @@ -90,7 +90,7 @@ source "amazon-ebs" "dataserver" { ami_regions = var.regions source_ami_filter { filters = { - name = "amzn2-ami-kernel-*-x86_64-gp2" + name = "al2023-ami-2023.6.20241010.0-kernel-6.1-x86_64" root-device-type = "ebs" virtualization-type = "hvm" } @@ -155,7 +155,7 @@ mv /tmp/envoy.yaml /etc/envoy/envoy.yaml chmod 555 /etc/envoy/{envoy.yaml,bidding_auction_servers_descriptor_set.pb} mv /tmp/vsockproxy.service /etc/systemd/system/vsockproxy.service -mkdir -p /opt/privacysandbox +sudo mkdir -p /opt/privacysandbox mv /tmp/proxy /opt/privacysandbox/proxy mv /tmp/server_enclave_image.eif /opt/privacysandbox/server_enclave_image.eif mv /tmp/otel_collector_config.yaml /opt/privacysandbox/otel_collector_config.yaml diff --git a/production/packaging/aws/common/ami/setup.sh b/production/packaging/aws/common/ami/setup.sh index 5f23a2bc..a1818091 100644 --- a/production/packaging/aws/common/ami/setup.sh +++ b/production/packaging/aws/common/ami/setup.sh @@ -16,11 +16,17 @@ set -o errexit set -o pipefail +declare -r arch=`arch` +if [[ ${arch} != "x86_64" && ${arch} != "aarch64" ]]; then + printf "Unrecognized or unsupported architecture for nitro: %s\n" "${arch}" &>/dev/stderr + return 1 +fi + # Install necessary dependencies -sudo yum update -y -sudo yum install -y \ - amazon-cloudwatch-agent \ - docker +sudo dnf update -y +sudo dnf install -y \ + "amazon-cloudwatch-agent-1.300044.0-1.amzn2023.${arch}" \ + "docker-25.0.6-1.amzn2023.0.2.${arch}" wget -O /tmp/otelcol-contrib_0.105.0_linux_amd64.rpm https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.105.0/otelcol-contrib_0.105.0_linux_amd64.rpm sudo yum localinstall -y /tmp/otelcol-contrib_0.105.0_linux_amd64.rpm @@ -28,7 +34,7 @@ ENV_FILE="/etc/otelcol-contrib/otelcol-contrib.conf" NEW_OTELCOL_OPTIONS="OTELCOL_OPTIONS=\"--config=/opt/privacysandbox/otel_collector_config.yaml\"" sudo bash -c "sudo sed -i 's|^OTELCOL_OPTIONS=.*|${NEW_OTELCOL_OPTIONS}|' $ENV_FILE" -sudo amazon-linux-extras install -y aws-nitro-enclaves-cli +sudo dnf install -y "aws-nitro-enclaves-cli-1.3.1-0.amzn2023.${arch}" sudo usermod -a -G docker ec2-user sudo usermod -a -G ne ec2-user diff --git a/services/auction_service/BUILD b/services/auction_service/BUILD index b3097ff8..db47dc2e 100644 --- a/services/auction_service/BUILD +++ b/services/auction_service/BUILD @@ -118,6 +118,7 @@ cc_test( "//services/auction_service/benchmarking:score_ads_no_op_logger", "//services/auction_service/reporting:reporting_helper_test", "//services/auction_service/reporting:reporting_test_util", + "//services/auction_service/utils:proto_utils", "//services/common/constants:common_service_flags", "//services/common/encryption:key_fetcher_factory", "//services/common/encryption:mock_crypto_client_wrapper", @@ -131,6 +132,7 @@ cc_test( "@com_google_absl//absl/time", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//src/core/test/utils", "@google_privacysandbox_servers_common//src/encryption/key_fetcher/mock:mock_key_fetcher_manager", "@google_privacysandbox_servers_common//src/logger:request_context_logger", ], @@ -272,6 +274,7 @@ cc_binary( "//services/auction_service/data:runtime_config", "//services/auction_service/udf_fetcher:auction_code_fetch_config_cc_proto", "//services/auction_service/udf_fetcher:seller_udf_fetch_manager", + "//services/common:feature_flags", "//services/common/clients/config:config_client_util", "//services/common/clients/http:multi_curl_http_fetcher_async", "//services/common/encryption:crypto_client_factory", diff --git a/services/auction_service/auction_main.cc b/services/auction_service/auction_main.cc index 576bc000..76dcdd2b 100644 --- a/services/auction_service/auction_main.cc +++ b/services/auction_service/auction_main.cc @@ -47,6 +47,7 @@ #include "services/common/clients/http/multi_curl_http_fetcher_async.h" #include "services/common/encryption/crypto_client_factory.h" #include "services/common/encryption/key_fetcher_factory.h" +#include "services/common/feature_flags.h" #include "services/common/telemetry/configure_telemetry.h" #include "services/common/util/tcmalloc_utils.h" #include "src/concurrent/event_engine_executor.h" @@ -320,7 +321,9 @@ absl::Status RunServer() { .default_code_version = default_code_version, .enable_seller_and_buyer_udf_isolation = enable_seller_and_buyer_udf_isolation, - .enable_private_aggregate_reporting = enable_private_aggregate_reporting}; + .enable_private_aggregate_reporting = enable_private_aggregate_reporting, + .enable_cancellation = absl::GetFlag(FLAGS_enable_cancellation), + .enable_kanon = absl::GetFlag(FLAGS_enable_kanon)}; AuctionService auction_service( std::move(score_ads_reactor_factory), CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr), diff --git a/services/auction_service/auction_service_integration_test.cc b/services/auction_service/auction_service_integration_test.cc index 1a2e60a2..027e0c07 100644 --- a/services/auction_service/auction_service_integration_test.cc +++ b/services/auction_service/auction_service_integration_test.cc @@ -389,7 +389,7 @@ void BuildScoreAdsRequestForReporting( R"json({"renderUrls":{"placeholder_url":[123])json"; for (int i = 0; i < test_score_ads_request_config.desired_ad_count; i++) { auto ad = MakeARandomAdWithBidMetadata(/*min_bid=*/1, /*max_bid=*/1); - if (test_score_ads_request_config.buyer_reporting_id.has_value()) { + if (test_score_ads_request_config.buyer_reporting_id) { ad.set_buyer_reporting_id( test_score_ads_request_config.buyer_reporting_id.value()); } diff --git a/services/auction_service/auction_service_integration_test_util.cc b/services/auction_service/auction_service_integration_test_util.cc index a54e6f11..8be34faf 100644 --- a/services/auction_service/auction_service_integration_test_util.cc +++ b/services/auction_service/auction_service_integration_test_util.cc @@ -71,10 +71,14 @@ AdWithBidMetadata GetTestAdWithBidMetadata( MakeAnAd(MakeARandomString(), MakeARandomString(), 2)); ad.set_bid(1.0); ad.add_ad_components("adComponent.com"); - if (test_score_ads_request_config.buyer_reporting_id.has_value()) { + if (test_score_ads_request_config.buyer_reporting_id) { ad.set_buyer_reporting_id( *test_score_ads_request_config.buyer_reporting_id); } + if (test_score_ads_request_config.buyer_and_seller_reporting_id) { + ad.set_buyer_and_seller_reporting_id( + *test_score_ads_request_config.buyer_and_seller_reporting_id); + } ad.set_bid_currency(kEurosIsoCode); ad.set_ad_cost( test_score_ads_request_config.test_buyer_reporting_signals.ad_cost); diff --git a/services/auction_service/auction_service_integration_test_util.h b/services/auction_service/auction_service_integration_test_util.h index 21ff0e37..73bc3023 100644 --- a/services/auction_service/auction_service_integration_test_util.h +++ b/services/auction_service/auction_service_integration_test_util.h @@ -44,10 +44,11 @@ constexpr inline char kTestGenerationId[] = "generationId"; constexpr inline char kExpectedComponentReportWinUrl[] = "http://test.com?seller=http://" "seller.com&interestGroupName=undefined&buyerReportingId=buyerReportingId&" - "adCost=2&highestScoringOtherBid=0&madeHighestScoringOtherBid=false&" - "signalsForWinner={\"testSignal\":\"testValue\"}&perBuyerSignals=1,test,2&" - "auctionSignals=3,test,4&desirability=" - "undefined&topLevelSeller=topLevelSeller&modifiedBid=undefined"; + "buyerAndSellerReportingId=undefined&adCost=2&highestScoringOtherBid=0&" + "madeHighestScoringOtherBid=false&signalsForWinner=" + "{\"testSignal\":\"testValue\"}&perBuyerSignals=1,test,2&auctionSignals=" + "3,test,4&desirability=undefined&topLevelSeller=topLevelSeller&" + "modifiedBid=undefined"; constexpr inline char kTestInteractionEvent[] = "clickEvent"; constexpr inline char kTestInteractionReportingUrl[] = "http://click.com"; constexpr char kTestIgOwner[] = "barStandardAds.com"; @@ -103,6 +104,7 @@ struct TestScoreAdsRequestConfig { std::string top_level_seller = ""; bool is_consented = false; std::optional buyer_reporting_id; + std::optional buyer_and_seller_reporting_id; std::string interest_group_owner = ""; TestComponentAuctionResultData component_auction_data; }; diff --git a/services/auction_service/auction_service_reporting_integration_test.cc b/services/auction_service/auction_service_reporting_integration_test.cc index d43c10e0..fd59c91a 100644 --- a/services/auction_service/auction_service_reporting_integration_test.cc +++ b/services/auction_service/auction_service_reporting_integration_test.cc @@ -45,17 +45,26 @@ constexpr absl::string_view kExpectedReportResultUrl = constexpr absl::string_view kExpectedReportWinUrl = "http://test.com?seller=http://" "seller.com&interestGroupName=undefined&buyerReportingId=buyerReportingId&" - "adCost=2&highestScoringOtherBid=0&madeHighestScoringOtherBid=false&" - "signalsForWinner={\"testSignal\":\"testValue\"}&perBuyerSignals=1,test,2&" - "auctionSignals=3,test,4&desirability=undefined&topLevelSeller=undefined&" + "buyerAndSellerReportingId=undefined&adCost=2&highestScoringOtherBid=0" + "&madeHighestScoringOtherBid=false&signalsForWinner=" + "{\"testSignal\":\"testValue\"}&perBuyerSignals=1,test,2&auctionSignals=" + "3,test,4&desirability=undefined&topLevelSeller=undefined&" "modifiedBid=undefined"; constexpr absl::string_view kExpectedReportWinUrlWithNullSignalsForWinner = "http://test.com?seller=http://" "seller.com&interestGroupName=undefined&buyerReportingId=buyerReportingId&" + "buyerAndSellerReportingId=undefined&adCost=2&highestScoringOtherBid=0&" + "madeHighestScoringOtherBid=false&signalsForWinner=null&" + "perBuyerSignals=1,test,2&auctionSignals=3,test,4&desirability=undefined&" + "topLevelSeller=undefined&modifiedBid=undefined"; +constexpr absl::string_view kExpectedReportWinUrlWithBuyerAndSellerReportingId = + "http://test.com?seller=http://" + "seller.com&interestGroupName=undefined&buyerReportingId=undefined&" + "buyerAndSellerReportingId=buyerAndSellerReportingId&" "adCost=2&highestScoringOtherBid=0&madeHighestScoringOtherBid=false&" - "signalsForWinner=null&perBuyerSignals=1,test,2&auctionSignals=3,test,4&" - "desirability=undefined&topLevelSeller=undefined&modifiedBid=undefined"; - + "signalsForWinner={\"testSignal\":\"testValue\"}&perBuyerSignals=1,test,2&" + "auctionSignals=3,test,4&desirability=undefined&topLevelSeller=undefined&" + "modifiedBid=undefined"; constexpr absl::string_view kTestTopLevelReportResultUrl = "http://" "test.com&bid=1&bidCurrency=undefined&highestScoringOtherBid=undefined&" @@ -337,5 +346,47 @@ TEST_F(AuctionServiceReportingIntegrationTest, 0); } +TEST_F(AuctionServiceReportingIntegrationTest, + ReportingSuccessWithCodeIsolationAndBuyerAndSellerReportingId) { + ScoreAdsResponse response; + AuctionServiceRuntimeConfig runtime_config = { + .enable_report_result_url_generation = true, + .enable_report_win_url_generation = true, + .enable_seller_and_buyer_udf_isolation = true}; + TestBuyerReportingSignals test_buyer_reporting_signals; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .buyer_reporting_id = kBuyerReportingId, + .buyer_and_seller_reporting_id = kBuyerAndSellerReportingId, + .interest_group_owner = kTestIgOwner}; + LoadAndRunScoreAdsForPA(runtime_config, test_score_ads_request_config, + kTestReportWinUdfWithValidation, kSellerBaseCode, + response); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& score_ad = raw_response.ad_score(); + EXPECT_GT(score_ad.desirability(), 0); + const auto& top_level_seller_reporting_urls = + score_ad.win_reporting_urls().top_level_seller_reporting_urls(); + const auto& component_buyer_reporting_urls = + score_ad.win_reporting_urls().buyer_reporting_urls(); + EXPECT_EQ(top_level_seller_reporting_urls.reporting_url(), + kExpectedReportResultUrl); + ASSERT_TRUE( + top_level_seller_reporting_urls.interaction_reporting_urls().contains( + kTestInteractionEvent)); + EXPECT_EQ(top_level_seller_reporting_urls.interaction_reporting_urls().at( + kTestInteractionEvent), + kTestInteractionReportingUrl); + EXPECT_EQ(component_buyer_reporting_urls.reporting_url(), + kExpectedReportWinUrlWithBuyerAndSellerReportingId); + ASSERT_TRUE( + component_buyer_reporting_urls.interaction_reporting_urls().contains( + kTestInteractionEvent)); + EXPECT_EQ(component_buyer_reporting_urls.interaction_reporting_urls().at( + kTestInteractionEvent), + kTestInteractionReportingUrl); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/buyer_reporting_test_constants.h b/services/auction_service/code_wrapper/buyer_reporting_test_constants.h index 90d39575..7ac67ef9 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_test_constants.h +++ b/services/auction_service/code_wrapper/buyer_reporting_test_constants.h @@ -24,8 +24,8 @@ constexpr absl::string_view kTestReportWinUdfWithValidation = console.error("Missing seller in input to reportWin") return } - if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId){ - console.error("Missing both interestGroupName and buyerReportingId in input to reportWin") + if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId && !buyerReportingSignals.buyerAndSellerReportingId){ + console.error("Missing all interestGroupName, buyerReportingId and buyerAndSellerReportingId in input to reportWin") return } if(!buyerReportingSignals.adCost || buyerReportingSignals.adCost < 1){ @@ -37,6 +37,7 @@ constexpr absl::string_view kTestReportWinUdfWithValidation = reportWinUrl = "http://test.com?seller="+buyerReportingSignals.seller+ "&interestGroupName="+buyerReportingSignals.interestGroupName+ "&buyerReportingId="+buyerReportingSignals.buyerReportingId+ + "&buyerAndSellerReportingId="+buyerReportingSignals.buyerAndSellerReportingId+ "&adCost="+buyerReportingSignals.adCost+ "&highestScoringOtherBid="+buyerReportingSignals.highestScoringOtherBid+ "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ @@ -119,8 +120,8 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep console.error("Missing seller in input to reportWin") return } - if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId){ - console.error("Missing both interestGroupName and buyerReportingId in input to reportWin") + if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId && !buyerReportingSignals.buyerAndSellerReportingId){ + console.error("Missing all interestGroupName, buyerReportingId and buyerAndSellerReportingId in input to reportWin") return } if(!buyerReportingSignals.adCost || buyerReportingSignals.adCost < 1){ @@ -132,6 +133,7 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep reportWinUrl = "http://test.com?seller="+buyerReportingSignals.seller+ "&interestGroupName="+buyerReportingSignals.interestGroupName+ "&buyerReportingId="+buyerReportingSignals.buyerReportingId+ + "&buyerAndSellerReportingId="+buyerReportingSignals.buyerAndSellerReportingId+ "&adCost="+buyerReportingSignals.adCost+ "&highestScoringOtherBid="+buyerReportingSignals.highestScoringOtherBid+ "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ @@ -214,8 +216,8 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep console.error("Missing seller in input to reportWin") return } - if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId){ - console.error("Missing both interestGroupName and buyerReportingId in input to reportWin") + if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId && !buyerReportingSignals.buyerAndSellerReportingId){ + console.error("Missing all interestGroupName, buyerReportingId and buyerAndSellerReportingId in input to reportWin") return } if(!buyerReportingSignals.adCost || buyerReportingSignals.adCost < 1){ @@ -227,6 +229,7 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep reportWinUrl = "http://test.com?seller="+buyerReportingSignals.seller+ "&interestGroupName="+buyerReportingSignals.interestGroupName+ "&buyerReportingId="+buyerReportingSignals.buyerReportingId+ + "&buyerAndSellerReportingId="+buyerReportingSignals.buyerAndSellerReportingId+ "&adCost="+buyerReportingSignals.adCost+ "&highestScoringOtherBid="+buyerReportingSignals.highestScoringOtherBid+ "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ @@ -261,6 +264,7 @@ reportWin = function( var reportWinUrl = 'http://test.com?seller=' + buyerReportingSignals.seller + '&interestGroupName=' + buyerReportingSignals.interestGroupName + '&buyerReportingId=' + buyerReportingSignals.buyerReportingId + + '&buyerAndSellerReportingId=' + buyerReportingSignals.buyerAndSellerReportingId + '&adCost=' + buyerReportingSignals.adCost + '&highestScoringOtherBid=' + buyerReportingSignals.highestScoringOtherBid + '&madeHighestScoringOtherBid=' + diff --git a/services/auction_service/data/runtime_config.h b/services/auction_service/data/runtime_config.h index 9985d051..f8c0bed3 100644 --- a/services/auction_service/data/runtime_config.h +++ b/services/auction_service/data/runtime_config.h @@ -59,6 +59,9 @@ struct AuctionServiceRuntimeConfig { bool enable_seller_and_buyer_udf_isolation = false; // Enables private aggregate reporting. bool enable_private_aggregate_reporting = false; + + bool enable_cancellation = false; + bool enable_kanon = false; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/private_aggregation/private_aggregation_manager.cc b/services/auction_service/private_aggregation/private_aggregation_manager.cc index 3021f6c9..54815345 100644 --- a/services/auction_service/private_aggregation/private_aggregation_manager.cc +++ b/services/auction_service/private_aggregation/private_aggregation_manager.cc @@ -227,7 +227,7 @@ absl::StatusOr ParseAndProcessContribution( } PS_ASSIGN_IF_PRESENT(int_value, value_it->value, kSignalValueIntValue, Int64); - if (int_value.has_value()) { + if (int_value) { contribution.mutable_value()->set_int_value(*int_value); } else if (!value_it->value.HasMember(kSignalValueExtendedValue)) { return absl::InvalidArgumentError(kInvalidPrivateAggregationValueType); diff --git a/services/auction_service/reporting/buyer/buyer_reporting_helper.cc b/services/auction_service/reporting/buyer/buyer_reporting_helper.cc index e73f06a6..fe424d7a 100644 --- a/services/auction_service/reporting/buyer/buyer_reporting_helper.cc +++ b/services/auction_service/reporting/buyer/buyer_reporting_helper.cc @@ -28,6 +28,34 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { constexpr absl::string_view kReportWinUDFName = "reportWin"; } // namespace + +void SetBuyerReportingIds( + const BuyerReportingDispatchRequestData& buyer_reporting_metadata, + rapidjson::Document& seller_device_signals) { + // if buyerAndSellerReportingId is present, + // it will be used as a substitute for the IG Name. else if buyerReportingId + // is present, it will be used as a substitute for the IG Name. else, the + // implementation will fallback to using IG name. Reference: + // https://github.com/WICG/turtledove/blob/main/FLEDGE.md#12-interest-group-attributes + if (buyer_reporting_metadata.buyer_and_seller_reporting_id) { + rapidjson::GenericStringRef buyer_and_seller_reporting_id( + (buyer_reporting_metadata.buyer_and_seller_reporting_id)->c_str()); + seller_device_signals.AddMember(kBuyerAndSellerReportingIdTag, + buyer_and_seller_reporting_id, + seller_device_signals.GetAllocator()); + } else if (buyer_reporting_metadata.buyer_reporting_id) { + rapidjson::GenericStringRef buyer_reporting_id( + (buyer_reporting_metadata.buyer_reporting_id)->c_str()); + seller_device_signals.AddMember(kBuyerReportingIdTag, buyer_reporting_id, + seller_device_signals.GetAllocator()); + } else { + rapidjson::GenericStringRef interest_group_name( + buyer_reporting_metadata.interest_group_name.c_str()); + seller_device_signals.AddMember(kInterestGroupNameTag, interest_group_name, + seller_device_signals.GetAllocator()); + } +} + absl::StatusOr> GenerateBuyerDeviceSignals( const BuyerReportingDispatchRequestData& buyer_reporting_metadata, rapidjson::Document& seller_device_signals) { @@ -40,7 +68,7 @@ absl::StatusOr> GenerateBuyerDeviceSignals( kMadeHighestScoringOtherBid, buyer_reporting_metadata.made_highest_scoring_other_bid, seller_device_signals.GetAllocator()); - if (buyer_reporting_metadata.join_count.has_value()) { + if (buyer_reporting_metadata.join_count) { int join_count = *buyer_reporting_metadata.join_count; absl::StatusOr noised_join_count = NoiseAndBucketJoinCount(join_count); if (noised_join_count.ok()) { @@ -54,7 +82,7 @@ absl::StatusOr> GenerateBuyerDeviceSignals( "reporting"; } } - if (buyer_reporting_metadata.recency.has_value()) { + if (buyer_reporting_metadata.recency) { long recency = *buyer_reporting_metadata.recency; absl::StatusOr noised_recency = NoiseAndBucketRecency(recency); if (noised_recency.ok()) { @@ -67,7 +95,7 @@ absl::StatusOr> GenerateBuyerDeviceSignals( << "Error with noising recency in buyerDeviceSignals for reporting"; } } - if (buyer_reporting_metadata.modeling_signals.has_value()) { + if (buyer_reporting_metadata.modeling_signals) { int modeling_signals = *buyer_reporting_metadata.modeling_signals; absl::StatusOr noised_modeling_signals = NoiseAndMaskModelingSignals(modeling_signals); @@ -93,20 +121,8 @@ absl::StatusOr> GenerateBuyerDeviceSignals( seller_device_signals.AddMember(kAdCostTag, *ad_cost, seller_device_signals.GetAllocator()); } - // If buyer_reporting_id is present, interestGroupName should not be set. - // Reference: - // https://github.com/WICG/turtledove/blob/main/FLEDGE.md#12-interest-group-attributes - if (buyer_reporting_metadata.buyer_reporting_id.has_value()) { - rapidjson::GenericStringRef buyer_reporting_id( - (buyer_reporting_metadata.buyer_reporting_id)->c_str()); - seller_device_signals.AddMember(kBuyerReportingIdTag, buyer_reporting_id, - seller_device_signals.GetAllocator()); - } else { - rapidjson::GenericStringRef interest_group_name( - buyer_reporting_metadata.interest_group_name.c_str()); - seller_device_signals.AddMember(kInterestGroupNameTag, interest_group_name, - seller_device_signals.GetAllocator()); - } + SetBuyerReportingIds(buyer_reporting_metadata, seller_device_signals); + return SerializeJsonDoc(seller_device_signals, buyer_reporting_metadata.buyer_signals.size()); } diff --git a/services/auction_service/reporting/buyer/buyer_reporting_helper.h b/services/auction_service/reporting/buyer/buyer_reporting_helper.h index f11b161d..c98e7619 100644 --- a/services/auction_service/reporting/buyer/buyer_reporting_helper.h +++ b/services/auction_service/reporting/buyer/buyer_reporting_helper.h @@ -43,6 +43,12 @@ absl::StatusOr> GenerateBuyerDeviceSignals( absl::StatusOr ParseReportWinResponse( const ReportingDispatchRequestConfig& dispatch_request_config, absl::string_view response, RequestLogContext& log_context); + +// Sets buyerReportingId, buyerAndSellerReportingId, +// or IG name as necessary +void SetBuyerReportingIds( + const BuyerReportingDispatchRequestData& buyer_reporting_metadata, + rapidjson::Document& seller_device_signals); } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_HELPER_H_ diff --git a/services/auction_service/reporting/buyer/buyer_reporting_helper_test.cc b/services/auction_service/reporting/buyer/buyer_reporting_helper_test.cc index 5a9efbee..dad5ef66 100644 --- a/services/auction_service/reporting/buyer/buyer_reporting_helper_test.cc +++ b/services/auction_service/reporting/buyer/buyer_reporting_helper_test.cc @@ -106,6 +106,7 @@ BuyerReportingDispatchRequestData GetTestBuyerReportingDispatchRequestData( .interest_group_name = kTestInterestGroupName, .ad_cost = kTestAdCost, .buyer_reporting_id = kTestBuyerReportingId, + .buyer_and_seller_reporting_id = "", .made_highest_scoring_other_bid = true, .log_context = log_context}; } @@ -123,6 +124,22 @@ TEST(GetBuyerDeviceSignals, ReturnsBuyerReportingSignalsWithBuyerReportingId) { reporting_dispatch_data); } +TEST(GetBuyerDeviceSignals, + ReturnsBuyerReportingSignalsWithBuyerAndSellerReportingId) { + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); + BuyerReportingDispatchRequestData reporting_dispatch_data = + GetTestBuyerReportingDispatchRequestData(log_context); + reporting_dispatch_data.buyer_and_seller_reporting_id = + kTestBuyerAndSellerReportingId; + rapidjson::Document document(rapidjson::kObjectType); + absl::StatusOr> buyer_device_signals = + GenerateBuyerDeviceSignals(reporting_dispatch_data, document); + ASSERT_TRUE(buyer_device_signals.ok()); + VerifyPABuyerReportingSignalsJson(*buyer_device_signals, + reporting_dispatch_data); +} + TEST(GetBuyerDeviceSignals, ReturnsBuyerReportingSignalsForComponentAuctions) { RequestLogContext log_context(/*context_map=*/{}, server_common::ConsentedDebugConfiguration()); diff --git a/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.cc b/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.cc index c96875a1..8cca6ba3 100644 --- a/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.cc +++ b/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.cc @@ -53,8 +53,8 @@ inline std::vector> GetReportWinInput( PS_VLOG(kDispatch, dispatch_request_data.log_context) << "\n\nReportWin Input Args:" << "\nAuction Config:\n" << *(input[PAReportWinArgIndex(PAReportWinArgs::kAuctionConfig)]) - << "\nBuyer Reporting Signals:\n" - << *(input[PAReportWinArgIndex(PAReportWinArgs::kBuyerReportingSignals)]) + << "\nPer buyer signals:\n" + << *(input[PAReportWinArgIndex(PAReportWinArgs::kPerBuyerSignals)]) << "\nSignals for winner:\n" << *(input[PAReportWinArgIndex(PAReportWinArgs::kSignalsForWinner)]) << "\nBuyer reporting signals:\n" diff --git a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc index 0555c4b6..23c7542e 100644 --- a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc +++ b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc @@ -62,9 +62,8 @@ inline std::vector> GetPASReportWinInput( PS_VLOG(kDispatch, dispatch_request_data.log_context) << "\n\nReportWin Input Args:" << "\nAuction Config:\n" << *(input[PASReportWinArgIndex(PASReportWinArgs::kAuctionConfig)]) - << "\nBuyer Reporting Signals:\n" - << *(input[PASReportWinArgIndex( - PASReportWinArgs::kBuyerReportingSignals)]) + << "\nPer buyer signals:\n" + << *(input[PASReportWinArgIndex(PASReportWinArgs::kPerBuyerSignals)]) << "\nSignals for winner:\n" << *(input[PASReportWinArgIndex(PASReportWinArgs::kSignalsForWinner)]) << "\nBuyer reporting signals:\n" diff --git a/services/auction_service/reporting/reporting_helper.cc b/services/auction_service/reporting/reporting_helper.cc index 7bc65248..e20c8ee5 100644 --- a/services/auction_service/reporting/reporting_helper.cc +++ b/services/auction_service/reporting/reporting_helper.cc @@ -297,7 +297,7 @@ std::string GetBuyerMetadataJson( kMadeHighestScoringOtherBid, dispatch_request_data.post_auction_signals.made_highest_scoring_other_bid, buyer_reporting_signals_obj.GetAllocator()); - if (dispatch_request_data.buyer_reporting_metadata.join_count.has_value()) { + if (dispatch_request_data.buyer_reporting_metadata.join_count) { int join_count = dispatch_request_data.buyer_reporting_metadata.join_count.value(); if (dispatch_request_config.enable_report_win_input_noising) { @@ -310,7 +310,7 @@ std::string GetBuyerMetadataJson( buyer_reporting_signals_obj.AddMember( kJoinCount, join_count, buyer_reporting_signals_obj.GetAllocator()); } - if (dispatch_request_data.buyer_reporting_metadata.recency.has_value()) { + if (dispatch_request_data.buyer_reporting_metadata.recency) { PS_VLOG(kNoisyInfo, dispatch_request_data.log_context) << "BuyerReportingMetadata: Recency:" << dispatch_request_data.buyer_reporting_metadata.recency.value(); diff --git a/services/auction_service/reporting/reporting_helper.h b/services/auction_service/reporting/reporting_helper.h index d2b5b4be..36009de5 100644 --- a/services/auction_service/reporting/reporting_helper.h +++ b/services/auction_service/reporting/reporting_helper.h @@ -76,6 +76,8 @@ inline constexpr char kBuyerOriginTag[] = "buyerOrigin"; inline constexpr char kSellerTag[] = "seller"; inline constexpr char kAdCostTag[] = "adCost"; inline constexpr char kBuyerReportingIdTag[] = "buyerReportingId"; +inline constexpr char kBuyerAndSellerReportingIdTag[] = + "buyerAndSellerReportingId"; inline constexpr char kMadeHighestScoringOtherBid[] = "madeHighestScoringOtherBid"; inline constexpr char kInteractionReportingUrlsWrapperResponse[] = @@ -187,6 +189,7 @@ struct BuyerReportingDispatchRequestData { std::string interest_group_name; double ad_cost; std::optional buyer_reporting_id; + std::optional buyer_and_seller_reporting_id; bool made_highest_scoring_other_bid; RequestLogContext& log_context; std::string buyer_origin; diff --git a/services/auction_service/reporting/reporting_helper_test_constants.h b/services/auction_service/reporting/reporting_helper_test_constants.h index f44b1772..efb13b06 100644 --- a/services/auction_service/reporting/reporting_helper_test_constants.h +++ b/services/auction_service/reporting/reporting_helper_test_constants.h @@ -19,6 +19,8 @@ namespace privacy_sandbox::bidding_auction_servers { constexpr char kTestReportResultUrl[] = "http://reportResultUrl.com"; constexpr char kTestReportWinUrl[] = "http://reportWinUrl.com"; constexpr char kTestBuyerReportingId[] = "testBuyerReportingId"; +constexpr char kTestBuyerAndSellerReportingId[] = + "testBuyerAndSellerReportingId"; constexpr char kTestReportWinUrlWithBuyerReportingId[] = "http://reportWinUrl.com&buyerReportingId=testId"; constexpr char kTestLog[] = "testLog"; diff --git a/services/auction_service/reporting/reporting_test_util.cc b/services/auction_service/reporting/reporting_test_util.cc index 949dbfc0..16454341 100644 --- a/services/auction_service/reporting/reporting_test_util.cc +++ b/services/auction_service/reporting/reporting_test_util.cc @@ -156,7 +156,10 @@ rapidjson::Document GenerateTestSellerDeviceSignals( void VerifyBuyerReportingSignals( BuyerReportingDispatchRequestData& observed_buyer_device_signals, const BuyerReportingDispatchRequestData& expected_buyer_device_signals) { - if (expected_buyer_device_signals.buyer_reporting_id.has_value()) { + if (expected_buyer_device_signals.buyer_and_seller_reporting_id) { + EXPECT_EQ(*observed_buyer_device_signals.buyer_and_seller_reporting_id, + *expected_buyer_device_signals.buyer_and_seller_reporting_id); + } else if (expected_buyer_device_signals.buyer_reporting_id) { EXPECT_EQ(*observed_buyer_device_signals.buyer_reporting_id, *expected_buyer_device_signals.buyer_reporting_id); } else { @@ -200,7 +203,7 @@ void VerifyBuyerReportingSignals( } EXPECT_EQ(observed_buyer_device_signals.made_highest_scoring_other_bid, expected_buyer_device_signals.made_highest_scoring_other_bid); - if (observed_buyer_device_signals.egress_payload.has_value()) { + if (observed_buyer_device_signals.egress_payload) { EXPECT_EQ(*observed_buyer_device_signals.egress_payload, *expected_buyer_device_signals.egress_payload); } @@ -219,6 +222,8 @@ void ParseBuyerReportingSignals( String); PS_ASSIGN_IF_PRESENT(reporting_dispatch_data.buyer_reporting_id, document, kBuyerReportingIdTag, String); + PS_ASSIGN_IF_PRESENT(reporting_dispatch_data.buyer_and_seller_reporting_id, + document, kBuyerAndSellerReportingIdTag, String); PS_ASSIGN_IF_PRESENT(reporting_dispatch_data.interest_group_name, document, kInterestGroupNameTag, String); PS_ASSIGN_IF_PRESENT(reporting_dispatch_data.ad_cost, document, kAdCostTag, @@ -337,6 +342,7 @@ BuyerReportingDispatchRequestData GetTestBuyerDispatchRequestData( .interest_group_name = kTestInterestGroupName, .ad_cost = kTestAdCost, .buyer_reporting_id = kTestBuyerReportingId, + .buyer_and_seller_reporting_id = "", .made_highest_scoring_other_bid = true, .log_context = log_context, .buyer_origin = kTestInterestGroupOwner, diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index 0615bd50..0feb0451 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -15,6 +15,7 @@ #include "score_ads_reactor.h" #include +#include #include #include #include @@ -110,10 +111,11 @@ long DebugReportUrlsLength(const ScoreAdsResponse::AdScore& ad_score) { // If the bid's original currency matches the seller currency, the // incomingBidInSellerCurrency must be unchanged by scoreAd(). If it is // changed, reject the bid. -bool IsIncomingBidInSellerCurrencyIllegallyModified( +inline bool IsIncomingBidInSellerCurrencyIllegallyModified( absl::string_view seller_currency, absl::string_view ad_with_bid_currency, float incoming_bid_in_seller_currency, float buyer_bid) { - return // First check if the original currency matches the seller currency. + return // First check if the original currency matches the seller + // currency. !seller_currency.empty() && !ad_with_bid_currency.empty() && seller_currency == ad_with_bid_currency && // Then check that the incomingBidInSellerCurrency was set (non-zero). @@ -134,6 +136,8 @@ ComponentReportingDataInAuctionResult GetComponentReportingDataInAuctionResult( std::move(*auction_result.mutable_win_reporting_urls()); component_reporting_data_in_auction_result.buyer_reporting_id = auction_result.buyer_reporting_id(); + component_reporting_data_in_auction_result.buyer_and_seller_reporting_id = + auction_result.buyer_and_seller_reporting_id(); return component_reporting_data_in_auction_result; } @@ -254,13 +258,103 @@ void SetSellerReportingUrlsInResponseForComponentAuction( mutable_interaction_url->try_emplace(event, interactionReportingUrl); } } + +// Grouping of data derived from *AdWithBidMetadata that is needed to score +// the ads. +struct ScoringAdWithBidMetadata { + float buyer_bid = 0.0; + absl::string_view ad_with_bid_currency = ""; + absl::string_view interest_group_name = ""; + absl::string_view interest_group_owner = ""; + absl::string_view interest_group_origin = ""; + bool k_anon_status = true; + AdType ad_type; +}; + +ScoringAdWithBidMetadata GetScoringDataFromAd(const AdWithBidMetadata& ad) { + return {.buyer_bid = ad.bid(), + .ad_with_bid_currency = ad.bid_currency(), + .interest_group_name = ad.interest_group_name(), + .interest_group_owner = ad.interest_group_owner(), + .interest_group_origin = ad.interest_group_origin(), + .k_anon_status = ad.k_anon_status(), + .ad_type = AdType::AD_TYPE_PROTECTED_AUDIENCE_AD}; +} + +ScoringAdWithBidMetadata GetScoringDataFromAd( + const ProtectedAppSignalsAdWithBidMetadata& ad) { + return {.buyer_bid = ad.bid(), + .ad_with_bid_currency = ad.bid_currency(), + .interest_group_owner = ad.owner(), + .k_anon_status = ad.k_anon_status(), + .ad_type = AdType::AD_TYPE_PROTECTED_APP_SIGNALS_AD}; +} + +// Populates relevant data in the AdScore object. The input data is sourced +// from (ProtectedAppSignals)AdWithBidMetadata object. +ScoringAdWithBidMetadata PopulateAdScoreData(ScoredAdData& parsed_ad) { + ScoringAdWithBidMetadata scoring_ad_with_bid_metadata; + ScoreAdsResponse::AdScore& ad_score = parsed_ad.ad_score; + if (parsed_ad.protected_audience_ad_with_bid) { + scoring_ad_with_bid_metadata = + GetScoringDataFromAd(*parsed_ad.protected_audience_ad_with_bid); + } else { + scoring_ad_with_bid_metadata = + GetScoringDataFromAd(*parsed_ad.protected_app_signals_ad_with_bid); + } + ad_score.set_interest_group_name( + scoring_ad_with_bid_metadata.interest_group_name); + ad_score.set_interest_group_owner( + scoring_ad_with_bid_metadata.interest_group_owner); + ad_score.set_interest_group_origin( + scoring_ad_with_bid_metadata.interest_group_origin); + ad_score.set_ad_type(scoring_ad_with_bid_metadata.ad_type); + ad_score.set_buyer_bid(scoring_ad_with_bid_metadata.buyer_bid); + ad_score.set_buyer_bid_currency( + scoring_ad_with_bid_metadata.ad_with_bid_currency); + return scoring_ad_with_bid_metadata; +} + +// Populates candidates for winning ads as well as ghost winning ads. +inline void AddIfCandsEmptyOrHasEqLastDesirability( + int ind_to_add, int desirability, + const std::vector& parsed_ads, std::vector& cands) { + if (cands.empty() || + parsed_ads[cands.back()].ad_score.desirability() == desirability) { + cands.push_back(ind_to_add); + } +} + +std::vector ChooseRandomElements(const std::vector& to_sample_from, + int num_elements_to_get) { + DCHECK(num_elements_to_get <= to_sample_from.size()); + static std::random_device random_device; + static std::mt19937 generator(random_device()); + std::vector sampled; + sampled.reserve(num_elements_to_get); + std::sample(to_sample_from.begin(), to_sample_from.end(), + std::back_inserter(sampled), num_elements_to_get, generator); + return sampled; +} + } // namespace -void ScoringData::UpdateWinner(int index, - const ScoreAdsResponse::AdScore& ad_score) { - winning_ad = ad_score; - index_of_most_desirable_ad = index; - desirability_of_most_desirable_ad = ad_score.desirability(); +void ScoringData::ChooseWinnerAndGhostWinners(size_t max_ghost_winners) { + if (!winner_cand_indices.empty()) { + std::vector winner_ind = + ChooseRandomElements(winner_cand_indices, /*num_elements_to_get=*/1); + winner_index = winner_ind[0]; + } + + if (!ghost_winner_cand_indices.empty()) { + if (ghost_winner_cand_indices.size() < max_ghost_winners) { + ghost_winner_indices = ghost_winner_cand_indices; + } else { + ghost_winner_indices = ChooseRandomElements( + ghost_winner_cand_indices, + std::min(max_ghost_winners, ghost_winner_cand_indices.size())); + } + } } ScoreAdsReactor::ScoreAdsReactor( @@ -274,7 +368,8 @@ ScoreAdsReactor::ScoreAdsReactor( : CodeDispatchReactor( - request, response, key_fetcher_manager, crypto_client), + request, response, key_fetcher_manager, crypto_client, + runtime_config.enable_cancellation, runtime_config.enable_kanon), context_(context), dispatcher_(dispatcher), benchmarking_logger_(std::move(benchmarking_logger)), @@ -303,7 +398,9 @@ ScoreAdsReactor::ScoreAdsReactor( auction_scope_ != AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER), enable_seller_and_buyer_code_isolation_( - runtime_config.enable_seller_and_buyer_udf_isolation) { + runtime_config.enable_seller_and_buyer_udf_isolation), + enable_enforce_kanon_(runtime_config.enable_kanon && + raw_request_.enforce_kanon()) { CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, metric::AuctionContextMap()->Remove(request_)); @@ -581,7 +678,7 @@ void ScoreAdsReactor::Execute() { int js_execution_time_ms = (absl::Now() - start_js_execution_time) / absl::Milliseconds(1); LogIfError( - metric_context_->LogHistogram( + metric_context_->LogHistogram( js_execution_time_ms)); ScoreAdsCallback(result, enable_debug_reporting); }, @@ -616,6 +713,28 @@ void ScoreAdsReactor::InitializeBuyerReportingDispatchRequestData( post_auction_signals_.made_highest_scoring_other_bid; } +void ScoreAdsReactor::SetBuyerReportingIdsInRawResponse( + const std::unique_ptr& ad) { + if (!ad->buyer_reporting_id().empty()) { + // Set buyer_reporting_id in the response. + // If this id doesn't match with the buyerReportingId on-device, reportWin + // urls will be dropped. + raw_response_.mutable_ad_score()->set_buyer_reporting_id( + ad->buyer_reporting_id()); + buyer_reporting_dispatch_request_data_.buyer_reporting_id = + ad->buyer_reporting_id(); + } + if (!ad->buyer_and_seller_reporting_id().empty()) { + // Set buyer_and_seller_reporting_id in the response. + // If this id doesn't match with the buyerAndSellerReportingId on-device, + // reportWin urls will be dropped. + raw_response_.mutable_ad_score()->set_buyer_and_seller_reporting_id( + ad->buyer_and_seller_reporting_id()); + buyer_reporting_dispatch_request_data_.buyer_and_seller_reporting_id = + ad->buyer_and_seller_reporting_id(); + } +} + void ScoreAdsReactor::PerformReportingWithSellerAndBuyerCodeIsolation( const ScoreAdsResponse::AdScore& winning_ad_score, absl::string_view id) { reporting_dispatch_request_config_ = { @@ -645,15 +764,8 @@ void ScoreAdsReactor::PerformReportingWithSellerAndBuyerCodeIsolation( buyer_reporting_dispatch_request_data_.ad_cost = ad->ad_cost(); buyer_reporting_dispatch_request_data_.winning_ad_render_url = ad->render(); - if (!ad->buyer_reporting_id().empty()) { - // Set buyer_reporting_id in the response. - // If this id doesn't match with the buyerReportingId on-device, reportWin - // urls will be dropped. - raw_response_.mutable_ad_score()->set_buyer_reporting_id( - ad->buyer_reporting_id()); - buyer_reporting_dispatch_request_data_.buyer_reporting_id = - ad->buyer_reporting_id(); - } + SetBuyerReportingIdsInRawResponse(std::move(ad)); + if (auction_scope_ == AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER || auction_scope_ == @@ -757,16 +869,15 @@ void ScoreAdsReactor::PerformReporting( } } -void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, - absl::string_view ad_with_bid_currency, - absl::string_view interest_group_name, - absl::string_view interest_group_owner, - absl::string_view interest_group_origin, - const rapidjson::Document& response_json, - AdType ad_type, - ScoreAdsResponse::AdScore& ad_score, - ScoringData& scoring_data, - const std::string& dispatch_response_id) { +ScoreAdsReactor::OptionalAdRejectionReason +ScoreAdsReactor::GetAdRejectionReason( + const rapidjson::Document& response_json, + const ScoreAdsResponse::AdScore& ad_score) { + absl::string_view interest_group_owner = ad_score.interest_group_owner(); + absl::string_view interest_group_name = ad_score.interest_group_name(); + absl::string_view ad_with_bid_currency = ad_score.buyer_bid_currency(); + absl::string_view bid_currency = ad_score.bid_currency(); + float bid = ad_score.bid(); // Get ad rejection reason before updating the scoring data. std::optional ad_rejection_reason; @@ -776,112 +887,39 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, response_json, interest_group_owner, interest_group_name, log_context_); } - ad_score.set_interest_group_name(interest_group_name); - ad_score.set_interest_group_owner(interest_group_owner); - ad_score.set_interest_group_origin(interest_group_origin); - ad_score.set_ad_type(ad_type); - ad_score.set_buyer_bid(buyer_bid); - ad_score.set_buyer_bid_currency(ad_with_bid_currency); - // Used for debug reporting. - // Should include bids rejected by bid currency mismatch - // and those not allowed in component auctions. - ad_scores_.emplace(dispatch_response_id, - std::make_unique(ad_score)); - - if (CheckAndUpdateModifiedBid(auction_scope_, buyer_bid, ad_with_bid_currency, - &ad_score)) { - PS_VLOG(kNoisyInfo, log_context_) - << "Setting modified bid value to original buyer bid value (and " - "currency) as the " - "modified bid is not set. For interest group: " - << interest_group_name << ": " << ad_score.DebugString(); - } - if (IsBidCurrencyMismatched(auction_scope_, raw_request_.seller_currency(), - ad_score.bid(), ad_score.bid_currency())) { - ad_rejection_reason = ScoreAdsResponse::AdScore::AdRejectionReason{}; - ad_rejection_reason->set_interest_group_name( - ad_score.interest_group_name()); - ad_rejection_reason->set_interest_group_owner( - ad_score.interest_group_owner()); - ad_rejection_reason->set_rejection_reason( - SellerRejectionReason::BID_FROM_SCORE_AD_FAILED_CURRENCY_CHECK); + bid, bid_currency)) { PS_VLOG(kNoisyInfo, log_context_) << "Skipping component bid as it does not match seller_currency: " << interest_group_name << ": " << ad_score.DebugString(); + return BuildAdRejectionReason( + interest_group_owner, interest_group_name, + SellerRejectionReason::BID_FROM_SCORE_AD_FAILED_CURRENCY_CHECK); } if (IsIncomingBidInSellerCurrencyIllegallyModified( raw_request_.seller_currency(), ad_with_bid_currency, - ad_score.incoming_bid_in_seller_currency(), buyer_bid)) { - ad_rejection_reason = ScoreAdsResponse::AdScore::AdRejectionReason{}; - ad_rejection_reason->set_interest_group_name( - ad_score.interest_group_name()); - ad_rejection_reason->set_interest_group_owner( - ad_score.interest_group_owner()); - ad_rejection_reason->set_rejection_reason( - SellerRejectionReason::INVALID_BID); + ad_score.incoming_bid_in_seller_currency(), ad_score.buyer_bid())) { PS_VLOG(kNoisyInfo, log_context_) << "Skipping ad_score as its incomingBidInSellerCurrency was " - "modified " - "when it should not have been: " - << interest_group_name << ": " << ad_score.DebugString(); - } - - const bool is_valid_ad = - !ad_rejection_reason.has_value() || - ad_rejection_reason->rejection_reason() == - SellerRejectionReason::SELLER_REJECTION_REASON_NOT_AVAILABLE; - // Consider only ads that are not explicitly rejected and the ones that have - // a positive desirability score. - if (is_valid_ad && ad_score.desirability() > - scoring_data.desirability_of_most_desirable_ad) { - if (auction_scope_ == - AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER && - !ad_score.allow_component_auction()) { - // Ignore component level winner if it is not allowed to - // participate in the top level auction. - // TODO(b/311234165): Add metric for rejected component ads. - PS_VLOG(kNoisyInfo, log_context_) - << "Skipping component bid as it is not allowed for " - << interest_group_name << ": " << ad_score.DebugString(); - return; - } - scoring_data.UpdateWinner(index, ad_score); - } - - if (is_valid_ad && ad_score.desirability() > 0) { - // Consider scored ad as valid (i.e. not rejected) when it has a - // positive desirability and either: - // 1. scoreAd returned a number. - // 2. scoreAd returned an object but the reject reason was not - // populated. - // 3. scoreAd returned an object and the reject reason was explicitly - // set to "not-available". - // Only consider valid bids for populating other highest bids. - scoring_data.score_ad_map[ad_score.desirability()].push_back(index); - return; + << "modified when it should not have been: " << interest_group_name + << ": " << ad_score.DebugString(); + return BuildAdRejectionReason(interest_group_owner, interest_group_name, + SellerRejectionReason::INVALID_BID); } // Populate a default rejection reason if needed when we didn't get a positive // desirability. - if (ad_score.desirability() <= 0 && !ad_rejection_reason.has_value()) { + if (ad_score.desirability() <= 0) { PS_VLOG(5, log_context_) << "Non-positive desirability for ad and no rejection reason " - "populated " - "by seller, providing a default rejection reason"; - ad_rejection_reason = ScoreAdsResponse::AdScore::AdRejectionReason{}; - ad_rejection_reason->set_interest_group_owner(interest_group_owner); - ad_rejection_reason->set_interest_group_name(interest_group_name); - ad_rejection_reason->set_rejection_reason( + "populated by seller, providing a default rejection reason"; + return BuildAdRejectionReason( + interest_group_owner, interest_group_name, SellerRejectionReason::SELLER_REJECTION_REASON_NOT_AVAILABLE); } - scoring_data.ad_rejection_reasons.push_back(*ad_rejection_reason); - scoring_data.seller_rejected_bid_count += 1; - LogIfError( - metric_context_->AccumulateMetric( - 1, ToSellerRejectionReasonString( - ad_rejection_reason->rejection_reason()))); + + return ad_rejection_reason; } void ScoreAdsReactor::FindScoredAdType( @@ -899,32 +937,30 @@ void ScoreAdsReactor::FindScoredAdType( } } -ScoringData ScoreAdsReactor::FindWinningAd( +std::vector ScoreAdsReactor::CollectValidRomaResponses( const std::vector>& responses) { - ScoringData scoring_data; + std::vector valid_scored_ads; int64_t current_all_debug_urls_chars = 0; for (int index = 0; index < responses.size(); ++index) { const auto& response = responses[index]; if (!response.ok()) { PS_LOG(WARNING, log_context_) << "Invalid execution (possibly invalid input): " - << responses[index].status().ToString( + << response.status().ToString( absl::StatusToStringMode::kWithEverything); LogIfError( - metric_context_->LogUpDownCounter(1)); + metric_context_->AccumulateMetric( + 1)); continue; } - absl::StatusOr response_json = - ParseAndGetScoreAdResponseJson(enable_adtech_code_logging_, - response->resp, log_context_); - // Determine what type of ad was scored in this response. - AdWithBidMetadata* ad = nullptr; + AdWithBidMetadata* protected_audience_ad_with_bid = nullptr; ProtectedAppSignalsAdWithBidMetadata* protected_app_signals_ad_with_bid = nullptr; - FindScoredAdType(response->id, &ad, &protected_app_signals_ad_with_bid); - if (!ad && !protected_app_signals_ad_with_bid) { + FindScoredAdType(response->id, &protected_audience_ad_with_bid, + &protected_app_signals_ad_with_bid); + if (!protected_audience_ad_with_bid && !protected_app_signals_ad_with_bid) { // This should never happen but we log here in case there is a bug in // our implementation. PS_LOG(ERROR, log_context_) @@ -935,13 +971,20 @@ ScoringData ScoreAdsReactor::FindWinningAd( continue; } + absl::StatusOr response_json = + ParseAndGetScoreAdResponseJson(enable_adtech_code_logging_, + response->resp, log_context_); + if (!response_json.ok()) { - LogWarningForBadResponse(response_json.status(), *response, ad, - log_context_); + LogWarningForBadResponse(response_json.status(), *response, + protected_audience_ad_with_bid, log_context_); + LogIfError( + metric_context_->AccumulateMetric( + 1)); continue; } - auto ad_score = ParseScoreAdResponse( + auto ad_score = ScoreAdResponseJsonToProto( *response_json, max_allowed_size_debug_url_chars_, max_allowed_size_all_debug_urls_chars_, (auction_scope_ == @@ -949,34 +992,206 @@ ScoringData ScoreAdsReactor::FindWinningAd( auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_COMPONENT_MULTI_SELLER), current_all_debug_urls_chars); + if (!ad_score.ok()) { - LogWarningForBadResponse(ad_score.status(), *response, ad, log_context_); + LogWarningForBadResponse(ad_score.status(), *response, + protected_audience_ad_with_bid, log_context_); continue; } + current_all_debug_urls_chars += DebugReportUrlsLength(*ad_score); + ScoredAdData scored_ad_data = { + .ad_score = *std::move(ad_score), + .response_json = *std::move(response_json), + .protected_audience_ad_with_bid = protected_audience_ad_with_bid, + .protected_app_signals_ad_with_bid = protected_app_signals_ad_with_bid, + .id = response->id}; + ScoringAdWithBidMetadata scoring_ad_with_bid_metadata = + PopulateAdScoreData(scored_ad_data); + // If k-anon is not enabled or enforced, we consider the ad k-anonymous + // by default. + scored_ad_data.k_anon_status = + !enable_enforce_kanon_ || scoring_ad_with_bid_metadata.k_anon_status; + valid_scored_ads.push_back(std::move(scored_ad_data)); + } - if (ad) { - HandleScoredAd(index, ad->bid(), ad->bid_currency(), - ad->interest_group_name(), ad->interest_group_owner(), - ad->interest_group_origin(), *response_json, - AdType::AD_TYPE_PROTECTED_AUDIENCE_AD, *ad_score, - scoring_data, response->id); - } else { - HandleScoredAd(index, protected_app_signals_ad_with_bid->bid(), - protected_app_signals_ad_with_bid->bid_currency(), - /*interest_group_name=*/"", - protected_app_signals_ad_with_bid->owner(), - /*interest_group_origin=*/"", *response_json, - AdType::AD_TYPE_PROTECTED_APP_SIGNALS_AD, *ad_score, - scoring_data, response->id); + return valid_scored_ads; +} + +void ScoredAdData::Swap(ScoredAdData& other) { + ad_score.Swap(&other.ad_score); + response_json.Swap(other.response_json); + id.swap(other.id); + + auto* tmp_protected_audience_ad_with_bid = + other.protected_audience_ad_with_bid; + other.protected_audience_ad_with_bid = protected_audience_ad_with_bid; + protected_audience_ad_with_bid = tmp_protected_audience_ad_with_bid; + + std::swap(protected_app_signals_ad_with_bid, + other.protected_app_signals_ad_with_bid); + std::swap(k_anon_status, other.k_anon_status); +} + +bool ScoredAdData::operator>(const ScoredAdData& other) const { + const auto& other_ad_score = other.ad_score; + if (ad_score.desirability() > other_ad_score.desirability()) { + return true; + } + + if (ad_score.desirability() == other_ad_score.desirability()) { + if (ad_score.buyer_bid() > other_ad_score.buyer_bid()) { + return true; + } + + if (ad_score.buyer_bid() == other_ad_score.buyer_bid()) { + if (k_anon_status && !other.k_anon_status) { + return true; + } + + return false; + } + + return false; + } + + return false; +} + +ScoringData ScoreAdsReactor::FindWinningAd( + std::vector& parsed_ads) { + ScoringData scoring_data; + + int index = 0; + int end = parsed_ads.size(); + while (index < end) { + auto& parsed_ad = parsed_ads[index]; + + // Used for debug reporting. + // Should include bids rejected by bid currency mismatch + // and those not allowed in component auctions. + auto& ad_score = parsed_ad.ad_score; + ad_scores_.emplace(parsed_ad.id, + std::make_unique(ad_score)); + if (CheckAndUpdateModifiedBid(auction_scope_, ad_score.buyer_bid(), + ad_score.buyer_bid_currency(), &ad_score)) { + PS_VLOG(kNoisyInfo, log_context_) + << "Setting modified bid value to original buyer bid value (and " + "currency) as the " + "modified bid is not set. For interest group: " + << ad_score.interest_group_name() << ": " << ad_score.DebugString(); + } + OptionalAdRejectionReason ad_rejection_reason = + GetAdRejectionReason(parsed_ad.response_json, ad_score); + if (ad_rejection_reason && + ad_rejection_reason->rejection_reason() != + SellerRejectionReason::SELLER_REJECTION_REASON_NOT_AVAILABLE) { + scoring_data.seller_rejected_bid_count += 1; + LogIfError( + metric_context_->AccumulateMetric( + 1, ToSellerRejectionReasonString( + ad_rejection_reason->rejection_reason()))); + *scoring_data.ad_rejection_reasons.Add() = + *std::move(ad_rejection_reason); + + // Move the rejected ad towards the end of the vector. + // Don't enhance index becuase we can have a potentially valid ad on this + // index after swap. + --end; + parsed_ads[end].Swap(parsed_ads[index]); + continue; + } + // Filter out component ads if not allowed. + if (auction_scope_ == + AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER && + !ad_score.allow_component_auction()) { + // Ignore component level ads if it is not allowed to + // participate in the top level auction. + // TODO(b/311234165): Add metric for rejected component ads. + PS_VLOG(kNoisyInfo, log_context_) + << "Skipping component bid as it is not allowed for " + << "owner: " << ad_score.interest_group_owner() + << ", render: " << ad_score.render(); + --end; + parsed_ads[end].Swap(parsed_ads[index]); + continue; + } + ++index; + } + + // Drop all the rejected bids from the vector. + parsed_ads.resize(end); + PS_VLOG(kStats, log_context_) + << "Number of valid positive score ads: " << parsed_ads.size(); + + // Sorts based on score/desirability, bid and k-anon status. + std::sort(parsed_ads.begin(), parsed_ads.end(), std::greater<>()); + + for (int ind = 0; ind < parsed_ads.size(); ++ind) { + const ScoredAdData& parsed_ad = parsed_ads[ind]; + const ScoreAdsResponse::AdScore& ad_score = parsed_ad.ad_score; + auto& winner_indices = scoring_data.winner_cand_indices; + auto& ghost_winner_indices = scoring_data.ghost_winner_cand_indices; + PS_VLOG(kStats, log_context_) + << "Parsed ad score at index: " << ind + << " (in order) k-anon status: " << parsed_ad.k_anon_status << ", " + << ad_score.DebugString() + << ", winner_cand_indices: " << winner_indices.size() + << ", ghost_winner_cand_indices: " << ghost_winner_indices.size(); + // Consider only ads that are not explicitly rejected and the ones that have + // a positive desirability score. + if (ad_score.desirability() <= 0) { + continue; + } + + if (parsed_ad.k_anon_status) { + // Winner is chosen randomly from top-scoring ads that are k-anonymous. + AddIfCandsEmptyOrHasEqLastDesirability(ind, ad_score.desirability(), + parsed_ads, winner_indices); + } else if (winner_indices.empty()) { + // Any ads that have higher scores than first k-anon ad are ghost winner + // candidates. + AddIfCandsEmptyOrHasEqLastDesirability(ind, ad_score.desirability(), + parsed_ads, ghost_winner_indices); + } + } + + const auto& ghost_winner_cand_indices = + scoring_data.ghost_winner_cand_indices; + PS_VLOG(kStats, log_context_) + << "Num winner candidates: " << scoring_data.winner_cand_indices.size() + << ", num ghost winner candidates: " << ghost_winner_cand_indices.size(); + // TODO(b/372097452): If we end up using num_allowd_ghost_winners > 1, then we + // should revise the random selection to prefer high scoring ghost winning ads + // over low scoring ghost winning ads. + scoring_data.ChooseWinnerAndGhostWinners( + raw_request_.num_allowed_ghost_winners()); + + absl::flat_hash_set ghost_winner_cands(ghost_winner_cand_indices.begin(), + ghost_winner_cand_indices.end()); + for (int ind = 0; ind < parsed_ads.size(); ++ind) { + const ScoredAdData& parsed_ad = parsed_ads[ind]; + const ScoreAdsResponse::AdScore& ad_score = parsed_ad.ad_score; + // Consider scored ad as valid (i.e. not rejected) when it has a + // positive desirability and either: + // 1. scoreAd returned a number. + // 2. scoreAd returned an object but the reject reason was not + // populated. + // 3. scoreAd returned an object and the reject reason was explicitly + // set to "not-available". + // 4. Ad under consideration is not one of k-anon ghost winners. + // Only consider valid bids for populating other highest bids. + if (!ghost_winner_cands.contains(ind)) { + scoring_data.score_ad_map[ad_score.desirability()].push_back(ind); } } return scoring_data; } void ScoreAdsReactor::PopulateRelevantFieldsInResponse( - int index_of_most_desirable_ad, absl::string_view request_id, - ScoreAdsResponse::AdScore& winning_ad) { + int index, std::vector& parsed_ads) { + auto& parsed_ad = parsed_ads[index]; + absl::string_view request_id = parsed_ad.id; AdWithBidMetadata* ad = nullptr; ProtectedAppSignalsAdWithBidMetadata* protected_app_signals_ad_with_bid = nullptr; @@ -986,18 +1201,19 @@ void ScoreAdsReactor::PopulateRelevantFieldsInResponse( // here. DCHECK(ad || protected_app_signals_ad_with_bid); + auto& ad_score = parsed_ad.ad_score; if (ad) { - winning_ad.set_render(ad->render()); - winning_ad.mutable_component_renders()->Swap(ad->mutable_ad_components()); + ad_score.set_render(ad->render()); + ad_score.mutable_component_renders()->Swap(ad->mutable_ad_components()); } else { - winning_ad.set_render(protected_app_signals_ad_with_bid->render()); + ad_score.set_render(protected_app_signals_ad_with_bid->render()); } } void ScoreAdsReactor::PopulateHighestScoringOtherBidsData( int index_of_most_desirable_ad_score, const absl::flat_hash_map>& score_ad_map, - const std::vector>& responses, + const std::vector& responses, ScoreAdsResponse::AdScore& winning_ad_score) { if (auction_scope_ == AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { return; @@ -1029,7 +1245,7 @@ void ScoreAdsReactor::PopulateHighestScoringOtherBidsData( AdWithBidMetadata* ad_with_bid_metadata_from_buyer = nullptr; ProtectedAppSignalsAdWithBidMetadata* protected_app_signals_ad_with_bid = nullptr; - FindScoredAdType(responses[current_index]->id, + FindScoredAdType(responses[current_index].id, &ad_with_bid_metadata_from_buyer, &protected_app_signals_ad_with_bid); DCHECK(ad_with_bid_metadata_from_buyer || @@ -1047,7 +1263,7 @@ void ScoreAdsReactor::PopulateHighestScoringOtherBidsData( owner = protected_app_signals_ad_with_bid->owner(); } if (!raw_request_.seller_currency().empty()) { - auto ad_score_it = ad_scores_.find(responses[current_index]->id); + auto ad_score_it = ad_scores_.find(responses[current_index].id); if (ad_score_it != ad_scores_.end()) { bid = ad_score_it->second->incoming_bid_in_seller_currency(); } @@ -1058,6 +1274,60 @@ void ScoreAdsReactor::PopulateHighestScoringOtherBidsData( } } +void ScoreAdsReactor::AddGhostWinnersToResponse( + const std::vector& ghost_winner_indices, + std::vector& parsed_responses) { + for (int ghost_winner_ind : ghost_winner_indices) { + PopulateRelevantFieldsInResponse(ghost_winner_ind, parsed_responses); + *raw_response_.mutable_ghost_winning_ad_scores()->Add() = + std::move(parsed_responses[ghost_winner_ind].ad_score); + } +} + +void ScoreAdsReactor::AddWinnerToResponse( + int winner_index, ScoringData& scoring_data, + std::vector& parsed_responses) { + auto& winning_ad = parsed_responses[winner_index].ad_score; + // Set the render URL in overall response for the winning ad. + PopulateRelevantFieldsInResponse(winner_index, parsed_responses); + // TODO: Check if this needs an adjustment based on k-anon status. + PopulateHighestScoringOtherBidsData(winner_index, scoring_data.score_ad_map, + parsed_responses, winning_ad); + *winning_ad.mutable_ad_rejection_reasons() = + std::move(scoring_data.ad_rejection_reasons); + *raw_response_.mutable_ad_score() = std::move(winning_ad); +} + +void ScoreAdsReactor::DoWinAndDebugReporting( + bool enable_debug_reporting, int winner_index, + const std::vector& parsed_responses) { + const ScoreAdsResponse::AdScore& winning_ad = raw_response_.ad_score(); + if (enable_debug_reporting) { + PerformDebugReporting(winning_ad); + } + + absl::string_view winner_id = parsed_responses[winner_index].id; + if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { + PopulateComponentReportingUrlsInTopLevelResponse( + winner_id, raw_response_, component_level_reporting_data_); + } + + if (!enable_report_result_url_generation_) { + EncryptAndFinishOK(); + return; + } + + // Todo(b/357146634): Execute reportWin even if + // enable_report_result_url_generation is disabled or reportResult execution + // fails. + if (enable_seller_and_buyer_code_isolation_) { + PerformReportingWithSellerAndBuyerCodeIsolation(winning_ad, winner_id); + return; + } + + PerformReporting(winning_ad, winner_id); +} + // Handles the output of the code execution dispatch. // Note that the dispatch response value is expected to be a json string // conforming to the scoreAd function output described here: @@ -1070,59 +1340,35 @@ void ScoreAdsReactor::ScoreAdsCallback( int total_bid_count = static_cast(responses.size()); LogIfError(metric_context_->AccumulateMetric( total_bid_count)); - ScoringData scoring_data = FindWinningAd(responses); + auto parsed_responses = CollectValidRomaResponses(responses); + ScoringData scoring_data = FindWinningAd(parsed_responses); LogIfError(metric_context_->LogHistogram( (static_cast(scoring_data.seller_rejected_bid_count)) / total_bid_count)); - auto& winning_ad = scoring_data.winning_ad; - // No Ad won. - if (!winning_ad.has_value()) { + + // Add ghost winners to the response (if any). + AddGhostWinnersToResponse(scoring_data.ghost_winner_indices, + parsed_responses); + + const int winner_index = scoring_data.winner_index; + const bool has_winning_ad = winner_index != -1; + // No ad won. + if (!has_winning_ad) { + PS_LOG(WARNING, log_context_) << "No ad was selected as most desirable and " + << "no ghost winners found"; LogIfError(metric_context_ ->AccumulateMetric( 1, metric::kAuctionScoreAdsNoAdSelected)); - PS_LOG(WARNING, log_context_) << "No ad was selected as most desirable"; if (enable_debug_reporting) { - PerformDebugReporting(winning_ad); + PerformDebugReporting(/*winning_ad_score=*/absl::nullopt); } EncryptAndFinishOK(); return; } - // Set the render URL in overall response for the winning ad. - const int index_of_most_desirable_ad = - scoring_data.index_of_most_desirable_ad; - const auto& id = responses[index_of_most_desirable_ad]->id; - PopulateRelevantFieldsInResponse(index_of_most_desirable_ad, id, *winning_ad); - - PopulateHighestScoringOtherBidsData(index_of_most_desirable_ad, - scoring_data.score_ad_map, responses, - *winning_ad); - - const auto& ad_rejection_reasons = scoring_data.ad_rejection_reasons; - winning_ad->mutable_ad_rejection_reasons()->Assign( - ad_rejection_reasons.begin(), ad_rejection_reasons.end()); - - if (enable_debug_reporting) { - PerformDebugReporting(winning_ad); - } - *raw_response_.mutable_ad_score() = *winning_ad; - if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { - PopulateComponentReportingUrlsInTopLevelResponse( - id, raw_response_, component_level_reporting_data_); - } - // Todo(b/357146634): Execute reportWin even if - // enable_report_result_url_generation is disabled or reportResult execution - // fails. - if (enable_report_result_url_generation_) { - if (enable_seller_and_buyer_code_isolation_) { - PerformReportingWithSellerAndBuyerCodeIsolation(*winning_ad, id); - return; - } - PerformReporting(*winning_ad, id); - return; - } - EncryptAndFinishOK(); - return; + AddWinnerToResponse(winner_index, scoring_data, parsed_responses); + DoWinAndDebugReporting(enable_debug_reporting, winner_index, + parsed_responses); } void ScoreAdsReactor::ReportingCallback( @@ -1341,7 +1587,7 @@ void ScoreAdsReactor::CancellableReportResultCallback( return; } absl::Status report_win_status; - if (!buyer_reporting_dispatch_request_data_.egress_payload.has_value()) { + if (!buyer_reporting_dispatch_request_data_.egress_payload) { report_win_status = PerformPAReportWin( reporting_dispatch_request_config_, buyer_reporting_dispatch_request_data_, seller_device_signals_, @@ -1578,6 +1824,8 @@ void ScoreAdsReactor::DispatchReportResultRequestForTopLevelAuction( // urls will be dropped. raw_response_.mutable_ad_score()->set_buyer_reporting_id( ad_it->second.buyer_reporting_id); + raw_response_.mutable_ad_score()->set_buyer_and_seller_reporting_id( + ad_it->second.buyer_and_seller_reporting_id); } else { PS_VLOG(kDispatch, log_context_) << "Could not find component reporting data " diff --git a/services/auction_service/score_ads_reactor.h b/services/auction_service/score_ads_reactor.h index 699b58a0..9633ffae 100644 --- a/services/auction_service/score_ads_reactor.h +++ b/services/auction_service/score_ads_reactor.h @@ -56,22 +56,30 @@ inline constexpr char kNoValidComponentAuctions[] = struct ScoringData { // Index of the most desirable ad. This helps us to set the overall response // object just once. - int index_of_most_desirable_ad = 0; + int winner_index = -1; // Count of rejected bids. int seller_rejected_bid_count = 0; // Map of all the desirability/scores and corresponding scored ad's index in // the response from the scoreAd's UDF. absl::flat_hash_map> score_ad_map; - // Saving the desirability allows us to compare desirability between ads - // without re-parsing the current most-desirable ad every time. - float desirability_of_most_desirable_ad = 0; // List of rejection reasons provided by seller. - std::vector + google::protobuf::RepeatedPtrField< + ScoreAdsResponse::AdScore::AdRejectionReason> ad_rejection_reasons; - // The most desirable ad. - std::optional winning_ad; - - void UpdateWinner(int index, const ScoreAdsResponse::AdScore& ad_score); + // The scored bids are sorted in descending order first. The highest scored + // ad(s) are added to this list. Note that there can be multiple highest + // scored ads that can have the same score -- in such a case the winner is + // chosen randomly among these candidates to ensure fairness. + std::vector winner_cand_indices; + // Indices of ghost winner candidates. Currently, we populate ghost winner + // candidates that have the highest and equal scores. If multiple such + // candidates are present, and we have to choose a subset from them then + // we choose a random sample from these to ensure fairness. + std::vector ghost_winner_cand_indices; + // Indices of ghost winners. + std::vector ghost_winner_indices; + + void ChooseWinnerAndGhostWinners(size_t max_ghost_winners); }; // Fields needed from component level AuctionResult for @@ -80,6 +88,39 @@ struct ComponentReportingDataInAuctionResult { std::string component_seller; WinReportingUrls win_reporting_urls; std::string buyer_reporting_id; + std::string buyer_and_seller_reporting_id; +}; + +// Group of data associated with the ad scored via dispatch to Roma. +struct ScoredAdData { + // Response returned from Roma. + ScoreAdsResponse::AdScore ad_score; + + // Parsed JSON response object. + rapidjson::Document response_json; + + // Populated to a non-null value only if the scored ad was of type Protected + // Audience. Note: Underlying object is stored in `ad_data_` + ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata* + protected_audience_ad_with_bid = nullptr; + + // Populated to a non-null value only if the scored ad was of type Protected + // App Signals. Note: Underlying object is stored in + // `protected_app_signals_ad_data_` + ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata* + protected_app_signals_ad_with_bid = nullptr; + + // ID of the request/response to/from Roma. + absl::string_view id; + + // K-anon status of the ad (derived from incoming AdWithBidMetadata) + bool k_anon_status = false; + + // Swaps the data memberwise. + void Swap(ScoredAdData& other); + + // Compares based on score, bid and k-anon status. + bool operator>(const ScoredAdData& other) const; }; // This is a gRPC reactor that serves a single ScoreAdsRequest. @@ -108,6 +149,8 @@ class ScoreAdsReactor ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; using ProtectedAppSignalsAdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata; + using OptionalAdRejectionReason = + std::optional; // Finds the ad type of the scored ad and set it. After the function call, // expect one of the input pointers to be populated. void FindScoredAdType(absl::string_view response_id, @@ -116,21 +159,27 @@ class ScoreAdsReactor protected_app_signals_ad_with_bid_metadata); // Populates the ad render URL and other ad type specific data in the ad score // response to be sent back to SFE. - void PopulateRelevantFieldsInResponse(int index_of_most_desirable_ad, - absl::string_view request_id, - ScoreAdsResponse::AdScore& winning_ad); + void PopulateRelevantFieldsInResponse(int index, + std::vector& parsed_ads); + + // Filters out invalid Roma responses while updating the corresponding error + // metrics. + // + // Returns valid responses. + std::vector CollectValidRomaResponses( + const std::vector>& responses); + // Finds the winning ad (if one exists) among the responses returned by Roma. // Returns all the data associated with scoring that can then be later used // for finding second highest bid (among other things). - ScoringData FindWinningAd( - const std::vector>& responses); + ScoringData FindWinningAd(std::vector& parsed_ads); // Populates the data about the highest second other bid in the response to // be returned to SFE. void PopulateHighestScoringOtherBidsData( int index_of_most_desirable_ad, const absl::flat_hash_map>& score_ad_map, - const std::vector>& responses, + const std::vector& responses, ScoreAdsResponse::AdScore& winning_ad); // Asynchronous callback used by the v8 code executor to return a result. This @@ -178,6 +227,12 @@ class ScoreAdsReactor void InitializeBuyerReportingDispatchRequestData( const ScoreAdsResponse::AdScore& winning_ad_score); + // Initializes buyer_reporting_dispatch_request_data_ and + // raw_response_ with buyer reporting IDs included in + // AdWithBidMetadata. + void SetBuyerReportingIdsInRawResponse( + const std::unique_ptr& ad); + // Performs reportResult and reportWin udf execution with seller and buyer // code isolation void PerformReportingWithSellerAndBuyerCodeIsolation( @@ -252,19 +307,24 @@ class ScoreAdsReactor google::protobuf::RepeatedPtrField& protected_app_signals_ad_bids); - // Sets the required fields in the passed AdScore object and populates - // scoring data. - // The AdScore fields that need to be parsed from ROMA response - // must be populated separately before this is called. - void HandleScoredAd(int index, float buyer_bid, - absl::string_view ad_with_bid_currency, - absl::string_view interest_group_name, - absl::string_view interest_group_owner, - absl::string_view interest_group_origin, - const rapidjson::Document& response_json, AdType ad_type, - ScoreAdsResponse::AdScore& score_ads_response, - ScoringData& scoring_data, - const std::string& dispatch_response_id); + // Validates the ad returned by Roma ScoreAd Response (e.g. validates + // currency) and sets a rejection reason if the ad is not valid. + OptionalAdRejectionReason GetAdRejectionReason( + const rapidjson::Document& response_json, + const ScoreAdsResponse::AdScore& ad_score); + + // Adds winner ad to the GRPC response. + void AddWinnerToResponse(int winner_index, ScoringData& scoring_data, + std::vector& parsed_responses); + + // Adds ghost winner ads to the GRPC response. + void AddGhostWinnersToResponse(const std::vector& ghost_winner_indices, + std::vector& parsed_responses); + + // Performs win and debug reporting (forDebuggingOnly). + void DoWinAndDebugReporting( + bool enable_debug_reporting, int winner_index, + const std::vector& parsed_responses); CLASS_CANCELLATION_WRAPPER(ReportResultCallback, enable_cancellation_, context_, FinishWithStatus) @@ -334,6 +394,7 @@ class ScoreAdsReactor empty_ad_component_render_urls; return empty_ad_component_render_urls; } + const bool enable_enforce_kanon_; }; } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_SCORE_ADS_REACTOR_H_ diff --git a/services/auction_service/score_ads_reactor_test.cc b/services/auction_service/score_ads_reactor_test.cc index 46d3f497..a9798e48 100644 --- a/services/auction_service/score_ads_reactor_test.cc +++ b/services/auction_service/score_ads_reactor_test.cc @@ -38,8 +38,10 @@ #include "services/auction_service/reporting/reporting_helper.h" #include "services/auction_service/reporting/reporting_helper_test_constants.h" #include "services/auction_service/reporting/reporting_test_util.h" +#include "services/auction_service/reporting/seller/seller_reporting_manager.h" #include "services/auction_service/score_ads_reactor_test_util.h" #include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/auction_service/utils/proto_utils.h" #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/constants/common_service_flags.h" #include "services/common/encryption/key_fetcher_factory.h" @@ -49,7 +51,9 @@ #include "services/common/test/mocks.h" #include "services/common/test/random.h" #include "services/common/test/utils/test_init.h" +#include "services/common/util/json_util.h" #include "services/common/util/proto_util.h" +#include "src/core/test/utils/proto_test_utils.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" #include "src/encryption/key_fetcher/mock/mock_key_fetcher_manager.h" #include "src/logger/request_context_logger.h" @@ -175,6 +179,48 @@ constexpr char kComponentWithCurrencyAndIncomingAndRejectReasAdScore[] = R"( } )"; +// Ad Scores +constexpr float kTestDesirability1 = 1.0; +constexpr float kTestDesirability2 = 2.0; + +// IDs +constexpr absl::string_view kTestScoredAdDataId1 = "id1"; +constexpr absl::string_view kTestScoredAdDataId2 = "id2"; + +// Buyer bids +constexpr float kTestBuyerBid1 = 3.0; +constexpr float kTestBuyerBid2 = 4.0; + +// K-anon status +const bool kTestAnonStatusFalse = false; +const bool kTestAnonStatusTrue = true; + +constexpr absl::string_view kHighScoringInterestGroupName1 = "IG1"; +constexpr absl::string_view kHighScoringInterestGroupName2 = "IG2"; +constexpr absl::string_view kLowScoringInterestGroupName = "IG"; +constexpr absl::string_view kHighScoringInterestGroupOwner1 = "IGOwner1"; +constexpr absl::string_view kHighScoringInterestGroupOwner2 = "IGOwner2"; +constexpr absl::string_view kLowScoringInterestGroupOwner = "IGOwner"; +constexpr absl::string_view kHighScoringInterestGroupOrigin1 = "IGOrigin1"; +constexpr absl::string_view kHighScoringInterestGroupOrigin2 = "IGOrigin2"; +constexpr absl::string_view kLowScoringInterestGroupOrigin = "IGOrigin"; +constexpr absl::string_view kHighScoringRenderUrl1 = "RenderUrl1"; +constexpr absl::string_view kHighScoringRenderUrl2 = "RenderUrl2"; +constexpr absl::string_view kLowScoringRenderUrl = "RenderUrl"; +constexpr absl::string_view kHighestScoringOtherBidRenderUrl = "HsobRenderUrl"; +constexpr absl::string_view kHighestScoringOtherBidInterestGroupName = "HsobIG"; +constexpr absl::string_view kHighestScoringOtherBidInterestGroupOrigin = + "HsobIGOrigin"; +// Bids can be used to break ties among the winners with equal scores and +// hence setting the same bid for high scored ad. +constexpr double kHighScoredBid = 1.0; +constexpr double kLowScoredBid = 3.0; +constexpr double kHighestScoringOtherBidsBidValue = 0.5; +// Winner is based on the score provided by ScoreAd. +constexpr float kHighScore = 10.0; +constexpr float kLowScore = 2.0; +constexpr float kHighestScoringOtherBidsScore = 1.0; + using ::google::protobuf::TextFormat; using ::testing::AnyNumber; using ::testing::HasSubstr; @@ -186,32 +232,59 @@ using AdWithBidMetadata = using ProtectedAppSignalsAdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata; using ::google::protobuf::util::MessageDifferencer; +using ::google::scp::core::test::EqualsProto; constexpr char kInterestGroupOwnerOfBarBidder[] = "barStandardAds.com"; constexpr char kInterestGroupOrigin[] = "candy_smush"; +constexpr absl::string_view kTestRenderUrl = + "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0"; +constexpr float kTestBid = 2.10; +constexpr int kTestNumOfComponentAds = 3; +constexpr absl::string_view kTestInterestGroupOwner = "https://fooAds.com"; + +struct AdWithBidMetadataParams { + absl::string_view render_url = kTestRenderUrl; + float bid = kTestBid; + absl::string_view interest_group_name = kTestInterestGroupName; + absl::string_view interest_group_owner = kTestInterestGroupOwner; + absl::string_view interest_group_origin = kInterestGroupOrigin; + double ad_cost = kTestAdCost; + absl::string_view buyer_reporting_id = ""; + int number_of_component_ads = kTestNumOfComponentAds; + bool k_anon_status = false; +}; -void GetTestAdWithBidFoo(AdWithBidMetadata& foo, - absl::string_view buyer_reporting_id = "") { - int number_of_component_ads = 3; - foo.mutable_ad()->mutable_struct_value()->MergeFrom( - MakeAnAd("https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0", - "arbitraryMetadataKey", 2)); +AdWithBidMetadata BuildTestAdWithBidMetadata( + const AdWithBidMetadataParams& params) { + AdWithBidMetadata ad_with_bid_metadata; + ad_with_bid_metadata.mutable_ad()->mutable_struct_value()->MergeFrom( + MakeAnAd(params.render_url, "arbitraryMetadataKey", 2)); // This must have an entry in kTestScoringSignals. - foo.set_render( - "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0"); - foo.set_bid(2.10); - foo.set_interest_group_name(kTestInterestGroupName); - foo.set_interest_group_owner("https://fooAds.com"); - foo.set_interest_group_origin(kInterestGroupOrigin); - for (int i = 0; i < number_of_component_ads; i++) { - foo.add_ad_components( + ad_with_bid_metadata.set_render(params.render_url); + ad_with_bid_metadata.set_bid(params.bid); + ad_with_bid_metadata.set_interest_group_name(params.interest_group_name); + ad_with_bid_metadata.set_interest_group_owner(params.interest_group_owner); + ad_with_bid_metadata.set_interest_group_origin(params.interest_group_origin); + for (int i = 0; i < params.number_of_component_ads; i++) { + ad_with_bid_metadata.add_ad_components( absl::StrCat("adComponent.com/foo_components/id=", i)); } - if (!buyer_reporting_id.empty()) { - foo.set_buyer_reporting_id(buyer_reporting_id); + if (!params.buyer_reporting_id.empty()) { + ad_with_bid_metadata.set_buyer_reporting_id(params.buyer_reporting_id); } - foo.set_ad_cost(kTestAdCost); + ad_with_bid_metadata.set_ad_cost(params.ad_cost); + ad_with_bid_metadata.set_k_anon_status(params.k_anon_status); + PS_VLOG(5) << "Generated ad bid with metadata: " << ad_with_bid_metadata; + return ad_with_bid_metadata; +} + +void GetTestAdWithBidFoo(AdWithBidMetadata& foo, + absl::string_view buyer_reporting_id = "") { + foo = BuildTestAdWithBidMetadata( + {kTestRenderUrl, kTestBid, kTestInterestGroupName, + kTestInterestGroupOwner, kInterestGroupOrigin, kTestAdCost, + buyer_reporting_id}); } void PopulateTestAdWithBidMetdata( @@ -233,6 +306,8 @@ void PopulateTestAdWithBidMetdata( absl::StrCat("adComponent.com/foo_components/id=", i)); } foo.set_buyer_reporting_id(*buyer_dispatch_data.buyer_reporting_id); + foo.set_buyer_and_seller_reporting_id( + *buyer_dispatch_data.buyer_and_seller_reporting_id); foo.set_interest_group_name(buyer_dispatch_data.interest_group_name); foo.set_ad_cost(kTestAdCost); foo.set_bid(post_auction_signals.winning_bid); @@ -1098,10 +1173,10 @@ TEST_F(ScoreAdsReactorTest, // barbecue to kNotAnyOriginalBid will thus cause rejection for // bar, (as bar.incomingBidInSellerCurrency does not match // bar.bid) whereas setting to 0 is acceptable. foo_two has a - // lower score thann bar but will win because bar will be rejected + // lower score than bar but will win because bar will be rejected // for modified bid currency. barbecue will be regarded as valid // but lose the auction. barbecue.incomingBidInSellerCurrency will - // be recored as the highestScoringOtherBid. + // be recorded as the highestScoringOtherBid. (request.id == foo_two.render()) ? 0.0f : kNotAnyOriginalBid, (allowComponentAuction) ? "true" : "false")); } @@ -2638,7 +2713,6 @@ TEST_F(ScoreAdsReactorTest, IgnoresUnknownFieldsFromScoreAdResponse) { BuildRawRequest({foo}, kTestSellerSignals, kTestAuctionSignals, kTestScoringSignals, kTestPublisherHostName, raw_request, true); - RawRequest raw_request_copy = raw_request; EXPECT_CALL(dispatcher, BatchExecute) .WillOnce([](std::vector& batch, BatchDispatchDoneCallback done_callback) { @@ -2757,7 +2831,7 @@ TEST_F(ScoreAdsReactorTest, CaptureRejectionReasonsForRejectedAds) { // seller currency below, the AdScore for bar must have an // incomingBidInSellerCurrency which is unmodified (that is, which matches // bar.bid()). This test will deliberately violate this requirement below to - // test the seller rejectionn reason setting. + // test the seller rejection reason setting. GetTestAdWithBidBar(bar); // This AdWithBid is intended to be the sole non-rejected AwB. GetTestAdWithBidBarbecue(barbecue); @@ -2833,31 +2907,35 @@ TEST_F(ScoreAdsReactorTest, CaptureRejectionReasonsForRejectedAds) { // Assertion avoids out-of-bounds array accesses below. ASSERT_EQ(scored_ad.ad_rejection_reasons().size(), 3); - // Note that the batch responses are in reverse order. - - const auto& boots_ad_rejection_reason = scored_ad.ad_rejection_reasons()[0]; - EXPECT_EQ(boots_ad_rejection_reason.interest_group_name(), - boots.interest_group_name()); - EXPECT_EQ(boots_ad_rejection_reason.interest_group_owner(), - boots.interest_group_owner()); - EXPECT_EQ(boots_ad_rejection_reason.rejection_reason(), - SellerRejectionReason::BID_FROM_SCORE_AD_FAILED_CURRENCY_CHECK); - - const auto& bar_ad_rejection_reason = scored_ad.ad_rejection_reasons()[1]; - EXPECT_EQ(bar_ad_rejection_reason.interest_group_name(), - bar.interest_group_name()); - EXPECT_EQ(bar_ad_rejection_reason.interest_group_owner(), - bar.interest_group_owner()); - EXPECT_EQ(bar_ad_rejection_reason.rejection_reason(), - SellerRejectionReason::INVALID_BID); - - const auto& foo_ad_rejection_reason = scored_ad.ad_rejection_reasons()[2]; - EXPECT_EQ(foo_ad_rejection_reason.interest_group_name(), - foo.interest_group_name()); - EXPECT_EQ(foo_ad_rejection_reason.interest_group_owner(), - foo.interest_group_owner()); - EXPECT_EQ(foo_ad_rejection_reason.rejection_reason(), - SellerRejectionReason::INVALID_BID); + int num_matching = 0; + // Note that the batch responses can be in any order. + for (const auto& ad_rejection_reason : scored_ad.ad_rejection_reasons()) { + PS_VLOG(5) << "Found ad rejection reason: " + << ad_rejection_reason.DebugString(); + if (ad_rejection_reason.interest_group_name() == + boots.interest_group_name()) { + ++num_matching; + EXPECT_EQ(ad_rejection_reason.interest_group_owner(), + boots.interest_group_owner()); + EXPECT_EQ(ad_rejection_reason.rejection_reason(), + SellerRejectionReason::BID_FROM_SCORE_AD_FAILED_CURRENCY_CHECK); + } else if (ad_rejection_reason.interest_group_name() == + bar.interest_group_name()) { + ++num_matching; + EXPECT_EQ(ad_rejection_reason.interest_group_owner(), + bar.interest_group_owner()); + EXPECT_EQ(ad_rejection_reason.rejection_reason(), + SellerRejectionReason::INVALID_BID); + } else if (ad_rejection_reason.interest_group_name() == + foo.interest_group_name()) { + ++num_matching; + EXPECT_EQ(ad_rejection_reason.interest_group_owner(), + foo.interest_group_owner()); + EXPECT_EQ(ad_rejection_reason.rejection_reason(), + SellerRejectionReason::INVALID_BID); + } + } + ASSERT_EQ(num_matching, 3); } TEST_F(ScoreAdsReactorTest, @@ -3755,6 +3833,939 @@ TEST_F(ScoreAdsReactorTest, RespectConfiguredDebugUrlLimits) { EXPECT_TRUE(scored_ad.debug_report_urls().auction_debug_loss_url().empty()); } -} // namespace +ScoredAdData BuildScoredAdData( + float desirability = 1.0, float buyer_bid = 1.0, bool k_anon_status = true, + AdWithBidMetadata* protected_audience_ad_with_bid = nullptr, + ProtectedAppSignalsAdWithBidMetadata* protected_app_signals_ad_with_bid = + nullptr, + absl::string_view id = "test_id") { + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); + absl::StatusOr response = ParseAndGetScoreAdResponseJson( + /*enable_ad_tech_code_logging=*/true, + absl::Substitute( + R"JSON({ + "response" : { + "desirability" : $0, + "bid": $1 + }, + "logs": [] + })JSON", + desirability, buyer_bid), + /*log_context=*/log_context); + CHECK_OK(response); + ScoredAdData scored_ad_data = { + .response_json = *std::move(response), + .protected_audience_ad_with_bid = protected_audience_ad_with_bid, + .protected_app_signals_ad_with_bid = protected_app_signals_ad_with_bid, + .id = id, + .k_anon_status = k_anon_status}; + auto& ad_score = scored_ad_data.ad_score; + ad_score.set_desirability(desirability); + ad_score.set_buyer_bid(buyer_bid); + return scored_ad_data; +} + +TEST_F(ScoreAdsReactorTest, ChoosesWinnerFromHighScoringAdsRandomly) { + absl::SetFlag(&FLAGS_enable_kanon, false); + MockV8DispatchClient dispatcher; + // Create two ads that have highest (and equal) score and another one with + // lower score. + std::vector scores = {kHighScore, kLowScore, kHighScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata({kHighScoringRenderUrl1, kHighScoredBid, + kHighScoringInterestGroupName1, + kHighScoringInterestGroupOwner1, + kHighScoringInterestGroupOrigin1}), + BuildTestAdWithBidMetadata( + {kLowScoringRenderUrl, kLowScoredBid, kLowScoringInterestGroupName, + kLowScoringInterestGroupOwner, kLowScoringInterestGroupOrigin}), + BuildTestAdWithBidMetadata({kHighScoringRenderUrl2, kHighScoredBid, + kHighScoringInterestGroupName2, + kHighScoringInterestGroupOwner2, + kHighScoringInterestGroupOrigin2}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1], + "$1": [2], + "$2": [3] + } + })JSON", + kHighScoringRenderUrl1, kLowScoringRenderUrl, kHighScoringRenderUrl2); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + absl::flat_hash_map score_logic; + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic[ad_with_bid_metadata.render()] = + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid()); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + absl::flat_hash_map + ig_owner_win_count; + for (int i = 0; i < 50; ++i) { + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, + kTestPublisherHostName, raw_request); + raw_request.set_enforce_kanon(true); + AuctionServiceRuntimeConfig runtime_config; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& ad_score = raw_response.ad_score(); + // Highest scored ad should win. + ASSERT_EQ(ad_score.desirability(), kHighScore) + << "Highest scored ad not selected as winner in " << i + << "-th iteration"; + ig_owner_win_count[ad_score.interest_group_owner()] += 1; + } + ASSERT_EQ(ig_owner_win_count.size(), 2); + for (const auto& [ig_owner, win_count] : ig_owner_win_count) { + ASSERT_TRUE(ig_owner == kHighScoringInterestGroupOwner1 || + ig_owner == kHighScoringInterestGroupOwner2); + EXPECT_GT(win_count, 0) + << "Expected IG owner to win as well but it didn't: " << ig_owner; + } +} + +TEST_F(ScoreAdsReactorTest, AllAdsConsideredKAnonymousWhenKAnonNotEnforced) { + absl::SetFlag(&FLAGS_enable_kanon, true); + MockV8DispatchClient dispatcher; + // Create an ad that is not k-anonymous. + std::vector scores = {kHighScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata({.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .k_anon_status = false}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1] + } + })JSON", + kHighScoringRenderUrl1); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector score_logic; + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic.push_back( + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid())); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(false); + AuctionServiceRuntimeConfig runtime_config; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& ad_score = raw_response.ad_score(); + // Highest scored ad should win since k-anon is not enforced. + EXPECT_EQ(ad_score.desirability(), kHighScore); +} + +TEST_F(ScoreAdsReactorTest, ChaffIfNoAdKAnonymousAndNoGhostWinners) { + MockV8DispatchClient dispatcher; + // Create an ad that is not k-anonymous. + std::vector scores = {kHighScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata({.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .k_anon_status = false}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1] + } + })JSON", + kHighScoringRenderUrl1); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector score_logic; + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic.push_back( + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid())); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(true); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + EXPECT_FALSE(raw_response.has_ad_score()); +} +// Verifies that if there is a high scoring ad that is k-anonymous then we use +// it as a winner and there are no ghost winners. +TEST_F(ScoreAdsReactorTest, WinnerPrecedesGhostWinnerCandidates) { + MockV8DispatchClient dispatcher; + // Create an ad that is not k-anonymous. + std::vector scores = {kHighScore, kLowScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata({.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .k_anon_status = true}), + BuildTestAdWithBidMetadata({.render_url = kHighScoringRenderUrl2, + .bid = kLowScoredBid, + .k_anon_status = false}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1] + } + })JSON", + kHighScoringRenderUrl1); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector score_logic; + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic.push_back( + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid())); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(true); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + ASSERT_TRUE(raw_response.has_ad_score()); + const auto& ad_score = raw_response.ad_score(); + EXPECT_EQ(ad_score.desirability(), kHighScore); + EXPECT_EQ(raw_response.ghost_winning_ad_scores_size(), 0); +} + +// Verifies that if there is a high scoring ad that is not k-anonymous then it +// is returned as a ghost winner and if there is a low scoring ad that is +// k-anonymous, it is used as the winner. +TEST_F(ScoreAdsReactorTest, GhostWinnerPresentIfTopScoringAdIsNotKAnonymous) { + MockV8DispatchClient dispatcher; + // Create: 1. A high scoring non-k anonymous ad which becomes a candidate for + // ghost winner and 2. A low scoring k-anonymous ad which becomes the winnner. + std::vector scores = {kHighScore, kLowScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata( + {.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .interest_group_name = kHighScoringInterestGroupName1, + .interest_group_owner = kHighScoringInterestGroupOwner1, + .interest_group_origin = kHighScoringInterestGroupOrigin1, + .k_anon_status = false}), + BuildTestAdWithBidMetadata( + {.render_url = kLowScoringRenderUrl, + .bid = kLowScoredBid, + .interest_group_name = kLowScoringInterestGroupName, + .interest_group_owner = kLowScoringInterestGroupOwner, + .interest_group_origin = kLowScoringInterestGroupOrigin, + .k_anon_status = true}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1], + "$1": [2] + } + })JSON", + kHighScoringRenderUrl1, kLowScoringRenderUrl); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + absl::flat_hash_map score_logic; + score_logic.reserve(scores.size()); + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic[ad_with_bid_metadata.render()] = + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid()); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(true); + raw_request.set_num_allowed_ghost_winners(1); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + ASSERT_TRUE(raw_response.has_ad_score()); + const auto& ad_score = raw_response.ad_score(); + EXPECT_EQ(ad_score.desirability(), kLowScore); + EXPECT_EQ(raw_response.ghost_winning_ad_scores_size(), 1); +} + +// Verifies that if there are high scoring ads that are not k-anonymous then +// they are returned as ghost winner (even in complete absence of a real +// winner). +TEST_F(ScoreAdsReactorTest, GhostWinnersAreReturnedEvenIfthereIsNoWinner) { + MockV8DispatchClient dispatcher; + // Create two non-k anonymous ad which becomes a candidate for + // ghost winner. Eventually we will choose the winner with the highest score. + std::vector scores = {kHighScore, kLowScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata( + {.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .interest_group_name = kHighScoringInterestGroupName1, + .interest_group_owner = kHighScoringInterestGroupOwner1, + .interest_group_origin = kHighScoringInterestGroupOrigin1, + .k_anon_status = false}), + BuildTestAdWithBidMetadata( + {.render_url = kLowScoringRenderUrl, + .bid = kLowScoredBid, + .interest_group_name = kLowScoringInterestGroupName, + .interest_group_owner = kLowScoringInterestGroupOwner, + .interest_group_origin = kLowScoringInterestGroupOrigin, + .k_anon_status = false}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1], + "$1": [2] + } + })JSON", + kHighScoringRenderUrl1, kLowScoringRenderUrl); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + absl::flat_hash_map score_logic; + score_logic.reserve(scores.size()); + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic[ad_with_bid_metadata.render()] = + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid()); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(true); + raw_request.set_num_allowed_ghost_winners(2); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + ASSERT_FALSE(raw_response.has_ad_score()); + ASSERT_EQ(raw_response.ghost_winning_ad_scores_size(), 1); + // Only ghost winner with highest desirability is returned. + EXPECT_EQ(raw_response.ghost_winning_ad_scores()[0].desirability(), + kHighScore); +} + +// Verifies that if there are multiple high scoring ads that are not k-anonymous +// then we choose ghost winners randomly from them. +TEST_F(ScoreAdsReactorTest, GhostWinnersAreChosenRandomly) { + MockV8DispatchClient dispatcher; + // Create 2 non-k-anonymous ads which will be candidate for ghost winners. + // With all things equal in the ads (score, bid, k-anon status), expect the + // ads to be chosen randomly as a ghost winner. + std::vector scores = {kHighScore, kHighScore}; + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata( + {.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .interest_group_name = kHighScoringInterestGroupName1, + .interest_group_owner = kHighScoringInterestGroupOwner1, + .interest_group_origin = kHighScoringInterestGroupOrigin1, + .k_anon_status = false}), + BuildTestAdWithBidMetadata( + {.render_url = kLowScoringRenderUrl, + .bid = kHighScoredBid, + .interest_group_name = kLowScoringInterestGroupName, + .interest_group_owner = kLowScoringInterestGroupOwner, + .interest_group_origin = kLowScoringInterestGroupOrigin, + .k_anon_status = false}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1], + "$1": [2] + } + })JSON", + kHighScoringRenderUrl1, kLowScoringRenderUrl); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + absl::flat_hash_map score_logic; + score_logic.reserve(scores.size()); + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic[ad_with_bid_metadata.render()] = + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid()); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + absl::flat_hash_map owner_count; + for (int i = 0; i < 50; ++i) { + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, + kTestPublisherHostName, raw_request); + raw_request.set_enforce_kanon(true); + raw_request.set_num_allowed_ghost_winners(1); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + ASSERT_FALSE(raw_response.has_ad_score()); + ASSERT_EQ(raw_response.ghost_winning_ad_scores_size(), 1); + // Only ghost winner with highest desirability is returned. + const auto& ghost_winner = raw_response.ghost_winning_ad_scores()[0]; + EXPECT_EQ(ghost_winner.desirability(), kHighScore); + owner_count[ghost_winner.interest_group_owner()] += 1; + } + ASSERT_EQ(owner_count.size(), 2); + for (const auto& [owner, count] : owner_count) { + EXPECT_TRUE(owner == kHighScoringInterestGroupOwner1 || + owner == kLowScoringInterestGroupOwner); + EXPECT_GT(count, 0); + } +} + +// Verifies that the upper limit of max ghost winners is respected. +TEST_F(ScoreAdsReactorTest, RespectsMaxGhostWinnersLimit) { + MockV8DispatchClient dispatcher; + std::vector scores = {kHighScore, kLowScore}; + // Create: 1. A high scoring non-k anonymous ad which becomes a candidate for + // ghost winner and 2. A low scoring k-anonymous ad which becomes the winnner. + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata( + {.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .interest_group_name = kHighScoringInterestGroupName1, + .interest_group_owner = kHighScoringInterestGroupOwner1, + .interest_group_origin = kHighScoringInterestGroupOrigin1, + .k_anon_status = false}), + BuildTestAdWithBidMetadata( + {.render_url = kLowScoringRenderUrl, + .bid = kLowScoredBid, + .interest_group_name = kLowScoringInterestGroupName, + .interest_group_owner = kLowScoringInterestGroupOwner, + .interest_group_origin = kLowScoringInterestGroupOrigin, + .k_anon_status = true}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1], + "$1": [2] + } + })JSON", + kHighScoringRenderUrl1, kLowScoringRenderUrl); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + absl::flat_hash_map score_logic; + score_logic.reserve(scores.size()); + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic[ad_with_bid_metadata.render()] = + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid()); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(true); + // Set a limit of zero so that the ghost winner is trimmed out due to this + // limit. + raw_request.set_num_allowed_ghost_winners(0); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + ASSERT_TRUE(raw_response.has_ad_score()); + const auto& ad_score = raw_response.ad_score(); + EXPECT_EQ(ad_score.desirability(), kLowScore); + EXPECT_EQ(raw_response.ghost_winning_ad_scores_size(), 0); +} + +// Verifies that the highes_scoring_other_bid never contains ghost winning ad. +TEST_F(ScoreAdsReactorTest, GhostWinnerNotCandidatesForHighestScoringOtherBid) { + MockV8DispatchClient dispatcher; + std::vector scores = {kHighScore, kLowScore, + kHighestScoringOtherBidsScore}; + ASSERT_GE(kLowScoredBid, kHighestScoringOtherBidsScore) + << "Highest scoring other bid score should be lesser than equal to the " + "winner's score"; + // Creates 3 ads. The highest scoring ad will be used as ghost winner since + // it is not k-anonymous, the second highest scoring ad will be the winner + // and the lowest scored ad will become the highest scoring other bid. + std::vector ads_with_bid_metadata = { + BuildTestAdWithBidMetadata( + {.render_url = kHighScoringRenderUrl1, + .bid = kHighScoredBid, + .interest_group_name = kHighScoringInterestGroupName1, + .interest_group_owner = kHighScoringInterestGroupOwner1, + .interest_group_origin = kHighScoringInterestGroupOrigin1, + .k_anon_status = false}), + BuildTestAdWithBidMetadata( + {.render_url = kLowScoringRenderUrl, + .bid = kLowScoredBid, + .interest_group_name = kLowScoringInterestGroupName, + .interest_group_owner = kLowScoringInterestGroupOwner, + .interest_group_origin = kLowScoringInterestGroupOrigin, + .k_anon_status = true}), + // k-anon status doesn't matter for HSOB. + BuildTestAdWithBidMetadata( + {.render_url = kHighestScoringOtherBidRenderUrl, + .bid = kHighestScoringOtherBidsBidValue, + .interest_group_name = kHighestScoringOtherBidInterestGroupName, + .interest_group_owner = kLowScoringInterestGroupOwner, + .interest_group_origin = kHighestScoringOtherBidInterestGroupOrigin, + .k_anon_status = false}), + }; + std::string scoring_signals = absl::Substitute( + R"JSON( + { + "renderUrls": { + "$0": [1], + "$1": [2], + "$2": [3] + } + })JSON", + kHighScoringRenderUrl1, kLowScoringRenderUrl, + kHighestScoringOtherBidRenderUrl); + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([scores, ads_with_bid_metadata]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + absl::flat_hash_map score_logic; + score_logic.reserve(scores.size()); + for (int i = 0; i < scores.size(); ++i) { + AdWithBidMetadata ad_with_bid_metadata = ads_with_bid_metadata[i]; + score_logic[ad_with_bid_metadata.render()] = + absl::Substitute(R"({ + "response" : { + "desirability" : $0, + "render": "$1", + "interest_group_name": "$2", + "interest_group_owner": "$3", + "interest_group_origin": "$4", + "bid" : $5 + }, + "logs":[]})", + scores[i], ad_with_bid_metadata.render(), + ad_with_bid_metadata.interest_group_name(), + ad_with_bid_metadata.interest_group_owner(), + ad_with_bid_metadata.interest_group_origin(), + ad_with_bid_metadata.bid()); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), true); + }); + RawRequest raw_request; + BuildRawRequest(ads_with_bid_metadata, kTestSellerSignals, + kTestAuctionSignals, scoring_signals, kTestPublisherHostName, + raw_request); + raw_request.set_enforce_kanon(true); + raw_request.set_num_allowed_ghost_winners(1); + AuctionServiceRuntimeConfig runtime_config = {.enable_kanon = true}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + ASSERT_TRUE(raw_response.has_ad_score()); + const auto& ad_score = raw_response.ad_score(); + EXPECT_EQ(ad_score.desirability(), kLowScore); + const auto& ig_owner_highest_scoring_other_bids_map = + ad_score.ig_owner_highest_scoring_other_bids_map(); + ASSERT_EQ(ig_owner_highest_scoring_other_bids_map.size(), 1); + auto it = ig_owner_highest_scoring_other_bids_map.find( + kLowScoringInterestGroupOwner); + ASSERT_NE(it, ig_owner_highest_scoring_other_bids_map.end()); + const auto& highest_scoring_bid_values = it->second.values(); + ASSERT_EQ(highest_scoring_bid_values.size(), 1); + EXPECT_EQ(highest_scoring_bid_values[0].number_value(), + kHighestScoringOtherBidsBidValue); + + ASSERT_EQ(raw_response.ghost_winning_ad_scores_size(), 1); + const auto& ghost_winner = raw_response.ghost_winning_ad_scores()[0]; + EXPECT_EQ(ghost_winner.desirability(), kHighScore); +} + +class ScoredAdDataTest : public ::testing::Test { + protected: + AdWithBidMetadata protected_audience_ad_with_bid_1_; + ProtectedAppSignalsAdWithBidMetadata protected_app_signals_ad_with_bid_1_; + AdWithBidMetadata protected_audience_ad_with_bid_2_; + ProtectedAppSignalsAdWithBidMetadata protected_app_signals_ad_with_bid_2_; +}; + +TEST_F(ScoredAdDataTest, SwapsAllData) { + ScoredAdData scored_ad_data_1 = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusFalse, + &protected_audience_ad_with_bid_1_, &protected_app_signals_ad_with_bid_1_, + kTestScoredAdDataId1); + ScoredAdData scored_ad_data_2 = BuildScoredAdData( + kTestDesirability2, kTestBuyerBid2, kTestAnonStatusTrue, + &protected_audience_ad_with_bid_2_, &protected_app_signals_ad_with_bid_2_, + kTestScoredAdDataId2); + + // Keep tabs on the previous state before swap happens. + ScoreAdsResponse::AdScore prev_ad_score_1 = scored_ad_data_1.ad_score; + ScoreAdsResponse::AdScore prev_ad_score_2 = scored_ad_data_2.ad_score; + + // Keep track of previous response JSONs. + rapidjson::Document prev_response_json_1; + prev_response_json_1.CopyFrom(scored_ad_data_1.response_json, + prev_response_json_1.GetAllocator()); + rapidjson::Document prev_response_json_2; + prev_response_json_2.CopyFrom(scored_ad_data_2.response_json, + prev_response_json_2.GetAllocator()); + + // Swap now and test the data is swapped. + scored_ad_data_2.Swap(scored_ad_data_1); + EXPECT_THAT(scored_ad_data_1.ad_score, EqualsProto(prev_ad_score_2)); + EXPECT_THAT(scored_ad_data_2.ad_score, EqualsProto(prev_ad_score_1)); + EXPECT_EQ(scored_ad_data_1.ad_score.desirability(), kTestDesirability2); + EXPECT_EQ(scored_ad_data_2.ad_score.desirability(), kTestDesirability1); + EXPECT_EQ(scored_ad_data_1.protected_audience_ad_with_bid, + &protected_audience_ad_with_bid_2_); + EXPECT_EQ(scored_ad_data_2.protected_audience_ad_with_bid, + &protected_audience_ad_with_bid_1_); + EXPECT_EQ(scored_ad_data_1.protected_app_signals_ad_with_bid, + &protected_app_signals_ad_with_bid_2_); + EXPECT_EQ(scored_ad_data_2.protected_app_signals_ad_with_bid, + &protected_app_signals_ad_with_bid_1_); + EXPECT_EQ(scored_ad_data_1.id, kTestScoredAdDataId2); + EXPECT_EQ(scored_ad_data_2.id, kTestScoredAdDataId1); + EXPECT_EQ(scored_ad_data_1.k_anon_status, kTestAnonStatusTrue); + EXPECT_EQ(scored_ad_data_2.k_anon_status, kTestAnonStatusFalse); + EXPECT_EQ(scored_ad_data_1.response_json, prev_response_json_2); + EXPECT_EQ(scored_ad_data_2.response_json, prev_response_json_1); +} + +TEST_F(ScoredAdDataTest, ScoredAdDataWithHighScoreIsConsideredBigger) { + ScoredAdData ad_with_high_score_less_bid = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusFalse, + &protected_audience_ad_with_bid_1_, &protected_app_signals_ad_with_bid_1_, + kTestScoredAdDataId1); + + ScoredAdData ad_with_low_score_high_bid = BuildScoredAdData( + kTestDesirability1 - 1.0, kTestBuyerBid1 + 10, kTestAnonStatusTrue, + &protected_audience_ad_with_bid_2_, &protected_app_signals_ad_with_bid_2_, + kTestScoredAdDataId2); + + EXPECT_GT(ad_with_high_score_less_bid, ad_with_low_score_high_bid); +} + +TEST_F(ScoredAdDataTest, ScoredAdDataWithHighBidIsConsideredBigger) { + ScoredAdData ad_with_same_score_less_bid = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusTrue, + &protected_audience_ad_with_bid_1_, &protected_app_signals_ad_with_bid_1_, + kTestScoredAdDataId1); + + ScoredAdData ad_with_same_score_high_bid = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1 + 10, kTestAnonStatusFalse, + &protected_audience_ad_with_bid_2_, &protected_app_signals_ad_with_bid_2_, + kTestScoredAdDataId2); + + EXPECT_GT(ad_with_same_score_high_bid, ad_with_same_score_less_bid); +} + +TEST_F(ScoredAdDataTest, ScoredAdDataMeetingKAnonThresholdIsBigger) { + ScoredAdData ad_with_same_score_bid_k_anon = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusTrue, + &protected_audience_ad_with_bid_1_, &protected_app_signals_ad_with_bid_1_, + kTestScoredAdDataId1); + + ScoredAdData ad_with_same_score_bid_non_k_anon = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusFalse, + &protected_audience_ad_with_bid_2_, &protected_app_signals_ad_with_bid_2_, + kTestScoredAdDataId2); + + EXPECT_GT(ad_with_same_score_bid_k_anon, ad_with_same_score_bid_non_k_anon); +} + +TEST_F(ScoredAdDataTest, ScoredAdDataWithSameScoreBidAndKAnonAreEqual) { + ScoredAdData ad_with_same_score_bid_k_anon_1 = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusTrue, + &protected_audience_ad_with_bid_1_, &protected_app_signals_ad_with_bid_1_, + kTestScoredAdDataId1); + + ScoredAdData ad_with_same_score_bid_k_anon_2 = BuildScoredAdData( + kTestDesirability1, kTestBuyerBid1, kTestAnonStatusTrue, + &protected_audience_ad_with_bid_2_, &protected_app_signals_ad_with_bid_2_, + kTestScoredAdDataId2); + + EXPECT_FALSE(ad_with_same_score_bid_k_anon_1 > + ad_with_same_score_bid_k_anon_2); + EXPECT_FALSE(ad_with_same_score_bid_k_anon_2 > + ad_with_same_score_bid_k_anon_1); +} + +TEST_F(ScoreAdsReactorTest, + ReportingSuccessWithCodeIsolationWithBuyerAndSellerReportingId) { + MockV8DispatchClient dispatcher; + AuctionServiceRuntimeConfig runtime_config = { + .enable_report_result_url_generation = true, + .enable_report_win_url_generation = true, + .enable_seller_and_buyer_udf_isolation = true}; + RequestLogContext log_context({}, + server_common::ConsentedDebugConfiguration()); + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kEuroIsoCode); + post_auction_signals.highest_scoring_other_bid = 0; + post_auction_signals.made_highest_scoring_other_bid = false; + post_auction_signals.highest_scoring_other_bid_currency = + kUnknownBidCurrencyCode; + post_auction_signals.winning_ad_render_url = kTestAdRenderUrl; + + SellerReportingDispatchRequestData seller_dispatch_data = + GetTestSellerDispatchRequestData(post_auction_signals, log_context); + BuyerReportingDispatchRequestData buyer_dispatch_data = + GetTestBuyerDispatchRequestData(log_context); + buyer_dispatch_data.buyer_reporting_id = ""; + buyer_dispatch_data.buyer_and_seller_reporting_id = + kBuyerAndSellerReportingId; + buyer_dispatch_data.made_highest_scoring_other_bid = false; + RawRequest raw_request; + AdWithBidMetadata ad; + PopulateTestAdWithBidMetdata(post_auction_signals, buyer_dispatch_data, ad); + ad.set_bid_currency(kEurosIsoCode); + ad.set_bid(1.0); + BuildRawRequest({ad}, kTestSellerSignals, kTestAuctionSignals, + kTestScoringSignals, kTestPublisherHostName, raw_request); + absl::StatusOr seller_version = GetDefaultSellerUdfVersion(); + absl::StatusOr buyer_version = GetBuyerReportWinVersion( + ad.interest_group_owner(), AuctionType::kProtectedAudience); + { + InSequence s; + EXPECT_CALL(dispatcher, BatchExecute) + .WillOnce([&seller_version](std::vector& batch, + BatchDispatchDoneCallback done_callback) { + EXPECT_EQ(batch.size(), 1); + const auto& request = batch[0]; + EXPECT_EQ(request.handler_name, "scoreAdEntryFunction"); + EXPECT_EQ(request.version_string, *seller_version); + std::vector> responses; + DispatchResponse response = { + .id = request.id, + .resp = absl::StrFormat(kTestScoreAdResponseTemplate, + kTestDesirability)}; + done_callback({response}); + return absl::OkStatus(); + }); + EXPECT_CALL(dispatcher, BatchExecute) + .WillOnce([&seller_version](std::vector& batch, + BatchDispatchDoneCallback done_callback) { + EXPECT_EQ(batch.size(), 1); + const auto& request = batch[0]; + EXPECT_EQ(request.handler_name, kReportResultEntryFunction); + EXPECT_EQ(request.version_string, *seller_version); + absl::StatusOr seller_signals = + ParseJsonString(*request.input[ReportResultArgIndex( + ReportResultArgs::kSellerReportingSignals)]); + std::string buyer_and_seller_id; + PS_ASSIGN_IF_PRESENT(buyer_and_seller_id, seller_signals.value(), + kBuyerAndSellerReportingIdTag, String); + EXPECT_TRUE(buyer_and_seller_id.empty()); + DispatchResponse response; + response.resp = kTestReportResultResponseJson; + response.id = request.id; + done_callback({response}); + return absl::OkStatus(); + }); + EXPECT_CALL(dispatcher, BatchExecute) + .WillOnce([&buyer_version, &seller_dispatch_data, &buyer_dispatch_data]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + EXPECT_EQ(batch.size(), 1); + + const DispatchRequest& request = batch[0]; + absl::StatusOr buyer_signals = + ParseJsonString(*request.input[PAReportWinArgIndex( + PAReportWinArgs::kBuyerReportingSignals)]); + std::string buyer_and_seller_id; + PS_ASSIGN_IF_PRESENT(buyer_and_seller_id, buyer_signals.value(), + kBuyerAndSellerReportingIdTag, String); + EXPECT_EQ(buyer_and_seller_id, kBuyerAndSellerReportingId); + VerifyReportWinInput(request, *buyer_version, seller_dispatch_data, + buyer_dispatch_data); + DispatchResponse response; + response.resp = kTestReportWinResponseJson; + response.id = request.id; + done_callback({response}); + return absl::OkStatus(); + }); + } + + const auto& response = + ExecuteScoreAds(raw_request, dispatcher, runtime_config); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + VerifyReportingUrlsInScoreAdsResponse(raw_response); +} +} // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/score_ads_reactor_test_util.cc b/services/auction_service/score_ads_reactor_test_util.cc index b3d243ea..1d715880 100644 --- a/services/auction_service/score_ads_reactor_test_util.cc +++ b/services/auction_service/score_ads_reactor_test_util.cc @@ -116,6 +116,36 @@ ScoreAdsResponse ScoreAdsReactorTestHelper::ExecuteScoreAds( return response; } +absl::Status FakeExecute( + std::vector& batch, + BatchDispatchDoneCallback done_callback, + absl::flat_hash_map + json_ad_scores, + const bool call_wrapper_method, const bool enable_adtech_code_logging) { + std::vector> responses; + + for (const auto& request : batch) { + if (std::strcmp(request.handler_name.c_str(), + kReportingDispatchHandlerFunctionName) != 0 && + std::strcmp(request.handler_name.c_str(), kReportResultEntryFunction) != + 0 && + std::strcmp(request.handler_name.c_str(), kReportWinEntryFunction) != + 0) { + EXPECT_EQ(request.handler_name, "scoreAdEntryFunction"); + } + DispatchResponse dispatch_response; + auto it = json_ad_scores.find(request.id); + CHECK(it != json_ad_scores.end()); + dispatch_response.resp = it->second; + dispatch_response.id = request.id; + PS_VLOG(5) << ">>>> Associated id: " << dispatch_response.id + << ", with response: " << dispatch_response.resp; + responses.emplace_back(std::move(dispatch_response)); + } + done_callback(responses); + return absl::OkStatus(); +} + absl::Status FakeExecute(std::vector& batch, BatchDispatchDoneCallback done_callback, std::vector json_ad_scores, @@ -133,10 +163,10 @@ absl::Status FakeExecute(std::vector& batch, 0) { EXPECT_EQ(request.handler_name, "scoreAdEntryFunction"); } - DispatchResponse dispatch_response = {}; + DispatchResponse dispatch_response; dispatch_response.resp = *json_ad_score_itr++; dispatch_response.id = request.id; - responses.emplace_back(dispatch_response); + responses.emplace_back(std::move(dispatch_response)); } done_callback(responses); return absl::OkStatus(); diff --git a/services/auction_service/score_ads_reactor_test_util.h b/services/auction_service/score_ads_reactor_test_util.h index 9032db34..e9c7142f 100644 --- a/services/auction_service/score_ads_reactor_test_util.h +++ b/services/auction_service/score_ads_reactor_test_util.h @@ -82,6 +82,14 @@ absl::Status FakeExecute(std::vector& batch, const bool call_wrapper_method = false, const bool enable_adtech_code_logging = false); +absl::Status FakeExecute( + std::vector& batch, + BatchDispatchDoneCallback done_callback, + absl::flat_hash_map + json_ad_scores, + const bool call_wrapper_method = false, + const bool enable_adtech_code_logging = false); + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_SCORE_ADS_REACTOR_TEST_UTIL_H_ diff --git a/services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc b/services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc index cd3d2a64..bea5b598 100644 --- a/services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc +++ b/services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc @@ -38,7 +38,7 @@ absl::Status BuyerReportingFetcher::Start() { } void BuyerReportingFetcher::End() { - if (task_id_.has_value()) { + if (task_id_) { executor_.Cancel(*task_id_); task_id_ = absl::nullopt; } diff --git a/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc index 577affbd..56c9599c 100644 --- a/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc +++ b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc @@ -85,7 +85,7 @@ absl::Status BuyerReportingUdfFetchManager::Start() { } void BuyerReportingUdfFetchManager::End() { - if (task_id_.has_value()) { + if (task_id_) { executor_.Cancel(*task_id_); task_id_ = absl::nullopt; } diff --git a/services/auction_service/utils/proto_utils.cc b/services/auction_service/utils/proto_utils.cc index a1ab0e1d..9c6e1e16 100644 --- a/services/auction_service/utils/proto_utils.cc +++ b/services/auction_service/utils/proto_utils.cc @@ -434,6 +434,17 @@ absl::StatusOr ParseAndGetScoreAdResponseJson( return response_obj; } +ScoreAdsResponse::AdScore::AdRejectionReason BuildAdRejectionReason( + absl::string_view interest_group_owner, + absl::string_view interest_group_name, + SellerRejectionReason seller_rejection_reason) { + ScoreAdsResponse::AdScore::AdRejectionReason ad_rejection_reason; + ad_rejection_reason.set_interest_group_owner(interest_group_owner); + ad_rejection_reason.set_interest_group_name(interest_group_name); + ad_rejection_reason.set_rejection_reason(seller_rejection_reason); + return ad_rejection_reason; +} + std::optional ParseAdRejectionReason(const rapidjson::Document& score_ad_resp, absl::string_view interest_group_owner, @@ -447,16 +458,11 @@ ParseAdRejectionReason(const rapidjson::Document& score_ad_resp, } std::string rejection_reason_str = score_ad_resp[kRejectReasonPropertyForScoreAd].GetString(); - SellerRejectionReason rejection_reason = - ToSellerRejectionReason(rejection_reason_str); - ScoreAdsResponse::AdScore::AdRejectionReason ad_rejection_reason; - ad_rejection_reason.set_interest_group_owner(interest_group_owner); - ad_rejection_reason.set_interest_group_name(interest_group_name); - ad_rejection_reason.set_rejection_reason(rejection_reason); - return ad_rejection_reason; + return BuildAdRejectionReason(interest_group_owner, interest_group_name, + ToSellerRejectionReason(rejection_reason_str)); } -absl::StatusOr ParseScoreAdResponse( +absl::StatusOr ScoreAdResponseJsonToProto( const rapidjson::Document& score_ad_resp, int max_allowed_size_debug_url_chars, int64_t max_allowed_size_all_debug_urls_chars, diff --git a/services/auction_service/utils/proto_utils.h b/services/auction_service/utils/proto_utils.h index 53ca9f3a..c30dea50 100644 --- a/services/auction_service/utils/proto_utils.h +++ b/services/auction_service/utils/proto_utils.h @@ -72,13 +72,18 @@ absl::StatusOr ParseAndGetScoreAdResponseJson( bool enable_ad_tech_code_logging, const std::string& response, RequestLogContext& log_context); +ScoreAdsResponse::AdScore::AdRejectionReason BuildAdRejectionReason( + absl::string_view interest_group_owner, + absl::string_view interest_group_name, + SellerRejectionReason seller_rejection_reason); + std::optional ParseAdRejectionReason(const rapidjson::Document& score_ad_resp, absl::string_view interest_group_owner, absl::string_view interest_group_name, RequestLogContext& log_context); -absl::StatusOr ParseScoreAdResponse( +absl::StatusOr ScoreAdResponseJsonToProto( const rapidjson::Document& score_ad_resp, int max_allowed_size_debug_url_chars, int64_t max_allowed_size_all_debug_urls_chars, diff --git a/services/auction_service/utils/proto_utils_test.cc b/services/auction_service/utils/proto_utils_test.cc index b17c83ee..8911cbe4 100644 --- a/services/auction_service/utils/proto_utils_test.cc +++ b/services/auction_service/utils/proto_utils_test.cc @@ -34,6 +34,9 @@ namespace { constexpr char kTestPublisher[] = "publisher.com"; constexpr char kTestIGOwner[] = "interest_group_owner"; +constexpr char kTestIGName[] = "interest_group_name"; +constexpr SellerRejectionReason kTestSellerRejectionReason = + SellerRejectionReason::INVALID_BID; constexpr char kTestRenderUrl[] = "https://render_url"; constexpr char kTestComponentSeller[] = "component_seller_origin"; constexpr char kTestTopLevelSeller[] = "top_level_seller_origin"; @@ -348,6 +351,15 @@ TEST(BuildScoreAdRequestTest, PopulatesFeatureFlagsForAdWithBidMetadata) { } } +TEST(BuildAdRejectionReasonTest, BuildsAdRejectionReason) { + ScoreAdsResponse::AdScore::AdRejectionReason ad_rejection_reason = + BuildAdRejectionReason(kTestIGOwner, kTestIGName, + kTestSellerRejectionReason); + EXPECT_EQ(ad_rejection_reason.interest_group_owner(), kTestIGOwner); + EXPECT_EQ(ad_rejection_reason.interest_group_name(), kTestIGName); + EXPECT_EQ(ad_rejection_reason.rejection_reason(), kTestSellerRejectionReason); +} + TEST(BuildScoreAdRequestTest, PopulatesFeatureFlagsForPASAdWithBidMetadata) { std::vector flag_values = {true, false}; for (auto flag_1 : flag_values) { @@ -380,7 +392,7 @@ TEST(ScoreAdsTest, ParsesScoreAdResponseRespectsDebugUrlLimits) { long_loss_url, long_win_url)); CHECK_OK(scored_ad); int64_t current_all_debug_urls_chars = 0; - auto parsed_response = ParseScoreAdResponse( + auto parsed_response = ScoreAdResponseJsonToProto( *scored_ad, /*max_allowed_size_debug_url_chars=*/65536, /*max_allowed_size_all_debug_urls_chars=*/1024, /*device_component_auction=*/false, current_all_debug_urls_chars); diff --git a/services/bidding_service/BUILD b/services/bidding_service/BUILD index 4c679d61..3e11dc39 100644 --- a/services/bidding_service/BUILD +++ b/services/bidding_service/BUILD @@ -96,7 +96,6 @@ cc_library( "//services/bidding_service/data:runtime_config", "//services/bidding_service/utils:egress", "//services/bidding_service/utils:validation", - "//services/common:feature_flags", "//services/common/clients/code_dispatcher:v8_dispatch_client", "//services/common/clients/kv_server:kv_async_client", "//services/common/code_dispatch:code_dispatch_reactor", @@ -345,6 +344,7 @@ cc_binary( "//services/common/encryption:key_fetcher_factory", "//services/common/telemetry:configure_telemetry", "//services/common/util:file_util", + "//services/common/util:read_system", "//services/common/util:request_response_constants", "//services/common/util:tcmalloc_utils", "@com_github_grpc_grpc//:grpc++", diff --git a/services/bidding_service/base_generate_bids_reactor.h b/services/bidding_service/base_generate_bids_reactor.h index 23503fdb..70aa5510 100644 --- a/services/bidding_service/base_generate_bids_reactor.h +++ b/services/bidding_service/base_generate_bids_reactor.h @@ -51,7 +51,8 @@ class BaseGenerateBidsReactor server_common::KeyFetcherManagerInterface* key_fetcher_manager, CryptoClientWrapperInterface* crypto_client) : CodeDispatchReactor( - request, response, key_fetcher_manager, crypto_client), + request, response, key_fetcher_manager, crypto_client, + runtime_config.enable_cancellation, runtime_config.enable_kanon), enable_buyer_debug_url_generation_( runtime_config.enable_buyer_debug_url_generation), roma_timeout_ms_(runtime_config.roma_timeout_ms), @@ -62,8 +63,10 @@ class BaseGenerateBidsReactor log_context_( GetLoggingContext(this->raw_request_), this->raw_request_.consented_debug_config(), - [this]() { return this->raw_response_.mutable_debug_info(); }), - enable_adtech_code_logging_(log_context_.is_consented()), + [this]() { return this->raw_response_.mutable_debug_info(); }, + this->raw_request_.is_sampled_for_debug()), + enable_adtech_code_logging_(log_context_.is_consented() || + this->raw_request_.is_sampled_for_debug()), max_allowed_size_debug_url_chars_( runtime_config.max_allowed_size_debug_url_bytes), max_allowed_size_all_debug_urls_chars_( diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index b9dbaec8..61c059a0 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -63,6 +63,7 @@ #include "services/common/feature_flags.h" #include "services/common/telemetry/configure_telemetry.h" #include "services/common/util/file_util.h" +#include "services/common/util/read_system.h" #include "services/common/util/request_response_constants.h" #include "services/common/util/tcmalloc_utils.h" #include "src/concurrent/event_engine_executor.h" @@ -330,6 +331,8 @@ DispatchConfig GetV8DispatchConfig( inference::SandboxExecutor& inference_executor = inference::Executor(); CHECK_EQ(inference_executor.StartSandboxee().code(), absl::StatusCode::kOk); + server_common::SystemMetrics::SetInferencePid( + inference_executor.GetPid()); } return config; }(); @@ -528,6 +531,9 @@ absl::Status RunServer() { "specify at least one."); } + const bool enable_temporary_unlimited_egress = + absl::GetFlag(FLAGS_enable_temporary_unlimited_egress); + BiddingServiceRuntimeConfig runtime_config = { .tee_ad_retrieval_kv_server_addr = std::move(tee_ad_retrieval_kv_server_addr), @@ -551,7 +557,10 @@ absl::Status RunServer() { config_client.GetBooleanParameter(AD_RETRIEVAL_KV_SERVER_EGRESS_TLS), .kv_server_egress_tls = config_client.GetBooleanParameter(KV_SERVER_EGRESS_TLS), - .enable_private_aggregate_reporting = enable_private_aggregate_reporting}; + .enable_private_aggregate_reporting = enable_private_aggregate_reporting, + .enable_cancellation = absl::GetFlag(FLAGS_enable_cancellation), + .enable_kanon = absl::GetFlag(FLAGS_enable_kanon), + .enable_temporary_unlimited_egress = enable_temporary_unlimited_egress}; if (udf_config.fetch_mode() == blob_fetch::FETCH_MODE_BUCKET) { if (enable_protected_audience) { @@ -575,8 +584,6 @@ absl::Status RunServer() { " (Invalid JSON provided?)"; PS_VLOG(5) << "Fetched egress schema fetch config: " << egress_schema_fetch_config.DebugString(); - const bool enable_temporary_unlimited_egress = - absl::GetFlag(FLAGS_enable_temporary_unlimited_egress); absl::StatusOr egress_info = absl::UnimplementedError(""); if (enable_temporary_unlimited_egress && enable_protected_app_signals) { PS_LOG(INFO) << "Temporary egress feature is enabled in the binary"; diff --git a/services/bidding_service/bidding_service_integration_test.cc b/services/bidding_service/bidding_service_integration_test.cc index df19c6ac..aa0e182d 100644 --- a/services/bidding_service/bidding_service_integration_test.cc +++ b/services/bidding_service/bidding_service_integration_test.cc @@ -50,6 +50,7 @@ constexpr char kSecret[] = "secret"; constexpr char kAdRenderUrlPrefixForTest[] = "https://advertising.net/ad?"; constexpr char kTopLevelSeller[] = "top_level_seller"; constexpr char kTestConsentToken[] = "testConsentToken"; +constexpr int kTestMultiBidLimit = 10; // While Roma demands JSON input and enforces it strictly, we follow the // javascript style guide for returning objects here, so object keys are @@ -400,6 +401,46 @@ constexpr absl::string_view kJsCodeWithComponentBidNotAllowed = R"JS_CODE( } )JS_CODE"; +constexpr absl::string_view js_code_with_multiple_bids_template = R"JS_CODE( + function generateBid(interest_group, + auction_signals, + buyer_signals, + trusted_bidding_signals, + device_signals) { + + forDebuggingOnly.reportAdAuctionLoss("https://example-dsp.com/debugLoss"); + forDebuggingOnly.reportAdAuctionWin("https://example-dsp.com/debugWin"); + + // Reshaped into an AdWithBid. + return [ + { + render: "render_1", + ad: {"arbitraryMetadataField": 1}, + bid: 1, + allowComponentAuction: false + }, + { + render: "render_2", + ad: {"arbitraryMetadataField": 2}, + bid: 2, + allowComponentAuction: false + }, + { + render: "render_3", + ad: {"arbitraryMetadataField": 3}, + bid: 3, + allowComponentAuction: false + }, + { + render: "render_4", + ad: {"arbitraryMetadataField": 4}, + bid: 4, + allowComponentAuction: false + } + ]; + } + )JS_CODE"; + SignalBucket GetExpectedSignalBucket() { BucketOffset bucket_offset; bucket_offset.add_value(1); // Add first 64-bit value @@ -509,7 +550,8 @@ BuildGenerateBidsRequestFromBrowser( absl::flat_hash_map>* interest_group_to_ad, int desired_bid_count = 5, bool set_enable_debug_reporting = false, - bool enable_adtech_code_logging = false) { + bool enable_adtech_code_logging = false, + int multi_bid_limit = kTestMultiBidLimit) { GenerateBidsRequest::GenerateBidsRawRequest raw_request; raw_request.set_enable_debug_reporting(set_enable_debug_reporting); for (int i = 0; i < desired_bid_count; i++) { @@ -525,6 +567,7 @@ BuildGenerateBidsRequestFromBrowser( raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); raw_request.mutable_consented_debug_config()->set_is_consented(true); } + raw_request.set_multi_bid_limit(multi_bid_limit); return raw_request; } @@ -1082,6 +1125,7 @@ TEST_F(GenerateBidsReactorIntegrationTest, GeneratesBidsFromDevice) { interest_group.ad_render_ids().end())); *raw_request.mutable_interest_group_for_bidding()->Add() = std::move(interest_group); + raw_request.set_multi_bid_limit(kTestMultiBidLimit); // This fails in production, the user Bidding Signals are not being set. // use logging to figure out why. } @@ -1165,6 +1209,7 @@ struct GenerateBidHelperConfig { int desired_bid_count = 5; bool component_auction = false; bool enable_private_aggregate_reporting = false; + int multi_bid_limit = kTestMultiBidLimit; }; void GenerateBidCodeWrapperTestHelper( @@ -1182,7 +1227,7 @@ void GenerateBidCodeWrapperTestHelper( auto req = BuildGenerateBidsRequestFromBrowser( &interest_group_to_ad, test_config.desired_bid_count, test_config.enable_debug_reporting, - test_config.enable_adtech_code_logging); + test_config.enable_adtech_code_logging, test_config.multi_bid_limit); ASSERT_TRUE(req.ok()) << req.status(); if (test_config.component_auction) { req.value().set_top_level_seller(kTopLevelSeller); @@ -1573,5 +1618,37 @@ TEST_F(GenerateBidsReactorIntegrationTest, EXPECT_EQ(raw_response.bids_size(), 0); } +TEST_F(GenerateBidsReactorIntegrationTest, DropsAllBidsIfOverMultiBidLimit) { + GenerateBidsResponse response; + GenerateBidHelperConfig test_config = {.desired_bid_count = 1, + .multi_bid_limit = 3}; + GenerateBidCodeWrapperTestHelper( + &response, js_code_with_multiple_bids_template, test_config); + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + EXPECT_EQ(raw_response.bids_size(), 0); +} + +TEST_F(GenerateBidsReactorIntegrationTest, ReturnsBidsArrayWithDebugURL) { + GenerateBidsResponse response; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = true, + .enable_buyer_debug_url_generation = true, + .enable_adtech_code_logging = true, + .desired_bid_count = 1}; + GenerateBidCodeWrapperTestHelper( + &response, js_code_with_multiple_bids_template, test_config); + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + EXPECT_GT(raw_response.bids_size(), 0); + for (const auto& ad_with_bid : raw_response.bids()) { + EXPECT_GT(ad_with_bid.bid(), 0); + EXPECT_EQ(ad_with_bid.debug_report_urls().auction_debug_win_url(), + "https://example-dsp.com/debugWin"); + EXPECT_EQ(ad_with_bid.debug_report_urls().auction_debug_loss_url(), + "https://example-dsp.com/debugLoss"); + } +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper.h b/services/bidding_service/code_wrapper/buyer_code_wrapper.h index 54151f3d..0bd5dce8 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper.h @@ -55,8 +55,18 @@ std::string GetProtectedAppSignalsGenericBuyerWrappedCode( //- Exporting logs to Bidding Service using console.log //- Hooks in wasm module inline constexpr absl::string_view kEntryFunction = R"JS_CODE( + function checkMultiBidLimit(generate_bid_response, multiBidLimit, featureFlags) { + // Drop all the bids if generateBid response exceed multiBidLimit. + if (generate_bid_response.length > multiBidLimit) { + generate_bid_response = []; + if (featureFlags.enable_logging) { + console.error("[Error: more bids returned by generateBid than multiBidLimit]"); + } + } + return generate_bid_response; + } var ps_response = { - response: {}, + response: [], logs: [], errors: [], warnings: [] @@ -87,20 +97,40 @@ inline constexpr absl::string_view kEntryFunction = R"JS_CODE( try { generate_bid_response = await generateBid($0); - //Add private aggregate contributions to the response. - //If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. - if(ps_response.paapicontributions){ - generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; - } - ps_response.response = generate_bid_response - if( featureFlags.enable_debug_url_generation && - (forDebuggingOnly_auction_loss_url - || forDebuggingOnly_auction_win_url)) { - ps_response.response.debug_report_urls = { + if (generate_bid_response instanceof Array) { + generate_bid_response = checkMultiBidLimit(generate_bid_response, multiBidLimit, featureFlags); + + // Add private aggregate contributions and debug urls to each response. + // If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. + generate_bid_response.forEach((response) => { + if (featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + response.debug_report_urls = { auction_debug_loss_url: forDebuggingOnly_auction_loss_url, auction_debug_win_url: forDebuggingOnly_auction_win_url - } + }; + } + if (ps_response.paapicontributions) { + response.private_aggregation_contributions = ps_response.paapicontributions; } + }); + ps_response.response = generate_bid_response; + } else { + // generateBid returned a single AdWithBid object. + if (featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + generate_bid_response.debug_report_urls = { + auction_debug_loss_url: forDebuggingOnly_auction_loss_url, + auction_debug_win_url: forDebuggingOnly_auction_win_url + }; + } + if (ps_response.paapicontributions) { + generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; + } + ps_response.response.push(generate_bid_response); + } } catch({error, message}) { if (featureFlags.enable_logging) { console.error("[Error: " + error + "; Message: " + message + "]"); diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h index 94137cb8..5fc3f962 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h @@ -94,13 +94,23 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; + function checkMultiBidLimit(generate_bid_response, multiBidLimit, featureFlags) { + // Drop all the bids if generateBid response exceed multiBidLimit. + if (generate_bid_response.length > multiBidLimit) { + generate_bid_response = []; + if (featureFlags.enable_logging) { + console.error("[Error: more bids returned by generateBid than multiBidLimit]"); + } + } + return generate_bid_response; + } var ps_response = { - response: {}, + response: [], logs: [], errors: [], warnings: [] }; - async function generateBidEntryFunction(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals, featureFlags){ + async function generateBidEntryFunction(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals, multiBidLimit, featureFlags){ if(featureFlags.enable_logging){ console.log = function(...args) { ps_response.logs.push(JSON.stringify(args)) @@ -125,21 +135,41 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( globalThis.forDebuggingOnly = forDebuggingOnly; try { - generate_bid_response = await generateBid(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals); - //Add private aggregate contributions to the response. - //If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. - if(ps_response.paapicontributions){ - generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; - } - ps_response.response = generate_bid_response - if( featureFlags.enable_debug_url_generation && - (forDebuggingOnly_auction_loss_url - || forDebuggingOnly_auction_win_url)) { - ps_response.response.debug_report_urls = { + generate_bid_response = await generateBid(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals, multiBidLimit); + if (generate_bid_response instanceof Array) { + generate_bid_response = checkMultiBidLimit(generate_bid_response, multiBidLimit, featureFlags); + + // Add private aggregate contributions and debug urls to each response. + // If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. + generate_bid_response.forEach((response) => { + if (featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + response.debug_report_urls = { auction_debug_loss_url: forDebuggingOnly_auction_loss_url, auction_debug_win_url: forDebuggingOnly_auction_win_url - } + }; + } + if (ps_response.paapicontributions) { + response.private_aggregation_contributions = ps_response.paapicontributions; } + }); + ps_response.response = generate_bid_response; + } else { + // generateBid returned a single AdWithBid object. + if (featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + generate_bid_response.debug_report_urls = { + auction_debug_loss_url: forDebuggingOnly_auction_loss_url, + auction_debug_win_url: forDebuggingOnly_auction_win_url + }; + } + if (ps_response.paapicontributions) { + generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; + } + ps_response.response.push(generate_bid_response); + } } catch({error, message}) { if (featureFlags.enable_logging) { console.error("[Error: " + error + "; Message: " + message + "]"); @@ -173,13 +203,23 @@ constexpr absl::string_view const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; + function checkMultiBidLimit(generate_bid_response, multiBidLimit, featureFlags) { + // Drop all the bids if generateBid response exceed multiBidLimit. + if (generate_bid_response.length > multiBidLimit) { + generate_bid_response = []; + if (featureFlags.enable_logging) { + console.error("[Error: more bids returned by generateBid than multiBidLimit]"); + } + } + return generate_bid_response; + } var ps_response = { - response: {}, + response: [], logs: [], errors: [], warnings: [] }; - async function generateBidEntryFunction(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, featureFlags){ + async function generateBidEntryFunction(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, multiBidLimit, featureFlags){ if(featureFlags.enable_logging){ console.log = function(...args) { ps_response.logs.push(JSON.stringify(args)) @@ -212,21 +252,41 @@ constexpr absl::string_view globalThis.forDebuggingOnly = forDebuggingOnly; try { - generate_bid_response = await generateBid(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion); - //Add private aggregate contributions to the response. - //If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. - if(ps_response.paapicontributions){ - generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; - } - ps_response.response = generate_bid_response - if( featureFlags.enable_debug_url_generation && - (forDebuggingOnly_auction_loss_url - || forDebuggingOnly_auction_win_url)) { - ps_response.response.debug_report_urls = { + generate_bid_response = await generateBid(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, multiBidLimit); + if (generate_bid_response instanceof Array) { + generate_bid_response = checkMultiBidLimit(generate_bid_response, multiBidLimit, featureFlags); + + // Add private aggregate contributions and debug urls to each response. + // If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. + generate_bid_response.forEach((response) => { + if (featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + response.debug_report_urls = { auction_debug_loss_url: forDebuggingOnly_auction_loss_url, auction_debug_win_url: forDebuggingOnly_auction_win_url - } + }; + } + if (ps_response.paapicontributions) { + response.private_aggregation_contributions = ps_response.paapicontributions; } + }); + ps_response.response = generate_bid_response; + } else { + // generateBid returned a single AdWithBid object. + if (featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + generate_bid_response.debug_report_urls = { + auction_debug_loss_url: forDebuggingOnly_auction_loss_url, + auction_debug_win_url: forDebuggingOnly_auction_win_url + }; + } + if (ps_response.paapicontributions) { + generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; + } + ps_response.response.push(generate_bid_response); + } } catch({error, message}) { if (featureFlags.enable_logging) { console.error("[Error: " + error + "; Message: " + message + "]"); diff --git a/services/bidding_service/constants.h b/services/bidding_service/constants.h index 23dc90ec..ae93b58f 100644 --- a/services/bidding_service/constants.h +++ b/services/bidding_service/constants.h @@ -37,10 +37,11 @@ inline constexpr char kPrepareDataForAdRetrievalArgs[] = "sellerAuctionSignals, contextualSignals"; constexpr absl::string_view kProtectedAudienceGenerateBidsArgs = "interest_group, auction_signals, buyer_signals, trusted_bidding_signals, " - "device_signals"; + "device_signals, multiBidLimit"; constexpr absl::string_view kProtectedAppSignalsGenerateBidsArgs = "ads, sellerAuctionSignals, buyerSignals, " - "preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion"; + "preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, " + "multiBidLimit"; constexpr absl::string_view kEncodedProtectedAppSignalsHandler = R"JS_CODE(if (encodedOnDeviceSignals) { @@ -66,7 +67,7 @@ enum class PrepareDataForRetrievalUdfArgs { }; // Params related to the UDF to use to generate bids for the protected signals. -inline constexpr int kNumGenerateBidsUdfArgs = 9; +inline constexpr int kNumGenerateBidsUdfArgs = 10; enum class GenerateBidsUdfArgs { kAds = 0, kAuctionSignals, @@ -74,7 +75,8 @@ enum class GenerateBidsUdfArgs { kPreProcessedDataForRetrieval, kProtectedAppSignals, kProtectedAppSignalsVersion, - kFeatureFlags, + kMultiBidLimit, + kFeatureFlags }; inline constexpr char kUnexpectedNumberOfRomaResponses[] = diff --git a/services/bidding_service/data/runtime_config.h b/services/bidding_service/data/runtime_config.h index b99eace9..e4becf37 100644 --- a/services/bidding_service/data/runtime_config.h +++ b/services/bidding_service/data/runtime_config.h @@ -67,6 +67,10 @@ struct BiddingServiceRuntimeConfig { kPrepareDataForAdRetrievalBlobVersion; // Enables private aggregate reporting. bool enable_private_aggregate_reporting = false; + + bool enable_cancellation = false; + bool enable_kanon = false; + bool enable_temporary_unlimited_egress = false; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/egress_features/boolean_feature.cc b/services/bidding_service/egress_features/boolean_feature.cc index e69cf1e5..cadb08a9 100644 --- a/services/bidding_service/egress_features/boolean_feature.cc +++ b/services/bidding_service/egress_features/boolean_feature.cc @@ -34,7 +34,7 @@ absl::StatusOr> BooleanFeature::Serialize() { } std::optional value; PS_ASSIGN_IF_PRESENT(value, value_, "value", Bool); - if (!value.has_value()) { + if (!value) { return absl::InvalidArgumentError( "Value in the egress payload didn't have a bool"); } diff --git a/services/bidding_service/egress_features/histogram_feature.cc b/services/bidding_service/egress_features/histogram_feature.cc index b7669f36..e79c4b93 100644 --- a/services/bidding_service/egress_features/histogram_feature.cc +++ b/services/bidding_service/egress_features/histogram_feature.cc @@ -39,7 +39,7 @@ absl::StatusOr> HistogramFeature::Copy() const { } uint32_t HistogramFeature::Size() const { - if (!histogram_size_.has_value()) { + if (!histogram_size_) { uint32_t total_size = 0; if (auto histogram_schema = GetArrayMember(*schema_value_, "value"); histogram_schema.ok()) { diff --git a/services/bidding_service/egress_features/signed_int_feature.cc b/services/bidding_service/egress_features/signed_int_feature.cc index 75819acd..9a64eb66 100644 --- a/services/bidding_service/egress_features/signed_int_feature.cc +++ b/services/bidding_service/egress_features/signed_int_feature.cc @@ -43,7 +43,7 @@ absl::StatusOr> SignedIntFeature::Serialize() { } std::optional value; PS_ASSIGN_IF_PRESENT(value, value_, "value", Int); - if (!value.has_value()) { + if (!value) { return absl::InvalidArgumentError( "Int feature not found in the egress payload"); } diff --git a/services/bidding_service/egress_features/unsigned_int_feature.cc b/services/bidding_service/egress_features/unsigned_int_feature.cc index c4d83cf8..159c905d 100644 --- a/services/bidding_service/egress_features/unsigned_int_feature.cc +++ b/services/bidding_service/egress_features/unsigned_int_feature.cc @@ -45,7 +45,7 @@ absl::StatusOr> UnsignedIntFeature::Serialize() { } std::optional value; PS_ASSIGN_IF_PRESENT(value, value_, "value", Uint); - if (!value.has_value()) { + if (!value) { return absl::InvalidArgumentError( "Int feature not found in the egress payload"); } diff --git a/services/bidding_service/generate_bids_binary_reactor.cc b/services/bidding_service/generate_bids_binary_reactor.cc index 00f54d83..693c73c7 100644 --- a/services/bidding_service/generate_bids_binary_reactor.cc +++ b/services/bidding_service/generate_bids_binary_reactor.cc @@ -57,10 +57,8 @@ BuildProtectedAudienceBidRequest(RawRequest& raw_request, std::move(*ig_for_bidding.mutable_user_bidding_signals()); // Populate auction, buyer, and bidding signals. - *bid_request.mutable_auction_signals() = - std::move(*raw_request.mutable_auction_signals()); - *bid_request.mutable_per_buyer_signals() = - std::move(*raw_request.mutable_buyer_signals()); + bid_request.set_auction_signals(raw_request.auction_signals()); + bid_request.set_per_buyer_signals(raw_request.buyer_signals()); *bid_request.mutable_trusted_bidding_signals() = std::move(*ig_for_bidding.mutable_trusted_bidding_signals()); @@ -69,18 +67,14 @@ BuildProtectedAudienceBidRequest(RawRequest& raw_request, ig_for_bidding.android_signals().IsInitialized()) { roma_service::ProtectedAudienceAndroidSignals* android_signals = bid_request.mutable_android_signals(); - *android_signals->mutable_top_level_seller() = - std::move(*raw_request.mutable_top_level_seller()); + android_signals->set_top_level_seller(raw_request.top_level_seller()); } else if (ig_for_bidding.has_browser_signals() && ig_for_bidding.browser_signals().IsInitialized()) { roma_service::ProtectedAudienceBrowserSignals* browser_signals = bid_request.mutable_browser_signals(); - *browser_signals->mutable_top_window_hostname() = - std::move(*raw_request.mutable_publisher_name()); - *browser_signals->mutable_seller() = - std::move(*raw_request.mutable_seller()); - *browser_signals->mutable_top_level_seller() = - std::move(*raw_request.mutable_top_level_seller()); + browser_signals->set_top_window_hostname(raw_request.publisher_name()); + browser_signals->set_seller(raw_request.seller()); + browser_signals->set_top_level_seller(raw_request.top_level_seller()); browser_signals->set_join_count( ig_for_bidding.browser_signals().join_count()); browser_signals->set_bid_count( @@ -245,6 +239,7 @@ void GenerateBidsBinaryReactor::Execute() { ads_with_bids_by_ig_.resize(ig_count); // Send execution requests for each interest group in parallel. + start_binary_execution_time_ = absl::Now(); for (int ig_index = 0; ig_index < ig_count; ++ig_index) { executor_->Run([this, ig_index]() { ExecuteForInterestGroup(ig_index); }); } @@ -273,6 +268,8 @@ void GenerateBidsBinaryReactor::ExecuteForInterestGroup(int ig_index) { LogIfError(metric_context_ ->AccumulateMetric( 1, metric::kBiddingGenerateBidsDispatchResponseError)); + LogIfError( + metric_context_->AccumulateMetric(1)); async_task_tracker_.TaskCompleted(TaskStatus::ERROR); return; } @@ -314,6 +311,11 @@ void GenerateBidsBinaryReactor::OnAllBidsDone(bool any_successful_bids) { return; } + int binary_execution_time_ms = + (absl::Now() - start_binary_execution_time_) / absl::Milliseconds(1); + LogIfError(metric_context_->LogHistogram( + binary_execution_time_ms)); + // Handle cancellation. if (enable_cancellation_ && context_->IsCancelled()) { EncryptResponseAndFinish( diff --git a/services/bidding_service/generate_bids_binary_reactor.h b/services/bidding_service/generate_bids_binary_reactor.h index e054ee4c..86afbfa1 100644 --- a/services/bidding_service/generate_bids_binary_reactor.h +++ b/services/bidding_service/generate_bids_binary_reactor.h @@ -99,6 +99,10 @@ class GenerateBidsBinaryReactor // Used to log metric, same life time as reactor. std::unique_ptr metric_context_; + // The timestamp at which the first ROMA execution dispatch was started. Used + // to calculate udf_execution.duration_ms metric. + absl::Time start_binary_execution_time_; + // Specifies whether this is a single seller or component auction. // Impacts the parsing of generateBid output. AuctionScope auction_scope_; diff --git a/services/bidding_service/generate_bids_binary_reactor_test.cc b/services/bidding_service/generate_bids_binary_reactor_test.cc index bf8c0644..abbd502c 100644 --- a/services/bidding_service/generate_bids_binary_reactor_test.cc +++ b/services/bidding_service/generate_bids_binary_reactor_test.cc @@ -514,6 +514,40 @@ TEST_F(GenerateBidsBinaryReactorTest, CheckGenerateBids(raw_request, expected_raw_response); } +TEST_F(GenerateBidsBinaryReactorTest, CreatesPABidRequestsForMultipleIGs) { + GenerateBidsRawRequest raw_request; + IGForBidding ig_for_bidding_1 = + MakeARandomInterestGroupForBiddingFromAndroid(); + IGForBidding ig_for_bidding_2 = + MakeARandomInterestGroupForBiddingFromAndroid(); + BuildGenerateBidsRawRequest({ig_for_bidding_1, ig_for_bidding_2}, + kTestAuctionSignals, kTestBuyerSignals, + raw_request, false, true); + ASSERT_EQ(raw_request.interest_group_for_bidding_size(), 2); + + GenerateBidsRawResponse expected_raw_response; + + EXPECT_CALL(byob_client_, Execute) + .WillOnce( + [&ig_for_bidding_1, &raw_request]( + const roma_service::GenerateProtectedAudienceBidRequest& request, + absl::Duration timeout) { + CheckBasicFieldsEqual(request, raw_request, ig_for_bidding_1); + return std::make_unique< + roma_service::GenerateProtectedAudienceBidResponse>(); + }) + .WillOnce( + [&ig_for_bidding_2, &raw_request]( + const roma_service::GenerateProtectedAudienceBidRequest& request, + absl::Duration timeout) { + CheckBasicFieldsEqual(request, raw_request, ig_for_bidding_2); + return std::make_unique< + roma_service::GenerateProtectedAudienceBidResponse>(); + }); + ExpectRun(raw_request.interest_group_for_bidding_size()); + CheckGenerateBids(raw_request, expected_raw_response); +} + TEST_F(GenerateBidsBinaryReactorTest, GeneratesBidForSingleIG) { std::unique_ptr bid_response = std::make_unique< diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index a767f7d2..f254d084 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -42,7 +42,7 @@ using ::google::protobuf::TextFormat; using RawRequest = GenerateBidsRequest::GenerateBidsRawRequest; using IGForBidding = GenerateBidsRequest::GenerateBidsRawRequest::InterestGroupForBidding; -constexpr int kArgsSizeWithWrapper = 6; +constexpr int kArgsSizeWithWrapper = 7; absl::StatusOr ProtoToJson( const google::protobuf::Message& proto) { @@ -235,6 +235,9 @@ absl::StatusOr BuildGenerateBidRequest( generate_bid_request.input[ArgIndex(GenerateBidArgs::kDeviceSignals)] = std::make_shared(kEmptyDeviceSignals); } + generate_bid_request.input[ArgIndex(GenerateBidArgs::kMultiBidLimit)] = + std::make_shared( + absl::StrCat(raw_request.multi_bid_limit())); generate_bid_request.input[ArgIndex(GenerateBidArgs::kFeatureFlags)] = std::make_shared( GetFeatureFlagJson(enable_adtech_code_logging, @@ -385,7 +388,7 @@ void GenerateBidsReactor::Execute() { int js_execution_time_ms = (absl::Now() - start_js_execution_time) / absl::Milliseconds(1); LogIfError( - metric_context_->LogHistogram( + metric_context_->LogHistogram( js_execution_time_ms)); GenerateBidsCallback(result); EncryptResponseAndFinish(grpc::Status::OK); @@ -450,54 +453,57 @@ void GenerateBidsReactor::GenerateBidsCallback( << result.status().ToString( absl::StatusToStringMode::kWithEverything); LogIfError( - metric_context_->LogUpDownCounter(1)); + metric_context_->AccumulateMetric( + 1)); continue; } // Parse JSON response from the result. - AdWithBid bid; - absl::StatusOr generate_bid_response = - enable_adtech_code_logging_ - ? ParseAndGetResponseJson(enable_adtech_code_logging_, result->resp, - log_context_) - : result->resp; - if (!generate_bid_response.ok()) { + absl::StatusOr> generate_bid_responses = + ParseAndGetResponseJsonArray(enable_adtech_code_logging_, result->resp, + log_context_); + if (!generate_bid_responses.ok()) { PS_LOG(ERROR, log_context_) << "Failed to parse response from Roma " - << generate_bid_response.status().ToString( + << generate_bid_responses.status().ToString( absl::StatusToStringMode::kWithEverything); + continue; } // Convert JSON response to AdWithBid proto. google::protobuf::json::ParseOptions parse_options; parse_options.ignore_unknown_fields = true; - auto valid = google::protobuf::util::JsonStringToMessage( - generate_bid_response.value(), &bid, parse_options); const std::string interest_group_name = result->id; - if (!valid.ok()) { - PS_LOG(ERROR, log_context_) - << "Invalid json output from code execution for interest_group " - << interest_group_name << ": " << result->resp; - continue; - } - // Validate AdWithBid proto. - if (absl::Status validation_status = - IsValidProtectedAudienceBid(bid, auction_scope_); - !validation_status.ok()) { - PS_VLOG(kNoisyWarn, log_context_) << validation_status.message(); - if (validation_status.code() == absl::StatusCode::kInvalidArgument) { - zero_bid_count += 1; - LogIfError( - metric_context_->AccumulateMetric(1)); + for (const auto& bid_response : *generate_bid_responses) { + AdWithBid bid; + auto valid = google::protobuf::util::JsonStringToMessage( + bid_response, &bid, parse_options); + if (!valid.ok()) { + PS_LOG(ERROR, log_context_) + << "Invalid json output from code execution for interest_group " + << interest_group_name << ": " << bid_response; + continue; } - continue; + // Validate AdWithBid proto. + if (absl::Status validation_status = + IsValidProtectedAudienceBid(bid, auction_scope_); + !validation_status.ok()) { + PS_VLOG(kNoisyWarn, log_context_) << validation_status.message(); + if (validation_status.code() == absl::StatusCode::kInvalidArgument) { + zero_bid_count += 1; + LogIfError( + metric_context_->AccumulateMetric( + 1)); + } + continue; + } + // Trim debug URLs for validated AdWithBid proto and add it to + // GenerateBidsResponse. + total_debug_urls_chars += + TrimAndReturnDebugUrlsSize(bid, max_allowed_size_debug_url_chars_, + max_allowed_size_all_debug_urls_chars_, + total_debug_urls_chars, log_context_); + bid.set_interest_group_name(interest_group_name); + *raw_response_.add_bids() = std::move(bid); } - // Trim debug URLs for validated AdWithBid proto and add it to - // GenerateBidsResponse. - total_debug_urls_chars += - TrimAndReturnDebugUrlsSize(bid, max_allowed_size_debug_url_chars_, - max_allowed_size_all_debug_urls_chars_, - total_debug_urls_chars, log_context_); - bid.set_interest_group_name(interest_group_name); - *raw_response_.add_bids() = std::move(bid); } LogIfError(metric_context_->LogHistogram( (static_cast(zero_bid_count)) / total_bid_count)); diff --git a/services/bidding_service/generate_bids_reactor.h b/services/bidding_service/generate_bids_reactor.h index f9bac6ee..868fa3ba 100644 --- a/services/bidding_service/generate_bids_reactor.h +++ b/services/bidding_service/generate_bids_reactor.h @@ -40,6 +40,7 @@ enum class GenerateBidArgs : int { kBuyerSignals, kTrustedBiddingSignals, kDeviceSignals, + kMultiBidLimit, kFeatureFlags }; diff --git a/services/bidding_service/generate_bids_reactor_test.cc b/services/bidding_service/generate_bids_reactor_test.cc index c3fe9035..b4be289b 100644 --- a/services/bidding_service/generate_bids_reactor_test.cc +++ b/services/bidding_service/generate_bids_reactor_test.cc @@ -86,10 +86,10 @@ std::string GetTestResponse(absl::string_view render, float bid, bool enable_adtech_code_logging = false) { if (enable_adtech_code_logging) { return absl::Substitute(R"JSON({ - "response": { + "response": [{ "render": "$0", "bid": $1 - }, + }], "logs": ["test log"], "errors": ["test.error"], "warnings":["test.warn"] @@ -97,10 +97,10 @@ std::string GetTestResponse(absl::string_view render, float bid, render, bid); } - return absl::Substitute(R"JSON({ + return absl::Substitute(R"JSON([{ "render": "$0", "bid": $1 - })JSON", + }])JSON", render, bid); } @@ -116,22 +116,22 @@ std::string GetTestResponseWithPAgg( if (enable_adtech_code_logging) { return absl::Substitute(R"JSON({ - "response": { + "response": [{ "render": "$0", "bid": $1, - "private_aggregation_contributions": [$2], - }, + "private_aggregation_contributions": [$2] + }], "logs": ["test log"], "errors": ["test.error"], "warnings":["test.warn"] })JSON", render, bid, json_contribution); } - return absl::Substitute(R"JSON({ + return absl::Substitute(R"JSON([{ "render": "$0", "bid": $1, "private_aggregation_contributions": [$2], - })JSON", + }])JSON", render, bid, json_contribution); } @@ -140,11 +140,11 @@ std::string GetTestResponseWithBuyerReportingId( bool enable_adtech_code_logging = false) { if (enable_adtech_code_logging) { return absl::Substitute(R"JSON({ - "response": { + "response": [{ "render": "$0", "bid": $1, "buyerReportingId": "$2" - }, + }], "logs": [], "errors": [], "warnings":[] @@ -152,11 +152,11 @@ std::string GetTestResponseWithBuyerReportingId( render, bid, buyer_reporting_id); } - return absl::Substitute(R"JSON({ + return absl::Substitute(R"JSON([{ "render": "$0", "bid": $1, "buyerReportingId": "$2" - })JSON", + }])JSON", render, bid, buyer_reporting_id); } @@ -165,11 +165,11 @@ std::string GetTestResponseWithUnknownField( bool enable_adtech_code_logging = false) { if (enable_adtech_code_logging) { return absl::Substitute(R"JSON({ - "response": { + "response": [{ "render": "$0", "bid": $1, "buyer_reporting_ids": "abcdef" - }, + }], "logs": [], "errors": [], "warnings":[] @@ -177,11 +177,11 @@ std::string GetTestResponseWithUnknownField( render, bid); } - return absl::Substitute(R"JSON({ + return absl::Substitute(R"JSON([{ "render": "$0", "bid": $1, "buyer_reporting_ids": "abcdef" - })JSON", + }])JSON", render, bid); } @@ -190,11 +190,11 @@ std::string GetComponentAuctionResponse( bool enable_adtech_code_logging = false) { if (enable_adtech_code_logging) { return absl::Substitute(R"JSON({ - "response": { + "response": [{ "render": "$0", "bid": $1, "allowComponentAuction": $2 - }, + }], "logs": ["test log"], "errors": ["test.error"], "warnings":["test.warn"] @@ -202,11 +202,11 @@ std::string GetComponentAuctionResponse( render, bid, allow_component_auction); } - return absl::Substitute(R"JSON({ + return absl::Substitute(R"JSON([{ "render": "$0", "bid": $1, "allowComponentAuction": $2 - })JSON", + }])JSON", render, bid, allow_component_auction); } @@ -359,7 +359,8 @@ void BuildRawRequest(const std::vector& interest_groups_to_add, absl::string_view auction_signals, absl::string_view buyer_signals, RawRequest& raw_request, bool enable_debug_reporting = false, - bool enable_adtech_code_logging = false) { + bool enable_adtech_code_logging = false, + int multi_bid_limit = kDefaultMultiBidLimit) { for (int i = 0; i < interest_groups_to_add.size(); i++) { *raw_request.mutable_interest_group_for_bidding()->Add() = interest_groups_to_add[i]; @@ -373,6 +374,7 @@ void BuildRawRequest(const std::vector& interest_groups_to_add, raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); raw_request.mutable_consented_debug_config()->set_is_consented(true); } + raw_request.set_multi_bid_limit(multi_bid_limit); } void BuildRawRequestForComponentAuction( @@ -415,7 +417,7 @@ TEST_F(GenerateBidsReactorTest, GenerateBidSuccessfulWithCodeWrapper) { TEST_F(GenerateBidsReactorTest, PrivateAggregationObjectSetInResponse) { bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; - bool enable_adtech_code_logging = false; + bool enable_adtech_code_logging = true; PrivateAggregateContribution pAggContribution = CreateTestPAggContribution(EVENT_TYPE_WIN, /* event_name = */ ""); @@ -668,7 +670,7 @@ TEST_F(GenerateBidsReactorTest, CreatesGenerateBidInputsInCorrectOrder) { .WillOnce([response_json](std::vector& batch, BatchDispatchDoneCallback batch_callback) { auto input = batch.at(0).input; - EXPECT_EQ(input.size(), 6); + EXPECT_EQ(input.size(), 7); if (input.size() == 6) { CheckCorrectnessOfIg(*input[0], GetIGForBiddingBar()); EXPECT_EQ(*input[1], R"JSON({"auction_signal": "test 1"})JSON"); @@ -704,7 +706,7 @@ TEST_F(GenerateBidsReactorTest, .WillOnce([response_json](std::vector& batch, BatchDispatchDoneCallback batch_callback) { auto input = batch.at(0).input; - EXPECT_EQ(input.size(), 6); + EXPECT_EQ(input.size(), 7); if (input.size() == 6) { CheckCorrectnessOfIg(*input[0], GetIGForBiddingBar()); EXPECT_EQ(*input[1], R"JSON({"auction_signal": "test 1"})JSON"); @@ -886,14 +888,14 @@ TEST_F(GenerateBidsReactorTest, GenerateBidResponseWithDebugUrls) { bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; const std::string response_json = R"JSON( - { + [{ "render": "https://adTech.com/ad?id=123", "bid": 1, "debug_report_urls": { "auction_debug_loss_url": "test.com/debugLoss", "auction_debug_win_url": "test.com/debugWin" } - } + }] )JSON"; AdWithBid bid; diff --git a/services/bidding_service/generate_bids_reactor_test_utils.cc b/services/bidding_service/generate_bids_reactor_test_utils.cc index 2e326df7..2663886b 100644 --- a/services/bidding_service/generate_bids_reactor_test_utils.cc +++ b/services/bidding_service/generate_bids_reactor_test_utils.cc @@ -98,7 +98,7 @@ GenerateProtectedAppSignalsBidsRawRequest CreateRawProtectedAppSignalsRequest( *raw_request.mutable_protected_app_signals() = protected_app_signals; raw_request.set_seller(seller); raw_request.set_publisher_name(publisher_name); - if (contextual_pas_data.has_value()) { + if (contextual_pas_data) { *raw_request.mutable_contextual_protected_app_signals_data() = *std::move(contextual_pas_data); } @@ -162,13 +162,13 @@ std::string CreateGenerateBidsUdfResponse( absl::string_view temporary_egress_payload_string) { std::string base64_encoded_features_bytes; return absl::Substitute(R"JSON( - { + [{ "render": "$0", "bid": $1, "egressPayload": "$2", "debugReportUrls": $3, "temporaryUnlimitedEgressPayload": "$4" - } + }] )JSON", render, bid, egress_payload_string, debug_reporting_urls, diff --git a/services/bidding_service/inference/periodic_model_fetcher.cc b/services/bidding_service/inference/periodic_model_fetcher.cc index ddf19388..8a151131 100644 --- a/services/bidding_service/inference/periodic_model_fetcher.cc +++ b/services/bidding_service/inference/periodic_model_fetcher.cc @@ -161,7 +161,8 @@ void PeriodicModelFetcher::InternalModelFetchAndRegistration() { absl::StatusOr model_checksum = ComputeChecksumForBlobs(blob_views); if (!model_checksum.ok() || *model_checksum != metadata.checksum()) { - PS_LOG(ERROR) << "Model rejected due to incorrect checksum."; + PS_LOG(ERROR) << "Model rejected due to incorrect checksum." + << " model_path=" << model_path; failure_models.push_back(model_path); ModelFetcherMetric::IncrementModelRegistrationFailedCountByStatus( absl::StatusCode::kFailedPrecondition); @@ -206,7 +207,7 @@ void PeriodicModelFetcher::InternalPeriodicModelFetchAndRegistration() { } void PeriodicModelFetcher::End() { - if (task_id_.has_value()) { + if (task_id_) { executor_.Cancel(*task_id_); task_id_ = absl::nullopt; } diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc index dcfa69dd..3f83f7c1 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc @@ -200,7 +200,9 @@ ProtectedAppSignalsGenerateBidsReactor::ProtectedAppSignalsGenerateBidsReactor( runtime_config.default_protected_app_signals_generate_bid_version), ad_retrieval_version_(runtime_config.default_ad_retrieval_version), egress_schema_cache_(egress_schema_cache), - limited_egress_schema_cache_(limited_egress_schema_cache) { + limited_egress_schema_cache_(limited_egress_schema_cache), + enable_temporary_unlimited_egress_( + runtime_config.enable_temporary_unlimited_egress) { DCHECK(ad_retrieval_async_client_) << "Missing: KV server Async GRPC client"; CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, @@ -233,7 +235,7 @@ ProtectedAppSignalsGenerateBidsReactor::CreateAdsRetrievalRequest( absl::optional ad_render_ids) { std::vector ad_render_ids_list; - if (ad_render_ids.has_value()) { + if (ad_render_ids) { ad_render_ids_list = std::vector(ad_render_ids->begin(), ad_render_ids->end()); } @@ -336,6 +338,9 @@ ProtectedAppSignalsGenerateBidsReactor::CreateGenerateBidsRequest( enable_buyer_debug_url_generation_ && raw_request_.enable_debug_reporting()), ArgIndex(GenerateBidsUdfArgs::kFeatureFlags), input); + PopulateArgInRomaRequest(absl::StrCat(raw_request_.multi_bid_limit()), + ArgIndex(GenerateBidsUdfArgs::kMultiBidLimit), + input); DispatchRequest request = { .id = raw_request_.log_context().generation_id(), .version_string = protected_app_signals_generate_bid_version_, @@ -347,26 +352,26 @@ ProtectedAppSignalsGenerateBidsReactor::CreateGenerateBidsRequest( return request; } -absl::StatusOr +absl::StatusOr> ProtectedAppSignalsGenerateBidsReactor:: ParseProtectedSignalsGenerateBidsResponse(const std::string& response) { PS_VLOG(8, log_context_) << __func__; - std::string generate_bid_response; - if (enable_adtech_code_logging_) { - PS_ASSIGN_OR_RETURN( - generate_bid_response, - ParseAndGetResponseJson(enable_adtech_code_logging_, response, - log_context_), - _ << "Failed to parse ProtectedAppSignalsAdWithBid JSON " - "response from Roma"); - } else { - generate_bid_response = response; + std::vector generate_bid_responses; + PS_ASSIGN_OR_RETURN(generate_bid_responses, + ParseAndGetResponseJsonArray(enable_adtech_code_logging_, + response, log_context_), + _ << "Failed to parse ProtectedAppSignalsAdWithBid JSON " + "response from Roma"); + + std::vector bids; + bids.reserve(generate_bid_responses.size()); + for (const auto& response : generate_bid_responses) { + ProtectedAppSignalsAdWithBid bid; + PS_RETURN_IF_ERROR( + google::protobuf::util::JsonStringToMessage(response, &bid)); + bids.push_back(std::move(bid)); } - - ProtectedAppSignalsAdWithBid bid; - PS_RETURN_IF_ERROR( - google::protobuf::util::JsonStringToMessage(generate_bid_response, &bid)); - return bid; + return bids; } absl::StatusOr @@ -431,54 +436,57 @@ void ProtectedAppSignalsGenerateBidsReactor::OnFetchAdsDataDone( dispatch_requests_.emplace_back(CreateGenerateBidsRequest( std::move(result), prepare_data_for_ads_retrieval_response)); absl::Time start_js_execution_time = absl::Now(); - ExecuteRomaRequests( + ExecuteRomaRequests>( dispatch_requests_, kDispatchHandlerFunctionNameWithCodeWrapper, [this, start_js_execution_time](const std::string& response) { int js_execution_time_ms = (absl::Now() - start_js_execution_time) / absl::Milliseconds(1); LogIfError( metric_context_ - ->LogHistogram( + ->LogHistogram( js_execution_time_ms)); return ParseProtectedSignalsGenerateBidsResponse(response); }, - [this](const ProtectedAppSignalsAdWithBid& bid) { - LogIfError( - metric_context_->AccumulateMetric( - 1)); - if (absl::Status validation_status = IsValidProtectedAppSignalsBid(bid); - !validation_status.ok()) { + [this](const std::vector& bids) { + for (auto& bid : bids) { LogIfError( - metric_context_->LogHistogram( - 1.0)); - PS_VLOG(kNoisyWarn, log_context_) << validation_status.message(); - } else { - PS_VLOG(kNoisyInfo, log_context_) - << "Successful non-zero protected app signals bid received"; - auto* added_bid = raw_response_.add_bids(); - *added_bid = bid; - const int limited_egress_bits = - absl::GetFlag(FLAGS_limited_egress_bits); - if (limited_egress_bits <= 0) { - PS_VLOG(5) << "Allowed limited egress bits: " << limited_egress_bits - << ", skipping it"; - added_bid->clear_egress_payload(); + metric_context_->AccumulateMetric( + 1)); + if (absl::Status validation_status = + IsValidProtectedAppSignalsBid(bid); + !validation_status.ok()) { + LogIfError( + metric_context_->LogHistogram( + 1.0)); + PS_VLOG(kNoisyWarn, log_context_) << validation_status.message(); } else { - PopulateSerializedEgressPayload( - kDefaultEgressSchemaVersion, - *added_bid->mutable_egress_payload(), - *limited_egress_schema_cache_, limited_egress_bits); - } - if (!raw_request_.enable_unlimited_egress() || - !absl::GetFlag(FLAGS_enable_temporary_unlimited_egress)) { - PS_VLOG(5) << "Either request doesn't allow unlimited egress or " - << "feature is disabled by the platform"; - added_bid->clear_temporary_unlimited_egress_payload(); - } else { - PopulateSerializedEgressPayload( - kDefaultEgressSchemaVersion, - *added_bid->mutable_temporary_unlimited_egress_payload(), - *egress_schema_cache_); + PS_VLOG(kNoisyInfo, log_context_) + << "Successful non-zero protected app signals bid received"; + auto* added_bid = raw_response_.add_bids(); + *added_bid = bid; + const int limited_egress_bits = + absl::GetFlag(FLAGS_limited_egress_bits); + if (limited_egress_bits <= 0) { + PS_VLOG(5) << "Allowed limited egress bits: " + << limited_egress_bits << ", skipping it"; + added_bid->clear_egress_payload(); + } else { + PopulateSerializedEgressPayload( + kDefaultEgressSchemaVersion, + *added_bid->mutable_egress_payload(), + *limited_egress_schema_cache_, limited_egress_bits); + } + if (!raw_request_.enable_unlimited_egress() || + !enable_temporary_unlimited_egress_) { + PS_VLOG(5) << "Either request doesn't allow unlimited egress or " + << "feature is disabled by the platform"; + added_bid->clear_temporary_unlimited_egress_payload(); + } else { + PopulateSerializedEgressPayload( + kDefaultEgressSchemaVersion, + *added_bid->mutable_temporary_unlimited_egress_payload(), + *egress_schema_cache_); + } } } EncryptResponseAndFinish(grpc::Status::OK); @@ -539,7 +547,7 @@ void ProtectedAppSignalsGenerateBidsReactor::StartNonContextualAdsRetrieval() { int js_execution_time_ms = (absl::Now() - start_js_execution_time) / absl::Milliseconds(1); LogIfError(metric_context_->LogHistogram< - metric::kPASPrepareDataForRetrievalJSExecutionDuration>( + metric::kPASPrepareDataForRetrievalUdfExecutionDuration>( js_execution_time_ms)); PS_ASSIGN_OR_RETURN(rapidjson::Document document, ParseJsonString(response)); @@ -551,7 +559,7 @@ void ProtectedAppSignalsGenerateBidsReactor::StartNonContextualAdsRetrieval() { } bool ProtectedAppSignalsGenerateBidsReactor::IsContextualRetrievalRequest() { - if (is_contextual_retrieval_request_.has_value()) { + if (is_contextual_retrieval_request_) { return *is_contextual_retrieval_request_; } diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.h b/services/bidding_service/protected_app_signals_generate_bids_reactor.h index 77d3c9fa..e0b60d0e 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.h +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.h @@ -103,7 +103,7 @@ class ProtectedAppSignalsGenerateBidsReactor absl::Status ValidateRomaResponse( const std::vector>& result); - absl::StatusOr + absl::StatusOr> ParseProtectedSignalsGenerateBidsResponse(const std::string& response); // Populates the serialized payload in egress_payload output variable, if @@ -218,6 +218,8 @@ class ProtectedAppSignalsGenerateBidsReactor // Used to log metric, same life time as reactor. std::unique_ptr metric_context_; + + const bool enable_temporary_unlimited_egress_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc index de3ccf85..c625b43c 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc @@ -151,7 +151,7 @@ class GenerateBidsReactorTest : public ::testing::Test { RawResponse RunReactorWithRequest(const RawRequest& raw_request, std::optional runtime_config = std::nullopt) { - if (!runtime_config.has_value()) { + if (!runtime_config) { runtime_config = { .enable_buyer_debug_url_generation = false, }; @@ -693,31 +693,16 @@ TEST_F(GenerateBidsReactorTest, KvInputIsCorrect) { } TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorGetsPopulated) { - absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); int num_roma_dispatches = 0; SetupProtectedAppSignalsRomaExpectations( dispatcher_, num_roma_dispatches, /*prepare_data_for_ad_retrieval_udf_response=*/absl::nullopt, absl::Substitute(R"JSON( - { - "bid" : $0, - "render": "$1", - "temporaryUnlimitedEgressPayload" : - "{\"features\": - [ - {\"name\": \"unsigned-integer-feature\", \"value\": 2}, - {\"name\": \"boolean-feature\", \"value\": true}, - {\"name\": \"histogram-feature\", - \"value\": [ - { - \"name\": \"signed-integer-feature\", - \"value\": -1 - } - ] - } - ] - }" - } + [{ + "bid": $0, + "render": "$1", + "temporaryUnlimitedEgressPayload": "{\"features\":[{\"name\":\"unsigned-integer-feature\",\"value\":2},{\"name\":\"boolean-feature\",\"value\":true},{\"name\":\"histogram-feature\",\"value\":[{\"name\":\"signed-integer-feature\",\"value\":-1}]}]}" + }] )JSON", kTestWinningBid, kTestRenderUrl)); @@ -742,7 +727,10 @@ TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorGetsPopulated) { CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), kTestSeller, kTestPublisherName, /*contextual_pas_data=*/absl::nullopt, /*enable_unlimited_egress=*/true); - auto raw_response = RunReactorWithRequest(raw_request); + auto raw_response = RunReactorWithRequest( + raw_request, + BiddingServiceRuntimeConfig({.enable_buyer_debug_url_generation = false, + .enable_temporary_unlimited_egress = true})); // One dispatch to `preparedDataForAdRetrieval` and another to `generateBids` // is expected. @@ -758,7 +746,6 @@ TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorGetsPopulated) { TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorNotPopulatedWhenNotEnabled) { - absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); int num_roma_dispatches = 0; SetupProtectedAppSignalsRomaExpectations(dispatcher_, num_roma_dispatches); @@ -783,7 +770,10 @@ TEST_F(GenerateBidsReactorTest, CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), kTestSeller, kTestPublisherName, /*contextual_pas_data=*/absl::nullopt, /*enable_unlimited_egress=*/false); - auto raw_response = RunReactorWithRequest(raw_request); + auto raw_response = RunReactorWithRequest( + raw_request, + BiddingServiceRuntimeConfig({.enable_buyer_debug_url_generation = false, + .enable_temporary_unlimited_egress = true})); // One dispatch to `preparedDataForAdRetrieval` and another to `generateBids` // is expected. @@ -799,7 +789,6 @@ TEST_F(GenerateBidsReactorTest, TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorNotPopulatedWhenFeatureIsOff) { - absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, false); int num_roma_dispatches = 0; SetupProtectedAppSignalsRomaExpectations(dispatcher_, num_roma_dispatches); @@ -839,31 +828,16 @@ TEST_F(GenerateBidsReactorTest, } TEST_F(GenerateBidsReactorTest, SerializesEgressVector) { - absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); int num_roma_dispatches = 0; SetupProtectedAppSignalsRomaExpectations( dispatcher_, num_roma_dispatches, /*prepare_data_for_ad_retrieval_udf_response=*/absl::nullopt, absl::Substitute(R"JSON( - { + [{ "bid" : $0, "render": "$1", - "temporaryUnlimitedEgressPayload" : - "{\"features\": - [ - {\"name\": \"unsigned-integer-feature\", \"value\": 2}, - {\"name\": \"boolean-feature\", \"value\": true}, - {\"name\": \"histogram-feature\", - \"value\": [ - { - \"name\": \"signed-integer-feature\", - \"value\": -1 - } - ] - } - ] - }" - } + "temporaryUnlimitedEgressPayload": "{\"features\":[{\"name\": \"unsigned-integer-feature\", \"value\": 2},{\"name\": \"boolean-feature\", \"value\": true},{\"name\": \"histogram-feature\",\"value\": [{\"name\": \"signed-integer-feature\",\"value\": -1}]}]}" + }] )JSON", kTestWinningBid, kTestRenderUrl)); @@ -888,7 +862,10 @@ TEST_F(GenerateBidsReactorTest, SerializesEgressVector) { CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), kTestSeller, kTestPublisherName, /*contextual_pas_data=*/absl::nullopt, /*enable_unlimited_egress=*/true); - auto raw_response = RunReactorWithRequest(raw_request); + auto raw_response = RunReactorWithRequest( + raw_request, + BiddingServiceRuntimeConfig({.enable_buyer_debug_url_generation = false, + .enable_temporary_unlimited_egress = true})); ASSERT_EQ(raw_response.bids().size(), 1); const auto& generated_bid = raw_response.bids()[0]; @@ -916,24 +893,11 @@ TEST_F(GenerateBidsReactorTest, SerializesMultipleFeatures) { dispatcher_, num_roma_dispatches, /*prepare_data_for_ad_retrieval_udf_response=*/absl::nullopt, absl::Substitute(R"JSON( - { + [{ "bid" : $0, "render": "$1", - "egressPayload" : - "{\"features\": [ - {\"name\": \"boolean-feature\", \"value\": true}, - {\"name\": \"unsigned-integer-feature\", \"value\": 127}, - {\"name\": \"histogram-feature\", - \"value\": [ - { - \"name\": \"signed-integer-feature\", - \"value\": -1 - } - ] - } - ] - }" - } + "egressPayload" : "{\"features\": [{\"name\": \"boolean-feature\", \"value\": true},{\"name\": \"unsigned-integer-feature\", \"value\": 127},{\"name\": \"histogram-feature\",\"value\": [{\"name\": \"signed-integer-feature\",\"value\": -1}]}]}" + }] )JSON", kTestWinningBid, kTestRenderUrl)); @@ -987,11 +951,11 @@ TEST_F(GenerateBidsReactorTest, EgressClearedIfOverLimit) { dispatcher_, num_roma_dispatches, /*prepare_data_for_ad_retrieval_udf_response=*/absl::nullopt, absl::Substitute(R"JSON( - { + [{ "bid" : $0, "render": "$1", "egressPayload" : "{\"features\": [{\"name\": \"boolean-feature\", \"value\": true}, {\"name\": \"unsigned-integer-feature\", \"value\": 127}]}" - } + }] )JSON", kTestWinningBid, kTestRenderUrl)); @@ -1027,18 +991,17 @@ TEST_F(GenerateBidsReactorTest, EgressClearedIfOverLimit) { } TEST_F(GenerateBidsReactorTest, ReturnsEmptyEgressVectorWhenNonePresent) { - absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); int num_roma_dispatches = 0; SetupProtectedAppSignalsRomaExpectations( dispatcher_, num_roma_dispatches, /*prepare_data_for_ad_retrieval_udf_response=*/absl::nullopt, absl::Substitute(R"JSON( - { + [{ "bid" : $0, "render": "$1", "temporaryUnlimitedEgressPayload" : "", "egressPayload": "" - } + }] )JSON", kTestWinningBid, kTestRenderUrl)); @@ -1063,7 +1026,10 @@ TEST_F(GenerateBidsReactorTest, ReturnsEmptyEgressVectorWhenNonePresent) { CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), kTestSeller, kTestPublisherName, /*contextual_pas_data=*/absl::nullopt, /*enable_unlimited_egress=*/true); - auto raw_response = RunReactorWithRequest(raw_request); + auto raw_response = RunReactorWithRequest( + raw_request, + BiddingServiceRuntimeConfig({.enable_buyer_debug_url_generation = false, + .enable_temporary_unlimited_egress = true})); ASSERT_EQ(raw_response.bids().size(), 1); const auto& generated_bid = raw_response.bids()[0]; diff --git a/services/buyer_frontend_service/BUILD b/services/buyer_frontend_service/BUILD index 23376b34..f4031d0c 100644 --- a/services/buyer_frontend_service/BUILD +++ b/services/buyer_frontend_service/BUILD @@ -35,14 +35,15 @@ cc_library( "get_bids_unary_reactor.h", ], deps = [ + ":kv_buyer_signals_adapter", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", - "//services/buyer_frontend_service/providers:bidding_signals_providers", + "//services/buyer_frontend_service/providers:http_bidding_signals_providers", "//services/buyer_frontend_service/util:bidding_signals", "//services/buyer_frontend_service/util:buyer_frontend_utils", - "//services/common:feature_flags", "//services/common/chaffing:transcoding_utils", "//services/common/clients/bidding_server:async_client", + "//services/common/clients/kv_server:kv_async_client", "//services/common/constants:user_error_strings", "//services/common/loggers:benchmarking_logger", "//services/common/loggers:build_input_process_response_benchmarking_logger", @@ -76,9 +77,10 @@ cc_library( ":get_bids_unary_reactor", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", - "//services/buyer_frontend_service/providers:bidding_signals_providers", + "//services/buyer_frontend_service/providers:http_bidding_signals_providers", "//services/buyer_frontend_service/util:bidding_signals", "//services/buyer_frontend_service/util:buyer_frontend_utils", + "//services/common:feature_flags", "//services/common/clients/bidding_server:async_client", "//services/common/clients/http:multi_curl_http_fetcher_async", "//services/common/metric:server_definition", @@ -125,7 +127,7 @@ cc_binary( ":runtime_flags", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", - "//services/buyer_frontend_service/providers:bidding_signals_providers", + "//services/buyer_frontend_service/providers:http_bidding_signals_providers", "//services/common/clients/config:config_client", "//services/common/clients/config:config_client_util", "//services/common/concurrent:local_cache", @@ -174,4 +176,32 @@ cc_test( ], ) -3 +cc_library( + name = "kv_buyer_signals_adapter", + srcs = [ + "kv_buyer_signals_adapter.cc", + ], + hdrs = [ + "kv_buyer_signals_adapter.h", + ], + deps = [ + "//services/buyer_frontend_service/providers:bidding_signals_providers", + "//services/common/util:json_util", + "@com_google_absl//absl/status", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@rapidjson", + "@service_value_key_fledge_privacysandbox//public/query/v2:get_values_v2_cc_proto", + ], +) + +cc_test( + name = "kv_buyer_signals_adapter_test", + size = "small", + srcs = [ + "kv_buyer_signals_adapter_test.cc", + ], + deps = [ + ":kv_buyer_signals_adapter", + "@google_privacysandbox_servers_common//src/core/test/utils", + ], +) diff --git a/services/buyer_frontend_service/buyer_frontend_main.cc b/services/buyer_frontend_service/buyer_frontend_main.cc index 66dcb044..5a0b5f17 100644 --- a/services/buyer_frontend_service/buyer_frontend_main.cc +++ b/services/buyer_frontend_service/buyer_frontend_main.cc @@ -39,6 +39,7 @@ #include "services/common/clients/http_kv_server/buyer/fake_buyer_key_value_async_http_client.h" #include "services/common/encryption/crypto_client_factory.h" #include "services/common/encryption/key_fetcher_factory.h" +#include "services/common/feature_flags.h" #include "services/common/telemetry/configure_telemetry.h" #include "services/common/util/tcmalloc_utils.h" #include "src/concurrent/event_engine_executor.h" @@ -168,6 +169,7 @@ absl::StatusOr GetConfigClient( config_client.SetFlag(FLAGS_bfe_tcmalloc_max_total_thread_cache_bytes, BFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES); config_client.SetFlag(FLAGS_enable_chaffing, ENABLE_CHAFFING); + config_client.SetFlag(FLAGS_debug_sample_rate_micro, DEBUG_SAMPLE_RATE_MICRO); if (absl::GetFlag(FLAGS_init_config_client)) { PS_RETURN_IF_ERROR(config_client.Init(config_param_prefix)).LogError() @@ -271,6 +273,9 @@ absl::Status RunServer() { config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS), config_client.GetBooleanParameter(ENABLE_PROTECTED_AUDIENCE), config_client.GetBooleanParameter(ENABLE_CHAFFING), + absl::GetFlag(FLAGS_enable_cancellation), + absl::GetFlag(FLAGS_enable_kanon), + config_client.GetIntParameter(DEBUG_SAMPLE_RATE_MICRO), }, enable_buyer_frontend_benchmarking); diff --git a/services/buyer_frontend_service/buyer_frontend_service.cc b/services/buyer_frontend_service/buyer_frontend_service.cc index af78f3c5..fb41f1f4 100644 --- a/services/buyer_frontend_service/buyer_frontend_service.cc +++ b/services/buyer_frontend_service/buyer_frontend_service.cc @@ -52,6 +52,8 @@ BuyerFrontEndService::BuyerFrontEndService( key_fetcher_manager_.get(), crypto_client_.get(), client_config, stub_.get()); } + CHECK(config_.debug_sample_rate_micro >= 0 && + config_.debug_sample_rate_micro <= 1'000'000); } BuyerFrontEndService::BuyerFrontEndService(ClientRegistry client_registry, diff --git a/services/buyer_frontend_service/data/bidding_signals.h b/services/buyer_frontend_service/data/bidding_signals.h index b25cd9fe..eaf6a10c 100644 --- a/services/buyer_frontend_service/data/bidding_signals.h +++ b/services/buyer_frontend_service/data/bidding_signals.h @@ -28,6 +28,7 @@ namespace privacy_sandbox::bidding_auction_servers { // sourced such as from a gRPC Key Value Service. struct BiddingSignals { std::unique_ptr trusted_signals; + uint32_t data_version; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/data/get_bids_config.h b/services/buyer_frontend_service/data/get_bids_config.h index 5f8cd014..ded5ba70 100644 --- a/services/buyer_frontend_service/data/get_bids_config.h +++ b/services/buyer_frontend_service/data/get_bids_config.h @@ -33,6 +33,10 @@ struct GetBidsConfig { bool is_protected_audience_enabled; // Whether chaffing is enabled. bool is_chaffing_enabled; + bool enable_cancellation = false; + bool enable_kanon = false; + // Sample rate for debug request. + int debug_sample_rate_micro; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.cc b/services/buyer_frontend_service/get_bids_unary_reactor.cc index fdc51c7c..8a55a2df 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor.cc @@ -27,6 +27,8 @@ #include "services/common/util/request_metadata.h" #include "services/common/util/request_response_constants.h" +#include "kv_buyer_signals_adapter.h" + namespace privacy_sandbox::bidding_auction_servers { namespace { @@ -38,6 +40,10 @@ using GenerateProtectedAppSignalsBidsRawRequest = using GenerateProtectedAppSignalsBidsRawResponse = GenerateProtectedAppSignalsBidsResponse:: GenerateProtectedAppSignalsBidsRawResponse; + +using KVLookUpResult = + absl::StatusOr>; + inline constexpr int kNumDefaultOutboundBiddingCalls = 1; template @@ -145,18 +151,32 @@ GetBidsUnaryReactor::GetBidsUnaryReactor( key_fetcher_manager_(key_fetcher_manager), crypto_client_(crypto_client), chaffing_enabled_(config_.is_chaffing_enabled), - log_context_([this]() { + is_sampled_for_debug_([this]() { decrypt_status_ = DecryptRequest(); + bool is_debug_eligible = raw_request_.is_debug_eligible() && + config_.debug_sample_rate_micro > 0; + if ((chaffing_enabled_ && raw_request_.is_chaff()) || + is_debug_eligible) { + generator_ = std::mt19937(std::hash{}( + raw_request_.log_context().generation_id())); + } else { + generator_ = std::nullopt; + } + return is_debug_eligible && + RandomSample(config_.debug_sample_rate_micro, *generator_); + }()), + log_context_([this]() { return RequestLogContext( GetLoggingContext(), raw_request_.consented_debug_config(), - [this]() { return get_bids_raw_response_->mutable_debug_info(); }); + [this]() { return get_bids_raw_response_->mutable_debug_info(); }, + is_sampled_for_debug_); }()), async_task_tracker_(kNumDefaultOutboundBiddingCalls, log_context_, [this](bool any_successful_bid) { OnAllBidsDone(any_successful_bid); }), - enable_cancellation_(absl::GetFlag(FLAGS_enable_cancellation)), - enable_enforce_kanon_(absl::GetFlag(FLAGS_enable_kanon) && + enable_cancellation_(config.enable_cancellation), + enable_enforce_kanon_(config.enable_kanon && raw_request_.enforce_kanon()) { if (enable_benchmarking) { std::string request_id = FormatTime(absl::Now()); @@ -178,13 +198,6 @@ GetBidsUnaryReactor::GetBidsUnaryReactor( DCHECK(!config_.is_protected_app_signals_enabled || protected_app_signals_bidding_async_client_ != nullptr) << "PAS is enabled but no PAS bidding async client available"; - - if (chaffing_enabled_ && raw_request_.is_chaff()) { - // The RNG is only needed for chaff requests. - std::size_t hash = - std::hash{}(raw_request_.log_context().generation_id()); - generator_ = std::mt19937(hash); - } } GetBidsUnaryReactor::GetBidsUnaryReactor( @@ -317,7 +330,7 @@ grpc::Status GetBidsUnaryReactor::DecryptRequest() { std::optional private_key = key_fetcher_manager_->GetPrivateKey(request_->key_id()); - if (!private_key.has_value()) { + if (!private_key) { return {grpc::StatusCode::INVALID_ARGUMENT, kInvalidKeyIdError}; } @@ -464,6 +477,8 @@ void GetBidsUnaryReactor::MayGetProtectedSignalsBids() { auto protected_app_signals_bid_request = CreateGenerateProtectedAppSignalsBidsRawRequest(raw_request_, enable_enforce_kanon_); + protected_app_signals_bid_request->set_is_sampled_for_debug( + is_sampled_for_debug_); grpc::ClientContext* client_context = client_contexts_.Add(bidding_metadata_); @@ -537,24 +552,11 @@ void GetBidsUnaryReactor::MayGetProtectedSignalsBids() { } } -void GetBidsUnaryReactor::MayGetProtectedAudienceBids() { - if (!config_.is_protected_audience_enabled) { - PS_VLOG(kNoisyWarn, log_context_) - << "Protected Audience is not enabled, skipping bids fetching for PA"; - return; - } - - if (raw_request_.buyer_input().interest_groups().empty()) { - PS_VLOG(kNoisyWarn, log_context_) - << "No interest groups found, skipping bidding for protected audience"; - return; - } - - BiddingSignalsRequest bidding_signals_request(raw_request_, kv_metadata_); +void GetBidsUnaryReactor::MayGetProtectedAudienceBidsV1( + const BiddingSignalsRequest& bidding_signals_request) { auto kv_request = metric::MakeInitiatedRequest(metric::kKv, metric_context_.get()) .release(); - // Get Bidding Signals. bidding_signals_async_provider_->Get( bidding_signals_request, @@ -600,11 +602,107 @@ void GetBidsUnaryReactor::MayGetProtectedAudienceBids() { {log_context_}); } +void GetBidsUnaryReactor::HandleV2Failure(const absl::Status& status, + const std::string& error_message) { + LogIfError( + metric_context_->AccumulateMetric( + 1, metric::kBfeBiddingSignalsResponseError)); + LogInitiatedRequestErrorMetrics(metric::kKv, status); + // Return error to client. + PS_LOG(ERROR, log_context_) << error_message << status; + async_task_tracker_.TaskCompleted(TaskStatus::ERROR, [this, &status]() { + bid_errors_.push_back(status.ToString()); + }); +} + +void GetBidsUnaryReactor::MayGetProtectedAudienceBidsV2( + const BiddingSignalsRequest& bidding_signals_request) { + auto kv_request = + metric::MakeInitiatedRequest(metric::kKv, metric_context_.get()) + .release(); + grpc::ClientContext* client_context = client_contexts_.Add(); + auto maybe_bidding_signals_request = + CreateV2BiddingRequest(bidding_signals_request); + if (!maybe_bidding_signals_request.ok()) { + PS_VLOG(kNoisyWarn, log_context_) << "Failed creating TKV bidding request. " + << maybe_bidding_signals_request.status(); + return; + } + auto status = kv_async_client_->ExecuteInternal( + *std::move(maybe_bidding_signals_request), client_context, + CancellationWrapper( + context_, enable_cancellation_, + [this, kv_request](KVLookUpResult kv_look_up_result, + ResponseMetadata response_metadata) mutable { + { + // Only logs KV request and response sizes if fetching signals + // succeeds. + if (kv_look_up_result.ok()) { + kv_request->SetRequestSize(response_metadata.request_size); + kv_request->SetResponseSize(response_metadata.response_size); + } + // destruct kv_request, destructor measures request time + delete kv_request; + } + if (!kv_look_up_result.ok()) { + HandleV2Failure(kv_look_up_result.status(), + "GetBiddingSignals request failed with status:"); + return; + } + auto signals = + ConvertV2BiddingSignalsToV1(*std::move(kv_look_up_result)); + if (!signals.ok()) { + HandleV2Failure(signals.status(), + "Failed converting TKV response. "); + return; + } + // Sends protected audience bid request to bidding service. + PrepareAndGenerateProtectedAudienceBid(*std::move(signals)); + }, + [this, kv_request]() { + delete kv_request; + async_task_tracker_.TaskCompleted(TaskStatus::CANCELLED); + }), + absl::Milliseconds(ad_bids_retrieval_timeout_ms_)); + if (!status.ok()) { + PS_VLOG(kNoisyWarn, log_context_) + << "Failed to execute ads metadata KV lookup request: " << status; + async_task_tracker_.TaskCompleted(TaskStatus::ERROR, [this, &status]() { + bid_errors_.push_back(status.ToString()); + }); + } +} + +void GetBidsUnaryReactor::MayGetProtectedAudienceBids() { + if (!config_.is_protected_audience_enabled) { + PS_VLOG(kNoisyWarn, log_context_) + << "Protected Audience is not enabled, skipping bids fetching for PA"; + return; + } + + if (raw_request_.buyer_input().interest_groups().empty()) { + PS_VLOG(kNoisyWarn, log_context_) + << "No interest groups found, skipping bidding for protected audience"; + return; + } + + BiddingSignalsRequest bidding_signals_request(raw_request_, kv_metadata_); + // TODO: this will be coming from GetBidsConfig, and ultimately from terraform + // or command line flag. + bool use_v2 = false; + if (use_v2) { + MayGetProtectedAudienceBidsV2(bidding_signals_request); + } else { + MayGetProtectedAudienceBidsV1(bidding_signals_request); + } +} + // Process Outputs from Actions to prepare bidding request. // All Preload actions must have completed before this is invoked. void GetBidsUnaryReactor::PrepareAndGenerateProtectedAudienceBid( std::unique_ptr bidding_signals) { auto start_deserialize_time = absl::Now(); + uint32_t data_version = bidding_signals->data_version; absl::StatusOr parsed_bidding_signals = ParseTrustedBiddingSignals(std::move(bidding_signals), raw_request_.buyer_input()); @@ -634,7 +732,9 @@ void GetBidsUnaryReactor::PrepareAndGenerateProtectedAudienceBid( raw_bidding_input = CreateGenerateBidsRawRequest( raw_request_, std::move(bidding_signal_json_components_.bidding_signals), - bidding_signal_json_components_.raw_size, enable_enforce_kanon_); + bidding_signal_json_components_.raw_size, data_version, + enable_enforce_kanon_); + raw_bidding_input->set_is_sampled_for_debug(is_sampled_for_debug_); PS_VLOG(kOriginated, log_context_) << "GenerateBidsRequest:\n" << raw_bidding_input->ShortDebugString(); if (raw_bidding_input->interest_group_for_bidding_size() == 0) { diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.h b/services/buyer_frontend_service/get_bids_unary_reactor.h index 03354575..49408a58 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.h +++ b/services/buyer_frontend_service/get_bids_unary_reactor.h @@ -35,8 +35,8 @@ #include "services/buyer_frontend_service/providers/bidding_signals_async_provider.h" #include "services/buyer_frontend_service/util/bidding_signals.h" #include "services/common/clients/bidding_server/bidding_async_client.h" +#include "services/common/clients/kv_server/kv_async_client.h" #include "services/common/encryption/crypto_client_wrapper_interface.h" -#include "services/common/feature_flags.h" #include "services/common/loggers/benchmarking_logger.h" #include "services/common/loggers/request_log_context.h" #include "services/common/metric/server_definition.h" @@ -169,6 +169,12 @@ class GetBidsUnaryReactor : public grpc::ServerUnaryReactor { // Helper classes for performing preload actions. // These are not owned by this class. const BiddingSignalsAsyncProvider* bidding_signals_async_provider_; + + // TODO: this will be initialized in the later commit. Note that this logic + // isn't used at the moment. + KVAsyncClient* kv_async_client_; + int ad_bids_retrieval_timeout_ms_; + BiddingAsyncClient* bidding_async_client_; // PAS bidding client should only be set by the caller if the feature is // enabled. @@ -187,6 +193,11 @@ class GetBidsUnaryReactor : public grpc::ServerUnaryReactor { // Whether the GetBids request follows the new SFE <> BFE request format. bool use_new_payload_encoding_ = false; + // Pseudo random number generator for chaffing and debug sampling. + std::optional generator_; + + bool is_sampled_for_debug_; + RequestLogContext log_context_; // Used to log metric, same life time as reactor. @@ -219,9 +230,12 @@ class GetBidsUnaryReactor : public grpc::ServerUnaryReactor { // server void LogInitiatedRequestErrorMetrics(absl::string_view server_name, const absl::Status& status); - - // Pseudo random number generator for use in chaffing. - std::optional generator_; + [[deprecated]] void MayGetProtectedAudienceBidsV1( + const BiddingSignalsRequest& bidding_signals_request); + void MayGetProtectedAudienceBidsV2( + const BiddingSignalsRequest& bidding_signals_request); + void HandleV2Failure(const absl::Status& status, + const std::string& error_message); // Compression used in the request object; the response will use the same. CompressionType compression_type_; diff --git a/services/buyer_frontend_service/kv_buyer_signals_adapter.cc b/services/buyer_frontend_service/kv_buyer_signals_adapter.cc new file mode 100644 index 00000000..2b880dec --- /dev/null +++ b/services/buyer_frontend_service/kv_buyer_signals_adapter.cc @@ -0,0 +1,188 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "kv_buyer_signals_adapter.h" + +#include + +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "services/common/util/json_util.h" +#include "src/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers { + +using kv_server::UDFArgument; +using std::vector; + +namespace { +inline constexpr char kKeyGroupOutputs[] = "keyGroupOutputs"; +inline constexpr char kTags[] = "tags"; +inline constexpr char kKeys[] = "keys"; +inline constexpr char kKeyValues[] = "keyValues"; +inline constexpr char kClient[] = "Bna.PA.Buyer.20240930"; +inline constexpr char kHostname[] = "hostname"; +inline constexpr char kClientType[] = "client_type"; +inline constexpr char kExperimentGroupId[] = "experiment_group_id"; + +kv_server::v2::GetValuesRequest GetRequest( + const GetBidsRequest::GetBidsRawRequest& get_bids_raw_request) { + kv_server::v2::GetValuesRequest req; + req.set_client_version(kClient); + auto& metadata = *(req.mutable_metadata()->mutable_fields()); + (metadata)[kHostname].set_string_value(get_bids_raw_request.publisher_name()); + (metadata)[kClientType].set_string_value( + absl::StrCat(get_bids_raw_request.client_type())); + if (get_bids_raw_request.has_buyer_kv_experiment_group_id()) { + (metadata)[kExperimentGroupId].set_string_value( + absl::StrCat(get_bids_raw_request.buyer_kv_experiment_group_id())); + } + return req; +} + +UDFArgument BuildArgument(vector keys) { + UDFArgument arg; + arg.mutable_tags()->add_values()->set_string_value(kKeys); + auto* key_list = arg.mutable_data()->mutable_list_value(); + for (auto& key : keys) { + key_list->add_values()->set_string_value(std::move(key)); + } + return arg; +} + +absl::Status ValidateInterestGroups(const BuyerInput& buyer_input) { + if (buyer_input.interest_groups().empty()) { + return absl::InvalidArgumentError("No interest groups in the buyer input"); + } + for (auto& ig : buyer_input.interest_groups()) { + if (!ig.bidding_signals_keys().empty()) { + return absl::OkStatus(); + } + } + return absl::InvalidArgumentError( + "Interest groups don't have any bidding signals"); +} + +} // namespace + +absl::StatusOr> ConvertV2BiddingSignalsToV1( + std::unique_ptr response) { + rapidjson::Document ig_signals(rapidjson::kObjectType); + std::vector docs; + docs.reserve(response->mutable_compression_groups()->size()); + + // ParseJsonString doesn't copy, it creates a Document that points to the + // underlying string. We no longer need the response object, so we are ok to + // directly move the strings from it to the `ig_signals`. However, a doc + // object for each compression group must exist until SerializeJsonDoc is + // called. Otherwise the chain of pointers is broken and we get undefined + // behavior. + for (auto& group : *(response->mutable_compression_groups())) { + PS_ASSIGN_OR_RETURN(auto json, ParseJsonString(group.content())); + docs.push_back(std::move(json)); + } + + int compression_group_index = 0; + for (auto& json : docs) { + if (!json.IsArray()) { + return absl::InvalidArgumentError(absl::StrFormat( + "Incorrrectly formed compression group. Array was expected. " + "Compression group id %i", + compression_group_index)); + } + for (auto& partition_output : json.GetArray()) { + if (!partition_output.IsObject()) { + return absl::InvalidArgumentError(absl::StrFormat( + "Incorrrectly formed compression group. Compression group id %i", + compression_group_index)); + } + auto&& po = partition_output.GetObject(); + auto key_group_outputs = po.FindMember(kKeyGroupOutputs); + if (!(key_group_outputs != po.MemberEnd() && + key_group_outputs->value.IsArray())) { + return absl::InvalidArgumentError(absl::StrFormat( + "Incorrrectly formed compression group. Compression group id %i", + compression_group_index)); + } + for (auto& key_group_output : key_group_outputs->value.GetArray()) { + auto tags = key_group_output.FindMember(kTags); + if (tags == key_group_output.MemberEnd() || tags->value.Size() != 1 || + std::strcmp(tags->value[0].GetString(), kKeys) != 0) { + continue; + } + auto key_values = key_group_output.FindMember(kKeyValues); + if (key_values == key_group_output.MemberEnd() || + !(key_values->value.IsObject())) { + return absl::InvalidArgumentError(absl::StrFormat( + "Incorrrectly formed compression group. Compression group id %i", + compression_group_index)); + } + for (auto& key_value_pair : key_values->value.GetObject()) { + if (ig_signals.FindMember(key_value_pair.name) == + ig_signals.MemberEnd()) { + ig_signals.AddMember(key_value_pair.name, + key_value_pair.value.Move(), + ig_signals.GetAllocator()); + } else { + PS_VLOG(8) + << __func__ + << "Key value response has multiple different values assosiated" + "with the same key: " + << key_value_pair.name.GetString(); + } + } + } + } + compression_group_index++; + } + rapidjson::Document top_level_doc(rapidjson::kObjectType); + top_level_doc.AddMember(kKeys, ig_signals.Move(), + top_level_doc.GetAllocator()); + PS_ASSIGN_OR_RETURN(auto trusted_signals, SerializeJsonDoc(top_level_doc)); + return std::make_unique(BiddingSignals{ + std::make_unique(std::move(trusted_signals))}); +} + +absl::StatusOr> +CreateV2BiddingRequest(const BiddingSignalsRequest& bidding_signals_request) { + auto& bids_request = bidding_signals_request.get_bids_raw_request_; + PS_RETURN_IF_ERROR(ValidateInterestGroups(bids_request.buyer_input())); + std::unique_ptr req = + std::make_unique( + GetRequest(bids_request)); + { + *req->mutable_consented_debug_config() = + bids_request.consented_debug_config(); + } + { *req->mutable_log_context() = bids_request.log_context(); } + int compression_and_partition_id = 0; + // TODO (b/369181315): this needs to be reworked to include multiple IGs's + // keys per partition. + for (auto& ig : bids_request.buyer_input().interest_groups()) { + kv_server::v2::RequestPartition* partition = req->add_partitions(); + for (auto& key : ig.bidding_signals_keys()) { + std::vector keys; + keys.push_back(key); + *partition->add_arguments() = BuildArgument(std::move(keys)); + partition->set_id(compression_and_partition_id); + partition->set_compression_group_id(compression_and_partition_id); + } + compression_and_partition_id++; + } + PS_VLOG(8) << __func__ + << " Created KV lookup request: " << req->DebugString(); + return req; +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/kv_buyer_signals_adapter.h b/services/buyer_frontend_service/kv_buyer_signals_adapter.h new file mode 100644 index 00000000..796ee318 --- /dev/null +++ b/services/buyer_frontend_service/kv_buyer_signals_adapter.h @@ -0,0 +1,60 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_BUYER_FRONTEND_SERVICE_KV_BUYER_SIGNALS_ADAPTER_H_ +#define SERVICES_BUYER_FRONTEND_SERVICE_KV_BUYER_SIGNALS_ADAPTER_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "public/query/v2/get_values_v2.pb.h" +#include "services/buyer_frontend_service/data/bidding_signals.h" +#include "services/buyer_frontend_service/providers/bidding_signals_async_provider.h" + +using kv_server::v2::GetValuesResponse; + +namespace privacy_sandbox::bidding_auction_servers { +// Convert a TKV V2 response to a serilized json string of format that +// `PrepareAndGenerateProtectedAudienceBid` expects. +// { +// "keys" : +// { +// "key_a": { "value": "value_a"}, +// "key_c": { "value": "value_b"}, +// .... +// } +// } +// This is done by going over all compression groups and partitions, and merging +// all keys, tagged as keys, together. +// Note that once B&A will no longer need to support both v1 and v2, the prepare +// function can be reworked to potentially not have this conversion or have an +// optimized version of it. +absl::StatusOr> ConvertV2BiddingSignalsToV1( + std::unique_ptr response); + +// Create a TKV v2 request given a list of IG keys. +// Current implementation creates a partition for each IG. This can be improved +// by combining multiple IGs into a single partition. This would result in a +// better performance on TKV side, since each partition is a single UDF +// execution. Each UDF execution has overhead. Note that unlike the current B&A +// v1 implementation, we are not passing IG names, since we don't really need +// those on TKV side to do the keys lookup. +absl::StatusOr> +CreateV2BiddingRequest(const BiddingSignalsRequest& bidding_signals_request); +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_BUYER_FRONTEND_SERVICE_KV_BUYER_SIGNALS_ADAPTER_H_ diff --git a/services/buyer_frontend_service/kv_buyer_signals_adapter_test.cc b/services/buyer_frontend_service/kv_buyer_signals_adapter_test.cc new file mode 100644 index 00000000..4be293de --- /dev/null +++ b/services/buyer_frontend_service/kv_buyer_signals_adapter_test.cc @@ -0,0 +1,436 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_buyer_signals_adapter.h" + +#include +#include + +#include "absl/strings/escaping.h" +#include "absl/strings/str_format.h" +#include "google/protobuf/text_format.h" +#include "gtest/gtest.h" +#include "src/core/test/utils/proto_test_utils.h" + +namespace kv_server::application_pas { +namespace { +using google::protobuf::TextFormat; +using google::scp::core::test::EqualsProto; +using privacy_sandbox::bidding_auction_servers::BiddingSignalsRequest; +using privacy_sandbox::bidding_auction_servers::BuyerInput; +using privacy_sandbox::bidding_auction_servers::ConvertV2BiddingSignalsToV1; +using privacy_sandbox::bidding_auction_servers::CreateV2BiddingRequest; +using privacy_sandbox::bidding_auction_servers::GetBidsRequest; + +std::string RemoveWhiteSpaces(std::string s) { + s.erase(std::remove_if(s.begin(), s.end(), ::isspace), s.end()); + return s; +} + +TEST(KvBuyerSignalsAdapter, Convert) { + kv_server::v2::GetValuesResponse response; + std::string compression_group = R"JSON( + [ + { + "id": 0, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello": { + "value": "world" + } + } + }, + { + "tags": [ + "structured", + "groupNames" + ], + "keyValues": { + "nohello": { + "value": "world" + } + } + } + ] + }, + { + "id": 1, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello2": { + "value": "world2" + } + } + } + ] + } + ])JSON"; + ASSERT_TRUE(TextFormat::ParseFromString( + absl::StrFormat(R"( + compression_groups { + compression_group_id : 33 + content : "%s" + })", + absl::CEscape(RemoveWhiteSpaces(compression_group))), + &response)); + auto result = ConvertV2BiddingSignalsToV1( + std::make_unique(response)); + CHECK_OK(result) << result.status(); + std::string expected_parsed_signals = + R"json( + { + "keys": { + "hello": { + "value": "world" + }, + "hello2": { + "value": "world2" + } + } + })json"; + ASSERT_EQ(*((*result)->trusted_signals), + RemoveWhiteSpaces(expected_parsed_signals)); +} + +TEST(KvBuyerSignalsAdapter, MultipleCompressionGroups) { + kv_server::v2::GetValuesResponse response; + std::string compression_group = R"JSON( + [ + { + "id": 0, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello": { + "value": "world" + } + } + }, + { + "tags": [ + "structured", + "groupNames" + ], + "keyValues": { + "nohello": { + "value": "world" + } + } + } + ] + }, + { + "id": 1, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello2": { + "value": "world2" + } + } + } + ] + } + ])JSON"; + + std::string compression_group_2 = R"JSON( + [ + { + "id": 3, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello44": { + "value": "world44" + } + } + }, + { + "tags": [ + "structured", + "groupNames" + ], + "keyValues": { + "blah": { + "value": "blah" + } + } + } + ] + }, + { + "id": 4, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello24": { + "value": "world24" + } + } + } + ] + } + ])JSON"; + + ASSERT_TRUE(TextFormat::ParseFromString( + absl::StrFormat(R"( + compression_groups { + compression_group_id : 33 + content : "%s" + } + compression_groups { + compression_group_id : 34 + content : "%s" + } + )", + absl::CEscape(RemoveWhiteSpaces(compression_group)), + absl::CEscape(RemoveWhiteSpaces(compression_group_2))), + &response)); + auto result = ConvertV2BiddingSignalsToV1( + std::make_unique(response)); + CHECK_OK(result) << result.status(); + std::string expected_parsed_signals = + R"json( + { + "keys": { + "hello": { + "value": "world" + }, + "hello2": { + "value": "world2" + }, + "hello44": { + "value": "world44" + }, + "hello24": { + "value": "world24" + } + } + })json"; + ASSERT_EQ(*((*result)->trusted_signals), + RemoveWhiteSpaces(expected_parsed_signals)); +} + +TEST(KvBuyerSignalsAdapter, MalformedJson) { + kv_server::v2::GetValuesResponse response; + std::string compression_group = R"JSON( + [ + { + "id": 0, + "keyGroupOutputsFAIL": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello": { + "value": "world" + } + } + }, + { + "tags": [ + "structured", + "groupNames" + ], + "keyValues": { + "nohello": { + "value": "world" + } + } + } + ] + }, + { + "id": 1, + "keyGroupOutputs": [ + { + "tags": [ + "keys" + ], + "keyValues": { + "hello2": { + "value": "world2" + } + } + } + ] + } + ])JSON"; + ASSERT_TRUE(TextFormat::ParseFromString( + absl::StrFormat(R"( + compression_groups { + compression_group_id : 33 + content : "%s" + })", + absl::CEscape(RemoveWhiteSpaces(compression_group))), + &response)); + auto result = ConvertV2BiddingSignalsToV1( + std::make_unique(response)); + ASSERT_FALSE(result.ok()); +} + +TEST(KvBuyerSignalsAdapter, EmptyJson) { + kv_server::v2::GetValuesResponse response; + ASSERT_TRUE(TextFormat::ParseFromString( + R"( + compression_groups { + compression_group_id : 33 + content : "" + })", + &response)); + auto result = ConvertV2BiddingSignalsToV1( + std::make_unique(response)); + ASSERT_FALSE(result.ok()); +} + +BuyerInput::InterestGroup MakeAnInterestGroup(const std::string& id, + int keys_number) { + BuyerInput::InterestGroup interest_group; + interest_group.set_name(absl::StrCat("ig_name_", id)); + for (int i = 0; i < keys_number; i++) { + interest_group.mutable_bidding_signals_keys()->Add( + absl::StrCat("bidding_signal_key_", id, i)); + } + return interest_group; +} + +TEST(KvBuyerSignalsAdapter, CreateV2BiddingRequestSuccess) { + v2::GetValuesRequest expected; + ASSERT_TRUE(TextFormat::ParseFromString( + R"pb( + client_version: "Bna.PA.Buyer.20240930" + metadata { + fields { + key: "client_type" + value { string_value: "1" } + } + fields { + key: "experiment_group_id" + value { string_value: "1689" } + } + fields { + key: "hostname" + value { string_value: "somepublisher.com" } + } + } + partitions { + id: 0 + compression_group_id: 0 + arguments { + tags { values { string_value: "keys" } } + data { + list_value { values { string_value: "bidding_signal_key_00" } } + } + } + arguments { + tags { values { string_value: "keys" } } + data { + list_value { values { string_value: "bidding_signal_key_01" } } + } + } + } + partitions { + id: 1 + compression_group_id: 1 + arguments { + tags { values { string_value: "keys" } } + data { + list_value { values { string_value: "bidding_signal_key_10" } } + } + } + arguments { + tags { values { string_value: "keys" } } + data { + list_value { values { string_value: "bidding_signal_key_11" } } + } + } + } + log_context { + generation_id: "generation_id" + adtech_debug_id: "debug_id" + } + consented_debug_config { is_consented: true token: "test_token" })pb", + &expected)); + privacy_sandbox::server_common::ConsentedDebugConfiguration + consented_debug_configuration; + consented_debug_configuration.set_is_consented(true); + consented_debug_configuration.set_token("test_token"); + privacy_sandbox::server_common::LogContext log_context; + log_context.set_generation_id("generation_id"); + log_context.set_adtech_debug_id("debug_id"); + GetBidsRequest::GetBidsRawRequest bids_request; + bids_request.set_buyer_kv_experiment_group_id(1689); + bids_request.set_client_type( + privacy_sandbox::bidding_auction_servers::CLIENT_TYPE_ANDROID); + bids_request.set_publisher_name("somepublisher.com"); + *bids_request.mutable_consented_debug_config() = + std::move(consented_debug_configuration); + *bids_request.mutable_log_context() = std::move(log_context); + for (int i = 0; i < 2; i++) { + *bids_request.mutable_buyer_input()->mutable_interest_groups()->Add() = + MakeAnInterestGroup(std::to_string(i), 2); + } + BiddingSignalsRequest bidding_signals_request(bids_request, {}); + auto maybe_result = CreateV2BiddingRequest(bidding_signals_request); + ASSERT_TRUE(maybe_result.ok()) << maybe_result.status(); + auto& request = *(*maybe_result); + EXPECT_THAT(request, EqualsProto(expected)); +} + +TEST(KvBuyerSignalsAdapter, CreateV2BiddingRequestCreationNoIGsFail) { + privacy_sandbox::server_common::ConsentedDebugConfiguration + consented_debug_configuration; + consented_debug_configuration.set_is_consented(true); + consented_debug_configuration.set_token("test_token"); + privacy_sandbox::server_common::LogContext log_context; + log_context.set_generation_id("generation_id"); + log_context.set_adtech_debug_id("debug_id"); + GetBidsRequest::GetBidsRawRequest bids_request; + bids_request.set_buyer_kv_experiment_group_id(1689); + bids_request.set_client_type( + privacy_sandbox::bidding_auction_servers::CLIENT_TYPE_ANDROID); + bids_request.set_publisher_name("somepublisher.com"); + *bids_request.mutable_consented_debug_config() = + std::move(consented_debug_configuration); + *bids_request.mutable_log_context() = std::move(log_context); + + BiddingSignalsRequest bidding_signals_request(bids_request, {}); + auto maybe_result = CreateV2BiddingRequest(bidding_signals_request); + ASSERT_FALSE(maybe_result.ok()); +} +} // namespace +} // namespace kv_server::application_pas diff --git a/services/buyer_frontend_service/providers/BUILD b/services/buyer_frontend_service/providers/BUILD index ca914d87..0d1513a9 100644 --- a/services/buyer_frontend_service/providers/BUILD +++ b/services/buyer_frontend_service/providers/BUILD @@ -18,6 +18,17 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "bidding_signals_providers", + hdrs = [ + "bidding_signals_async_provider.h", + ], + deps = [ + "//services/buyer_frontend_service/data:buyer_frontend_data", + "//services/common/providers:async_provider", + ], +) + +cc_library( + name = "http_bidding_signals_providers", srcs = [ "http_bidding_signals_async_provider.cc", ], @@ -26,6 +37,7 @@ cc_library( "http_bidding_signals_async_provider.h", ], deps = [ + "bidding_signals_providers", "//services/buyer_frontend_service/data:buyer_frontend_data", "//services/common/clients:client_factory_template", "//services/common/clients/http_kv_server/buyer:buyer_key_value_async_http_client", @@ -39,13 +51,13 @@ cc_library( ) cc_test( - name = "bidding_signals_providers_test", + name = "http_bidding_signals_providers_test", size = "small", srcs = [ "http_bidding_signals_async_provider_test.cc", ], deps = [ - "bidding_signals_providers", + "http_bidding_signals_providers", "//services/common/test:mocks", "//services/common/test:random", "@com_google_googletest//:gtest_main", diff --git a/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc b/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc index 90c65190..2fa030a1 100644 --- a/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc +++ b/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc @@ -58,9 +58,10 @@ void HttpBiddingSignalsAsyncProvider::Get( if (buyer_kv_output.ok()) { // TODO(b/258281777): Add ads and buyer signals from KV. res.value()->trusted_signals = - std::make_unique(buyer_kv_output.value()->result); - get_byte_size.request = buyer_kv_output.value()->request_size; - get_byte_size.response = buyer_kv_output.value()->response_size; + std::make_unique((*buyer_kv_output)->result); + res.value()->data_version = (*buyer_kv_output)->data_version; + get_byte_size.request = (*buyer_kv_output)->request_size; + get_byte_size.response = (*buyer_kv_output)->response_size; } else { res = buyer_kv_output.status(); } diff --git a/services/buyer_frontend_service/providers/http_bidding_signals_async_provider_test.cc b/services/buyer_frontend_service/providers/http_bidding_signals_async_provider_test.cc index ea0427f2..cc7d869a 100644 --- a/services/buyer_frontend_service/providers/http_bidding_signals_async_provider_test.cc +++ b/services/buyer_frontend_service/providers/http_bidding_signals_async_provider_test.cc @@ -198,7 +198,8 @@ TEST(HttpBiddingSignalsAsyncProviderTest, request.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( MakeARandomInterestGroupFromBrowser().release()); absl::Notification notification; - std::string expected_output = MakeARandomString(); + std::string expected_bidding_signals = MakeARandomString(); + const uint32_t expected_data_version = 1215; EXPECT_CALL( *mock_client, @@ -208,7 +209,8 @@ TEST(HttpBiddingSignalsAsyncProviderTest, GetBuyerValuesOutput>>) &&>>(), An(), _)) .WillOnce( - [&expected_output]( + [&expected_bidding_signals, + data_version_for_mock = expected_data_version]( std::unique_ptr input, const RequestMetadata& metadata, absl::AnyInvocable(); - output->result = expected_output; + output->result = expected_bidding_signals; + output->data_version = data_version_for_mock; (std::move(callback))(std::move(output)); return absl::OkStatus(); }); @@ -226,10 +229,12 @@ TEST(HttpBiddingSignalsAsyncProviderTest, BiddingSignalsRequest bidding_signals_request(request, {}); class_under_test.Get( bidding_signals_request, - [&expected_output, ¬ification]( + [&expected_bidding_signals, &expected_data_version, ¬ification]( absl::StatusOr> signals, GetByteSize get_byte_size) { - EXPECT_EQ(*(signals.value()->trusted_signals), expected_output); + EXPECT_EQ(*(signals.value()->trusted_signals), + expected_bidding_signals); + EXPECT_EQ(signals.value()->data_version, expected_data_version); notification.Notify(); }, absl::Milliseconds(100)); diff --git a/services/buyer_frontend_service/util/BUILD b/services/buyer_frontend_service/util/BUILD index 9fff7be8..9a1f4ac8 100644 --- a/services/buyer_frontend_service/util/BUILD +++ b/services/buyer_frontend_service/util/BUILD @@ -99,7 +99,7 @@ cc_library( ], deps = [ "//services/buyer_frontend_service/data:buyer_frontend_data", - "//services/buyer_frontend_service/providers:bidding_signals_providers", + "//services/buyer_frontend_service/providers:http_bidding_signals_providers", "//services/common/test:mocks", "//services/common/test/utils:test_utils", "@com_google_absl//absl/functional:any_invocable", diff --git a/services/buyer_frontend_service/util/buyer_frontend_test_utils.cc b/services/buyer_frontend_service/util/buyer_frontend_test_utils.cc index 22ce0dff..6dcb168b 100644 --- a/services/buyer_frontend_service/util/buyer_frontend_test_utils.cc +++ b/services/buyer_frontend_service/util/buyer_frontend_test_utils.cc @@ -53,11 +53,11 @@ void SetupBiddingProviderMock( on_done, absl::Duration timeout, RequestContext context) { GetByteSize get_byte_size; - if (server_error_to_return.has_value()) { + if (server_error_to_return) { std::move(on_done)(*server_error_to_return, get_byte_size); } else { auto bidding_signals = std::make_unique(); - if (bidding_signals_value.has_value()) { + if (bidding_signals_value) { bidding_signals->trusted_signals = std::make_unique(bidding_signals_value.value()); } diff --git a/services/buyer_frontend_service/util/proto_factory.cc b/services/buyer_frontend_service/util/proto_factory.cc index 959a89e0..f4b4dc1a 100644 --- a/services/buyer_frontend_service/util/proto_factory.cc +++ b/services/buyer_frontend_service/util/proto_factory.cc @@ -127,7 +127,7 @@ void CopyIGFromDeviceToIGForBidding( std::unique_ptr CreateGenerateBidsRawRequest( const GetBidsRequest::GetBidsRawRequest& get_bids_raw_request, std::unique_ptr bidding_signals_obj, - const size_t signal_size, const bool enable_kanon) { + const size_t signal_size, uint32_t data_version, const bool enable_kanon) { auto generate_bids_raw_request = std::make_unique(); const BuyerInput& buyer_input = get_bids_raw_request.buyer_input(); @@ -169,6 +169,8 @@ std::unique_ptr CreateGenerateBidsRawRequest( } } + generate_bids_raw_request->set_data_version(data_version); + // 2. Set auction signals. generate_bids_raw_request->set_auction_signals( get_bids_raw_request.auction_signals()); diff --git a/services/buyer_frontend_service/util/proto_factory.h b/services/buyer_frontend_service/util/proto_factory.h index d728a6d7..47097794 100644 --- a/services/buyer_frontend_service/util/proto_factory.h +++ b/services/buyer_frontend_service/util/proto_factory.h @@ -38,7 +38,8 @@ std::unique_ptr CreateGenerateBidsRawRequest( const GetBidsRequest::GetBidsRawRequest& get_bids_raw_request, std::unique_ptr bidding_signals_obj, - const size_t signal_size, const bool enable_kanon = false); + const size_t signal_size, uint32_t data_version, + const bool enable_kanon = false); // Creates a request to generate bid for protected app signals. std::unique_ptr(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -143,7 +143,8 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForAndroid) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size, /*enable_kanon=*/true); + (*parsed_bidding_signals).raw_size, expected_raw_output.data_version(), + /*enable_kanon=*/true); std::string difference; MessageDifferencer differencer; differencer.ReportDifferencesToString(&difference); @@ -154,8 +155,7 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForAndroid) { TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForBrowser) { GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -227,7 +227,7 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForBrowser) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, expected_raw_output.data_version()); EXPECT_TRUE(MessageDifferencer::Equals(expected_raw_output, *raw_output)); std::string difference; @@ -310,7 +310,7 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForTestIG) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, expected_raw_output.data_version()); ASSERT_GT(expected_raw_output.interest_group_for_bidding().size(), 0); ASSERT_GT(raw_output->interest_group_for_bidding().size(), 0); @@ -351,7 +351,7 @@ TEST(CreateGenerateBidsRequestTest, SkipsIGWithEmptyBiddingSignalsKeys) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_EQ(raw_output->interest_group_for_bidding().size(), 0); } @@ -381,7 +381,7 @@ TEST(CreateGenerateBidsRequestTest, HandlesIGWithDuplicateBiddingSignalsKeys) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_EQ(raw_output->interest_group_for_bidding(0) .trusted_bidding_signals_keys_size(), 1); @@ -411,7 +411,7 @@ TEST(CreateGenerateBidsRequestTest, auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_EQ(raw_output->interest_group_for_bidding(0) .trusted_bidding_signals_keys_size(), 1); @@ -422,8 +422,7 @@ TEST(CreateGenerateBidsRequestTest, TEST(CreateGenerateBidsRequestTest, SetsEnableEventLevelDebugReporting) { GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -434,15 +433,14 @@ TEST(CreateGenerateBidsRequestTest, SetsEnableEventLevelDebugReporting) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_TRUE(raw_output->enable_debug_reporting()); } TEST(CreateGenerateBidsRequestTest, SetsLogContext) { GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -454,7 +452,7 @@ TEST(CreateGenerateBidsRequestTest, SetsLogContext) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_EQ(raw_output->log_context().generation_id(), input.log_context().generation_id()); @@ -465,8 +463,7 @@ TEST(CreateGenerateBidsRequestTest, SetsLogContext) { TEST(CreateGenerateBidsRequestTest, SetsConsentedDebugConfig) { GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -479,7 +476,7 @@ TEST(CreateGenerateBidsRequestTest, SetsConsentedDebugConfig) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_EQ(raw_output->consented_debug_config().is_consented(), kIsConsentedDebug); EXPECT_EQ(raw_output->consented_debug_config().token(), kConsentedDebugToken); @@ -488,8 +485,7 @@ TEST(CreateGenerateBidsRequestTest, SetsConsentedDebugConfig) { TEST(CreateGenerateBidsRequestTest, SetsTopLevelSellerForComponentAuction) { GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -500,15 +496,14 @@ TEST(CreateGenerateBidsRequestTest, SetsTopLevelSellerForComponentAuction) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - (*parsed_bidding_signals).raw_size); + (*parsed_bidding_signals).raw_size, kTestDefaultDataVersion); EXPECT_EQ(input.top_level_seller(), raw_output->top_level_seller()); } TEST(CreateGenerateBidsRequestTest, SetsMultiBidLimit) { GenBidsRawReq expected_raw_output = MakeARandomGenerateBidsRequestForBrowser( /*enforce_kanon=*/true, /*multi_bid_limit=*/5); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -520,7 +515,8 @@ TEST(CreateGenerateBidsRequestTest, SetsMultiBidLimit) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - parsed_bidding_signals->raw_size, /*enable_kanon=*/true); + parsed_bidding_signals->raw_size, /*data_version=*/0, + /*enable_kanon=*/true); ASSERT_EQ(raw_output->enforce_kanon(), expected_raw_output.enforce_kanon()); EXPECT_EQ(raw_output->multi_bid_limit(), expected_raw_output.multi_bid_limit()); @@ -529,8 +525,7 @@ TEST(CreateGenerateBidsRequestTest, SetsMultiBidLimit) { TEST(CreateGenerateBidsRequestTest, MultiBidLimitDefaultsWithFalseFlags) { GenBidsRawReq expected_raw_output = MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - bidding_signals->trusted_signals = std::make_unique( + auto bidding_signals = std::make_unique( GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); auto parsed_bidding_signals = ParseTrustedBiddingSignals( std::move(bidding_signals), /*buyer_input*/ {}); @@ -541,7 +536,8 @@ TEST(CreateGenerateBidsRequestTest, MultiBidLimitDefaultsWithFalseFlags) { auto raw_output = CreateGenerateBidsRawRequest( input, std::move(parsed_bidding_signals->bidding_signals), - parsed_bidding_signals->raw_size, /*enable_kanon=*/false); + parsed_bidding_signals->raw_size, /*data_version=*/0, + /*enable_kanon=*/false); ASSERT_FALSE(raw_output->enforce_kanon()); EXPECT_EQ(raw_output->multi_bid_limit(), kDefaultMultiBidLimit); } diff --git a/services/common/clients/config/trusted_server_config_client.h b/services/common/clients/config/trusted_server_config_client.h index 53db0709..0ebfb956 100644 --- a/services/common/clients/config/trusted_server_config_client.h +++ b/services/common/clients/config/trusted_server_config_client.h @@ -100,7 +100,7 @@ class TrustedServersConfigClient { void SetFlag(const absl::Flag>& flag, absl::string_view config_name) { std::optional flag_value = absl::GetFlag(flag); - if (flag_value.has_value()) { + if (flag_value) { config_entries_map_[config_name] = absl::StrCat(*flag_value); } } @@ -108,7 +108,7 @@ class TrustedServersConfigClient { void SetFlag(const absl::Flag>& flag, absl::string_view config_name) { std::optional flag_value = absl::GetFlag(flag); - if (flag_value.has_value()) { + if (flag_value) { config_entries_map_[config_name] = *flag_value ? kTrue : kFalse; } } @@ -119,7 +119,7 @@ class TrustedServersConfigClient { absl::string_view config_name) { std::optional flag_value = absl::GetFlag(flag); - if (flag_value.has_value()) { + if (flag_value) { config_entries_map_[config_name] = AbslUnparseFlag(*flag_value); } } diff --git a/services/common/clients/http/http_fetcher_async.h b/services/common/clients/http/http_fetcher_async.h index 6758493c..91ab9602 100644 --- a/services/common/clients/http/http_fetcher_async.h +++ b/services/common/clients/http/http_fetcher_async.h @@ -86,6 +86,18 @@ class HttpFetcherAsync { virtual void FetchUrl(const HTTPRequest& http_request, int timeout_ms, OnDoneFetchUrl done_callback) = 0; + // Fetches the specified url with response metadata. + // + // http_request: The URL and headers for the HTTP GET request. + // timeout_ms: The request timeout + // done_callback: Output param. Invoked either on error or after finished + // receiving a response. Please note that done_callback will run in a + // threadpool and is not guaranteed to be the FetchUrl client's thread. + // Clients can expect done_callback to be called exactly once. + virtual void FetchUrlWithMetadata( + const HTTPRequest& http_request, int timeout_ms, + OnDoneFetchUrlWithMetadata done_callback) = 0; + // PUTs data to the specified url. // // http_request: The URL, headers, body for the HTTP PUT request. diff --git a/services/common/clients/http/multi_curl_http_fetcher_async.cc b/services/common/clients/http/multi_curl_http_fetcher_async.cc index c2b8bfd8..ee9bbc47 100644 --- a/services/common/clients/http/multi_curl_http_fetcher_async.cc +++ b/services/common/clients/http/multi_curl_http_fetcher_async.cc @@ -286,15 +286,16 @@ void MultiCurlHttpFetcherAsync::FetchUrlsWithMetadata( return; } for (int i = 0; i < requests.size(); i++) { - FetchUrl(requests.at(i), absl::ToInt64Milliseconds(timeout), - [i, shared_lifetime](absl::StatusOr result) { - absl::MutexLock lock_results(&shared_lifetime->results_mu); - shared_lifetime->results[i] = std::move(result); - if (--shared_lifetime->pending_results == 0) { - std::move(shared_lifetime->all_done_callback)( - std::move(shared_lifetime->results)); - } - }); + FetchUrlWithMetadata( + requests.at(i), absl::ToInt64Milliseconds(timeout), + [i, shared_lifetime](absl::StatusOr result) { + absl::MutexLock lock_results(&shared_lifetime->results_mu); + shared_lifetime->results[i] = std::move(result); + if (--shared_lifetime->pending_results == 0) { + std::move(shared_lifetime->all_done_callback)( + std::move(shared_lifetime->results)); + } + }); } } @@ -381,20 +382,20 @@ void MultiCurlHttpFetcherAsync::ExecuteCurlRequest( void MultiCurlHttpFetcherAsync::FetchUrl(const HTTPRequest& request, int timeout_ms, OnDoneFetchUrl done_callback) { - FetchUrl(request, timeout_ms, - [callback = std::move(done_callback)]( - absl::StatusOr response) mutable { - if (response.ok()) { - absl::StatusOr string_response = - std::move(response->body); - std::move(callback)(std::move(string_response)); - } else { - std::move(callback)(response.status()); - } - }); + FetchUrlWithMetadata(request, timeout_ms, + [callback = std::move(done_callback)]( + absl::StatusOr response) mutable { + if (response.ok()) { + absl::StatusOr string_response = + std::move(response->body); + std::move(callback)(std::move(string_response)); + } else { + std::move(callback)(response.status()); + } + }); } -void MultiCurlHttpFetcherAsync::FetchUrl( +void MultiCurlHttpFetcherAsync::FetchUrlWithMetadata( const HTTPRequest& request, int timeout_ms, OnDoneFetchUrlWithMetadata done_callback) { ExecuteCurlRequest(CreateCurlRequest(request, timeout_ms, keepalive_idle_sec_, diff --git a/services/common/clients/http/multi_curl_http_fetcher_async.h b/services/common/clients/http/multi_curl_http_fetcher_async.h index aca53c81..48085a48 100644 --- a/services/common/clients/http/multi_curl_http_fetcher_async.h +++ b/services/common/clients/http/multi_curl_http_fetcher_async.h @@ -129,6 +129,19 @@ class MultiCurlHttpFetcherAsync final : public HttpFetcherAsync { OnDoneFetchUrl done_callback) override ABSL_LOCKS_EXCLUDED(curl_handle_set_lock_); + // Fetches provided url and response metadata with libcurl. + // + // http_request: The URL and headers for the HTTP GET request. + // timeout_ms: The request timeout + // done_callback: Output param. Invoked either on error or after finished + // receiving a response. This method guarantees that the callback will be + // invoked once with the obtained result or error. + // Please note: done_callback will run in a threadpool and is not guaranteed + // to be the FetchUrl client's thread. + void FetchUrlWithMetadata(const HTTPRequest& request, int timeout_ms, + OnDoneFetchUrlWithMetadata done_callback) override + ABSL_LOCKS_EXCLUDED(curl_handle_set_lock_); + // PUTs data to the specified url. // // http_request: The URL, headers, body for the HTTP PUT request. diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc index 2d9dbe0b..bf47509c 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc @@ -93,25 +93,41 @@ absl::Status BuyerKeyValueAsyncHttpClient::Execute( }(); auto done_callback = [on_done = std::move(on_done), request_size, bid_signal = std::move(bid_signal), context]( - absl::StatusOr resultStr) mutable { - if (resultStr.ok()) { + absl::StatusOr httpResponse) mutable { + if (httpResponse.ok()) { PS_VLOG(kKVLog, context.log) << "BuyerKeyValueAsyncHttpClient response exported in EventMessage"; if (AllowAnyEventLogging(context.log)) { - bid_signal.set_response(resultStr.value()); + bid_signal.set_response(httpResponse->body); context.log.SetEventMessageField(std::move(bid_signal)); } - size_t response_size = resultStr->size(); + size_t response_size = httpResponse->body.size(); + uint32_t data_version_value = 0; + if (auto dv_header_it = + httpResponse->headers.find(kDataVersionResponseHeaderName); + dv_header_it != httpResponse->headers.end()) { + if (const auto& dv_statusor = dv_header_it->second; dv_statusor.ok()) { + if (!absl::SimpleAtoi(dv_statusor.value(), &data_version_value)) { + // Out param will be "left in an unspecified state" if parsing + // fails, so reset to 0. + data_version_value = 0; + PS_VLOG(kNoisyWarn, context.log) + << "BuyerKeyValueAsyncHttpClient received a non-int, negative, " + "or too-large data version header."; + } + } + } std::unique_ptr resultUPtr = - std::make_unique(GetBuyerValuesOutput( - {std::move(resultStr.value()), request_size, response_size})); + std::make_unique( + GetBuyerValuesOutput({std::move(httpResponse->body), request_size, + response_size, data_version_value})); std::move(on_done)(std::move(resultUPtr)); } else { PS_VLOG(kNoisyWarn, context.log) << "BuyerKeyValueAsyncHttpClient Failure Response: " - << resultStr.status(); + << httpResponse.status(); context.log.SetEventMessageField(std::move(bid_signal)); - std::move(on_done)(resultStr.status()); + std::move(on_done)(httpResponse.status()); } }; PS_VLOG(kKVLog, context.log) << "BTS Request Url:\n" @@ -119,7 +135,7 @@ absl::Status BuyerKeyValueAsyncHttpClient::Execute( for (const auto& header : request.headers) { PS_VLOG(kKVLog, context.log) << header; } - http_fetcher_async_->FetchUrl( + http_fetcher_async_->FetchUrlWithMetadata( request, static_cast(absl::ToInt64Milliseconds(timeout)), std::move(done_callback)); return absl::OkStatus(); diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.h b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.h index 9b8a2458..5fc41aca 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.h +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.h @@ -34,6 +34,8 @@ namespace privacy_sandbox::bidding_auction_servers { inline constexpr std::array kMandatoryHeaders{ {"X-BnA-Client-IP"}}; +inline constexpr char kDataVersionResponseHeaderName[] = "Data-Version"; + // The data used to build the Buyer KV look url suffix struct GetBuyerValuesInput { // [DSP] List of keys to query values for, under the namespace keys. @@ -61,6 +63,8 @@ struct GetBuyerValuesOutput { // Used for instrumentation purposes in upper layers. size_t request_size; size_t response_size; + // Optional, indicates version of the KV data. + uint32_t data_version; }; // This class fetches Key/Value pairs from a Buyer Key/Value Server instance diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc index 24dfa920..70f22826 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc @@ -25,6 +25,11 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { constexpr char kEgId[] = "1776"; +constexpr uint32_t kDataVersionHeaderValue = 1689; +constexpr absl::string_view kIvalidDataVersionHeaderStringValue = "abcxyz"; +// Don't check these. They are not calculated from the request or response. +constexpr size_t kRequestSizeDontCheck = 0; +constexpr size_t kResponseSizeDontCheck = 0; class KeyValueAsyncHttpClientTest : public testing::Test { public: @@ -75,17 +80,18 @@ class KeyValueAsyncHttpClientTest : public testing::Test { * 1. Define inputs * 2. Create the actual client and actually call it * 3. Actual client makes a url - * 4. Actual client calls FetchUrl with that url it made - * 5. Since FetchUrl is a mock, we get to assert that it is called with the url - * we expect (or in other words: assert that the actual URL the client created - * matches the expected URL we wrote) + * 4. Actual client calls FetchUrlWithMetadata with that url it made + * 5. Since FetchUrlWithMetadata is a mock, we get to assert that it is called + * with the url we expect (or in other words: assert that the actual URL the + * client created matches the expected URL we wrote) * 6. If and when all of that goes well: Since the HttpAsyncFetcher is a mock, * we define its behavior * 7. We specify that it returns a string like a KV server would - * 8. Then we pass that string into a callback exactly as the real FetchUrl - * would have done - * 9. THAT callback, into which the output of our mock FetchUrl is passed, is - * the one actually defined to be the actual callback made in the actual client. + * 8. Then we pass that string into a callback exactly as the real + * FetchUrlWithMetadata would have done + * 9. THAT callback, into which the output of our mock FetchUrlWithMetadata is + * passed, is the one actually defined to be the actual callback made in the + * actual client. * 10. It is THAT callback that WE define HERE in this test. * Specifically we define that it shall check that the actual output * matches an expected output And we define that expected output here. @@ -116,7 +122,7 @@ TEST_F(KeyValueAsyncHttpClientTest, "interestGroupNames=ig_name_likes_boots")); // Now we define what we expect to get back out of the client, which is a // GetBuyerValuesOutput struct. - const std::string expected_result = R"json({ + const std::string expected_response_body_json_string = R"json({ "keys": { "1j1043317685": { "constitution_author": "madison", @@ -136,8 +142,9 @@ TEST_F(KeyValueAsyncHttpClientTest, })json"; std::unique_ptr expectedOutputStructUPtr = - std::make_unique( - GetBuyerValuesOutput({expected_result})); + std::make_unique(GetBuyerValuesOutput( + {expected_response_body_json_string, kRequestSizeDontCheck, + kResponseSizeDontCheck, kDataVersionHeaderValue})); // Define the lambda function which is the callback. // Inside this callback, we will actually check that the client correctly @@ -152,30 +159,45 @@ TEST_F(KeyValueAsyncHttpClientTest, // This is what the client actually passes back absl::StatusOr> actualOutputStruct) { - ASSERT_TRUE(actualOutputStruct.ok()); - ASSERT_EQ(actualOutputStruct.value()->result, - expectedOutputStructUPtr->result); + // We can't make any assertions here because if we do and they fail the + // callback will never be notified. + EXPECT_TRUE(actualOutputStruct.ok()); + if (actualOutputStruct.ok()) { + EXPECT_EQ(actualOutputStruct.value()->result, + expectedOutputStructUPtr->result); + EXPECT_EQ(actualOutputStruct.value()->data_version, + expectedOutputStructUPtr->data_version); + } callback_invoked.Notify(); }; - // Assert that the mocked fetcher will have the method FetchUrl called on it, - // with the URL being expectedUrl. - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) - // If and when that happens: DEFINE that the FetchUrl function SHALL do - // the following: + // Assert that the mocked fetcher will have the method FetchUrlWithMetadata + // called on it, with the URL being expectedUrl. + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) + // If and when that happens: DEFINE that the FetchUrlWithMetadata function + // SHALL do the following: // (This part is NOT an assertion of expected behavior but rather a mock // defining what it shall be) - .WillOnce([actual_result = expected_result, &expected_urls]( + .WillOnce([actual_response_body_json_string = + expected_response_body_json_string, + &expected_urls]( const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { EXPECT_TRUE(expected_urls.contains(request.url)); - // Pack said string into a statusOr - absl::StatusOr resp = - absl::StatusOr(actual_result); + HTTPResponse actual_http_response; + actual_http_response.body = actual_response_body_json_string; + // Add a valid header. + absl::flat_hash_map> headers; + headers[kDataVersionResponseHeaderName] = absl::StatusOr( + absl::StrFormat("%d", kDataVersionHeaderValue)); + actual_http_response.headers = headers; + // Pack said response into a statusOr + absl::StatusOr actual_http_response_statusor = + absl::StatusOr(actual_http_response); // Now, call the callback (Note: we defined it above!) with the // 'response' from the 'server' - std::move(done_callback)(resp); + std::move(done_callback)(actual_http_response_statusor); }); // Finally, actually call the function to perform the test @@ -225,7 +247,7 @@ TEST_F(KeyValueAsyncHttpClientTest, // Now we define what we expect to get back out of the client, which is a // GetBuyerValuesOutput struct. // Note that this test is trivial; we use the same string - const std::string expectedResult = R"json({ + const std::string expected_response_body_json_string = R"json({ "keys": { "1j386134098": { "constitution_author": "madison", @@ -249,7 +271,7 @@ TEST_F(KeyValueAsyncHttpClientTest, } })json"; - const std::string actualResult = R"json({ + const std::string actual_response_body_json_string = R"json({ "keys": { "1j386134098": { "constitution_author": "Edmund Burke", @@ -274,7 +296,7 @@ TEST_F(KeyValueAsyncHttpClientTest, })json"; std::unique_ptr expectedOutputStructUPtr = std::make_unique( - GetBuyerValuesOutput({expectedResult})); + GetBuyerValuesOutput({expected_response_body_json_string})); absl::Notification callback_invoked; // Define the lambda function which is the callback. @@ -289,30 +311,42 @@ TEST_F(KeyValueAsyncHttpClientTest, // This is what the client actually passes back absl::StatusOr> actualOutputStruct) { - ASSERT_TRUE(actualOutputStruct.ok()); - ASSERT_FALSE(actualOutputStruct.value()->result == - expectedOutputStructUPtr->result); + EXPECT_TRUE(actualOutputStruct.ok()); + if (actualOutputStruct.ok()) { + EXPECT_NE(actualOutputStruct.value()->result, + expectedOutputStructUPtr->result); + EXPECT_EQ(actualOutputStruct.value()->data_version, 0); + } callback_invoked.Notify(); }; - // Assert that the mocked fetcher will have the method FetchUrl called on it, - // with the URL being expectedUrl. - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) - // If and when that happens: DEFINE that the FetchUrl function SHALL do - // the following: + // Assert that the mocked fetcher will have the method FetchUrlWithMetadata + // called on it, with the URL being expectedUrl. + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) + // If and when that happens: DEFINE that the FetchUrlWithMetadata function + // SHALL do the following: // (This part is NOT an assertion of expected behavior but rather a mock // defining what it shall be) - .WillOnce([actualResult, &expected_urls]( + .WillOnce([actual_response_body_json_string, &expected_urls]( const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { EXPECT_TRUE(expected_urls.contains(request.url)) << request.url; + HTTPResponse actual_http_response; + actual_http_response.body = actual_response_body_json_string; + // Add a header value that is INVALID. + // The idea is that it will NOT parse but will also NOT CRASH the + // client. + absl::flat_hash_map> headers; + headers[kDataVersionResponseHeaderName] = + absl::StatusOr(kIvalidDataVersionHeaderStringValue); + actual_http_response.headers = headers; // Pack said string into a statusOr - absl::StatusOr resp = - absl::StatusOr(actualResult); + absl::StatusOr actual_http_response_statusor = + absl::StatusOr(actual_http_response); // Now, call the callback (Note: we defined it above!) with the // 'response' from the 'server' - std::move(done_callback)(resp); + std::move(done_callback)(actual_http_response_statusor); }); // Finally, actually call the function to perform the test @@ -328,13 +362,12 @@ TEST_F(KeyValueAsyncHttpClientTest, MakesDSPUrlCorrectlyWithDuplicateKey) { std::make_unique(getValuesClientInput); const std::string expected_url = absl::StrCat(hostname_, "?hostname=www.usatoday.com&keys=url1"); - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce( - [expected_url](const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> - done_callback) { - EXPECT_EQ(expected_url, request.url); - }); + [expected_url]( + const HTTPRequest& request, int timeout_ms, + absl::AnyInvocable)&&> + done_callback) { EXPECT_EQ(expected_url, request.url); }); CheckGetValuesFromKeysViaHttpClient(std::move(input)); } @@ -344,13 +377,12 @@ TEST_F(KeyValueAsyncHttpClientTest, MakesDSPUrlCorrectlyWithNoKeys) { std::make_unique(getValuesClientInput); const std::string expected_url = absl::StrCat(hostname_, "?hostname=www.usatoday.com"); - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce( - [expected_url](const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> - done_callback) { - EXPECT_EQ(expected_url, request.url); - }); + [expected_url]( + const HTTPRequest& request, int timeout_ms, + absl::AnyInvocable)&&> + done_callback) { EXPECT_EQ(expected_url, request.url); }); CheckGetValuesFromKeysViaHttpClient(std::move(input)); } @@ -360,10 +392,10 @@ TEST_F(KeyValueAsyncHttpClientTest, {"lloyd_george", "clementine", "birkenhead"}, {}, ""}; std::unique_ptr input = std::make_unique(getValuesClientInput); - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce([expected_urls_1 = &(expected_urls_1)]( const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { EXPECT_TRUE(expected_urls_1->contains(request.url)); }); @@ -382,10 +414,10 @@ TEST_F(KeyValueAsyncHttpClientTest, // Note: if the client type is not CLIENT_TYPE_ANDROID, no client_type // param is attached to the url. This behavior will change after beta // testing to always include a client_type. - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce([expected_urls_1 = &(expected_urls_1)]( const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { EXPECT_TRUE(expected_urls_1->contains(request.url)); }); @@ -413,10 +445,10 @@ TEST_F(KeyValueAsyncHttpClientTest, MakesDSPUrlCorrectlyWithClientTypeAndroid) { "?client_type=1&keys=lloyd_george,clementine,birkenhead"), absl::StrCat(hostname_, "?client_type=1&keys=clementine,lloyd_george,birkenhead")}; - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce([&expected_urls]( const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { EXPECT_TRUE(expected_urls.contains(request.url)); }); @@ -440,12 +472,12 @@ TEST_F(KeyValueAsyncHttpClientTest, AddsMetadataToHeaders) { expectedHeaders.emplace_back( absl::StrCat(expected_metadatum.first, ":", expected_metadatum.second)); } - // Assert that the mocked fetcher will have the method FetchUrl called on it, - // with the metadata. - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + // Assert that the mocked fetcher will have the method FetchUrlWithMetadata + // called on it, with the metadata. + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce([&expectedHeaders]( HTTPRequest request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { std::sort(expectedHeaders.begin(), expectedHeaders.end()); std::sort(request.headers.begin(), request.headers.end()); @@ -471,12 +503,12 @@ TEST_F(KeyValueAsyncHttpClientTest, AddsMandatoryHeaders) { for (const auto& mandatory_header : kMandatoryHeaders) { expected_headers.push_back(absl::StrCat(mandatory_header, ":")); } - // Assert that the mocked fetcher will have the method FetchUrl called on it, - // with the metadata. - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) + // Assert that the mocked fetcher will have the method FetchUrlWithMetadata + // called on it, with the metadata. + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) .WillOnce([&expected_headers]( HTTPRequest request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { std::sort(expected_headers.begin(), expected_headers.end()); std::sort(request.headers.begin(), request.headers.end()); @@ -493,7 +525,7 @@ TEST_F(KeyValueAsyncHttpClientTest, AddsMandatoryHeaders) { TEST_F(KeyValueAsyncHttpClientTest, PrewarmsHTTPClient) { const std::string expectedUrl = absl::StrCat(hostname_, "?"); - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl).Times(1); + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata).Times(1); // Create the client. BuyerKeyValueAsyncHttpClient kvHttpClient( hostname_, std::move(mock_http_fetcher_async_), true); @@ -515,7 +547,7 @@ TEST_F(KeyValueAsyncHttpClientTest, SpacesInKeysGetEncoded) { // Now we define what we expect to get back out of the client, which is a // GetBuyerValuesOutput struct. // Note that this test is trivial; we use the same string - const std::string expected_result = R"json({ + const std::string expected_response_body_json_string = R"json({ "keys": { "1j1043317685": { "constitution_author": "madison", @@ -528,8 +560,9 @@ TEST_F(KeyValueAsyncHttpClientTest, SpacesInKeysGetEncoded) { })json"; std::unique_ptr expectedOutputStructUPtr = - std::make_unique( - GetBuyerValuesOutput({expected_result})); + std::make_unique(GetBuyerValuesOutput( + {expected_response_body_json_string, kRequestSizeDontCheck, + kResponseSizeDontCheck, kDataVersionHeaderValue})); // Define the lambda function which is the callback. // Inside this callback, we will actually check that the client correctly @@ -544,30 +577,42 @@ TEST_F(KeyValueAsyncHttpClientTest, SpacesInKeysGetEncoded) { // This is what the client actually passes back absl::StatusOr> actualOutputStruct) { - ASSERT_TRUE(actualOutputStruct.ok()); - ASSERT_EQ(actualOutputStruct.value()->result, - expectedOutputStructUPtr->result); + EXPECT_TRUE(actualOutputStruct.ok()); + if (actualOutputStruct.ok()) { + EXPECT_EQ(actualOutputStruct.value()->result, + expectedOutputStructUPtr->result); + EXPECT_EQ(actualOutputStruct.value()->data_version, + expectedOutputStructUPtr->data_version); + } callback_invoked.Notify(); }; - // Assert that the mocked fetcher will have the method FetchUrl called on it, - // with the URL being expectedUrl. - EXPECT_CALL(*mock_http_fetcher_async_, FetchUrl) - // If and when that happens: DEFINE that the FetchUrl function SHALL do - // the following: + // Assert that the mocked fetcher will have the method FetchUrlWithMetadata + // called on it, with the URL being expectedUrl. + EXPECT_CALL(*mock_http_fetcher_async_, FetchUrlWithMetadata) + // If and when that happens: DEFINE that the FetchUrlWithMetadata function + // SHALL do the following: // (This part is NOT an assertion of expected behavior but rather a mock // defining what it shall be) - .WillOnce([actual_result = expected_result, &expectedUrl]( + .WillOnce([actual_response_body_json_string = + expected_response_body_json_string, + &expectedUrl]( const HTTPRequest& request, int timeout_ms, - absl::AnyInvocable)&&> + absl::AnyInvocable)&&> done_callback) { EXPECT_EQ(request.url, expectedUrl); + HTTPResponse actual_http_response; + actual_http_response.body = actual_response_body_json_string; + absl::flat_hash_map> headers; + headers[kDataVersionResponseHeaderName] = absl::StatusOr( + absl::StrCat("", kDataVersionHeaderValue)); + actual_http_response.headers = headers; // Pack said string into a statusOr - absl::StatusOr resp = - absl::StatusOr(actual_result); + absl::StatusOr actual_http_response_statusor = + absl::StatusOr(actual_http_response); // Now, call the callback (Note: we defined it above!) with the // 'response' from the 'server' - std::move(done_callback)(resp); + std::move(done_callback)(actual_http_response_statusor); }); // Finally, actually call the function to perform the test diff --git a/services/common/clients/kv_server/BUILD b/services/common/clients/kv_server/BUILD index c5279d3c..b7aba566 100644 --- a/services/common/clients/kv_server/BUILD +++ b/services/common/clients/kv_server/BUILD @@ -38,9 +38,9 @@ cc_library( deps = [ "//services/common/clients:client_params_template", "//services/common/clients/async_grpc:default_async_grpc_client", - "//services/common/util:binary_http_utils", "//services/common/util:client_context_util", "//services/common/util:oblivious_http_utils", + "//services/seller_frontend_service/util:framing_utils", "@com_github_google_quiche//quiche:binary_http_unstable_api", "@com_github_google_quiche//quiche:oblivious_http_unstable_api", "@com_github_grpc_grpc//:grpc++", diff --git a/services/common/clients/kv_server/kv_async_client.cc b/services/common/clients/kv_server/kv_async_client.cc index 7a208e81..6fcc7eaa 100644 --- a/services/common/clients/kv_server/kv_async_client.cc +++ b/services/common/clients/kv_server/kv_async_client.cc @@ -19,12 +19,23 @@ #include "include/grpcpp/support/status_code_enum.h" #include "services/common/clients/async_grpc/default_async_grpc_client.h" #include "services/common/util/client_context_util.h" +#include "services/seller_frontend_service/util/framing_utils.h" #include "src/communication/encoding_utils.h" #include "src/public/cpio/interface/crypto_client/crypto_client_interface.h" #include "src/public/cpio/proto/public_key_service/v1/public_key_service.pb.h" namespace privacy_sandbox::bidding_auction_servers { +inline constexpr std::string_view kKVContentTypeHeader = "kv-content-type"; +inline constexpr std::string_view kContentEncodingProtoHeaderValue = + "message/ad-auction-trusted-signals-request+proto"; +// TODO: once the KV dependency is bumped, these constants are available and +// should be reused. +inline constexpr absl::string_view kKVOhttpRequestLabel = + "message/ad-auction-trusted-signals-request"; +inline constexpr absl::string_view kKVOhttpResponseLabel = + "message/ad-auction-trusted-signals-response"; + using ::google::cmrt::sdk::public_key_service::v1::PublicKey; KVAsyncGrpcClient::KVAsyncGrpcClient( @@ -62,20 +73,23 @@ void KVAsyncGrpcClient::SendRpc( return; } PS_VLOG(6) << "SendRPC completion status ok"; - auto plain_text_binary_http_response = - FromObliviousHTTPResponse(*params->ResponseRef()->mutable_data(), - *captured_oblivious_http_context); - if (!plain_text_binary_http_response.ok()) { + auto plain_text_http_response = FromObliviousHTTPResponse( + *params->ResponseRef()->mutable_data(), + *captured_oblivious_http_context, kKVOhttpResponseLabel); + if (!plain_text_http_response.ok()) { + // Passing an incorrect label will result in "SslErrorAsStatus("Failed + // to export secret.")" see + // https://github.com/google/quiche/blob/6fe69b2cf77d5fc175a729bc7a6c322a6388b8b6/quiche/oblivious_http/buffers/oblivious_http_response.cc#L265 PS_LOG(ERROR, SystemLogContext()) - << "KVAsyncGrpcClient failed to get binary HTTP response"; - params->OnDone(grpc::Status( - grpc::StatusCode::INVALID_ARGUMENT, - plain_text_binary_http_response.status().ToString())); + << "KVAsyncGrpcClient failed to get HTTP response"; + params->OnDone( + grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + plain_text_http_response.status().ToString())); return; } auto deframed_req = privacy_sandbox::server_common::DecodeRequestPayload( - *plain_text_binary_http_response); + *plain_text_http_response); if (!deframed_req.ok()) { PS_LOG(ERROR, SystemLogContext()) << "Unpadding response failed: " << deframed_req.status(); @@ -83,20 +97,18 @@ void KVAsyncGrpcClient::SendRpc( deframed_req.status().ToString())); return; } - auto response = FromBinaryHTTP( - deframed_req->compressed_data, /*from_json=*/false); - PS_VLOG(7) << "Retrieved proto response: " << response->DebugString(); - if (!response->has_single_partition()) { - PS_LOG(ERROR) - << "KVAsyncGrpcClient expected a single partition response, got: " - << response->DebugString(); - params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, - "Expected a single partition response")); + GetValuesResponse response; + if (!response.ParseFromString(deframed_req->compressed_data)) { + PS_LOG(ERROR, SystemLogContext()) + << "Unable to convert the response to proto"; + params->OnDone( + grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "Unable to convert the response to proto")); return; } - + PS_VLOG(7) << "Retrieved proto response: " << response.DebugString(); params->SetRawResponse( - std::make_unique(*std::move(response))); + std::make_unique(std::move(response))); PS_VLOG(6) << "Returning the decrypted response via callback"; params->OnDone(status); }); @@ -109,8 +121,7 @@ absl::Status KVAsyncGrpcClient::ExecuteInternal( on_done, absl::Duration timeout, RequestConfig request_config) { PS_VLOG(6) << "Raw request:\n" << raw_request->DebugString(); - PS_ASSIGN_OR_RETURN(std::string binary_http_msg, - ToBinaryHTTP(*raw_request, /*to_json=*/false)); + std::string serialized_req = raw_request->SerializeAsString(); PS_VLOG(5) << "Fetching public Key ..."; PS_ASSIGN_OR_RETURN(auto public_key, @@ -122,12 +133,22 @@ absl::Status KVAsyncGrpcClient::ExecuteInternal( absl::StrCat("Failed to base64 decode the fetched public key: ", public_key.public_key())); } - - PS_ASSIGN_OR_RETURN(auto oblivious_http_request, - ToObliviousHTTPRequest( - binary_http_msg, unescaped_public_key_bytes, - stoi(public_key.key_id()), kv_server::kKEMParameter, - kv_server::kKDFParameter, kv_server::kAEADParameter)); + auto encoded_data_size = GetEncodedDataSize(serialized_req.size()); + auto maybe_padded_request = + privacy_sandbox::server_common::EncodeResponsePayload( + privacy_sandbox::server_common::CompressionType::kUncompressed, + std::move(serialized_req), encoded_data_size); + if (!maybe_padded_request.ok()) { + PS_LOG(ERROR, SystemLogContext()) + << "Padding failed: " << maybe_padded_request.status().message(); + return maybe_padded_request.status(); + } + PS_ASSIGN_OR_RETURN( + auto oblivious_http_request, + ToObliviousHTTPRequest(*maybe_padded_request, unescaped_public_key_bytes, + stoi(public_key.key_id()), + kv_server::kKEMParameter, kv_server::kKDFParameter, + kv_server::kAEADParameter, kKVOhttpRequestLabel)); PS_VLOG(6) << "Encapsulating and serializing request"; std::string encrypted_request = oblivious_http_request.EncapsulateAndSerialize(); @@ -141,6 +162,8 @@ absl::Status KVAsyncGrpcClient::ExecuteInternal( ObliviousGetValuesRequest, google::api::HttpBody, GetValuesResponse>>( std::move(request), std::move(on_done)); context->set_deadline(GetClientContextDeadline(timeout, kMaxClientTimeout)); + context->AddMetadata(std::string(kKVContentTypeHeader), + std::string(kContentEncodingProtoHeaderValue)); PS_VLOG(5) << "Sending RPC ..."; SendRpc(std::make_unique( std::move(oblivious_http_request).ReleaseContext()), diff --git a/services/common/clients/kv_server/kv_async_client.h b/services/common/clients/kv_server/kv_async_client.h index 13b780d5..2664265b 100644 --- a/services/common/clients/kv_server/kv_async_client.h +++ b/services/common/clients/kv_server/kv_async_client.h @@ -39,7 +39,6 @@ #include "services/common/clients/client_params.h" #include "services/common/encryption/crypto_client_wrapper_interface.h" #include "services/common/loggers/request_log_context.h" -#include "services/common/util/binary_http_utils.h" #include "services/common/util/oblivious_http_utils.h" #include "src/encryption/key_fetcher/key_fetcher_manager.h" #include "src/encryption/key_fetcher/key_fetcher_utils.h" diff --git a/services/common/code_dispatch/BUILD b/services/common/code_dispatch/BUILD index 491d9f20..616b3e66 100644 --- a/services/common/code_dispatch/BUILD +++ b/services/common/code_dispatch/BUILD @@ -22,7 +22,6 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//api:bidding_auction_servers_cc_grpc_proto", - "//services/common:feature_flags", "//services/common/clients:async_client", "//services/common/constants:user_error_strings", "//services/common/encryption:crypto_client_wrapper_interface", diff --git a/services/common/code_dispatch/code_dispatch_reactor.h b/services/common/code_dispatch/code_dispatch_reactor.h index 5da5954e..1be63903 100644 --- a/services/common/code_dispatch/code_dispatch_reactor.h +++ b/services/common/code_dispatch/code_dispatch_reactor.h @@ -29,7 +29,6 @@ #include "services/common/clients/async_client.h" #include "services/common/constants/user_error_strings.h" #include "services/common/encryption/crypto_client_wrapper_interface.h" -#include "services/common/feature_flags.h" #include "services/common/loggers/request_log_context.h" #include "services/common/util/client_contexts.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" @@ -47,13 +46,14 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { explicit CodeDispatchReactor( const Request* request, Response* response, server_common::KeyFetcherManagerInterface* key_fetcher_manager, - CryptoClientWrapperInterface* crypto_client) + CryptoClientWrapperInterface* crypto_client, + bool enable_cancellation = false, bool enable_kanon = false) : request_(request), response_(response), key_fetcher_manager_(key_fetcher_manager), crypto_client_(crypto_client), - enable_cancellation_(absl::GetFlag(FLAGS_enable_cancellation)), - enable_kanon_(absl::GetFlag(FLAGS_enable_kanon)) { + enable_cancellation_(enable_cancellation), + enable_kanon_(enable_kanon) { PS_VLOG(5) << "Encryption is enabled, decrypting request now"; if (DecryptRequest()) { PS_VLOG(5) << "Decrypted request: " << raw_request_.DebugString(); @@ -100,7 +100,7 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { std::optional private_key = key_fetcher_manager_->GetPrivateKey(request_->key_id()); - if (!private_key.has_value()) { + if (!private_key) { PS_LOG(ERROR, SystemLogContext()) << "Unable to fetch private key from the key fetcher manager"; Finish( diff --git a/services/common/constants/common_service_flags.cc b/services/common/constants/common_service_flags.cc index 1bae1d89..5e5f7599 100644 --- a/services/common/constants/common_service_flags.cc +++ b/services/common/constants/common_service_flags.cc @@ -94,3 +94,8 @@ ABSL_FLAG(std::optional, enable_chaffing, false, "If true, chaff requests are sent out from the SFE. Chaff requests " "are requests sent to buyers not participating in an auction to mask " "the buyers associated with a client request."); +ABSL_FLAG(std::optional, debug_sample_rate_micro, 0, + "Set a value between 0 (never debug) and 1,000,000 (always debug) to " + "determine the sampling rate of requests eligible for debugging. " + "Chosen requests export" + "privacy_sandbox.bidding_auction_servers.EventMessage"); diff --git a/services/common/constants/common_service_flags.h b/services/common/constants/common_service_flags.h index af70ea48..20656da4 100644 --- a/services/common/constants/common_service_flags.h +++ b/services/common/constants/common_service_flags.h @@ -59,6 +59,7 @@ ABSL_DECLARE_FLAG(std::optional, ps_verbosity); ABSL_DECLARE_FLAG(std::optional, max_allowed_size_debug_url_bytes); ABSL_DECLARE_FLAG(std::optional, max_allowed_size_all_debug_urls_kb); ABSL_DECLARE_FLAG(std::optional, enable_chaffing); +ABSL_DECLARE_FLAG(std::optional, debug_sample_rate_micro); namespace privacy_sandbox::bidding_auction_servers { @@ -102,6 +103,7 @@ inline constexpr char MAX_ALLOWED_SIZE_DEBUG_URL_BYTES[] = inline constexpr char MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB[] = "MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB"; inline constexpr char ENABLE_CHAFFING[] = "ENABLE_CHAFFING"; +inline constexpr char DEBUG_SAMPLE_RATE_MICRO[] = "DEBUG_SAMPLE_RATE_MICRO"; inline constexpr absl::string_view kCommonServiceFlags[] = { PUBLIC_KEY_ENDPOINT, @@ -128,7 +130,9 @@ inline constexpr absl::string_view kCommonServiceFlags[] = { PS_VERBOSITY, MAX_ALLOWED_SIZE_DEBUG_URL_BYTES, MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB, - ENABLE_CHAFFING}; + ENABLE_CHAFFING, + DEBUG_SAMPLE_RATE_MICRO, +}; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/data_fetch/periodic_bucket_fetcher.cc b/services/common/data_fetch/periodic_bucket_fetcher.cc index 8536fc89..75e045a9 100644 --- a/services/common/data_fetch/periodic_bucket_fetcher.cc +++ b/services/common/data_fetch/periodic_bucket_fetcher.cc @@ -71,7 +71,7 @@ absl::Status PeriodicBucketFetcher::Start() { } void PeriodicBucketFetcher::End() { - if (task_id_.has_value()) { + if (task_id_) { executor_.Cancel(*task_id_); task_id_ = absl::nullopt; } diff --git a/services/common/data_fetch/periodic_url_fetcher.cc b/services/common/data_fetch/periodic_url_fetcher.cc index 417d61a7..59d64045 100644 --- a/services/common/data_fetch/periodic_url_fetcher.cc +++ b/services/common/data_fetch/periodic_url_fetcher.cc @@ -59,7 +59,7 @@ absl::Status PeriodicUrlFetcher::Start() { } void PeriodicUrlFetcher::End() { - if (task_id_.has_value()) { + if (task_id_) { executor_.Cancel(*task_id_); task_id_ = absl::nullopt; } diff --git a/services/common/loggers/BUILD b/services/common/loggers/BUILD index 38d05be0..3bfd236e 100644 --- a/services/common/loggers/BUILD +++ b/services/common/loggers/BUILD @@ -31,6 +31,8 @@ cc_library( deps = [ "//api:bidding_auction_servers_cc_proto", "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/random", + "@com_google_absl//absl/random:bit_gen_ref", "@google_privacysandbox_servers_common//src/logger:request_context_impl", ], ) diff --git a/services/common/loggers/request_log_context.h b/services/common/loggers/request_log_context.h index e7e1562f..e7f1579d 100644 --- a/services/common/loggers/request_log_context.h +++ b/services/common/loggers/request_log_context.h @@ -20,6 +20,9 @@ #include #include "absl/base/no_destructor.h" +#include "absl/random/bit_gen_ref.h" +#include "absl/random/discrete_distribution.h" +#include "absl/random/random.h" #include "absl/strings/string_view.h" #include "api/bidding_auction_servers.pb.h" #include "src/logger/request_context_impl.h" @@ -113,14 +116,14 @@ inline constexpr int kDispatch = 4; // UDF dispatch request and response inline constexpr int kOriginated = 5; // plaintext B&A request and response originated from server inline constexpr int kKVLog = 5; // KV request response -inline constexpr int kStats = 5; // Stats log like time , byte size, etc. +inline constexpr int kStats = 5; // Stats log e.g. time, byte size, etc. inline constexpr int kEncrypted = 6; inline bool AllowAnyEventLogging(RequestLogContext& log_context) { // if is_debug_response in non prod, it logs to debug kNoisyInfo // if is_consented, it logs to event message return (log_context.is_debug_response() && !server_common::log::IsProd()) || - log_context.is_consented(); + log_context.is_consented() || log_context.is_prod_debug(); } inline bool AllowAnyUdfLogging(RequestLogContext& log_context) { @@ -128,6 +131,12 @@ inline bool AllowAnyUdfLogging(RequestLogContext& log_context) { AllowAnyEventLogging(log_context); } +inline bool RandomSample(int sample_rate_micro, absl::BitGenRef bitgen) { + absl::discrete_distribution dist( + {1e6 - sample_rate_micro, (double)sample_rate_micro}); + return dist(bitgen); +} + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_LOGGERS_REQUEST_LOG_CONTEXT_H_ diff --git a/services/common/metric/server_definition.h b/services/common/metric/server_definition.h index 382829f6..842f89f0 100644 --- a/services/common/metric/server_definition.h +++ b/services/common/metric/server_definition.h @@ -77,32 +77,36 @@ inline constexpr server_common::metrics::Definition< inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, server_common::metrics::Instrument::kHistogram> - kJSExecutionDuration("js_execution.duration_ms", - "Time taken to execute the JS dispatcher", - server_common::metrics::kTimeHistogram, 300, 10); + kUdfExecutionDuration("udf_execution.duration_ms", + "End to end time taken from dispatching the UDF to " + "the execution of the callback (includes queue time)", + server_common::metrics::kTimeHistogram, 300, 10); inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, server_common::metrics::Instrument::kHistogram> - kPASGenerateBidJSExecutionDuration( - "pas_generate_bid_js_execution.duration_ms", - "Time taken to execute the PAS GenerateBid JS dispatcher", + kPASGenerateBidUdfExecutionDuration( + "pas_generate_bid_udf_execution.duration_ms", + "End to end time taken from dispatching the PAS GenerateBid UDF to the " + "execution of the callback (includes queue time)", server_common::metrics::kTimeHistogram, 300, 10); inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, server_common::metrics::Instrument::kHistogram> - kPASPrepareDataForRetrievalJSExecutionDuration( - "pas_prepare_data_for_retrieval_js_execution.duration_ms", - "Time taken to execute the PAS PrepareDataForAdRetrieval JS dispatcher", + kPASPrepareDataForRetrievalUdfExecutionDuration( + "pas_prepare_data_for_retrieval_udf_execution.duration_ms", + "End to end time taken from dispatching the PAS " + "PrepareDataForAdRetrieval UDF to the execution of the callback " + "(includes queue time)", server_common::metrics::kTimeHistogram, 300, 10); inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, server_common::metrics::Instrument::kUpDownCounter> - kJSExecutionErrorCount("js_execution.errors_count", - "No. of times js execution returned status != OK", 1, - 0); + kUdfExecutionErrorCount("udf_execution.errors_count", + "No. of times UDF execution returned status != OK", + 1, 0); inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, @@ -212,7 +216,7 @@ inline constexpr server_common::metrics::Definition< kBiddingInferenceRequestDuration( "bidding.inference.request.duration_ms", "Time taken by Roma callback to execute inference", - server_common::metrics::kTimeHistogram, 300, 0); + server_common::metrics::kTimeHistogram, 200, 0); inline constexpr server_common::metrics::Definition< double, server_common::metrics::Privacy::kNonImpacting, @@ -589,7 +593,7 @@ inline constexpr server_common::metrics::Definition< kInferenceRequestDuration( "inference.request.duration_ms", "Time taken by inference sidecar to execute inference", - server_common::metrics::kTimeHistogram, 300, 0); + server_common::metrics::kTimeHistogram, 200, 0); inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, @@ -643,7 +647,7 @@ inline constexpr server_common::metrics::Definition< /*partition_type*/ "model", /*max_partitions_contributed*/ 1, /*public_partitions*/ kDefaultDynamicPartition, - /*upper_bound*/ 300, + /*upper_bound*/ 200, /*lower_bound*/ 0); inline constexpr server_common::metrics::Definition< @@ -699,8 +703,8 @@ inline constexpr const server_common::metrics::DefinitionName* &kBiddingTotalBidsCount, &kBiddingZeroBidCount, &kBiddingZeroBidPercent, - &kJSExecutionDuration, - &kJSExecutionErrorCount, + &kUdfExecutionDuration, + &kUdfExecutionErrorCount, &kBiddingErrorCountByErrorCode, &kBiddingInferenceRequestDuration, &kInferenceCloudFetchSuccessCount, @@ -718,8 +722,8 @@ inline constexpr const server_common::metrics::DefinitionName* &kInferenceRequestCountByModel, &kInferenceRequestDurationByModel, &kInferenceRequestFailedCountByModel, - &kPASGenerateBidJSExecutionDuration, - &kPASPrepareDataForRetrievalJSExecutionDuration, + &kPASGenerateBidUdfExecutionDuration, + &kPASPrepareDataForRetrievalUdfExecutionDuration, &kInferenceRequestBatchCountByModel, }; @@ -823,8 +827,8 @@ inline constexpr const server_common::metrics::DefinitionName* &kAuctionTotalBidsCount, &kAuctionBidRejectedCount, &kAuctionBidRejectedPercent, - &kJSExecutionDuration, - &kJSExecutionErrorCount, + &kUdfExecutionDuration, + &kUdfExecutionErrorCount, &kAuctionErrorCountByErrorCode, }; @@ -960,10 +964,11 @@ std::unique_ptr> MakeInitiatedRequest( template inline void AddSystemMetric(T* context_map) { context_map->AddObserverable(server_common::metrics::kCpuPercent, - server_common::GetCpu); + server_common::SystemMetrics::GetCpu); context_map->AddObserverable(server_common::metrics::kMemoryKB, - server_common::GetMemory); - context_map->AddObserverable(metric::kThreadCount, server_common::GetThread); + server_common::SystemMetrics::GetMemory); + context_map->AddObserverable(metric::kThreadCount, + server_common::SystemMetrics::GetThread); context_map->AddObserverable( server_common::metrics::kKeyFetchFailureCount, server_common::KeyFetchResultCounter::GetKeyFetchFailureCount); diff --git a/services/common/private_aggregation/private_aggregation_post_auction_util.cc b/services/common/private_aggregation/private_aggregation_post_auction_util.cc index 585d0425..741a8490 100644 --- a/services/common/private_aggregation/private_aggregation_post_auction_util.cc +++ b/services/common/private_aggregation/private_aggregation_post_auction_util.cc @@ -50,7 +50,7 @@ absl::StatusOr GetPrivateAggregationBucketPostAuction( base_value_numerical_value = base_values.highest_scoring_other_bid; break; case BaseValue::BASE_VALUE_BID_REJECTION_REASON: - if (!base_values.reject_reason.has_value()) { + if (!base_values.reject_reason) { return absl::InvalidArgumentError(kBaseValuesRejectReasonNotAvailable); } base_value_numerical_value = base_values.reject_reason.value(); @@ -114,7 +114,7 @@ absl::StatusOr GetPrivateAggregationValuePostAuction( final_value = base_values.highest_scoring_other_bid; break; case BaseValue::BASE_VALUE_BID_REJECTION_REASON: - if (!base_values.reject_reason.has_value()) { + if (!base_values.reject_reason) { return absl::InvalidArgumentError(kBaseValuesRejectReasonNotAvailable); } final_value = base_values.reject_reason.value(); diff --git a/services/common/test/BUILD b/services/common/test/BUILD index 7831176c..d70acce0 100644 --- a/services/common/test/BUILD +++ b/services/common/test/BUILD @@ -58,6 +58,7 @@ cc_library( deps = [ "//api:bidding_auction_servers_cc_grpc_proto", "//api/udf:generate_bid_sdk_cc_proto", + "//services/buyer_frontend_service/data:buyer_frontend_data", "//services/common/test/utils:cbor_test_utils", "//services/common/util:request_response_constants", "//services/seller_frontend_service/test:app_test_utils", diff --git a/services/common/test/mocks.h b/services/common/test/mocks.h index d956d8de..5f28576c 100644 --- a/services/common/test/mocks.h +++ b/services/common/test/mocks.h @@ -202,6 +202,10 @@ class MockHttpFetcherAsync : public HttpFetcherAsync { (const HTTPRequest& http_request, int timeout_ms, OnDoneFetchUrl done_callback), (override)); + MOCK_METHOD(void, FetchUrlWithMetadata, + (const HTTPRequest& http_request, int timeout_ms, + OnDoneFetchUrlWithMetadata done_callback), + (override)); MOCK_METHOD(void, PutUrl, (const HTTPRequest& http_request, int timeout_ms, OnDoneFetchUrl done_callback), diff --git a/services/common/test/random.cc b/services/common/test/random.cc index 62d53f05..cb162ad1 100644 --- a/services/common/test/random.cc +++ b/services/common/test/random.cc @@ -18,6 +18,7 @@ #include #include "absl/strings/str_format.h" +#include "services/buyer_frontend_service/data/bidding_signals.h" namespace privacy_sandbox::bidding_auction_servers { @@ -274,7 +275,7 @@ std::string MakeTrustedBiddingSignalsForIG( return signals_dest; } -std::string GetBiddingSignalsFromGenerateBidsRequest( +BiddingSignals GetBiddingSignalsFromGenerateBidsRequest( const GenerateBidsRequest::GenerateBidsRawRequest& raw_request) { std::string signals_dest; @@ -342,7 +343,9 @@ std::string GetBiddingSignalsFromGenerateBidsRequest( } absl::StrAppend(&signals_dest, "}}"); - return signals_dest; + BiddingSignals bidding_signals{std::make_unique(signals_dest), + raw_request.data_version()}; + return bidding_signals; } InterestGroupForBidding MakeAnInterestGroupForBiddingSentFromDevice() { @@ -452,6 +455,8 @@ MakeARandomGenerateBidsRawRequestForAndroid(bool enforce_kanon, std::move(MakeARandomStructJsonString(MakeARandomInt(0, 100))).release()); raw_request.set_allocated_buyer_signals( std::move(MakeARandomStructJsonString(MakeARandomInt(0, 100))).release()); + // Android has an 8-bit limit on this field. + raw_request.set_data_version(MakeARandomInt(0, UINT8_MAX)); raw_request.set_enforce_kanon(enforce_kanon); raw_request.set_multi_bid_limit(multi_bid_limit); @@ -475,6 +480,7 @@ MakeARandomGenerateBidsRequestForBrowser(bool enforce_kanon, std::move(MakeARandomStructJsonString(MakeARandomInt(0, 10))).release()); raw_request.set_seller(MakeARandomString()); raw_request.set_publisher_name(MakeARandomString()); + raw_request.set_data_version(MakeARandomInt(0, UINT32_MAX)); raw_request.set_enforce_kanon(enforce_kanon); raw_request.set_multi_bid_limit(multi_bid_limit); diff --git a/services/common/test/random.h b/services/common/test/random.h index e136accf..a8187ff8 100644 --- a/services/common/test/random.h +++ b/services/common/test/random.h @@ -31,6 +31,7 @@ #include "absl/time/time.h" #include "api/bidding_auction_servers.pb.h" #include "api/udf/generate_bid_udf_interface.pb.h" +#include "services/buyer_frontend_service/data/bidding_signals.h" #include "services/common/test/utils/cbor_test_utils.h" #include "services/common/util/request_response_constants.h" #include "services/seller_frontend_service/test/app_test_utils.h" @@ -112,7 +113,7 @@ std::string MakeTrustedBiddingSignalsForIG( // "key2": ["val3"]}, // "perInterestGroupData": {"ig1": ["val1", "val2", "val3"]} // "ig2": ["val3"]}} -std::string GetBiddingSignalsFromGenerateBidsRequest( +BiddingSignals GetBiddingSignalsFromGenerateBidsRequest( const GenerateBidsRequest::GenerateBidsRawRequest& raw_request); InterestGroupForBidding MakeAnInterestGroupForBiddingSentFromDevice(); diff --git a/services/common/util/BUILD b/services/common/util/BUILD index 8b9565f4..8550bb03 100644 --- a/services/common/util/BUILD +++ b/services/common/util/BUILD @@ -304,36 +304,6 @@ cc_library( ], ) -cc_library( - name = "binary_http_utils", - hdrs = [ - "binary_http_utils.h", - ], - deps = [ - "//services/common/loggers:request_log_context", - "//services/common/util:request_response_constants", - "@com_github_google_quiche//quiche:binary_http_unstable_api", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_protobuf//:protobuf", - "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", - ], -) - -cc_test( - name = "binary_http_utils_test", - size = "small", - srcs = [ - "binary_http_utils_test.cc", - ], - deps = [ - ":binary_http_utils", - "@com_google_googletest//:gtest_main", - "@service_value_key_fledge_privacysandbox//public/query/v2:get_values_v2_cc_grpc", - "@service_value_key_fledge_privacysandbox//public/query/v2:get_values_v2_cc_proto", - ], -) - cc_library( name = "auction_scope_util", srcs = [ diff --git a/services/common/util/async_task_tracker.cc b/services/common/util/async_task_tracker.cc index 3a1c6865..403e9977 100644 --- a/services/common/util/async_task_tracker.cc +++ b/services/common/util/async_task_tracker.cc @@ -51,7 +51,7 @@ void AsyncTaskTracker::TaskCompleted( << "Unexpected call (indicates either a bug in the initialization or " "the usage)"; - if (on_single_task_done.has_value()) { + if (on_single_task_done) { (*on_single_task_done)(); } diff --git a/services/common/util/binary_http_utils.h b/services/common/util/binary_http_utils.h deleted file mode 100644 index 091cfac4..00000000 --- a/services/common/util/binary_http_utils.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SERVICES_COMMON_UTIL_BINARY_HTTP_UTILS_H_ -#define SERVICES_COMMON_UTIL_BINARY_HTTP_UTILS_H_ - -#include -#include - -#include "absl/status/statusor.h" -#include "google/protobuf/util/json_util.h" -#include "quiche/binary_http/binary_http_message.h" -#include "services/common/loggers/request_log_context.h" -#include "services/common/util/request_response_constants.h" -#include "src/util/status_macro/status_macros.h" - -namespace privacy_sandbox::bidding_auction_servers { - -using ::google::protobuf::json::MessageToJsonString; -using ::google::protobuf::util::JsonStringToMessage; - -inline constexpr char kContentType[] = "Content-Type"; -inline constexpr char kJsonContentType[] = "application/json"; -inline constexpr char kProtoContentType[] = "application/protobuf"; - -// Converts the incoming proto to Binary HTTP request. -// If to_json is set, the proto will be serialized to a JSON before encoding -// as binary HTTP, otherwise the proto is directly serialized into bytes string -// before binary HTTP encoding. -template -absl::StatusOr ToBinaryHTTP(const ProtoMessageType& request_proto, - bool to_json = true) { - static_assert( - std::is_base_of::value, - "Request should be a google::protobuf::Message."); - PS_VLOG(5) << "Converting request to binary HTTP ..."; - quiche::BinaryHttpRequest binary_http_request({}); - std::string serialized_request; - if (to_json) { - PS_RETURN_IF_ERROR(MessageToJsonString(request_proto, &serialized_request)); - PS_VLOG(6) << "JSON request: " << serialized_request; - binary_http_request.AddHeaderField({kContentType, kJsonContentType}); - } else { - serialized_request = request_proto.SerializeAsString(); - binary_http_request.AddHeaderField({kContentType, kProtoContentType}); - } - binary_http_request.set_body(std::move(serialized_request)); - return binary_http_request.Serialize(); -} - -// Converts the incoming Binary HTTP response to the given proto type. -// If from_json is set, the data embedded in Binary HTTP response is expected -// to be a JSON, otherwise the encapsulated data is assumed to be bytes array -// representation of the proto. -template -absl::StatusOr FromBinaryHTTP( - absl::string_view binary_http_response, bool from_json = true) { - static_assert( - std::is_base_of::value, - "Response type should be a google::protobuf::Message."); - PS_ASSIGN_OR_RETURN(auto retrieved_binary_http_response, - quiche::BinaryHttpResponse::Create(binary_http_response)); - std::string retrieved_binary_http_response_body; - retrieved_binary_http_response.swap_body(retrieved_binary_http_response_body); - ProtoMessageType response_proto; - PS_VLOG(5) << "Converting the binary HTTP response to proto"; - if (from_json) { - PS_RETURN_IF_ERROR(JsonStringToMessage(retrieved_binary_http_response_body, - &response_proto)); - } else { - if (!response_proto.ParseFromString(retrieved_binary_http_response_body)) { - return absl::InvalidArgumentError( - "Unable to convert the Binary HTTP response to proto"); - } - PS_VLOG(kNoisyInfo) << "Converted the http request to proto: " - << response_proto.DebugString(); - } - return response_proto; -} - -} // namespace privacy_sandbox::bidding_auction_servers - -#endif // SERVICES_COMMON_UTIL_BINARY_HTTP_UTILS_H_ diff --git a/services/common/util/binary_http_utils_test.cc b/services/common/util/binary_http_utils_test.cc deleted file mode 100644 index dcd00eb7..00000000 --- a/services/common/util/binary_http_utils_test.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "services/common/util/binary_http_utils.h" - -#include - -#include "absl/log/check.h" -#include "absl/strings/substitute.h" -#include "google/protobuf/text_format.h" -#include "include/gtest/gtest.h" -#include "public/query/v2/get_values_v2.grpc.pb.h" -#include "public/query/v2/get_values_v2.pb.h" - -namespace privacy_sandbox::bidding_auction_servers { -namespace { - -using ::google::protobuf::TextFormat; -using kv_server::v2::GetValuesRequest; -using kv_server::v2::GetValuesResponse; - -inline constexpr char kTestArgumentValue[] = "test_data"; - -std::string TestGetValuesRequest() { - return absl::Substitute( - R"pb( - partitions { arguments { data { string_value: "$0" } } } - )pb", - kTestArgumentValue); -} - -TEST(BinaryHttpRequest, CreatesRequestWithJsonPayload) { - GetValuesRequest get_values_request; - CHECK( - TextFormat::ParseFromString(TestGetValuesRequest(), &get_values_request)) - << "Unable to parse text proto"; - auto binary_http_request = ToBinaryHTTP(get_values_request, /*to_json=*/true); - CHECK_OK(binary_http_request); - - EXPECT_EQ(absl::BytesToHexString(*binary_http_request), - "00000000001e0c636f6e74656e742d74797065106170706c69636174696f6e2f6a" - "736f6e357b22706172746974696f6e73223a5b7b22617267756d656e7473223a5b" - "7b2264617461223a22746573745f64617461227d5d7d5d7d"); -} -TEST(BinaryHttpRequest, CreatesRequestWithProtoPayload) { - GetValuesRequest get_values_request; - CHECK( - TextFormat::ParseFromString(TestGetValuesRequest(), &get_values_request)) - << "Unable to parse text proto"; - auto binary_http_request = - ToBinaryHTTP(get_values_request, /*to_json=*/false); - CHECK_OK(binary_http_request); - - EXPECT_EQ(absl::BytesToHexString(*binary_http_request), - "0000000000220c636f6e74656e742d74797065146170706c69636174696f6e2f70" - "726f746f627566111a0f2a0d120b1a09746573745f64617461"); -} - -TEST(BinaryHttpResponse, RetrievesResponseFromJsonPayload) { - std::string test_binary_http_response_hex = - "0140c8003d7b2273696e676c65506172746974696f6e223a7b22737472696e674f757470" - "7574223a225b7b5c2263617465676f72795c223a5c22305c227d5d227d7d"; - auto get_values_response = FromBinaryHTTP( - absl::HexStringToBytes(test_binary_http_response_hex), - /*from_json=*/true); - CHECK_OK(get_values_response); - - ASSERT_TRUE(get_values_response->has_single_partition()); - ASSERT_TRUE(get_values_response->single_partition().has_string_output()); -} - -} // namespace - -} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/hpke_utils.cc b/services/common/util/hpke_utils.cc index 344f3fdc..134bbb85 100644 --- a/services/common/util/hpke_utils.cc +++ b/services/common/util/hpke_utils.cc @@ -62,7 +62,7 @@ absl::StatusOr HpkeDecrypt( } std::optional private_key = key_fetcher_manager.GetPrivateKey(key_id.data()); - if (!private_key.has_value()) { + if (!private_key) { return absl::Status(absl::StatusCode::kInvalidArgument, kInvalidKeyIdError); } auto decrypt_response = crypto_client.HpkeDecrypt(*private_key, ciphertext); diff --git a/services/common/util/json_util.h b/services/common/util/json_util.h index 9134b9c1..631eb3ae 100644 --- a/services/common/util/json_util.h +++ b/services/common/util/json_util.h @@ -19,6 +19,7 @@ #include #include +#include #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" @@ -81,9 +82,10 @@ inline absl::StatusOr ParseJsonString( rapidjson::ParseResult parse_result = doc.Parse(str.data()); if (parse_result.IsError()) { - return absl::InvalidArgumentError( - absl::StrCat("JSON Parse Error: ", - rapidjson::GetParseError_En(parse_result.Code()))); + return absl::InvalidArgumentError(absl::StrCat( + "JSON Parse Error: ", rapidjson::GetParseError_En(parse_result.Code()), + " at offset: ", parse_result.Offset(), " troublesome prefix:\n", + str.substr(0, parse_result.Offset()))); } return doc; } @@ -138,6 +140,26 @@ inline absl::StatusOr SerializeJsonDoc( return absl::InternalError("Unknown JSON to string serialization error"); } +// Converts rapidjson::Value& to a vector. If any value in Array fails, returns +// an error. +inline absl::StatusOr> SerializeJsonArrayDocToVector( + const rapidjson::Value& document) { + if (!document.IsArray()) { + return absl::InternalError("Expected a JSON array."); + } + std::vector ads; + for (auto& value : document.GetArray()) { + rapidjson::StringBuffer string_buffer; + rapidjson::Writer writer(string_buffer); + if (value.Accept(writer)) { + ads.push_back(std::string(string_buffer.GetString())); + } else { + return absl::InternalError("Error converting inner JSON to String."); + } + } + return ads; +} + // Retrieves the string value of the specified member in the document. template inline absl::StatusOr GetStringMember( diff --git a/services/common/util/json_util_test.cc b/services/common/util/json_util_test.cc index 97ff3e45..fd3afd66 100644 --- a/services/common/util/json_util_test.cc +++ b/services/common/util/json_util_test.cc @@ -271,5 +271,37 @@ TEST(SerializeJsonDoc, GetBoolMember_ComplainsOnNonBoolType) { EXPECT_FALSE(actual_value.ok()) << actual_value.status(); } +TEST(SerializeJsonDoc, WorksForValidArrayDoc) { + std::string json_str = R"json({"key": ["bid1", "bid2", "bid3"]})json"; + const auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + absl::StatusOr> actualArray = + SerializeJsonArrayDocToVector(document.value()["key"]); + ASSERT_TRUE(actualArray.ok()) << actualArray.status(); + + const std::vector expectedArray = {"\"bid1\"", "\"bid2\"", + "\"bid3\""}; + EXPECT_EQ(*actualArray, expectedArray); +} + +TEST(SerializeJsonArrayDocToVector, ComplainsOnNonArrayType) { + std::string key = MakeARandomString(); + std::string value = MakeARandomString(); + std::string expected_output = "{\"" + key + "\":\"" + value + "\"}"; + + rapidjson::Document document; + document.SetObject(); + rapidjson::Value key_v; + key_v.SetString(key.c_str(), document.GetAllocator()); + rapidjson::Value val_v; + val_v.SetString(value.c_str(), document.GetAllocator()); + document.AddMember(key_v, val_v, document.GetAllocator()); + + absl::StatusOr> output = + SerializeJsonArrayDocToVector(key_v); + EXPECT_FALSE(output.ok()) << output.status(); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/read_system.cc b/services/common/util/read_system.cc index 32679d1d..c4199d99 100644 --- a/services/common/util/read_system.cc +++ b/services/common/util/read_system.cc @@ -28,8 +28,12 @@ namespace privacy_sandbox::server_common { -namespace { +void SystemMetrics::SetInferencePid(pid_t pid) { + absl::MutexLock lock(&mu_); + inference_pid_ = pid; +} +namespace { constexpr int kLogInterval = 600; std::vector SystemCpuTime() { @@ -46,31 +50,37 @@ std::vector SystemCpuTime() { return cpu_times; } -std::vector SelfStat() { - std::ifstream proc_self_stat("/proc/self/stat"); - if (!proc_self_stat) { +std::vector StatFields(absl::string_view pid) { + std::string path = absl::StrCat("/proc/", pid, "/stat"); + std::ifstream proc_stat(path); + if (!proc_stat) { ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) - << "failed to open /proc/self/stat; " << strerror(errno); + << "failed to open path: " << path << strerror(errno); return {}; } - std::vector self_stat_fields; - for (std::string field; proc_self_stat >> field; - self_stat_fields.push_back(field)) { + std::vector stat_fields; + for (std::string field; proc_stat >> field; stat_fields.push_back(field)) { } - return self_stat_fields; + return stat_fields; } } // namespace namespace internal { -Utilization ReadCpuTime(const std::vector& cpu_times, - const std::vector& self_stat_fields) { +Utilization ReadCpuTime( + const std::vector& cpu_times, + const std::vector& self_stat_fields, + std::optional> process_stat_fields) { static size_t idle_time = 0; static size_t total_time = 0; static size_t self_time = 0; - Utilization ret{0, 0}; + static size_t process_time = 0; + Utilization ret{0, 0, std::nullopt}; constexpr int kCpuIdle = 3; + constexpr int kUtime = 13, kStime = 14; + constexpr int kStatTotal = 52; + if (cpu_times.size() < kCpuIdle + 1) { ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) << "unknown /proc/stat format " << absl::StrJoin(cpu_times, " "); @@ -92,7 +102,6 @@ Utilization ReadCpuTime(const std::vector& cpu_times, ? 0 : 1.0 * (total_time_diff - idle_time_diff) / total_time_diff; - constexpr int kStatTotal = 52; if (self_stat_fields.size() < kStatTotal) { ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) << "unknown /proc/self/stat format " @@ -100,21 +109,29 @@ Utilization ReadCpuTime(const std::vector& cpu_times, return ret; } - constexpr int kUtime = 13, kStime = 14; - auto utime = std::stoull(self_stat_fields[kUtime]); - auto stime = std::stoull(self_stat_fields[kStime]); - // calculate and update self_time - size_t self_time_new = utime + stime; - size_t self_time_diff = self_time_new - self_time; - self_time = self_time_new; - ret.self = total_time_diff == 0 ? 0 : 1.0 * self_time_diff / total_time_diff; + auto CalculateUtilization = [&](size_t& process_time_to_update, + const std::vector& stat_fields) { + auto utime = std::stoull(stat_fields[kUtime]); + auto stime = std::stoull(stat_fields[kStime]); + size_t process_time_new = utime + stime; + size_t process_time_diff = process_time_new - process_time_to_update; + process_time_to_update = process_time_new; + return total_time_diff == 0 ? 0 : 1.0 * process_time_diff / total_time_diff; + }; + + ret.self = CalculateUtilization(self_time, self_stat_fields); + + if (process_stat_fields && process_stat_fields->size() >= kStatTotal) { + ret.process = CalculateUtilization(process_time, *process_stat_fields); + } return ret; } } // namespace internal -absl::flat_hash_map GetCpu() { +absl::flat_hash_map SystemMetrics::GetCpu() + ABSL_LOCKS_EXCLUDED(mu_) { struct sysinfo info; memset(&info, 0, sizeof(info)); absl::flat_hash_map ret; @@ -125,52 +142,78 @@ absl::flat_hash_map GetCpu() { ret["total load"] = (1.0 * info.loads[0] / (1 << SI_LOAD_SHIFT)) / get_nprocs(); } - internal::Utilization cpu_utilization = - internal::ReadCpuTime(SystemCpuTime(), SelfStat()); + + absl::MutexLock lock(&mu_); + internal::Utilization cpu_utilization = internal::ReadCpuTime( + SystemCpuTime(), StatFields("self"), + inference_pid_ == -1 ? std::optional>{} + : StatFields(std::to_string(inference_pid_))); + + if (cpu_utilization.process.has_value()) { + ret["inference process utilization"] = *cpu_utilization.process; + } ret["total utilization"] = cpu_utilization.total; ret["main process utilization"] = cpu_utilization.self; ret["total cpu cores"] = static_cast(get_nprocs()); + return ret; } -absl::flat_hash_map GetThread() { +absl::flat_hash_map SystemMetrics::GetThread() + ABSL_LOCKS_EXCLUDED(mu_) { absl::flat_hash_map ret; - FILE* fd = fopen("/proc/self/status", "r"); - if (fd == nullptr) { - ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) - << "Failed to open /proc/self/status; " << strerror(errno); - return ret; - } + auto ReadThreadCount = [&ret](absl::string_view pid, + absl::string_view label) { + std::string path = absl::StrCat("/proc/", pid, "/status"); + FILE* fd = fopen(path.c_str(), "r"); + if (fd == nullptr) { + ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) + << "Failed to open " << path << "; " << strerror(errno); + return; + } - char buff[256]; - while (fgets(buff, sizeof(buff), fd) != nullptr) { - if (absl::StartsWith(buff, "Threads:")) { - int thread_count; - if (sscanf(buff, "Threads: %d", &thread_count) == 1) { - ret["thread count"] = static_cast(thread_count); - } else { - ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) - << "Failed to parse thread count from /proc/self/status"; + char buff[256]; + + while (fgets(buff, sizeof(buff), fd) != nullptr) { + if (absl::StartsWith(buff, "Threads:")) { + int thread_count; + if (sscanf(buff, "Threads: %d", &thread_count) == 1) { + ret[std::string(label)] = static_cast(thread_count); + } else { + ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) + << "Failed to parse thread count from " << path; + } + break; } - break; } - } - fclose(fd); + fclose(fd); + }; + // Read thread count for the main process + ReadThreadCount("self", "main process thread count"); if (ret.empty()) { ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) << "Thread count information not found in /proc/self/status"; } + absl::MutexLock lock(&mu_); + // Optionally read thread count for the inference process + if (inference_pid_ != -1) { + ReadThreadCount(std::to_string(inference_pid_), + "inference process thread count"); + } return ret; } -absl::flat_hash_map GetMemory() { +absl::flat_hash_map SystemMetrics::GetMemory() + ABSL_LOCKS_EXCLUDED(mu_) { absl::flat_hash_map ret; char buff[256]; char name[64]; int64_t value = 0; char unit[16]; + + // Read general memory stats FILE* fd = fopen("/proc/meminfo", "r"); if (fd == nullptr) { ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) @@ -186,21 +229,37 @@ absl::flat_hash_map GetMemory() { } fclose(fd); - fd = fopen("/proc/self/status", "r"); - if (fd == nullptr) { - ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) - << "failed to open /proc/self/status; " << strerror(errno); - return ret; - } + auto ReadProcessMemory = [&ret](absl::string_view pid, + absl::string_view label) { + char buff[256]; + char name[64]; + int64_t value = 0; + char unit[16]; + std::string path = absl::StrCat("/proc/", pid, "/status"); + FILE* fd = fopen(path.c_str(), "r"); + if (fd == nullptr) { + ABSL_LOG_EVERY_N_SEC(ERROR, kLogInterval) + << "Failed to open " << path << "; " << strerror(errno); + return; + } - while (fgets(buff, sizeof(buff), fd) != nullptr) { - if (absl::StartsWith(buff, "VmRSS")) { - sscanf(buff, "%s %lu %s", name, &value, unit); - ret["main process"] = value; - break; + while (fgets(buff, sizeof(buff), fd) != nullptr) { + if (absl::StartsWith(buff, "VmRSS:")) { + sscanf(buff, "%s %lu %s", name, &value, unit); + ret[std::string(label)] = value; + break; + } } + fclose(fd); + }; + + // Read memory stats for the main process + ReadProcessMemory("self", "main process"); + absl::MutexLock lock(&mu_); + // Optionally read memory stats for the inference process + if (inference_pid_ != -1) { + ReadProcessMemory(std::to_string(inference_pid_), "inference process"); } - fclose(fd); return ret; } diff --git a/services/common/util/read_system.h b/services/common/util/read_system.h index 579f13a4..d30bea61 100644 --- a/services/common/util/read_system.h +++ b/services/common/util/read_system.h @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#ifndef SERVICES_COMMON_UTIL_READ_SYSTEM_H_ +#define SERVICES_COMMON_UTIL_READ_SYSTEM_H_ +#include #include #include @@ -21,25 +24,45 @@ namespace privacy_sandbox::server_common { -// Read system CPU metric, return map of attribute and value. -absl::flat_hash_map GetCpu(); +class SystemMetrics { + public: + // Sets the PID for inference sidecar + static void SetInferencePid(pid_t pid); -// Read system Memory metric, return map of attribute and value. -absl::flat_hash_map GetMemory(); + // Read system CPU metric, return map of attribute and value + static absl::flat_hash_map GetCpu() + ABSL_LOCKS_EXCLUDED(mu_); -// Read system Thread metric, return map of attribute and value. -absl::flat_hash_map GetThread(); + // Read system Memory metric, return map of attribute and value + static absl::flat_hash_map GetMemory() + ABSL_LOCKS_EXCLUDED(mu_); + + // Read system Thread metric, return map of attribute and value + static absl::flat_hash_map GetThread() + ABSL_LOCKS_EXCLUDED(mu_); + + private: + static inline absl::Mutex mu_; + static inline pid_t inference_pid_ ABSL_GUARDED_BY(mu_) = + -1; // Default value as -1 +}; namespace internal { struct Utilization { double total; double self; + std::optional process; }; // Use system cpu time from /proc/stat (`cpu_times`) /proc/self/stat -// (`self_stat_fields`) to calculate utilization -Utilization ReadCpuTime(const std::vector& cpu_times, - const std::vector& fields); +// (`self_stat_fields`) to calculate utilization & +// (`process_stat_fields`) to calculate utilization for inference sidecar +Utilization ReadCpuTime( + const std::vector& cpu_times, + const std::vector& self_fields, + std::optional> process_fields = std::nullopt); } // namespace internal } // namespace privacy_sandbox::server_common + +#endif // SERVICES_COMMON_UTIL_READ_SYSTEM_H_ diff --git a/services/common/util/read_system_test.cc b/services/common/util/read_system_test.cc index ff7c0bda..0d3ed164 100644 --- a/services/common/util/read_system_test.cc +++ b/services/common/util/read_system_test.cc @@ -24,26 +24,30 @@ namespace privacy_sandbox::server_common { namespace { TEST(ReadSystem, GetMemory) { - ABSL_LOG(INFO) << absl::StrJoin(GetMemory(), " ", absl::PairFormatter("->")); + ABSL_LOG(INFO) << absl::StrJoin(SystemMetrics::GetMemory(), " ", + absl::PairFormatter("->")); } TEST(ReadSystem, GetThread) { - ABSL_LOG(INFO) << absl::StrJoin(GetThread(), " ", absl::PairFormatter("->")); + ABSL_LOG(INFO) << absl::StrJoin(SystemMetrics::GetThread(), " ", + absl::PairFormatter("->")); } TEST(ReadSystem, ReadCpuTime) { - EXPECT_THAT(internal::ReadCpuTime({}, {}), testing::FieldsAre(0, 0)); - EXPECT_THAT(internal::ReadCpuTime({1, 1, 1, 1}, {}), - testing::FieldsAre(0.75, 0)); + EXPECT_THAT(internal::ReadCpuTime({}, {}, std::nullopt), + testing::FieldsAre(0, 0, testing::Eq(std::nullopt))); + EXPECT_THAT(internal::ReadCpuTime({1, 1, 1, 1}, {}, std::nullopt), + testing::FieldsAre(0.75, 0, testing::Eq(std::nullopt))); constexpr int kUtime = 13, kStime = 14; std::vector fields(52, ""); fields[kUtime] = "1"; fields[kStime] = "0"; - EXPECT_THAT(internal::ReadCpuTime({2, 1, 1, 2}, fields), - testing::FieldsAre(0.5, 0.5)); - ABSL_LOG(INFO) << absl::StrJoin(GetCpu(), " ", absl::PairFormatter("->")); + EXPECT_THAT(internal::ReadCpuTime({2, 1, 1, 2}, fields, std::nullopt), + testing::FieldsAre(0.5, 0.5, testing::Eq(std::nullopt))); + ABSL_LOG(INFO) << absl::StrJoin(SystemMetrics::GetCpu(), " ", + absl::PairFormatter("->")); } } // namespace diff --git a/services/common/util/reporting_util.cc b/services/common/util/reporting_util.cc index 95be134e..365a1bab 100644 --- a/services/common/util/reporting_util.cc +++ b/services/common/util/reporting_util.cc @@ -68,7 +68,7 @@ void AppendFeatureFlagValue(std::string& feature_flags, PostAuctionSignals GeneratePostAuctionSignalsForTopLevelSeller( const std::optional& winning_ad_score) { // If there is no winning ad, return with default signals values. - if (!winning_ad_score.has_value()) { + if (!winning_ad_score) { return {kDefaultWinningInterestGroupName, kDefaultWinningInterestGroupOwner, kDefaultWinningBid, @@ -105,7 +105,7 @@ PostAuctionSignals GeneratePostAuctionSignals( const std::optional& winning_ad_score, std::string seller_currency) { // If there is no winning ad, return with default signals values. - if (!winning_ad_score.has_value()) { + if (!winning_ad_score) { return {kDefaultWinningInterestGroupName, kDefaultWinningInterestGroupOwner, kDefaultWinningBid, @@ -348,11 +348,26 @@ void MayVlogAdTechCodeLogs(bool enable_ad_tech_code_logging, absl::StatusOr ParseAndGetResponseJson( bool enable_ad_tech_code_logging, const std::string& response, RequestLogContext& log_context) { + PS_VLOG(5) << "Parsing with logging enabled: " << enable_ad_tech_code_logging; PS_ASSIGN_OR_RETURN(rapidjson::Document document, ParseJsonString(response)); MayVlogAdTechCodeLogs(enable_ad_tech_code_logging, document, log_context); return SerializeJsonDoc(document["response"]); } +absl::StatusOr> ParseAndGetResponseJsonArray( + bool enable_ad_tech_code_logging, const std::string& response, + RequestLogContext& log_context) { + PS_ASSIGN_OR_RETURN(rapidjson::Document document, ParseJsonString(response)); + if (enable_ad_tech_code_logging) { + PS_VLOG(5) << "Parsing with logging enabled: " + << enable_ad_tech_code_logging; + MayVlogAdTechCodeLogs(enable_ad_tech_code_logging, document, log_context); + return SerializeJsonArrayDocToVector(document["response"]); + } else { + return SerializeJsonArrayDocToVector(document); + } +} + std::string GetFeatureFlagJson(bool enable_logging, bool enable_debug_url_generation) { std::string feature_flags = "{"; diff --git a/services/common/util/reporting_util.h b/services/common/util/reporting_util.h index 408143d3..a37acef7 100644 --- a/services/common/util/reporting_util.h +++ b/services/common/util/reporting_util.h @@ -17,6 +17,7 @@ #include #include +#include #include @@ -139,6 +140,13 @@ absl::StatusOr ParseAndGetResponseJson( bool enable_ad_tech_code_logging, const std::string& response, RequestLogContext& log_context); +// Parses the JSON array of string, conditionally prints the logs from the +// response and returns a serialized response vector retrieved from the +// underlying UDF. +absl::StatusOr> ParseAndGetResponseJsonArray( + bool enable_ad_tech_code_logging, const std::string& response, + RequestLogContext& log_context); + // Returns a JSON string for feature flags to be used by the wrapper script. std::string GetFeatureFlagJson(bool enable_logging, bool enable_debug_url_generation); diff --git a/services/common/util/tcmalloc_utils.h b/services/common/util/tcmalloc_utils.h index 0273a201..aa88a02e 100644 --- a/services/common/util/tcmalloc_utils.h +++ b/services/common/util/tcmalloc_utils.h @@ -27,7 +27,7 @@ namespace privacy_sandbox::bidding_auction_servers { inline void MaySetBackgroundReleaseRate( std::optional release_rate_bytes_per_second) { - if (!release_rate_bytes_per_second.has_value()) { + if (!release_rate_bytes_per_second) { PS_VLOG(4) << "No release_rate_bytes_per_second for TCMalloc " "specified, will use default"; return; @@ -42,7 +42,7 @@ inline void MaySetBackgroundReleaseRate( inline void MaySetMaxTotalThreadCacheBytes( std::optional max_total_thread_cache_bytes) { - if (!max_total_thread_cache_bytes.has_value()) { + if (!max_total_thread_cache_bytes) { PS_VLOG(4) << "No max_total_thread_cache_bytes for TCMalloc specified, " "will use default"; return; diff --git a/services/inference_sidecar/README.md b/services/inference_sidecar/README.md index 5e01f059..c6ea1e38 100644 --- a/services/inference_sidecar/README.md +++ b/services/inference_sidecar/README.md @@ -51,10 +51,10 @@ To disable Inference packaging Build: change .bazelrc files's `--//:inference_bu `tensorflow_v2_14_0`. - Set `INFERENCE_MODEL_BUCKET_NAME` to the name of the GCS bucket that you have created. Note that this _not_ a url. For example, the bucket name can be "test_models". - - Set `INFERENCE_MODEL_BUCKET_PATHS` to a comma separated list of model paths under the - bucket. There must be NO spaces between each comma separated model path. For example, within - the "test_models" bucket there are saved models "tensorflow/model1" and "tensorflow/model1". - To load these two models, this flag should be set to "tensorflow/model1,tensorflow/model2". + - Set `INFERENCE_MODEL_CONFIG_PATH` flag to the path to the model configuration file relative + to the model cloud bucket. This metadata file lists models to be fetched. + - Set `INFERENCE_MODEL_FETCH_PERIOD_MS` to the period that the bidding server checks updates + in the model configuration file and fetches models. - Set `INFERENCE_SIDECAR_RUNTIME_CONFIG` which has the following format: ```javascript @@ -97,10 +97,10 @@ To disable Inference packaging Build: change .bazelrc files's `--//:inference_bu `tensorflow_v2_14_0`. - Set `INFERENCE_MODEL_BUCKET_NAME` to the name of the S3 bucket that you have created. Note that this _not_ a url. For example, the bucket name can be "test_models". - - Set `INFERENCE_MODEL_BUCKET_PATHS` to a comma separated list of model paths under the - bucket. For example, within the "test_models" bucket there are saved models - "tensorflow/model1" and "tensorflow/model1". To load these two models, this flag should be - set to "tensorflow/model1,tensorflow/model2". + - Set `INFERENCE_MODEL_CONFIG_PATH` flag to the path to the model configuration file relative + to the model cloud bucket. This metadata file lists models to be fetched. + - Set `INFERENCE_MODEL_FETCH_PERIOD_MS` to the period that the bidding server checks updates + in the model configuration file and fetches models. - Set `INFERENCE_SIDECAR_RUNTIME_CONFIG` which has the following format: ```javascript diff --git a/services/inference_sidecar/common/benchmark/BUILD b/services/inference_sidecar/common/benchmark/BUILD index 6cf74dc8..1ba59eae 100644 --- a/services/inference_sidecar/common/benchmark/BUILD +++ b/services/inference_sidecar/common/benchmark/BUILD @@ -66,6 +66,7 @@ cc_binary( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", "@com_google_benchmark//:benchmark", "@com_google_benchmark//:benchmark_main", ], diff --git a/services/inference_sidecar/common/benchmark/module_benchmark.cc b/services/inference_sidecar/common/benchmark/module_benchmark.cc index 67319b11..3340116b 100644 --- a/services/inference_sidecar/common/benchmark/module_benchmark.cc +++ b/services/inference_sidecar/common/benchmark/module_benchmark.cc @@ -39,6 +39,8 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "absl/synchronization/barrier.h" +#include "absl/synchronization/notification.h" #include "benchmark/benchmark.h" #include "benchmark/request_utils.h" #include "modules/module_interface.h" @@ -81,7 +83,12 @@ static void ExportMetrics(benchmark::State& state) { class ModuleFixture : public benchmark::Fixture { public: void SetUp(::benchmark::State& state) { - if (state.thread_index() != 0) return; + if (state.thread_index() != 0) { + // Blocks until the model is loaded on the main thread. + model_loaded_.WaitForNotification(); + return; + } + completion_barrier_ = absl::make_unique(state.threads()); InferenceSidecarRuntimeConfig config; module_ = ModuleInterface::Create(config); @@ -90,15 +97,19 @@ class ModuleFixture : public benchmark::Fixture { .ok()); absl::StatusOr response = module_->RegisterModel(register_model_request_); CHECK(response.ok()) << response.status().message(); + model_loaded_.Notify(); } void TearDown(::benchmark::State& state) { - if (state.thread_index() != 0) return; + // The last thread reaching the barrier will perform the cleanup. + if (!completion_barrier_->Block()) return; module_.reset(); } protected: std::unique_ptr module_; RegisterModelRequest register_model_request_; + absl::Notification model_loaded_; + std::unique_ptr completion_barrier_; }; BENCHMARK_DEFINE_F(ModuleFixture, BM_Predict)(benchmark::State& state) { diff --git a/services/inference_sidecar/common/model/model_store.h b/services/inference_sidecar/common/model/model_store.h index 9097014e..1b326e15 100644 --- a/services/inference_sidecar/common/model/model_store.h +++ b/services/inference_sidecar/common/model/model_store.h @@ -93,14 +93,14 @@ class ModelStore { model_constructor_(config_, request)); absl::MutexLock model_data_lock(&model_data_mutex_); - model_data_map_[key] = request; - absl::MutexLock prod_model_lock(&prod_model_mutex_); - prod_model_map_[key] = std::move(prod_model); - absl::MutexLock consented_model_lock(&consented_model_mutex_); - consented_model_map_[key] = std::move(consented_model); + absl::MutexLock lock(&per_model_inference_count_mu_); + model_data_map_[key] = request; + prod_model_map_[key] = std::move(prod_model); + consented_model_map_[key] = std::move(consented_model); + per_model_inference_count_[key] = 0; return absl::OkStatus(); } @@ -122,15 +122,14 @@ class ModelStore { return it->second; } - // Reset a model entry using the model constructor. - // This method is thread-safe. + // Reset a model entry using the model constructor. If the model doesn't + // exist, return ok because there is no need to reset. This method is + // thread-safe. absl::Status ResetModel(absl::string_view key, bool is_consented = false) { absl::MutexLock model_data_lock(&model_data_mutex_); auto it = model_data_map_.find(key); if (it == model_data_map_.end()) { - return absl::NotFoundError( - absl::StrCat("Resetting model '", key, - "' fails because it has not been registered")); + return absl::OkStatus(); } const RegisterModelRequest& request = it->second; PS_ASSIGN_OR_RETURN(std::shared_ptr model, @@ -153,13 +152,39 @@ class ModelStore { return model_keys; } + // Deletes all resources associated with the given model key including both + // the consented and production copies of the model. + // This method is thread-safe. + absl::Status DeleteModel(absl::string_view key) { + absl::MutexLock model_data_lock(&model_data_mutex_); + absl::MutexLock prod_model_lock(&prod_model_mutex_); + absl::MutexLock consented_model_lock(&consented_model_mutex_); + absl::MutexLock lock(&per_model_inference_count_mu_); + auto it = model_data_map_.find(key); + if (it == model_data_map_.end()) { + return absl::NotFoundError( + "Failed to delete because the model is not found."); + } + model_data_map_.erase(it); + + prod_model_map_.erase(key); + consented_model_map_.erase(key); + per_model_inference_count_.erase(key); + + return absl::OkStatus(); + } + // Increments the inference count associated with a model key for model // resetting purposes. Note that model reset currently only happens for prod - // models so no consented flag is required for this method. + // models so no consented flag is required for this method. Also, if the model + // is not found, this function call has not effect. void IncrementModelInferenceCount(absl::string_view key) { absl::MutexLock lock(&per_model_inference_count_mu_); - ++per_model_inference_count_[std::string(key)]; - inference_notification_.Notify(); + if (auto it = per_model_inference_count_.find(key); + it != per_model_inference_count_.end()) { + ++(it->second); + inference_notification_.Notify(); + } } // Constructs and returns model by invoking the model constructor. @@ -189,9 +214,12 @@ class ModelStore { int count = 0; { absl::MutexLock lock(&per_model_inference_count_mu_); - count = per_model_inference_count_[model_key]; - // Sets the per-model counter to 0. - per_model_inference_count_[model_key] = 0; + if (auto it = per_model_inference_count_.find(model_key); + it != per_model_inference_count_.end()) { + count = it->second; + // Sets the per-model counter to 0. + it->second = 0; + } } if (count <= 0) continue; double random = absl::Uniform(bitgen_, 0.0, 1.0); diff --git a/services/inference_sidecar/common/model/model_store_test.cc b/services/inference_sidecar/common/model/model_store_test.cc index b638757f..0a3a4921 100644 --- a/services/inference_sidecar/common/model/model_store_test.cc +++ b/services/inference_sidecar/common/model/model_store_test.cc @@ -144,10 +144,9 @@ TEST_F(ModelStoreTest, ResetModelFailureDoesNotOverwrite) { EXPECT_EQ((*result2)->GetCounter(), 1); } -TEST_F(ModelStoreTest, ResetNonExistentModelReturnsNotFound) { +TEST_F(ModelStoreTest, ResetNonExistentModelReturnsOkStatus) { absl::Status status = store_.ResetModel(kNonExistModelName); - EXPECT_FALSE(status.ok()); - EXPECT_EQ(status.code(), absl::StatusCode::kNotFound); + EXPECT_TRUE(status.ok()); } TEST_F(ModelStoreTest, ListModels) { @@ -160,6 +159,24 @@ TEST_F(ModelStoreTest, ListModels) { EXPECT_EQ(models.front(), kTestModelName); } +TEST_F(ModelStoreTest, DeleteModelsSuccess) { + RegisterModelRequest request; + EXPECT_TRUE(store_.PutModel(kTestModelName, request).ok()); + EXPECT_TRUE(store_.GetModel(kTestModelName, /*is_consented=*/false).ok()); + EXPECT_TRUE(store_.GetModel(kTestModelName, /*is_consented=*/true).ok()); + + EXPECT_TRUE(store_.DeleteModel(kTestModelName).ok()); + + EXPECT_FALSE(store_.GetModel(kTestModelName, /*is_consented=*/false).ok()); + EXPECT_FALSE(store_.GetModel(kTestModelName, /*is_consented=*/true).ok()); +} + +TEST_F(ModelStoreTest, DeleteNonExistentModelReturnsNotFound) { + absl::Status result = store_.DeleteModel(kTestModelName); + ASSERT_FALSE(result.ok()); + EXPECT_EQ(result.code(), absl::StatusCode::kNotFound); +} + absl::StatusOr> MockModelConstructorWithInitCounter( const InferenceSidecarRuntimeConfig& config, const RegisterModelRequest& request) { @@ -223,6 +240,57 @@ TEST_F(ModelStoreConcurrencyTest, EXPECT_EQ(prod_model_counter, consented_model_counter); } +TEST_F(ModelStoreConcurrencyTest, ConcurrentPutAndDeleteModelSuccess) { + std::vector threads; + threads.reserve(kNumThreads * 2); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_ + .PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + })); + threads.push_back( + std::thread([this]() { this->store_.DeleteModel(kTestModelName); })); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +TEST_F(ModelStoreConcurrencyTest, ConcurrentGetAndDeleteModelSuccess) { + for (int i = 0; i < kNumThreads; ++i) { + EXPECT_TRUE(this->store_ + .PutModel(absl::StrCat(kTestModelName, i), + BuildMockModelRegisterModelRequest(1)) + .ok()); + } + + std::vector threads; + threads.reserve(kNumThreads * 2); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this, i]() { + auto get_result = this->store_.GetModel(absl::StrCat(kTestModelName, i)); + if (get_result.ok()) { + (*get_result)->Increment(); + store_.IncrementModelInferenceCount(absl::StrCat(kTestModelName, i)); + } + })); + threads.push_back(std::thread([this, i]() { + EXPECT_TRUE( + this->store_.DeleteModel(absl::StrCat(kTestModelName, i)).ok()); + })); + } + for (auto& thread : threads) { + thread.join(); + } + + for (int i = 0; i < kNumThreads; ++i) { + EXPECT_FALSE(this->store_.GetModel(absl::StrCat(kTestModelName, i)).ok()); + } +} + TEST_F(ModelStoreConcurrencyTest, ConcurrentGetModelSuccess) { EXPECT_TRUE( store_.PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) @@ -456,5 +524,53 @@ TEST_F(ModelStoreBackgroundModelResetTest, ResetSuccessWithStatefulModels) { } } +TEST_F(ModelStoreBackgroundModelResetTest, ResetSuccessWhileDeletingModels) { + for (int i = 0; i < kNumThreads; ++i) { + EXPECT_TRUE(store_ + .PutModel(absl::StrCat(kTestModelName, i), + BuildMockModelRegisterModelRequest(1)) + .ok()); + } + + std::vector threads; + threads.reserve(2 * kNumThreads); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this, i]() { + absl::StatusOr> prod_model = store_.GetModel( + absl::StrCat(kTestModelName, i), /*is_consented=*/false); + if (prod_model.ok()) { + EXPECT_EQ((*prod_model)->GetCounter(), 1); + (*prod_model)->Increment(); + store_.IncrementModelInferenceCount(absl::StrCat(kTestModelName, i)); + } + })); + // Delete all models except the (kNumThreads - 1)th model, which is reset. + if (i < kNumThreads - 1) { + threads.push_back(std::thread([this, i]() { + EXPECT_TRUE(store_.DeleteModel(absl::StrCat(kTestModelName, i)).ok()); + })); + } + } + + for (auto& thread : threads) { + thread.join(); + } + + absl::SleepFor(absl::Seconds(1)); + for (int i = 0; i < kNumThreads - 1; ++i) { + EXPECT_FALSE( + store_.GetModel(absl::StrCat(kTestModelName, 0), /*is_consented=*/false) + .ok()); + EXPECT_FALSE( + store_.GetModel(absl::StrCat(kTestModelName, 0), /*is_consented=*/true) + .ok()); + } + + absl::StatusOr> prod_model = store_.GetModel( + absl::StrCat(kTestModelName, kNumThreads - 1), /*is_consented=*/false); + ASSERT_TRUE(prod_model.ok()); + EXPECT_EQ((*prod_model)->GetCounter(), 1); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/sandbox/sandbox_executor.cc b/services/inference_sidecar/common/sandbox/sandbox_executor.cc index d2d0f0fc..13250a27 100644 --- a/services/inference_sidecar/common/sandbox/sandbox_executor.cc +++ b/services/inference_sidecar/common/sandbox/sandbox_executor.cc @@ -235,6 +235,21 @@ absl::Status SandboxExecutor::StartSandboxee() { return absl::OkStatus(); } +pid_t SandboxExecutor::GetPid() const { + if (!sandbox_) { + ABSL_LOG(ERROR) << "Sandbox object is null. Cannot retrieve PID."; + return -1; + } + + pid_t pid = sandbox_->pid(); + if (pid <= 0) { + ABSL_LOG(ERROR) << "Failed to obtain PID for the inference sidecar. PID: " + << pid; + return -1; + } + return pid; +} + absl::StatusOr SandboxExecutor::StopSandboxee() { if (sandboxee_state_ == SandboxeeState::kStopped) { return run_result_; diff --git a/services/inference_sidecar/common/sandbox/sandbox_executor.h b/services/inference_sidecar/common/sandbox/sandbox_executor.h index dce4e643..713f2de4 100644 --- a/services/inference_sidecar/common/sandbox/sandbox_executor.h +++ b/services/inference_sidecar/common/sandbox/sandbox_executor.h @@ -66,6 +66,9 @@ class SandboxExecutor { // You should not call after `StopSandboxee`. absl::Status StartSandboxee(); + // Get the pid of inference sidecar + pid_t GetPid() const; + // Stops the sandboxee instance. Returns an error status if the sandboxee // fails to stop. Returns `sandbox2::Result` if stopped successfully. You can // call many times safely. You should not call other operations like diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc index 41aa9980..a1a1f15a 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc @@ -188,8 +188,7 @@ absl::StatusOr PyTorchModule::Predict( return predict_response; } - AddMetric(predict_response, "kInferenceRequestCount", - parsed_requests->size()); + AddMetric(predict_response, "kInferenceRequestCount", 1); std::vector>> tasks( parsed_requests->size()); diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc index a937d09c..ae6021b6 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc @@ -62,10 +62,16 @@ namespace privacy_sandbox::bidding_auction_servers::inference { namespace { constexpr absl::string_view kRamFileSystemScheme = "ram://"; +// Tensorflow's ram filesystem allows file overwrite. Without this mutex, +// multiple model registration requests registering models with the same path +// could cause race conditions. +absl::Mutex tf_ram_fs_mu; -// TODO(b/316960066): Delete all files in RamFileSystem after model loading. // TODO(b/316960066): Move the code to a separate file with more unit tests. absl::Status SaveToRamFileSystem(const RegisterModelRequest& request) { + // Creates the top-level destination directory. + PS_RETURN_IF_ERROR(tsl::Env::Default()->RecursivelyCreateDir( + absl::StrCat(kRamFileSystemScheme, request.model_spec().model_path()))); for (const auto& pair : request.model_files()) { const std::string& path = absl::StrCat(kRamFileSystemScheme, pair.first); const std::string& bytes = pair.second; @@ -75,6 +81,23 @@ absl::Status SaveToRamFileSystem(const RegisterModelRequest& request) { return absl::OkStatus(); } +// Delete a directory at the given path. Deletion is not guaranteed to succeed. +// It can fail either when the path is non-existent or there are write-protected +// files under the path. With normal operations of the sidecar, these +// two scenarios will not happen. Crash the sidecar if deletion fails as this +// indicates abnormalities. +void DeleteFromRamFileSystem(const std::string& path) { + const std::string& ram_fs_path = absl::StrCat(kRamFileSystemScheme, path); + tensorflow::int64 undeleted_files; + tensorflow::int64 undeleted_dirs; + absl::Status status = tsl::Env::Default()->DeleteRecursively( + ram_fs_path, &undeleted_files, &undeleted_dirs); + CHECK_OK(status) << "DeleteFromRamFileSystem failed: " << status + << " #undeleted files = " << undeleted_files + << " #undeleted dirs =" << undeleted_dirs + << " path = " << ram_fs_path; +} + absl::StatusOr> PredictPerModel( std::shared_ptr model, const InferenceRequest& inference_request) { @@ -171,12 +194,23 @@ TensorFlowModelConstructor(const InferenceSidecarRuntimeConfig& config, const std::unordered_set tags = {"serve"}; const auto& model_path = request.model_spec().model_path(); auto model_bundle = std::make_shared(); - if (auto status = tensorflow::LoadSavedModel( - session_options, {}, absl::StrCat(kRamFileSystemScheme, model_path), - tags, model_bundle.get()); - !status.ok()) { - return absl::InternalError( - absl::StrCat("Error loading model: ", model_path)); + + { + absl::MutexLock tf_ram_fs_lock(&tf_ram_fs_mu); + // TODO(b/374168406): Improve Tensorflow model loading performance. + // Tensorflow needs to load models from a file system. + PS_RETURN_IF_ERROR(SaveToRamFileSystem(request)); + auto status = tensorflow::LoadSavedModel( + session_options, {}, absl::StrCat(kRamFileSystemScheme, model_path), + tags, model_bundle.get()); + // Note that the deletion function itself doesn't guarantee success. + // Deletion failure indicates abnormalities and so the sidecar is made to + // crash in that case. + DeleteFromRamFileSystem(model_path); + if (!status.ok()) { + return absl::InternalError( + absl::StrCat("Error loading model: ", model_path)); + } } // TODO(b/361373900): Freeze a model only once at RegisterModel(). @@ -236,8 +270,7 @@ absl::StatusOr TensorflowModule::Predict( return predict_response; } - AddMetric(predict_response, "kInferenceRequestCount", - parsed_requests->size()); + AddMetric(predict_response, "kInferenceRequestCount", 1); std::vector batch_outputs(parsed_requests->size()); std::vector>>> tasks( @@ -334,27 +367,16 @@ absl::StatusOr TensorflowModule::Predict( return predict_response; } -// TODO(b/346418962): Move the function into TensorFlowGraphValidator. -absl::Status IsModelAllowed(const RegisterModelRequest& request) { - tensorflow::SessionOptions session_options; - const std::unordered_set tags = {"serve"}; - const auto& model_path = request.model_spec().model_path(); - auto model_bundle = std::make_unique(); - if (auto status = tensorflow::LoadSavedModel( - session_options, {}, absl::StrCat(kRamFileSystemScheme, model_path), - tags, model_bundle.get()); - !status.ok()) { - return absl::InternalError( - absl::StrCat("Error loading model: ", model_path)); - } - - const tensorflow::GraphDef& graph_def = - model_bundle->meta_graph_def.graph_def(); +absl::Status TensorflowModule::IsModelAllowed( + const RegisterModelRequest& request) { + PS_ASSIGN_OR_RETURN(std::shared_ptr model, + store_->ConstructModel(request)); + const tensorflow::GraphDef& graph_def = model->meta_graph_def.graph_def(); if (!TensorFlowGraphValidator(graph_def).IsGraphAllowed()) { // TODO(b/368395202): Improve error messages by including the specific // operation that is disallowed. return absl::InternalError( - absl::StrCat("Error: model ", model_path, + absl::StrCat("Error: model ", request.model_spec().model_path(), " is not allowed due to using a disallowed operator")); } return absl::OkStatus(); @@ -371,16 +393,9 @@ absl::StatusOr TensorflowModule::RegisterModel( return absl::AlreadyExistsError( absl::StrCat("Model ", model_path, " has already been registered")); } - PS_RETURN_IF_ERROR(SaveToRamFileSystem(request)); - PS_RETURN_IF_ERROR(IsModelAllowed(request)); - RegisterModelRequest model_request; - *model_request.mutable_model_spec() = request.model_spec(); - if (!request.warm_up_batch_request_json().empty()) { - model_request.set_warm_up_batch_request_json( - request.warm_up_batch_request_json()); - } - PS_RETURN_IF_ERROR(store_->PutModel(model_path, model_request)); + PS_RETURN_IF_ERROR(IsModelAllowed(request)); + PS_RETURN_IF_ERROR(store_->PutModel(model_path, request)); return RegisterModelResponse(); } diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.h b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.h index 8269f2ba..2aee8ade 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.h +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.h @@ -39,6 +39,9 @@ class TensorflowModule final : public ModuleInterface { private: friend class NoFreezeTensorflowTest; + // TODO(b/346418962): Move the function into TensorFlowGraphValidator. + // Performs a graph traversal and checks if a model's operators are allowed. + absl::Status IsModelAllowed(const RegisterModelRequest& request); void SetModelStoreForTestOnly( std::unique_ptr> store) { store_ = std::move(store); diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index 2c390a46..ab6c8b2d 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -70,8 +70,8 @@ inline void RecordInterestGroupUpdates( SelectAdReactor::SelectAdReactor( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client, bool fail_fast, - int max_buyers_solicited) + const TrustedServersConfigClient& config_client, bool enable_cancellation, + bool enable_kanon, bool fail_fast, int max_buyers_solicited) : request_context_(context), request_(request), response_(response), @@ -96,8 +96,8 @@ SelectAdReactor::SelectAdReactor( max_buyers_solicited_(chaffing_enabled_ ? kMaxBuyersSolicitedChaffingEnabled : max_buyers_solicited), - enable_cancellation_(absl::GetFlag(FLAGS_enable_cancellation)), - enable_kanon_(absl::GetFlag(FLAGS_enable_kanon)), + enable_cancellation_(enable_cancellation), + enable_enforce_kanon_(enable_kanon), async_task_tracker_( request->auction_config().buyer_list_size(), log_context_, [this](bool successful) { OnAllBidsDone(successful); }) { @@ -160,8 +160,10 @@ AdWithBidMetadata SelectAdReactor::BuildAdWithBidMetadata( break; } } - if (enable_kanon_) { + if (enable_enforce_kanon_) { result.set_k_anon_status(k_anon_status); + } else { + result.set_k_anon_status(true); } if (!input.buyer_reporting_id().empty()) { result.set_buyer_reporting_id(input.buyer_reporting_id()); @@ -243,7 +245,7 @@ grpc::Status SelectAdReactor::DecryptRequest() { std::visit( [this](auto& input) { buyer_inputs_ = GetDecodedBuyerinputs(input.buyer_input()); - enforce_kanon_ = input.enforce_kanon(); + enable_enforce_kanon_ &= input.enforce_kanon(); }, protected_auction_input_); @@ -641,7 +643,7 @@ SelectAdReactor::CreateGetBidsRequest(const std::string& buyer_ig_owner, get_bids_request->set_buyer_signals( per_buyer_config_itr->second.buyer_signals()); } - if (enable_kanon_) { + if (enable_enforce_kanon_) { get_bids_request->set_multi_bid_limit( per_buyer_config_itr->second.per_buyer_multi_bid_limit()); } @@ -666,7 +668,7 @@ SelectAdReactor::CreateGetBidsRequest(const std::string& buyer_ig_owner, } get_bids_request->set_enable_unlimited_egress( protected_auction_input.enable_unlimited_egress()); - if (enable_kanon_) { + if (enable_enforce_kanon_) { get_bids_request->set_enforce_kanon( protected_auction_input.enforce_kanon()); } @@ -1068,7 +1070,7 @@ void SelectAdReactor::OnFetchScoringSignalsDone( } bool SelectAdReactor::GetKAnonStatusForAdWithBid(absl::string_view ad_key) { - if (!enable_kanon_ || !enforce_kanon_) { + if (!enable_enforce_kanon_) { PS_VLOG(5) << "k-anon is not enabled or not enforced"; return true; } @@ -1120,7 +1122,7 @@ SelectAdReactor::CreateScoreAdsRequest() { *raw_request->mutable_consented_debug_config() = protected_auction_input.consented_debug_config(); } - if (enable_kanon_) { + if (enable_enforce_kanon_) { raw_request->set_num_allowed_ghost_winners( kNumAllowedChromGhostWinners); raw_request->set_enforce_kanon( diff --git a/services/seller_frontend_service/select_ad_reactor.h b/services/seller_frontend_service/select_ad_reactor.h index 4a126053..e023a9b4 100644 --- a/services/seller_frontend_service/select_ad_reactor.h +++ b/services/seller_frontend_service/select_ad_reactor.h @@ -25,7 +25,6 @@ #include -#include "absl/flags/flag.h" #include "absl/status/statusor.h" #include "absl/synchronization/blocking_counter.h" #include "api/bidding_auction_servers.grpc.pb.h" @@ -89,7 +88,9 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { explicit SelectAdReactor( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client, bool fail_fast = true, + const TrustedServersConfigClient& config_client, + bool enable_cancellation = false, bool enable_kanon = false, + bool fail_fast = true, int max_buyers_solicited = metric::kMaxBuyersSolicited); // Initiate the asynchronous execution of the SelectAdRequest. @@ -401,9 +402,7 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { const bool enable_cancellation_; - const bool enable_kanon_; - - bool enforce_kanon_; + bool enable_enforce_kanon_; // Pseudo random number generator for use in chaffing. std::optional generator_; diff --git a/services/seller_frontend_service/select_ad_reactor_app.cc b/services/seller_frontend_service/select_ad_reactor_app.cc index 2d3768ed..901b7c43 100644 --- a/services/seller_frontend_service/select_ad_reactor_app.cc +++ b/services/seller_frontend_service/select_ad_reactor_app.cc @@ -69,15 +69,16 @@ T GetDecodedProtectedAuctionInputHelper(absl::string_view encoded_data, SelectAdReactorForApp::SelectAdReactorForApp( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client, bool fail_fast) + const TrustedServersConfigClient& config_client, bool enable_cancellation, + bool enable_kanon, bool fail_fast) : SelectAdReactor(context, request, response, clients, config_client, - fail_fast) {} + enable_cancellation, enable_kanon, fail_fast) {} absl::StatusOr SelectAdReactorForApp::GetNonEncryptedResponse( const std::optional& high_score, const std::optional& error) { AuctionResult auction_result; - if (high_score.has_value()) { + if (high_score) { auction_result = AdScoreToAuctionResult( high_score, GetBiddingGroups(shared_buyer_bids_map_, *buyer_inputs_), shared_ig_updates_map_, error, auction_scope_, @@ -219,6 +220,9 @@ SelectAdReactorForApp::CreateGetBidsRequest(const std::string& buyer_ig_owner, const BuyerInput& buyer_input) { auto request = SelectAdReactor::CreateGetBidsRequest(buyer_ig_owner, buyer_input); + + // ToDo(b/369159855): use is_debug_eligible from client. + request->set_is_debug_eligible(request->enable_unlimited_egress()); MayPopulateProtectedAppSignalsBuyerInput(buyer_ig_owner, request.get()); return request; } @@ -228,7 +232,7 @@ SelectAdReactorForApp::CreateScoreAdsRequest() { auto request = SelectAdReactor::CreateScoreAdsRequest(); std::visit( [&request, this](const auto& protected_auction_input) { - if (enable_kanon_) { + if (enable_enforce_kanon_) { request->set_num_allowed_ghost_winners( protected_auction_input.num_k_anon_ghost_winners()); } @@ -255,7 +259,7 @@ SelectAdReactorForApp::BuildProtectedAppSignalsAdWithBidMetadata( result.set_egress_payload(input.egress_payload()); result.set_temporary_unlimited_egress_payload( input.temporary_unlimited_egress_payload()); - if (enable_kanon_) { + if (enable_enforce_kanon_) { result.set_k_anon_status(k_anon_status); } return result; diff --git a/services/seller_frontend_service/select_ad_reactor_app.h b/services/seller_frontend_service/select_ad_reactor_app.h index 4060bdbd..f65bb6ae 100644 --- a/services/seller_frontend_service/select_ad_reactor_app.h +++ b/services/seller_frontend_service/select_ad_reactor_app.h @@ -33,7 +33,9 @@ class SelectAdReactorForApp : public SelectAdReactor { explicit SelectAdReactorForApp( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client, bool fail_fast = true); + const TrustedServersConfigClient& config_client, + bool enable_cancellation = false, bool enable_kanon = false, + bool fail_fast = true); virtual ~SelectAdReactorForApp() = default; // SelectAdReactorForApp is neither copyable nor movable. diff --git a/services/seller_frontend_service/select_ad_reactor_test.cc b/services/seller_frontend_service/select_ad_reactor_test.cc index 6d8a882d..a37b313e 100644 --- a/services/seller_frontend_service/select_ad_reactor_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_test.cc @@ -387,7 +387,8 @@ TYPED_TEST(SellerFrontEndServiceTest, std::move(async_reporter)}; Response response = RunRequest( - this->config_, clients, this->request_, /*max_buyers_solicited=*/3); + this->config_, clients, this->request_, /*max_buyers_solicited=*/3, + /*enable_kanon=*/false); // Hard limit is calling 3 buyers since we upped it. EXPECT_EQ(num_buyers_solicited, 3); } @@ -642,7 +643,7 @@ TYPED_TEST(SellerFrontEndServiceTest, * buyer or seller currency is specified, breaks nothing. */ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { - absl::SetFlag(&FLAGS_enable_kanon, true); + bool enable_kanon = true; this->SetupRequest(/*num_buyers=*/2, /*set_buyer_egid=*/false, /*set_seller_egid=*/false, @@ -754,8 +755,116 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { scoring_signals_provider, scoring_client, buyer_clients, this->key_fetcher_manager_, /* crypto_client = */ nullptr, std::move(async_reporter)}; - Response response = - RunRequest(this->config_, clients, this->request_); + Response response = RunRequest( + this->config_, clients, this->request_, /*max_buyers_solicited=*/2, + enable_kanon); +} + +TYPED_TEST(SellerFrontEndServiceTest, + WhenKAnonDisabledAdsAreConsideredKAnonByDefault) { + bool enable_kanon = false; + this->SetupRequest(/*num_buyers=*/2, + /*set_buyer_egid=*/false, + /*set_seller_egid=*/false, + /*seller_currency=*/"", + /*buyer_currency=*/"", + /*num_k_anon_ghost_winners=*/10, + /*enforce_kanon=*/true); + absl::flat_hash_map buyer_to_ad_url = + BuildBuyerWinningAdUrlMap(this->request_); + + // Buyer Clients + BuyerFrontEndAsyncClientFactoryMock buyer_clients; + int client_count = this->protected_auction_input_.buyer_input_size(); + EXPECT_EQ(client_count, 2); + absl::flat_hash_map bids; + BuyerBidsResponseMap expected_buyer_bids; + for (const auto& [buyer, unused] : + this->protected_auction_input_.buyer_input()) { + AdUrl url = buyer_to_ad_url.at(buyer); + // Set the bid currency on the AdWithBids. + GetBidsResponse::GetBidsRawResponse response = + BuildGetBidsResponseWithSingleAd( + /*ad_url = */ url, + /*interest_group_name = */ absl::nullopt, + /*bid_value = */ absl::nullopt, + /*enable_event_level_debug_reporting = */ false, + /*number_ad_component_render_urls = */ kDefaultNumAdComponents, + /*bid_currency = */ kEurosIsoCode); + bids.insert_or_assign(url, response.bids()[0]); + SetupBuyerClientMock(buyer, buyer_clients, response); + expected_buyer_bids.try_emplace( + buyer, std::make_unique(response)); + } + + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + + // Scoring signals provider + // Scoring signals must be nonzero for scoring to be attempted. + std::string scoring_signals_value = + R"JSON({"someAdRenderUrl":{"someKey":"someValue"}})JSON"; + MockAsyncProvider + scoring_signals_provider; + SetupScoringProviderMock(scoring_signals_provider, expected_buyer_bids, + scoring_signals_value); + // Scoring Client + ScoringAsyncClientMock scoring_client; + EXPECT_CALL(scoring_client, ExecuteInternal) + .WillOnce([select_ad_req = this->request_, + protected_auction_input = this->protected_auction_input_, + scoring_signals_value, bids]( + std::unique_ptr + score_ads_raw_request, + grpc::ClientContext* context, ScoreAdsDoneCallback on_done, + absl::Duration timeout, RequestConfig request_config) { + google::protobuf::util::MessageDifferencer diff; + std::string diff_output; + diff.ReportDifferencesToString(&diff_output); + + EXPECT_EQ(score_ads_raw_request->publisher_hostname(), + protected_auction_input.publisher_name()); + EXPECT_EQ(score_ads_raw_request->seller_signals(), + select_ad_req.auction_config().seller_signals()); + EXPECT_EQ(score_ads_raw_request->auction_signals(), + select_ad_req.auction_config().auction_signals()); + EXPECT_EQ(score_ads_raw_request->scoring_signals(), + scoring_signals_value); + EXPECT_EQ(score_ads_raw_request->per_buyer_signals().size(), 2); + // No ghost winner allowed since k-anon is not enabled. + EXPECT_EQ(score_ads_raw_request->num_allowed_ghost_winners(), 0); + for (const auto& actual_ad_with_bid_metadata : + score_ads_raw_request->ad_bids()) { + AdWithBid actual_ad_with_bid_for_test; + BuildAdWithBidFromAdWithBidMetadata(actual_ad_with_bid_metadata, + &actual_ad_with_bid_for_test); + EXPECT_TRUE( + diff.Compare(actual_ad_with_bid_for_test, + bids.at(actual_ad_with_bid_for_test.render()))); + // When k-anon is not being enforced, consider all the bids as k-anon. + EXPECT_TRUE(actual_ad_with_bid_metadata.k_anon_status()); + } + EXPECT_EQ(diff_output, ""); + EXPECT_EQ(score_ads_raw_request->seller(), + select_ad_req.auction_config().seller()); + EXPECT_EQ(request_config.chaff_request_size, 0); + std::move(on_done)( + std::make_unique(), {}); + return absl::OkStatus(); + }); + + // Reporting Client. + std::unique_ptr async_reporter = + std::make_unique( + std::make_unique()); + // Client Registry + ClientRegistry clients{ + scoring_signals_provider, scoring_client, buyer_clients, + this->key_fetcher_manager_, + /* crypto_client = */ nullptr, std::move(async_reporter)}; + Response response = RunRequest( + this->config_, clients, this->request_, /*max_buyers_solicited=*/2, + enable_kanon); } TYPED_TEST(SellerFrontEndServiceTest, DoesNotScoreAdsAfterGettingEmptySignals) { @@ -1772,7 +1881,7 @@ auto EqGetBidsRawRequestKAnonFields( } TYPED_TEST(SellerFrontEndServiceTest, VerifyKAnonFieldsPropagateToBuyers) { - absl::SetFlag(&FLAGS_enable_kanon, true); + bool enable_kanon = true; MockAsyncProvider scoring_signals_provider; ScoringAsyncClientMock scoring_client; @@ -1846,12 +1955,12 @@ TYPED_TEST(SellerFrontEndServiceTest, VerifyKAnonFieldsPropagateToBuyers) { buyer_config.set_buyer_signals(kSampleBuyerSignals); buyer_config.set_per_buyer_multi_bid_limit(10); - RunReactorRequest(this->config_, clients, request); + RunReactorRequest(this->config_, clients, request, + enable_kanon); } TYPED_TEST(SellerFrontEndServiceTest, VerifyKAnonFieldsDontPropagateToBuyersWhenFeatDisabled) { - absl::SetFlag(&FLAGS_enable_kanon, false); MockAsyncProvider scoring_signals_provider; ScoringAsyncClientMock scoring_client; diff --git a/services/seller_frontend_service/select_ad_reactor_web.cc b/services/seller_frontend_service/select_ad_reactor_web.cc index 6050ca06..2a40c1b7 100644 --- a/services/seller_frontend_service/select_ad_reactor_web.cc +++ b/services/seller_frontend_service/select_ad_reactor_web.cc @@ -62,10 +62,11 @@ T GetDecodedProtectedAuctionInputHelper(absl::string_view encoded_data, SelectAdReactorForWeb::SelectAdReactorForWeb( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client, bool fail_fast, - int max_buyers_solicited) + const TrustedServersConfigClient& config_client, bool enable_cancellation, + bool enable_kanon, bool fail_fast, int max_buyers_solicited) : SelectAdReactor(context, request, response, clients, config_client, - fail_fast, max_buyers_solicited) {} + enable_cancellation, enable_kanon, fail_fast, + max_buyers_solicited) {} absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( const std::optional& high_score, @@ -96,7 +97,7 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( AuctionScope::AUCTION_SCOPE_SERVER_COMPONENT_MULTI_SELLER) { // If this is server component auction, serialize as proto. AuctionResult auction_result; - if (high_score.has_value()) { + if (high_score) { auction_result = AdScoreToAuctionResult( high_score, GetBiddingGroups(shared_buyer_bids_map_, *buyer_inputs_), shared_ig_updates_map_, error, auction_scope_, diff --git a/services/seller_frontend_service/select_ad_reactor_web.h b/services/seller_frontend_service/select_ad_reactor_web.h index d957c838..b25f1858 100644 --- a/services/seller_frontend_service/select_ad_reactor_web.h +++ b/services/seller_frontend_service/select_ad_reactor_web.h @@ -33,8 +33,9 @@ class SelectAdReactorForWeb : public SelectAdReactor { explicit SelectAdReactorForWeb( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client, bool fail_fast = true, - int max_buyers_solicited = 2); + const TrustedServersConfigClient& config_client, + bool enable_cancellation = false, bool enable_kanon = false, + bool fail_fast = true, int max_buyers_solicited = 2); virtual ~SelectAdReactorForWeb() = default; // SelectAdReactorForWeb is neither copyable nor movable. diff --git a/services/seller_frontend_service/select_auction_result_reactor.cc b/services/seller_frontend_service/select_auction_result_reactor.cc index fbd298f3..fa3836ad 100644 --- a/services/seller_frontend_service/select_auction_result_reactor.cc +++ b/services/seller_frontend_service/select_auction_result_reactor.cc @@ -330,7 +330,7 @@ void SelectAuctionResultReactor::OnCancel() { client_contexts_.CancelAll(); } SelectAuctionResultReactor::SelectAuctionResultReactor( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client) + const TrustedServersConfigClient& config_client, bool enable_cancellation) : request_context_(context), request_(request), response_(response), @@ -341,7 +341,7 @@ SelectAuctionResultReactor::SelectAuctionResultReactor( log_context_({}, server_common::ConsentedDebugConfiguration(), [this]() { return response_->mutable_debug_info(); }), error_accumulator_(&log_context_), - enable_cancellation_(absl::GetFlag(FLAGS_enable_cancellation)) { + enable_cancellation_(enable_cancellation) { seller_domain_ = config_client_.GetStringParameter(SELLER_ORIGIN_DOMAIN); CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, diff --git a/services/seller_frontend_service/select_auction_result_reactor.h b/services/seller_frontend_service/select_auction_result_reactor.h index 67687840..34898f01 100644 --- a/services/seller_frontend_service/select_auction_result_reactor.h +++ b/services/seller_frontend_service/select_auction_result_reactor.h @@ -51,7 +51,8 @@ class SelectAuctionResultReactor : public grpc::ServerUnaryReactor { explicit SelectAuctionResultReactor( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client); + const TrustedServersConfigClient& config_client, + bool enable_cancellation = false); virtual ~SelectAuctionResultReactor() = default; // SelectAuctionResultReactor is neither copyable nor movable. diff --git a/services/seller_frontend_service/seller_frontend_service.cc b/services/seller_frontend_service/seller_frontend_service.cc index 99b9bcdd..bd9acba5 100644 --- a/services/seller_frontend_service/seller_frontend_service.cc +++ b/services/seller_frontend_service/seller_frontend_service.cc @@ -39,14 +39,17 @@ namespace { std::unique_ptr GetSelectAdReactor( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, - const TrustedServersConfigClient& config_client) { + const TrustedServersConfigClient& config_client, bool enable_cancellation, + bool enable_kanon) { switch (request->client_type()) { case CLIENT_TYPE_ANDROID: - return std::make_unique(context, request, response, - clients, config_client); + return std::make_unique( + context, request, response, clients, config_client, + enable_cancellation, enable_kanon); case CLIENT_TYPE_BROWSER: - return std::make_unique(context, request, response, - clients, config_client); + return std::make_unique( + context, request, response, clients, config_client, + enable_cancellation, enable_kanon); default: return std::make_unique( context, request, response, clients, config_client); @@ -81,12 +84,14 @@ grpc::ServerUnaryReactor* SellerFrontEndService::SelectAd( if (AuctionScope auction_scope = GetAuctionScope(*request); auction_scope == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { auto reactor = std::make_unique( - context, request, response, clients_, config_client_); + context, request, response, clients_, config_client_, + enable_cancellation_); reactor->Execute(); return reactor.release(); } std::unique_ptr reactor = - GetSelectAdReactor(context, request, response, clients_, config_client_); + GetSelectAdReactor(context, request, response, clients_, config_client_, + enable_cancellation_, enable_kanon_); reactor->Execute(); return reactor.release(); } diff --git a/services/seller_frontend_service/seller_frontend_service.h b/services/seller_frontend_service/seller_frontend_service.h index e0cb34ac..9939cc49 100644 --- a/services/seller_frontend_service/seller_frontend_service.h +++ b/services/seller_frontend_service/seller_frontend_service.h @@ -22,12 +22,14 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/flags/flag.h" #include "api/bidding_auction_servers.grpc.pb.h" #include "services/common/clients/auction_server/scoring_async_client.h" #include "services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h" #include "services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.h" #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/clients/http/multi_curl_http_fetcher_async.h" +#include "services/common/feature_flags.h" #include "services/common/reporters/async_reporter.h" #include "services/seller_frontend_service/providers/http_scoring_signals_async_provider.h" #include "services/seller_frontend_service/providers/scoring_signals_async_provider.h" @@ -120,7 +122,9 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { *key_fetcher_manager_, crypto_client_.get(), std::make_unique( - std::make_unique(executor_.get()))} { + std::make_unique(executor_.get()))}, + enable_cancellation_(absl::GetFlag(FLAGS_enable_cancellation)), + enable_kanon_(absl::GetFlag(FLAGS_enable_kanon)) { if (config_client_.HasParameter(SELLER_CLOUD_PLATFORMS_MAP)) { seller_cloud_platforms_map_ = ParseSellerCloudPlarformMap( config_client_.GetStringParameter(SELLER_CLOUD_PLATFORMS_MAP)); @@ -129,7 +133,10 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { SellerFrontEndService(const TrustedServersConfigClient* config_client, ClientRegistry clients) - : config_client_(*config_client), clients_(std::move(clients)) { + : config_client_(*config_client), + clients_(std::move(clients)), + enable_cancellation_(absl::GetFlag(FLAGS_enable_cancellation)), + enable_kanon_(absl::GetFlag(FLAGS_enable_kanon)) { if (config_client_.HasParameter(SELLER_CLOUD_PLATFORMS_MAP)) { seller_cloud_platforms_map_ = ParseSellerCloudPlarformMap( config_client_.GetStringParameter(SELLER_CLOUD_PLATFORMS_MAP)); @@ -188,6 +195,8 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { const ClientRegistry clients_; absl::flat_hash_map seller_cloud_platforms_map_; + const bool enable_cancellation_; + const bool enable_kanon_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/util/encryption_util.cc b/services/seller_frontend_service/util/encryption_util.cc index 2e01b28a..3f8b4cbb 100644 --- a/services/seller_frontend_service/util/encryption_util.cc +++ b/services/seller_frontend_service/util/encryption_util.cc @@ -58,7 +58,7 @@ DecryptOHTTPEncapsulatedHpkeCiphertext( std::optional private_key = key_fetcher_manager.GetPrivateKey(str_key_id); - if (!private_key.has_value()) { + if (!private_key) { PS_VLOG(kNoisyWarn, SystemLogContext()) << "Unable to retrieve private key for key ID: " << str_key_id; return absl::Status(absl::StatusCode::kNotFound, diff --git a/services/seller_frontend_service/util/proto_mapping_util.cc b/services/seller_frontend_service/util/proto_mapping_util.cc index 2cabc5ea..ea005f0c 100644 --- a/services/seller_frontend_service/util/proto_mapping_util.cc +++ b/services/seller_frontend_service/util/proto_mapping_util.cc @@ -50,9 +50,9 @@ AuctionResult MapAdScoreToAuctionResult( const std::optional& high_score, const std::optional& error) { AuctionResult auction_result; - if (error.has_value()) { + if (error) { *auction_result.mutable_error() = *error; - } else if (high_score.has_value()) { + } else if (high_score) { auction_result.set_is_chaff(false); if (high_score->bid() > 0) { auction_result.set_bid(high_score->bid()); @@ -200,7 +200,7 @@ AuctionResult AdScoreToAuctionResult( generation_id); auction_result.mutable_auction_params()->set_component_seller(seller); auction_result.set_top_level_seller(top_level_seller); - if (bidding_groups.has_value()) { + if (bidding_groups) { *auction_result.mutable_bidding_groups() = (*std::move(bidding_groups)); } diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc b/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc index 1ad015bf..a443eecf 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc @@ -122,7 +122,7 @@ void SetupBuyerClientMock( if (!top_level_seller.empty()) { EXPECT_EQ(get_values_request->top_level_seller(), top_level_seller); } - if (bid.has_value()) { + if (bid) { std::move(on_done)( std::make_unique(*bid), /* response_metadata= */ {}); @@ -195,13 +195,13 @@ AdWithBid BuildNewAdWithBid( bid.add_ad_components( absl::StrCat("https://fooAds.com/adComponents?id=", i)); } - if (bid_value.has_value()) { + if (bid_value) { bid.set_bid(bid_value.value()); } - if (interest_group_name.has_value()) { + if (interest_group_name) { bid.set_interest_group_name(*interest_group_name); } - if (bid_currency.has_value()) { + if (bid_currency) { bid.set_bid_currency(*bid_currency); } bid.set_ad_cost(kAdCost); @@ -234,10 +234,10 @@ ProtectedAppSignalsAdWithBid BuildNewPASAdWithBid( absl::optional bid_currency) { ProtectedAppSignalsAdWithBid pas_ad_with_bid; pas_ad_with_bid.set_render(ad_render_url); - if (bid_value.has_value()) { + if (bid_value) { pas_ad_with_bid.set_bid(bid_value.value()); } - if (bid_currency.has_value()) { + if (bid_currency) { pas_ad_with_bid.set_bid_currency(bid_currency.value()); } pas_ad_with_bid.set_ad_cost(kAdCost); @@ -304,11 +304,11 @@ void SetupScoringProviderMock( seller_egid); GetByteSize get_byte_size; - if (server_error_to_return.has_value()) { + if (server_error_to_return) { std::move(on_done)(*server_error_to_return, get_byte_size); } else { auto scoring_signals = std::make_unique(); - if (scoring_signals_value.has_value()) { + if (scoring_signals_value) { scoring_signals->scoring_signals = std::make_unique(scoring_signals_value.value()); std::move(on_done)(std::move(scoring_signals), get_byte_size); diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h index 73f52665..a939f4f8 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h @@ -162,10 +162,12 @@ template SelectAdResponse RunRequest(const TrustedServersConfigClient& config_client, const ClientRegistry& clients, const SelectAdRequest& request, - int max_buyers_solicited = 2) { + int max_buyers_solicited = 2, + bool enable_kanon = false) { grpc::CallbackServerContext context; SelectAdResponse response; T reactor(&context, &request, &response, clients, config_client, + /*enable_cancellation=*/false, enable_kanon, /*fail_fast=*/true, max_buyers_solicited); reactor.Execute(); return response; @@ -362,7 +364,7 @@ GetSelectAdRequestAndClientRegistryForTest( ad_render_urls, /*repeated_get_allowed=*/true); float bid_value = kNonZeroBidValue; - if (buyer_bid.has_value()) { + if (buyer_bid) { bid_value = *buyer_bid; } using ScoreAdsDoneCallback = @@ -456,11 +458,12 @@ template SelectAdResponse RunReactorRequest( const TrustedServersConfigClient& config_client, const ClientRegistry& clients, const SelectAdRequest& request, - bool fail_fast = false) { + bool enable_kanon = false, bool fail_fast = false) { metric::SfeContextMap()->Get(&request); grpc::CallbackServerContext context; SelectAdResponse response; - T reactor(&context, &request, &response, clients, config_client, fail_fast); + T reactor(&context, &request, &response, clients, config_client, + /*enable_cancellation=*/false, enable_kanon, fail_fast); reactor.Execute(); return response; } diff --git a/services/seller_frontend_service/util/web_utils.cc b/services/seller_frontend_service/util/web_utils.cc index 20c5d73e..d08bfa4f 100644 --- a/services/seller_frontend_service/util/web_utils.cc +++ b/services/seller_frontend_service/util/web_utils.cc @@ -1481,10 +1481,10 @@ absl::StatusOr Encode( ScopedCbor cbor_data_root(cbor_new_definite_map(kNumAuctionResultKeys)); auto* cbor_internal = cbor_data_root.get(); - if (error.has_value()) { + if (error) { PS_RETURN_IF_ERROR( CborSerializeError(*error, error_handler, *cbor_internal)); - } else if (high_score.has_value()) { + } else if (high_score) { PS_RETURN_IF_ERROR(CborSerializeScoreAdResponse( *high_score, bidding_group_map, update_group_map, kanon_auction_result_data, error_handler, *cbor_internal)); @@ -1513,10 +1513,10 @@ absl::StatusOr EncodeComponent( ScopedCbor cbor_data_root(cbor_new_definite_map(kNumAuctionResultKeys)); auto* cbor_internal = cbor_data_root.get(); - if (error.has_value()) { + if (error) { PS_RETURN_IF_ERROR( CborSerializeError(*error, error_handler, *cbor_internal)); - } else if (high_score.has_value()) { + } else if (high_score) { PS_RETURN_IF_ERROR(CborSerializeComponentScoreAdResponse( top_level_seller, *high_score, bidding_group_map, update_group_map, error_handler, *cbor_internal)); diff --git a/services/seller_frontend_service/util/web_utils_test.cc b/services/seller_frontend_service/util/web_utils_test.cc index 8f0b560d..5d056213 100644 --- a/services/seller_frontend_service/util/web_utils_test.cc +++ b/services/seller_frontend_service/util/web_utils_test.cc @@ -164,12 +164,12 @@ struct cbor_item_t* BuildSampleCborInterestGroup( BuildIntMapPair(kJoinCount, kSampleJoinCount))); EXPECT_TRUE(cbor_map_add(browser_signals, BuildIntMapPair(kBidCount, kSampleBidCount))); - if (cbor_interest_group_config.recency.has_value()) { + if (cbor_interest_group_config.recency) { EXPECT_TRUE(cbor_map_add( browser_signals, BuildIntMapPair(kRecency, cbor_interest_group_config.recency.value()))); } - if (cbor_interest_group_config.recency_ms.has_value()) { + if (cbor_interest_group_config.recency_ms) { EXPECT_TRUE(cbor_map_add( browser_signals, BuildIntMapPair(kRecencyMs, @@ -355,10 +355,10 @@ void TestDecode(const CborInterestGroupConfig& cbor_interest_group_config) { BrowserSignals* signals = expected_ig.mutable_browser_signals(); signals->set_join_count(kSampleJoinCount); signals->set_bid_count(kSampleBidCount); - if (cbor_interest_group_config.recency.has_value()) { + if (cbor_interest_group_config.recency) { signals->set_recency(cbor_interest_group_config.recency.value()); } - if (cbor_interest_group_config.recency_ms.has_value()) { + if (cbor_interest_group_config.recency_ms) { signals->set_recency_ms(cbor_interest_group_config.recency_ms.value()); } std::string prev_wins_json_str = absl::StrFormat( diff --git a/tools/debug/start_auction b/tools/debug/start_auction index 848e0183..70ca3b9e 100755 --- a/tools/debug/start_auction +++ b/tools/debug/start_auction @@ -15,6 +15,10 @@ source $(dirname "$0")/common +export AUCTION_JS_URL="" +export BUYER_REPORT_WIN_URL="" +export BUYER_REPORT_WIN_SCRIPT="" + export SERVER_START_CMD=$(cat << END /server/bin/server \ --init_config_client="false" \ diff --git a/tools/debug/start_bfe b/tools/debug/start_bfe index 3eca84fb..5e752dba 100755 --- a/tools/debug/start_bfe +++ b/tools/debug/start_bfe @@ -16,6 +16,8 @@ source $(dirname "$0")/common export EXTRA_DOCKER_RUN_ARGS="${COMMON_DOCKER_RUN_ARGS[@]} ${DOCKER_RUN_ARGS[@]}" +export BUYER_KV_SERVER_ADDR="" + export SERVER_START_CMD=$(cat << END /server/bin/server \ --init_config_client="false" --port=50051 \ diff --git a/tools/debug/start_bidding b/tools/debug/start_bidding index 16b8996c..cbdd02db 100755 --- a/tools/debug/start_bidding +++ b/tools/debug/start_bidding @@ -28,6 +28,7 @@ USAGE declare -i ENABLE_INFERENCE=0 declare -i USE_GDB=0 declare GENERATE_BID_FILEPATH="" +declare -i FETCH_MODE=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -41,6 +42,7 @@ while [[ $# -gt 0 ]]; do ;; --use-with-generateBid-file) GENERATE_BID_FILEPATH="$2" + FETCH_MODE=2 shift 2 ;; -h | --help) print_usage ;; @@ -67,12 +69,13 @@ function get_libcddl_path() { done } -export INFERENCE_BIDDING_JS_PATH="/src/workspace/services/inference_sidecar/common/tools/debug/generateBidRunInference.js" +export BIDDING_JS_URL="" +export EGRESS_SCHEMA_URL="" # Set default buyer code fetch config export BUYER_CODE_FETCH_CONFIG=$(cat << END { - "fetchMode": 0, + "fetchMode": ${FETCH_MODE}, "biddingJsPath": "/generateBid.js", "biddingJsUrl": "${BIDDING_JS_URL}", "protectedAppSignalsBiddingJsUrl": "${BIDDING_JS_URL}", @@ -97,6 +100,8 @@ fi # Update buyer code fetch config if enabling inference if [[ ${ENABLE_INFERENCE} -eq 1 ]]; then + GENERATE_BID_FILEPATH="services/inference_sidecar/common/tools/debug/generateBidRunInference.js" + INFERENCE_ARGS=$(cat << END --inference_sidecar_binary_path="/server/bin/inference_sidecar_pytorch_v2_1_1" \ --inference_sidecar_runtime_config='{ @@ -110,7 +115,7 @@ END BUYER_CODE_FETCH_CONFIG=$(cat << END { "fetchMode": 2, - "biddingJsPath": "${INFERENCE_BIDDING_JS_PATH}", + "biddingJsPath": "/generateBid.js", "urlFetchPeriodMs": 13000000, "urlFetchTimeoutMs": 30000, "enableBuyerDebugUrlGeneration": true diff --git a/tools/debug/start_bidding_byob b/tools/debug/start_bidding_byob index 67ea2fac..6100a05f 100755 --- a/tools/debug/start_bidding_byob +++ b/tools/debug/start_bidding_byob @@ -14,12 +14,12 @@ # limitations under the License. export BIDDING_EXECUTABLE_URL="" -export BIDDING_EXECUTABLE_PATH="/sample_udf/bin/sample_generate_bid_udf" +export BIDDING_EXECUTABLE_PATH="" -if [[ -z "$BIDDING_EXECUTABLE_URL" ]]; then - export FETCH_MODE=2 -else +if [[ -n "$BIDDING_EXECUTABLE_URL" ]]; then export FETCH_MODE=0 +else + export FETCH_MODE=2 fi export SERVER_START_CMD=$(cat << END diff --git a/tools/debug/start_sfe b/tools/debug/start_sfe index 63cbea65..0f2bd919 100755 --- a/tools/debug/start_sfe +++ b/tools/debug/start_sfe @@ -15,6 +15,10 @@ source $(dirname "$0")/common +export BUYER_SERVER_HOST="" +export KEY_VALUE_SIGNALS_HOST="" +export SELLER_ORIGIN_DOMAIN="" + export SERVER_START_CMD=$(cat << END /server/bin/server \ --init_config_client="false" --port=50053 \ diff --git a/tools/load_testing/README.md b/tools/load_testing/README.md index c2377ab3..33d71064 100644 --- a/tools/load_testing/README.md +++ b/tools/load_testing/README.md @@ -95,8 +95,8 @@ end services are deployed on to find the buyer and seller metrics. 1. Look at the round trip latency using the request.duration_ms metrics. -Note: _Verify the request count and make sure request.failed_count is 0. _ IGNORE the latency -metrics from the wrk2 tool. +Note: _Verify the request count and make sure request.failed_count is 0._ IGNORE the latency metrics +from the wrk2 tool. ## Recommended load testing tool diff --git a/tools/secure_invoke/payload_generator/payload_packaging.cc b/tools/secure_invoke/payload_generator/payload_packaging.cc index 1f7ba07b..73e8b328 100644 --- a/tools/secure_invoke/payload_generator/payload_packaging.cc +++ b/tools/secure_invoke/payload_generator/payload_packaging.cc @@ -101,12 +101,12 @@ ProtectedAuctionInput GetProtectedAuctionInput( // Enable debug reporting for all calls from this tool. protected_auction_input.set_enable_debug_reporting(enable_debug_reporting); - if (enable_debug_info.has_value()) { + if (enable_debug_info) { protected_auction_input.mutable_consented_debug_config() ->set_is_debug_info_in_response(*enable_debug_info); } - if (enable_unlimited_egress.has_value()) { + if (enable_unlimited_egress) { protected_auction_input.set_enable_unlimited_egress( *enable_unlimited_egress); } diff --git a/tools/secure_invoke/payload_generator/payload_packaging_test.cc b/tools/secure_invoke/payload_generator/payload_packaging_test.cc index 5eaf4d00..367110e5 100644 --- a/tools/secure_invoke/payload_generator/payload_packaging_test.cc +++ b/tools/secure_invoke/payload_generator/payload_packaging_test.cc @@ -352,7 +352,7 @@ TEST(PaylodPackagingTest, SetEnableDebugInfo) { })JSON"; std::string consent_config; - if (json_value.has_value()) { + if (json_value) { consent_config = absl::Substitute(R"JSON( "consented_debug_config": { "is_debug_info_in_response": $0 @@ -395,7 +395,7 @@ TEST(PaylodPackagingTest, SetEnableDebugInfo) { ASSERT_TRUE(protected_auction_input.ParseFromArray( decoded_request->compressed_data.data(), decoded_request->compressed_data.size())); - if (!json_value.has_value() && !enable_debug_info.has_value()) { + if (!json_value.has_value() && !enable_debug_info) { ASSERT_FALSE(protected_auction_input.has_consented_debug_config()) << protected_auction_input.DebugString(); } else { diff --git a/version.txt b/version.txt index 99eba4de..ef8d7569 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.1.0 \ No newline at end of file +4.2.0 \ No newline at end of file