diff --git a/.buildconfig.yml b/.buildconfig.yml index 201199aed9..150e6c90ef 100644 --- a/.buildconfig.yml +++ b/.buildconfig.yml @@ -1,4 +1,4 @@ -libraryVersion: 60.4.0 +libraryVersion: 60.5.0 groupId: org.mozilla.telemetry projects: glean: diff --git a/.circleci/config.yml b/.circleci/config.yml index f3db1ffc30..e500eb921f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,6 +90,10 @@ commands: name: Run Rust RLB delayed ping data test command: | glean-core/rlb/tests/test-delayed-ping-data.sh + - run: + name: Run Rust RLB flush test + command: | + glean-core/rlb/tests/test-ping-lifetime-flush.sh - run: name: Upload coverage report command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6145aeffb2..80802d9e51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Unreleased changes -[Full changelog](https://github.com/mozilla/glean/compare/v60.4.0...main) +[Full changelog](https://github.com/mozilla/glean/compare/v60.5.0...main) + +# v60.5.0 (2024-08-06) + +[Full changelog](https://github.com/mozilla/glean/compare/v60.4.0...v60.5.0) + +* General + * Make auto-flush behavior configurable and time-based ([#2871](https://github.com/mozilla/glean/pull/2871)) + * Require `glean_parser` v14.5.0 ([#2916](https://github.com/mozilla/glean/pull/2916)) +* Android + * Update to Gradle v8.9 ([#2909](https://github.com/mozilla/glean/pull/2909)) + * Fixed `GleanTestLocalServer` test rule to prevent leaking between tests([Bug 1787234](https://bugzilla.mozilla.org/show_bug.cgi?id=1787234)) +* Rust + * Remove cargo feature `preinit_million_queue` and set the default pre-init queue size to 10^6 for all consumers ([Bug 1909246](https://bugzilla.mozilla.org/show_bug.cgi?id=1909246)) # v60.4.0 (2024-07-23) diff --git a/Cargo.lock b/Cargo.lock index ec9b6f2a06..29eb29b15e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,7 +318,7 @@ dependencies = [ [[package]] name = "glean" -version = "60.4.0" +version = "60.5.0" dependencies = [ "crossbeam-channel", "env_logger", @@ -336,7 +336,7 @@ dependencies = [ [[package]] name = "glean-build" -version = "14.3.0" +version = "14.5.1" dependencies = [ "tempfile", "xshell-venv", @@ -359,7 +359,7 @@ dependencies = [ [[package]] name = "glean-core" -version = "60.4.0" +version = "60.5.0" dependencies = [ "android_logger", "bincode", @@ -776,6 +776,7 @@ dependencies = [ "flate2", "glean", "glean-build", + "serde_json", "tempfile", ] diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index e35ea224d8..fdb343a5ee 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -4951,9 +4951,9 @@ SOFTWARE. The following text applies to code linked from these dependencies: -* [glean-core 60.4.0]( https://github.com/mozilla/glean ) -* [glean-build 14.3.0]( https://github.com/mozilla/glean ) -* [glean 60.4.0]( https://github.com/mozilla/glean ) +* [glean-core 60.5.0]( https://github.com/mozilla/glean ) +* [glean-build 14.5.1]( https://github.com/mozilla/glean ) +* [glean 60.5.0]( https://github.com/mozilla/glean ) * [zeitstempel 0.1.1]( https://github.com/badboy/zeitstempel ) ``` diff --git a/Makefile b/Makefile index c135e355e6..aecf57a488 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ docs-python: build-python ## Build the Python documentation .PHONY: docs docs-rust docs-swift docs-metrics: setup-python ## Build the internal metrics documentation - $(GLEAN_PYENV)/bin/pip install glean_parser~=14.3 + $(GLEAN_PYENV)/bin/pip install glean_parser~=14.5 $(GLEAN_PYENV)/bin/glean_parser translate --allow-reserved \ -f markdown \ -o ./docs/user/user/collected-metrics \ diff --git a/docs/user/SUMMARY.md b/docs/user/SUMMARY.md index 101433b907..1362192e9c 100644 --- a/docs/user/SUMMARY.md +++ b/docs/user/SUMMARY.md @@ -18,12 +18,6 @@ - [Validating metrics](user/metrics/validation-checklist.md) - [Error reporting](user/metrics/error-reporting.md) - [Metrics collected by the Glean SDKs](user/collected-metrics/metrics.md) - - [Data Control Plane (Remote Metric Configuration)](user/metrics/data-control-plane/index.md) - - [Example Scenarios](user/metrics/data-control-plane/example-scenarios.md) - - [Product Integration](user/metrics/data-control-plane/product-integration.md) - - [Experimenter Configuration](user/metrics/data-control-plane/experimenter-configuration.md) - - [Advanced Topics](user/metrics/data-control-plane/advanced-topics.md) - - [Frequently Asked Questions](user/metrics/data-control-plane/faq.md) - [Pings](user/pings/index.md) - [Adding new custom pings](user/pings/custom.md) - [Testing custom pings](user/pings/testing-custom-pings.md) @@ -33,9 +27,18 @@ - [Events Ping](user/pings/events.md) - [Metrics Ping](user/pings/metrics.md) - [Schedules and timings overview](user/pings/ping-schedules-and-timings.md) - - [Data Control Plane (Remote Metric Configuration)](user/pings/data-control-plane/index.md) - - [Product Integration](user/pings/data-control-plane/product-integration.md) - - [Experimenter Configuration](user/pings/data-control-plane/experimenter-configuration.md) +- [Server Knobs: Glean Data Control Plane](user/server-knobs/index.md) + - [Controlling Metrics with Server Knobs](user/server-knobs/metrics/index.md) + - [Example Scenarios](user/server-knobs/metrics/example-scenarios.md) + - [Product Integration](user/server-knobs/metrics/product-integration.md) + - [Experimenter Configuration](user/server-knobs/metrics/experimenter-configuration.md) + - [Advanced Topics](user/server-knobs/metrics/advanced-topics.md) + - [Frequently Asked Questions](user/server-knobs/metrics/faq.md) + - [Controlling Pings with Server Knobs](user/server-knobs/pings/index.md) + - [Product Integration](user/server-knobs/pings/product-integration.md) + - [Experimenter Configuration](user/server-knobs/pings/experimenter-configuration.md) + - [Other Server Knobs](user/server-knobs/other/index.md) + - [Max Events per Ping](user/server-knobs/other/max-events.md) - [Debugging products using Glean](user/debugging/index.md) - [Android](user/debugging/android.md) - [iOS](user/debugging/ios.md) diff --git a/docs/user/reference/general/initializing.md b/docs/user/reference/general/initializing.md index d0aa3306d2..e3df475262 100644 --- a/docs/user/reference/general/initializing.md +++ b/docs/user/reference/general/initializing.md @@ -16,7 +16,7 @@ application start using Glean. ## Behavior when uninitialized Any API called before Glean is initialized is queued and applied at initialization. -To avoid unbounded memory growth the queue is bounded (currently to a maximum of 100 tasks), +To avoid unbounded memory growth the queue is bounded (currently to a maximum of 1 million tasks), and further calls are dropped. The number of calls dropped, if any, diff --git a/docs/user/reference/metrics/event.md b/docs/user/reference/metrics/event.md index 9f4bcd3fbe..2fe518db62 100644 --- a/docs/user/reference/metrics/event.md +++ b/docs/user/reference/metrics/event.md @@ -394,7 +394,8 @@ Each extra key contains additional metadata: * In Glean.js the default value for `maxEvents` is 1. In all other SDKs it is 500. * Once the `maxEvents` threshold is reached on the client an "events" ping is immediately sent. * The `extra_keys` allows for a maximum of 50 keys. -* The keys in the `extra_keys` list must be in dotted snake case, with a maximum length of 40 bytes, when encoded as UTF-8. +* The keys in the `extra_keys` list must be written using printable ASCII characters, + with a maximum length of 40 bytes, when encoded as UTF-8. * The values in the `extras` object have a maximum length of 500 bytes when serialized and encoded as UTF-8. Longer values are truncated, and an `invalid_overflow` error is recorded. diff --git a/docs/user/user/pings/data-control-plane/product-integration.md b/docs/user/user/pings/data-control-plane/product-integration.md deleted file mode 100644 index 86be259337..0000000000 --- a/docs/user/user/pings/data-control-plane/product-integration.md +++ /dev/null @@ -1,8 +0,0 @@ -# Product Integration - -Glean provides a general Nimbus feature named `glean` that can be used for configuration of pings. Simply select the `glean` feature along with your Nimbus feature from the Experimenter UI when configuring the experiment or rollout (see https://experimenter.info for more information on multi-feature experiments). - -The `glean` Nimbus feature requires the `gleanMetricConfiguration` variable to be used to provide the require metric configuration. The format of the configuration is defined in the [Experimenter Configuration](./experimenter-configuration.md) section. If a ping is not included, it will default to the value found in the pings.yaml. Note that this can also serve as an override for Glean builtin pings disabled using the Configuration property `enable_internal_pings=false` during initialization. - -[Nimbus]: https://experimenter.info -[Nimbus Desktop Feature API]: https://experimenter.info/desktop-feature-api \ No newline at end of file diff --git a/docs/user/user/server-knobs/index.md b/docs/user/user/server-knobs/index.md new file mode 100644 index 0000000000..e7e27326cd --- /dev/null +++ b/docs/user/user/server-knobs/index.md @@ -0,0 +1,25 @@ +# Server Knobs: Glean Data Control Plane + +Glean provides Server Knobs, a Data [Control Plane] through which Glean runtime settings can be changed remotely including the ability to enable, disable or throttle metrics and pings through a [Nimbus] rollout or experiment. + +Products can use this capability to control "data traffic", similar to how a network control plane controls "network traffic". + +Server Knobs provides the ability to do the following: + +- Allow runtime changes to data collection without needing to land code and ride release trains. +- Eliminate the need for manual creation and maintenance of feature flags specific to data collection. +- Sampling of measurements from a subset of the population so that we do not collect or ingest more data than is necessary from high traffic areas of an application instrumented with Glean metrics. +- Operational safety through being able to react to high-volume or unwanted data. +- Visibility into sampling and sampling rates for remotely configured metrics. + +## Contents + +- [Controlling Metrics with Server Knobs] +- [Controlling Pings with Server Knobs] +- [Other Server Knobs] + +[Control Plane]: https://en.wikipedia.org/wiki/Control_plane +[Nimbus]: https://experimenter.info +[Controlling Metrics with Server Knobs]: ./metrics/index.md +[Controlling Pings with Server Knobs]: ./pings/index.md +[Other Server Knobs]: ./other/index.md \ No newline at end of file diff --git a/docs/user/user/metrics/data-control-plane/advanced-topics.md b/docs/user/user/server-knobs/metrics/advanced-topics.md similarity index 100% rename from docs/user/user/metrics/data-control-plane/advanced-topics.md rename to docs/user/user/server-knobs/metrics/advanced-topics.md diff --git a/docs/user/user/metrics/data-control-plane/example-scenarios.md b/docs/user/user/server-knobs/metrics/example-scenarios.md similarity index 100% rename from docs/user/user/metrics/data-control-plane/example-scenarios.md rename to docs/user/user/server-knobs/metrics/example-scenarios.md diff --git a/docs/user/user/metrics/data-control-plane/experimenter-configuration.md b/docs/user/user/server-knobs/metrics/experimenter-configuration.md similarity index 100% rename from docs/user/user/metrics/data-control-plane/experimenter-configuration.md rename to docs/user/user/server-knobs/metrics/experimenter-configuration.md diff --git a/docs/user/user/metrics/data-control-plane/faq.md b/docs/user/user/server-knobs/metrics/faq.md similarity index 100% rename from docs/user/user/metrics/data-control-plane/faq.md rename to docs/user/user/server-knobs/metrics/faq.md diff --git a/docs/user/user/metrics/data-control-plane/index.md b/docs/user/user/server-knobs/metrics/index.md similarity index 95% rename from docs/user/user/metrics/data-control-plane/index.md rename to docs/user/user/server-knobs/metrics/index.md index a29c7b86c5..332692ddde 100644 --- a/docs/user/user/metrics/data-control-plane/index.md +++ b/docs/user/user/server-knobs/metrics/index.md @@ -23,4 +23,4 @@ For information on controlling pings with Server Knobs, see the metrics document [Control Plane]: https://en.wikipedia.org/wiki/Control_plane [Nimbus]: https://experimenter.info -[Server Knobs - Pings]: ../../pings/data-control-plane/index.md \ No newline at end of file +[Server Knobs - Pings]: ../pings/index.md \ No newline at end of file diff --git a/docs/user/user/metrics/data-control-plane/product-integration.md b/docs/user/user/server-knobs/metrics/product-integration.md similarity index 70% rename from docs/user/user/metrics/data-control-plane/product-integration.md rename to docs/user/user/server-knobs/metrics/product-integration.md index 409e8b8ffb..8ed72faa0c 100644 --- a/docs/user/user/metrics/data-control-plane/product-integration.md +++ b/docs/user/user/server-knobs/metrics/product-integration.md @@ -8,14 +8,18 @@ In order to make use of the remote metric configuration in a Firefox Desktop com ### Integration Option 1: -Glean provides a general Nimbus feature that can be used for configuration of metrics named `glean`. Simply select the `glean` feature along with your Nimbus feature from the Experimenter UI when configuring the experiment or rollout (see https://experimenter.info for more information on multi-feature experiments). +Glean provides a general Nimbus feature that can be used for configuration of metrics named `glean`. Simply select the `glean` feature along with your Nimbus feature from the Experimenter UI when configuring the experiment or rollout (see [Nimbus] documentation for more information on multi-feature experiments). -The `glean` Nimbus feature requires the `gleanMetricConfiguration` variable to be used to provide the require metric configuration. The format of the configuration is a map of the fully qualified metric name (category.name) of the metric to a boolean value representing whether the metric is enabled. If a metric is omitted from this map, it will default to the value found in the metrics.yaml. An example configuration for the `glean` feature can be found on the [Experimenter Configuration](./experimenter-configuration.md) page. +The `glean` Nimbus feature requires the `gleanMetricConfiguration` variable to be used to provide the required metric configuration. +The format of the configuration is a map of the fully qualified metric name (category.name) of the metric to a boolean value representing whether the metric is enabled. +If a metric is omitted from this map, it will default to the value found in the metrics.yaml. +An example configuration for the `glean` feature can be found on the [Experimenter Configuration] page. ### Integration Option 2 (Advanced use): - A second option that can give you more control over the metric configuration, especially if there are more than one experiments or rollouts that are currently using the `glean` feature from Option 1 above, is to add a Feature Variable to represent the Glean metric configuration in your own feature. This can be accomplished by modifying the FeatureManifest.yaml file, adding a variable through which to pass metric configurations. Glean will handle merging this configuration with other metrics configurations for you (See [Advanced Topics](./advanced-topics.md) for more info on this). An example feature manifest entry would look like the following: + A second option that can give you more control over the metric configuration, especially if there are more than one experiments or rollouts that are currently using the `glean` feature from Option 1 above, is to add a Feature Variable to represent the Glean metric configuration in your own feature. This can be accomplished by modifying the FeatureManifest.yaml file, adding a variable through which to pass metric configurations. Glean will handle merging this configuration with other metrics configurations for you (See [Advanced Topics] for more info on this). + An example feature manifest entry would look like the following: ```yaml variables: @@ -26,7 +30,8 @@ variables: "Glean metric configuration" ``` -This definition allows for configuration to be set in a Nimbus rollout or experiment and fetched by the client to be applied based on the enrollment. Once the Feature Variable has been defined, the final step is to fetch the configuration from Nimbus and supply it to the Glean API. This can be done during initialization and again any time afterwards, such as in response to receiving an updated configuration from Nimbus. Glean will merge this configuration with any other active configurations and enable or disable the metrics accordingly. An example call to set a configuration through your Nimbus Feature could look like this: +This definition allows for configuration to be set in a Nimbus rollout or experiment and fetched by the client to be applied based on the enrollment. Once the Feature Variable has been defined, the final step is to fetch the configuration from Nimbus and supply it to the Glean API. This can be done during initialization and again any time afterwards, such as in response to receiving an updated configuration from Nimbus. Glean will merge this configuration with any other active configurations and enable or disable the metrics accordingly. +An example call to set a configuration through your Nimbus Feature could look like this: ```JavaScript // Fetch the Glean metric configuration from your feature's Nimbus variable @@ -55,13 +60,18 @@ lazy.NimbusFeatures.yourNimbusFeatureName.onUpdate(() => { ### Integration Option 1: -Glean provides a general Nimbus feature that can be used for configuration of metrics named `glean`. Simply select the `glean` feature along with your Nimbus feature from the Experimenter UI when configuring the experiment or rollout (see https://experimenter.info for more information on multi-feature experiments). +Glean provides a general Nimbus feature that can be used for configuration of metrics named `glean`. Simply select the `glean` feature along with your Nimbus feature from the Experimenter UI when configuring the experiment or rollout (see [Nimbus] documentation for more information on multi-feature experiments). -The `glean` Nimbus feature requires the `gleanMetricConfiguration` variable to be used to provide the require metric configuration. The format of the configuration is a map of the fully qualified metric name (category.name) of the metric to a boolean value representing whether the metric is enabled. If a metric is omitted from this map, it will default to the value found in the metrics.yaml. An example configuration for the `glean` feature can be found on the [Experimenter Configuration](./experimenter-configuration.md) page. +The `glean` Nimbus feature requires the `gleanMetricConfiguration` variable to be used to provide the required metric configuration. +The format of the configuration is a map of the fully qualified metric name (category.name) of the metric to a boolean value representing whether the metric is enabled. +If a metric is omitted from this map, it will default to the value found in the metrics.yaml. +An example configuration for the `glean` feature can be found on the [Experimenter Configuration] page. ### Integration Option 2 (Advanced use): -A second option that can give you more control over the metric configuration, especially if there are more than one experiments or rollouts that are currently using the `glean` feature from Option 1 above, is to add a Feature Variable to represent the Glean metric configuration in your own feature. This can be accomplished by modifying the Nimbus Feature Manifest file, adding a variable through which to pass metric configurations. Glean will handle merging this configuration with other metrics configurations for you (See [Advanced Topics](./advanced-topics.md) for more info on this). An example feature manifest entry would look like the following: +A second option that can give you more control over the metric configuration, especially if there are more than one experiments or rollouts that are currently using the `glean` feature from Option 1 above, is to add a Feature Variable to represent the Glean metric configuration in your own feature. +This can be accomplished by modifying the Nimbus Feature Manifest file, adding a variable through which to pass metric configurations. Glean will handle merging this configuration with other metrics configurations for you (See [Advanced Topics] for more info on this). +An example feature manifest entry would look like the following: ```yaml features: @@ -77,7 +87,8 @@ features: default: "{}" ``` -Once the Feature Variable has been defined, the final step is to fetch the configuration from Nimbus and supply it to the Glean API. This can be done during initialization and again any time afterwards, such as in response to receiving an updated configuration from Nimbus. Only the latest configuration provided will be applied and any previously configured metrics that are omitted from the new configuration will not be changed. An example call to set a configuration from the “homescreen” Nimbus Feature could look like this: +Once the Feature Variable has been defined, the final step is to fetch the configuration from Nimbus and supply it to the Glean API. This can be done during initialization and again any time afterwards, such as in response to receiving an updated configuration from Nimbus. Only the latest configuration provided will be applied and any previously configured metrics that are omitted from the new configuration will not be changed. +An example call to set a configuration from the “homescreen” Nimbus Feature could look like this: ```Swift Glean.applyServerKnobsConfig(FxNimbus.features.homescreen.value().metricsEnabled) @@ -86,4 +97,6 @@ Glean.applyServerKnobsConfig(FxNimbus.features.homescreen.value().metricsEnabled Since mobile experiments only update on initialization of the application, it isn't necessary to register to listen for notifications for experiment updates. [Nimbus]: https://experimenter.info -[Nimbus Desktop Feature API]: https://experimenter.info/desktop-feature-api \ No newline at end of file +[Nimbus Desktop Feature API]: https://experimenter.info/desktop-feature-api +[Advanced Topics]: ./advanced-topics.md +[Experimenter Configuration]: ./experimenter-configuration.md diff --git a/docs/user/user/server-knobs/other/index.md b/docs/user/user/server-knobs/other/index.md new file mode 100644 index 0000000000..33b4145535 --- /dev/null +++ b/docs/user/user/server-knobs/other/index.md @@ -0,0 +1,15 @@ +# Other Server Knobs + +Below are additional Glean parameters and settings that are exposed via Server Knobs for use in a [Nimbus] experiment or rollout. + +## Contents +- [Max Events per Event Ping] + +Additional Glean settings will be added to Server Knobs as needed or by request. + +> *For information on controlling metrics and pings via Server Knobs, please refer to [Controlling Metrics with Server Knobs] and [Controlling Pings with Server Knobs].* + +[Nimbus]: https://experimenter.info +[Controlling Metrics with Server Knobs]: ../metrics/index.md +[Controlling Pings with Server Knobs]: ../pings/index.md +[Max Events per Event Ping]: ./max-events.md \ No newline at end of file diff --git a/docs/user/user/server-knobs/other/max-events.md b/docs/user/user/server-knobs/other/max-events.md new file mode 100644 index 0000000000..9d4b043ed3 --- /dev/null +++ b/docs/user/user/server-knobs/other/max-events.md @@ -0,0 +1,16 @@ +# Max Events + +By default, Glean batches events together to submit on a single events ping. +The `event_threshold` Server Knob controls how many events Glean will collect before submitting an events ping. + +For instance, if you wanted to disable batching in order to transmit an events ping after every event is recorded you could set `event_threshold: 1`. + +## Example Configuration: + +```json +{ + "gleanMetricConfiguration": { + "event_threshold": 1 + } +} +``` diff --git a/docs/user/user/pings/data-control-plane/experimenter-configuration.md b/docs/user/user/server-knobs/pings/experimenter-configuration.md similarity index 100% rename from docs/user/user/pings/data-control-plane/experimenter-configuration.md rename to docs/user/user/server-knobs/pings/experimenter-configuration.md diff --git a/docs/user/user/pings/data-control-plane/index.md b/docs/user/user/server-knobs/pings/index.md similarity index 94% rename from docs/user/user/pings/data-control-plane/index.md rename to docs/user/user/server-knobs/pings/index.md index 490d893aea..c6377323ec 100644 --- a/docs/user/user/pings/data-control-plane/index.md +++ b/docs/user/user/server-knobs/pings/index.md @@ -20,4 +20,4 @@ For information on controlling metrics with Server Knobs, see the metrics docume [Control Plane]: https://en.wikipedia.org/wiki/Control_plane [Nimbus]: https://experimenter.info -[Server Knobs - Metrics]: ../../metrics/data-control-plane/index.md \ No newline at end of file +[Server Knobs - Metrics]: ../metrics/index.md \ No newline at end of file diff --git a/docs/user/user/server-knobs/pings/product-integration.md b/docs/user/user/server-knobs/pings/product-integration.md new file mode 100644 index 0000000000..663ce79530 --- /dev/null +++ b/docs/user/user/server-knobs/pings/product-integration.md @@ -0,0 +1,11 @@ +# Product Integration + +Glean provides a general Nimbus feature named `glean` that can be used for configuration of pings. Simply select the `glean` feature along with your Nimbus feature from the Experimenter UI when configuring the experiment or rollout (see [Nimbus] documentation for more information on multi-feature experiments). + +The `glean` Nimbus feature requires the `gleanMetricConfiguration` variable to be used to provide the required metric configuration. The format of the configuration is defined in the [Experimenter Configuration] section. +If a ping is not included, it will default to the value found in the pings.yaml. +Note that this can also serve as an override for Glean builtin pings disabled using the Configuration property `enable_internal_pings=false` during initialization. + +[Experimenter Configuration]: ./experimenter-configuration.md +[Nimbus]: https://experimenter.info +[Nimbus Desktop Feature API]: https://experimenter.info/desktop-feature-api diff --git a/glean-core/Cargo.toml b/glean-core/Cargo.toml index 0e631d7998..4db51bcf48 100644 --- a/glean-core/Cargo.toml +++ b/glean-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "glean-core" -version = "60.4.0" +version = "60.5.0" authors = ["Jan-Erik Rediger ", "The Glean Team "] description = "A modern Telemetry library" repository = "https://github.com/mozilla/glean" @@ -21,7 +21,7 @@ include = [ rust-version = "1.66" [package.metadata.glean] -glean-parser = "14.3.0" +glean-parser = "14.5.1" [badges] circle-ci = { repository = "mozilla/glean", branch = "main" } @@ -60,7 +60,5 @@ ctor = "0.2.2" uniffi = { version = "0.27.0", default-features = false, features = ["build"] } [features] -# Increases the preinit queue limit to 10^6 -preinit_million_queue = [] # Enable `env_logger`. Only works on non-Android non-iOS targets. enable_env_logger = ["env_logger"] diff --git a/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt b/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt index c45d6e71d8..b104525f27 100644 --- a/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt +++ b/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt @@ -13,7 +13,6 @@ import android.os.Process import android.util.Log import androidx.annotation.VisibleForTesting import androidx.lifecycle.ProcessLifecycleOwner -import kotlinx.coroutines.Job import kotlinx.coroutines.MainScope import kotlinx.coroutines.launch import mozilla.telemetry.glean.GleanMetrics.GleanValidation @@ -260,6 +259,8 @@ open class GleanInternalAPI internal constructor() { experimentationId = configuration.experimentationId, enableInternalPings = configuration.enableInternalPings, pingSchedule = emptyMap(), + pingLifetimeThreshold = configuration.pingLifetimeThreshold.toULong(), + pingLifetimeMaxTime = configuration.pingLifetimeMaxTime.toULong(), ) val clientInfo = getClientInfo(configuration, buildInfo) val callbacks = OnGleanEventsImpl(this@GleanInternalAPI) @@ -466,9 +467,8 @@ open class GleanInternalAPI internal constructor() { * * @param pingName Name of the ping to submit. * @param reason The reason the ping is being submitted. - * @return The async [Job] performing the work of assembling the ping */ - internal fun submitPingByName(pingName: String, reason: String? = null) { + fun submitPingByName(pingName: String, reason: String? = null) { gleanSubmitPingByName(pingName, reason) } @@ -478,8 +478,6 @@ open class GleanInternalAPI internal constructor() { * If the tag is invalid it won't be set and this function will return `false`, * although if we are not initialized yet, there won't be any validation. * - * This is only meant to be used internally by the `GleanDebugActivity`. - * * @param value The value of the tag, which must be a valid HTTP header value. */ fun setDebugViewTag(value: String): Boolean { @@ -492,8 +490,6 @@ open class GleanInternalAPI internal constructor() { * If any of the tags is invalid nothing will be set and this function will * return `false`, although if we are not initialized yet, there won't be any validation. * - * This is only meant to be used internally by the `GleanDebugActivity`. - * * @param tags A list of tags, which must be valid HTTP header values. */ fun setSourceTags(tags: Set): Boolean { @@ -514,7 +510,7 @@ open class GleanInternalAPI internal constructor() { * Set configuration to override metrics' enabled/disabled state, typically from a remote_settings * experiment or rollout. * - * @param json Stringified JSON map of metrics and their associated `disabled` property. + * @param json Stringified JSON Server Knobs configuration. */ fun applyServerKnobsConfig(json: String) { gleanApplyServerKnobsConfig(json) @@ -524,8 +520,6 @@ open class GleanInternalAPI internal constructor() { * Set the logPing debug option, when this is `true` * the payload of assembled ping requests get logged. * - * This is only meant to be used internally by the `GleanDebugActivity`. - * * @param value The value of the option. */ fun setLogPings(value: Boolean) { diff --git a/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt b/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt index 4be7c5e2a3..d7537f20ad 100644 --- a/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt +++ b/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt @@ -25,6 +25,8 @@ import mozilla.telemetry.glean.net.PingUploader * to be sent with all pings. * @property enableInternalPings Whether to enable internal pings. * @property delayPingLifetimeIo Whether Glean should delay persistence of data from metrics with ping lifetime. + * @property pingLifetimeThreshold Write count threshold when to auto-flush. `0` disables it. + * @property pingLifetimeMaxTime After what time to auto-flush (in milliseconds). 0 disables it. */ data class Configuration @JvmOverloads constructor( val serverEndpoint: String = DEFAULT_TELEMETRY_ENDPOINT, @@ -40,6 +42,8 @@ data class Configuration @JvmOverloads constructor( val experimentationId: String? = null, val enableInternalPings: Boolean = true, val delayPingLifetimeIo: Boolean = true, + val pingLifetimeThreshold: Int = 1000, + val pingLifetimeMaxTime: Int = 0, ) { companion object { /** diff --git a/glean-core/android/src/main/java/mozilla/telemetry/glean/testing/GleanTestLocalServer.kt b/glean-core/android/src/main/java/mozilla/telemetry/glean/testing/GleanTestLocalServer.kt index ee3facf0dc..df5bb5513f 100644 --- a/glean-core/android/src/main/java/mozilla/telemetry/glean/testing/GleanTestLocalServer.kt +++ b/glean-core/android/src/main/java/mozilla/telemetry/glean/testing/GleanTestLocalServer.kt @@ -11,7 +11,9 @@ import androidx.work.testing.WorkManagerTestInitHelper import mozilla.telemetry.glean.Glean import org.junit.rules.TestWatcher import org.junit.runner.Description +import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit /** * This implements a JUnit rule for writing tests for Glean SDK metrics. @@ -39,6 +41,8 @@ class GleanTestLocalServer( val context: Context, private val localPort: Int, ) : TestWatcher() { + private val executor: ExecutorService = Executors.newSingleThreadExecutor() + /** * Invoked when a test is about to start. */ @@ -50,10 +54,24 @@ class GleanTestLocalServer( // executor which runs on the main thread as we cannot make background // upload tasks run on that thread. Otherwise the application will crash // with a "networking on the main thread" exception. - .setExecutor(Executors.newSingleThreadExecutor()) + .setExecutor(executor) .build() // Initialize WorkManager for instrumentation tests. WorkManagerTestInitHelper.initializeTestWorkManager(context, config) } + + override fun finished(description: Description?) { + executor.awaitTermination(GleanTestRuleConstants.TERMINATION_TIMEOUT, TimeUnit.SECONDS) + + // This closes the database to help prevent leaking it during tests. + // See Bug1719905 for more info. + WorkManagerTestInitHelper.closeWorkDatabase() + + super.finished(description) + } +} + +object GleanTestRuleConstants { + const val TERMINATION_TIMEOUT = 10L } diff --git a/glean-core/android/src/test/java/mozilla/telemetry/glean/GleanTest.kt b/glean-core/android/src/test/java/mozilla/telemetry/glean/GleanTest.kt index ae88d9d7cb..8bc9689154 100644 --- a/glean-core/android/src/test/java/mozilla/telemetry/glean/GleanTest.kt +++ b/glean-core/android/src/test/java/mozilla/telemetry/glean/GleanTest.kt @@ -633,6 +633,7 @@ class GleanTest { } @Test + @Ignore("oops, long running test see Bug 1911350 for more info") fun `overflowing the task queue records telemetry`() { delayMetricsPing(context) val server = getMockWebServer() @@ -651,7 +652,7 @@ class GleanTest { ), ) - repeat(1010) { + repeat(1000010) { counterMetric.add() } @@ -684,7 +685,7 @@ class GleanTest { ) assertEquals( "Ping payload: $jsonContent", - 1000, + 1000000, counters.getInt("telemetry.repeatedly"), ) } diff --git a/glean-core/build/Cargo.toml b/glean-core/build/Cargo.toml index 6089b0d00c..4765e9dc50 100644 --- a/glean-core/build/Cargo.toml +++ b/glean-core/build/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "glean-build" -version = "14.3.0" +version = "14.5.1" edition = "2021" description = "Glean SDK Rust build helper" repository = "https://github.com/mozilla/glean" diff --git a/glean-core/build/src/lib.rs b/glean-core/build/src/lib.rs index 23f6841135..35f2e978a2 100644 --- a/glean-core/build/src/lib.rs +++ b/glean-core/build/src/lib.rs @@ -39,7 +39,7 @@ use std::env; use xshell_venv::{Result, Shell, VirtualEnv}; -const GLEAN_PARSER_VERSION: &str = "14.3.0"; +const GLEAN_PARSER_VERSION: &str = "14.5.1"; /// A Glean Rust bindings generator. pub struct Builder { diff --git a/glean-core/ios/Glean/Config/Configuration.swift b/glean-core/ios/Glean/Config/Configuration.swift index 93f6828dbd..7576fc5cea 100644 --- a/glean-core/ios/Glean/Config/Configuration.swift +++ b/glean-core/ios/Glean/Config/Configuration.swift @@ -13,6 +13,8 @@ public struct Configuration { let enableEventTimestamps: Bool let experimentationId: String? let enableInternalPings: Bool + let pingLifetimeThreshold: Int + let pingLifetimeMaxTime: Int struct Constants { static let defaultTelemetryEndpoint = "https://incoming.telemetry.mozilla.org" @@ -32,6 +34,8 @@ public struct Configuration { /// * experimentationId An experimentation identifier derived by the application /// to be sent with all pings. /// * enableInternalPings Whether to enable internal pings. + /// * pingLifetimeThreshold Write count threshold when to auto-flush. `0` disables it. + /// * pingLifetimeMaxTime After what time to auto-flush (in milliseconds). 0 disables it. public init( maxEvents: Int32? = nil, channel: String? = nil, @@ -40,7 +44,9 @@ public struct Configuration { logLevel: LevelFilter? = nil, enableEventTimestamps: Bool = true, experimentationId: String? = nil, - enableInternalPings: Bool = true + enableInternalPings: Bool = true, + pingLifetimeThreshold: Int = 0, + pingLifetimeMaxTime: Int = 0 ) { self.serverEndpoint = serverEndpoint ?? Constants.defaultTelemetryEndpoint self.maxEvents = maxEvents @@ -50,5 +56,7 @@ public struct Configuration { self.enableEventTimestamps = enableEventTimestamps self.experimentationId = experimentationId self.enableInternalPings = enableInternalPings + self.pingLifetimeThreshold = pingLifetimeThreshold + self.pingLifetimeMaxTime = pingLifetimeMaxTime } } diff --git a/glean-core/ios/Glean/Glean.swift b/glean-core/ios/Glean/Glean.swift index 43424a7928..ea3829c82c 100644 --- a/glean-core/ios/Glean/Glean.swift +++ b/glean-core/ios/Glean/Glean.swift @@ -196,7 +196,9 @@ public class Glean { enableEventTimestamps: configuration.enableEventTimestamps, experimentationId: configuration.experimentationId, enableInternalPings: configuration.enableInternalPings, - pingSchedule: [:] + pingSchedule: [:], + pingLifetimeThreshold: UInt64(configuration.pingLifetimeThreshold), + pingLifetimeMaxTime: UInt64(configuration.pingLifetimeMaxTime) ) let clientInfo = getClientInfo(configuration, buildInfo: buildInfo) let callbacks = OnGleanEventsImpl(glean: self) diff --git a/glean-core/ios/sdk_generator.sh b/glean-core/ios/sdk_generator.sh index 9df4877759..05a9bd60ba 100755 --- a/glean-core/ios/sdk_generator.sh +++ b/glean-core/ios/sdk_generator.sh @@ -25,7 +25,7 @@ set -e -GLEAN_PARSER_VERSION=14.3 +GLEAN_PARSER_VERSION=14.5 # CMDNAME is used in the usage text below. # shellcheck disable=SC2034 diff --git a/glean-core/python/glean/__init__.py b/glean-core/python/glean/__init__.py index 4a34816964..5d60206e12 100644 --- a/glean-core/python/glean/__init__.py +++ b/glean-core/python/glean/__init__.py @@ -30,7 +30,7 @@ __email__ = "glean-team@mozilla.com" -GLEAN_PARSER_VERSION = "14.3.0" +GLEAN_PARSER_VERSION = "14.5.1" parser_version = VersionInfo.parse(GLEAN_PARSER_VERSION) parser_version_next_major = parser_version.bump_major() diff --git a/glean-core/python/glean/glean.py b/glean-core/python/glean/glean.py index b6ed757a2e..088306427f 100644 --- a/glean-core/python/glean/glean.py +++ b/glean-core/python/glean/glean.py @@ -235,6 +235,8 @@ def initialize( experimentation_id=configuration.experimentation_id, enable_internal_pings=configuration.enable_internal_pings, ping_schedule={}, + ping_lifetime_threshold=0, + ping_lifetime_max_time=0, ) _uniffi.glean_initialize(cfg, client_info, callbacks) diff --git a/glean-core/python/glean/net/ping_upload_worker.py b/glean-core/python/glean/net/ping_upload_worker.py index 008b07f719..6c00cbda15 100644 --- a/glean-core/python/glean/net/ping_upload_worker.py +++ b/glean-core/python/glean/net/ping_upload_worker.py @@ -120,6 +120,8 @@ def _process(data_dir: Path, application_id: str, configuration) -> bool: experimentation_id=None, enable_internal_pings=False, ping_schedule={}, + ping_lifetime_threshold=0, + ping_lifetime_max_time=0, ) if not glean_initialize_for_subprocess(cfg): log.error("Couldn't initialize Glean in subprocess") diff --git a/glean-core/python/requirements_dev.txt b/glean-core/python/requirements_dev.txt index bba7d59340..418c18cc45 100644 --- a/glean-core/python/requirements_dev.txt +++ b/glean-core/python/requirements_dev.txt @@ -13,6 +13,6 @@ semver==2.13.0 setuptools-git==1.2 twine==5.0.0 types-pkg_resources==0.1.3 -wheel==0.42.0 +wheel==0.44.0 maturin==1.6.0 patchelf>=0.17; sys_platform == "linux" diff --git a/glean-core/python/tests/data/events_with_types.yaml b/glean-core/python/tests/data/events_with_types.yaml index a33da488ab..142dbfa581 100644 --- a/glean-core/python/tests/data/events_with_types.yaml +++ b/glean-core/python/tests/data/events_with_types.yaml @@ -25,4 +25,7 @@ core: swapped: type: quantity description: "This is key three" + And1WithUnusualCASING: + type: boolean + description: "This is key four" expires: never diff --git a/glean-core/rlb/Cargo.toml b/glean-core/rlb/Cargo.toml index 42ba287a2b..bb2562a66e 100644 --- a/glean-core/rlb/Cargo.toml +++ b/glean-core/rlb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "glean" -version = "60.4.0" +version = "60.5.0" authors = ["Jan-Erik Rediger ", "The Glean Team "] description = "Glean SDK Rust language bindings" repository = "https://github.com/mozilla/glean" @@ -23,7 +23,7 @@ maintenance = { status = "actively-developed" } [dependencies.glean-core] path = ".." -version = "60.4.0" +version = "60.5.0" [dependencies] inherent = "1" @@ -40,5 +40,5 @@ libc = "0.2" serde_json = "1.0.44" tempfile = "3.1.0" -[features] -preinit_million_queue = ["glean-core/preinit_million_queue"] +[[example]] +name = "ping-lifetime-flush" diff --git a/glean-core/rlb/examples/ping-lifetime-flush.rs b/glean-core/rlb/examples/ping-lifetime-flush.rs new file mode 100644 index 0000000000..2aedbe20bd --- /dev/null +++ b/glean-core/rlb/examples/ping-lifetime-flush.rs @@ -0,0 +1,136 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fs::File; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::time::Duration; +use std::{env, process, thread}; + +use once_cell::sync::Lazy; + +use flate2::read::GzDecoder; +use glean::net; +use glean::{private::PingType, ClientInfoMetrics, ConfigurationBuilder}; + +pub mod glean_metrics { + use glean::{private::CounterMetric, CommonMetricData, Lifetime}; + + #[allow(non_upper_case_globals)] + pub static sample_counter: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(|| { + CounterMetric::new(CommonMetricData { + name: "sample_counter".into(), + category: "test.metrics".into(), + send_in_pings: vec!["prototype".into()], + disabled: false, + lifetime: Lifetime::Ping, + ..Default::default() + }) + }); +} + +#[derive(Debug)] +struct MovingUploader(PathBuf); + +impl MovingUploader { + fn new(mut path: PathBuf) -> Self { + path.push("sent_pings"); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl net::PingUploader for MovingUploader { + fn upload(&self, upload_request: net::PingUploadRequest) -> net::UploadResult { + let net::PingUploadRequest { + body, url, headers, .. + } = upload_request; + let mut gzip_decoder = GzDecoder::new(&body[..]); + let mut s = String::with_capacity(body.len()); + + let data = gzip_decoder + .read_to_string(&mut s) + .ok() + .map(|_| &s[..]) + .or_else(|| std::str::from_utf8(&body).ok()) + .unwrap(); + + let docid = url.rsplit('/').next().unwrap(); + let out_path = self.0.join(format!("{docid}.json")); + let mut fp = File::create(out_path).unwrap(); + + // pseudo-JSON, let's hope this works. + writeln!(fp, "{{").unwrap(); + writeln!(fp, " \"url\": {url},").unwrap(); + for (key, val) in headers { + writeln!(fp, " \"{key}\": \"{val}\",").unwrap(); + } + writeln!(fp, "}}").unwrap(); + writeln!(fp, "{data}").unwrap(); + + net::UploadResult::http_status(200) + } +} + +#[allow(non_upper_case_globals)] +pub static PrototypePing: Lazy = + Lazy::new(|| PingType::new("prototype", true, true, false, true, true, vec![], vec![])); + +fn main() { + env_logger::init(); + + let mut args = env::args().skip(1); + + let data_path = PathBuf::from(args.next().expect("need data path")); + let state = args.next().unwrap_or_default(); + + let uploader = MovingUploader::new(data_path.clone()); + let cfg = ConfigurationBuilder::new(true, data_path, "glean.pingflush") + .with_server_endpoint("invalid-test-host") + .with_use_core_mps(false) + .with_uploader(uploader) + .with_delay_ping_lifetime_io(true) + .with_ping_lifetime_threshold(1000) + .with_ping_lifetime_max_time(Duration::from_millis(2000)) + .build(); + + let client_info = ClientInfoMetrics { + app_build: env!("CARGO_PKG_VERSION").to_string(), + app_display_version: env!("CARGO_PKG_VERSION").to_string(), + channel: None, + locale: None, + }; + + glean::initialize(cfg, client_info); + + // Wait for init to finish, + // otherwise we might be to quick with calling `shutdown`. + let _ = glean_metrics::sample_counter.test_get_value(None); + + match &*state { + "accumulate_one_and_pretend_crash" => { + log::debug!("incrementing by 1. exiting without shutdown."); + glean_metrics::sample_counter.add(1) + } + "accumulate_ten_and_wait" => { + log::debug!("incrementing by 10, waiting, incrementing again. should trigger a flush."); + glean_metrics::sample_counter.add(10); + thread::sleep(Duration::from_millis(2500)); + glean_metrics::sample_counter.add(10); + // give it some time to work + thread::sleep(Duration::from_millis(100)); + } + "submit_ping" => { + log::info!("submitting PrototypePing"); + PrototypePing.submit(None); + + glean::shutdown(); + } + _ => { + eprintln!("unknown argument: {state}"); + process::exit(1); + } + } +} diff --git a/glean-core/rlb/src/configuration.rs b/glean-core/rlb/src/configuration.rs index 1f786254e1..12220009d3 100644 --- a/glean-core/rlb/src/configuration.rs +++ b/glean-core/rlb/src/configuration.rs @@ -8,6 +8,7 @@ use crate::net::PingUploader; use std::collections::HashMap; use std::path::PathBuf; +use std::time::Duration; /// The default server pings are sent to. pub(crate) const DEFAULT_GLEAN_ENDPOINT: &str = "https://incoming.telemetry.mozilla.org"; @@ -53,6 +54,10 @@ pub struct Configuration { /// Maps a ping name to a list of pings to schedule along with it. /// Only used if the ping's own ping schedule list is empty. pub ping_schedule: HashMap>, + /// Write count threshold when to auto-flush. `0` disables it. + pub ping_lifetime_threshold: usize, + /// After what time to auto-flush. 0 disables it. + pub ping_lifetime_max_time: Duration, } /// Configuration builder. @@ -105,6 +110,10 @@ pub struct Builder { /// Maps a ping name to a list of pings to schedule along with it. /// Only used if the ping's own ping schedule list is empty. pub ping_schedule: HashMap>, + /// Write count threshold when to auto-flush. `0` disables it. + pub ping_lifetime_threshold: usize, + /// After what time to auto-flush. 0 disables it. + pub ping_lifetime_max_time: Duration, } impl Builder { @@ -130,6 +139,8 @@ impl Builder { experimentation_id: None, enable_internal_pings: true, ping_schedule: HashMap::new(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: Duration::ZERO, } } @@ -151,6 +162,8 @@ impl Builder { experimentation_id: self.experimentation_id, enable_internal_pings: self.enable_internal_pings, ping_schedule: self.ping_schedule, + ping_lifetime_threshold: self.ping_lifetime_threshold, + ping_lifetime_max_time: self.ping_lifetime_max_time, } } @@ -213,4 +226,16 @@ impl Builder { self.ping_schedule = value; self } + + /// Write count threshold when to auto-flush. `0` disables it. + pub fn with_ping_lifetime_threshold(mut self, value: usize) -> Self { + self.ping_lifetime_threshold = value; + self + } + + /// After what time to auto-flush. 0 disables it. + pub fn with_ping_lifetime_max_time(mut self, value: Duration) -> Self { + self.ping_lifetime_max_time = value; + self + } } diff --git a/glean-core/rlb/src/lib.rs b/glean-core/rlb/src/lib.rs index 34b3670526..2aa2ba900b 100644 --- a/glean-core/rlb/src/lib.rs +++ b/glean-core/rlb/src/lib.rs @@ -124,6 +124,8 @@ fn initialize_internal(cfg: Configuration, client_info: ClientInfoMetrics) -> Op experimentation_id: cfg.experimentation_id, enable_internal_pings: cfg.enable_internal_pings, ping_schedule: cfg.ping_schedule, + ping_lifetime_threshold: cfg.ping_lifetime_threshold as u64, + ping_lifetime_max_time: cfg.ping_lifetime_max_time.as_millis() as u64, }; glean_core::glean_initialize(core_cfg, client_info.into(), callbacks); diff --git a/glean-core/rlb/src/private/object.rs b/glean-core/rlb/src/private/object.rs index f7403ec889..6e9a684bfd 100644 --- a/glean-core/rlb/src/private/object.rs +++ b/glean-core/rlb/src/private/object.rs @@ -189,4 +189,62 @@ mod test { let expected = json!([1, 2, 3]); assert_eq!(expected, data); } + + #[test] + fn set_string_api_complex() { + let _lock = lock_test(); + let _t = new_glean(None, true); + + #[derive( + Debug, Hash, Eq, PartialEq, traits::__serde::Deserialize, traits::__serde::Serialize, + )] + #[serde(crate = "traits::__serde")] + #[serde(deny_unknown_fields)] + struct StackTrace { + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + #[serde( + skip_serializing_if = "Vec::is_empty", + default = "Vec::new", + deserialize_with = "traits::__serde_helper::vec_null" + )] + modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + thread_info: Option, + } + + #[derive( + Debug, Hash, Eq, PartialEq, traits::__serde::Serialize, traits::__serde::Deserialize, + )] + #[serde(crate = "traits::__serde")] + #[serde(deny_unknown_fields)] + struct StackTraceThreadInfo { + base_address: Option, + } + + let metric: ObjectMetric = ObjectMetric::new(CommonMetricData { + name: "object".into(), + category: "test".into(), + send_in_pings: vec!["test1".into()], + ..Default::default() + }); + + let arr_str = json!({ + "error": "error", + "modules": null, + "thread_info": null, + }) + .to_string(); + metric.set_string(arr_str); + + let data = metric.test_get_value(None).expect("no object recorded"); + let expected = json!({ + "error": "error" + }); + assert_eq!(expected, data); + assert_eq!( + 0, + metric.test_get_num_recorded_errors(ErrorType::InvalidValue) + ); + } } diff --git a/glean-core/rlb/tests/overflowing_preinit.rs b/glean-core/rlb/tests/overflowing_preinit.rs index 6d4ec7f6ae..a82ec67955 100644 --- a/glean-core/rlb/tests/overflowing_preinit.rs +++ b/glean-core/rlb/tests/overflowing_preinit.rs @@ -54,6 +54,7 @@ mod metrics { /// The pre-init dispatcher queue records how many recordings over the limit it saw. /// /// This is an integration test to avoid dealing with resetting the dispatcher. +#[ignore] // oops, long running test see Bug 1911350 for more info #[test] fn overflowing_the_task_queue_records_telemetry() { common::enable_test_logging(); @@ -67,14 +68,14 @@ fn overflowing_the_task_queue_records_telemetry() { .build(); // Insert a bunch of tasks to overflow the queue. - for _ in 0..1010 { + for _ in 0..1000010 { metrics::rapid_counting.add(1); } // Now initialize Glean common::initialize(cfg); - assert_eq!(Some(1000), metrics::rapid_counting.test_get_value(None)); + assert_eq!(Some(1000000), metrics::rapid_counting.test_get_value(None)); // The metrics counts the total number of overflowing tasks, // (and the count of tasks in the queue when we overflowed: bug 1764573) diff --git a/glean-core/rlb/tests/test-ping-lifetime-flush.sh b/glean-core/rlb/tests/test-ping-lifetime-flush.sh new file mode 100755 index 0000000000..2de2edb335 --- /dev/null +++ b/glean-core/rlb/tests/test-ping-lifetime-flush.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Test harness for testing the RLB processes from the outside. +# +# Some behavior can only be observed when properly exiting the process running Glean, +# e.g. when an uploader runs in another thread. +# On exit the threads will be killed, regardless of their state. + +# Remove the temporary data path on all exit conditions +cleanup() { + if [ -n "$datapath" ]; then + rm -r "$datapath" + fi +} +trap cleanup INT ABRT TERM EXIT + +tmp="${TMPDIR:-/tmp}" +datapath=$(mktemp -d "${tmp}/glean_ping_lifetime_flush.XXXX") + +cmd="cargo run -p glean --example ping-lifetime-flush -- $datapath" + +# First run "crashes" -> no increment stored +$cmd accumulate_one_and_pretend_crash +count=$(ls -1q "$datapath/sent_pings" | wc -l) +if [[ "$count" -ne 0 ]]; then + echo "test result: FAILED." + exit 101 +fi + +# Second run increments, waits, increments -> increment flushed to disk. +# No ping is sent. +$cmd accumulate_ten_and_wait +count=$(ls -1q "$datapath/sent_pings" | wc -l) +if [[ "$count" -ne 0 ]]; then + echo "test result: FAILED." + exit 101 +fi + +# Third run sends the ping. +$cmd submit_ping +count=$(ls -1q "$datapath/sent_pings" | wc -l) +if [[ "$count" -ne 1 ]]; then + echo "test result: FAILED." + exit 101 +fi + +if ! grep -q '"test.metrics.sample_counter":20' "$datapath"/sent_pings/*; then + echo "test result: FAILED." + exit 101 +fi + +echo "test result: ok." +exit 0 diff --git a/glean-core/src/core/mod.rs b/glean-core/src/core/mod.rs index a9a37ba390..b38db1bd76 100644 --- a/glean-core/src/core/mod.rs +++ b/glean-core/src/core/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use chrono::{DateTime, FixedOffset}; use once_cell::sync::OnceCell; @@ -122,6 +123,8 @@ where /// experimentation_id: None, /// enable_internal_pings: true, /// ping_schedule: Default::default(), +/// ping_lifetime_threshold: 1000, +/// ping_lifetime_max_time: 2000, /// }; /// let mut glean = Glean::new(cfg).unwrap(); /// let ping = PingType::new("sample", true, false, true, true, true, vec![], vec![]); @@ -250,7 +253,14 @@ impl Glean { // Creating the data store creates the necessary path as well. // If that fails we bail out and don't initialize further. let data_path = Path::new(&cfg.data_path); - glean.data_store = Some(Database::new(data_path, cfg.delay_ping_lifetime_io)?); + let ping_lifetime_threshold = cfg.ping_lifetime_threshold as usize; + let ping_lifetime_max_time = Duration::from_millis(cfg.ping_lifetime_max_time); + glean.data_store = Some(Database::new( + data_path, + cfg.delay_ping_lifetime_io, + ping_lifetime_threshold, + ping_lifetime_max_time, + )?); // Set experimentation identifier (if any) if let Some(experimentation_id) = &cfg.experimentation_id { @@ -329,6 +339,8 @@ impl Glean { experimentation_id: None, enable_internal_pings, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }; let mut glean = Self::new(cfg).unwrap(); @@ -594,7 +606,13 @@ impl Glean { /// Gets the maximum number of events to store before sending a ping. pub fn get_max_events(&self) -> usize { - self.max_events as usize + let remote_settings_config = self.remote_settings_config.lock().unwrap(); + + if let Some(max_events) = remote_settings_config.event_threshold { + max_events as usize + } else { + self.max_events as usize + } } /// Gets the next task for an uploader. @@ -783,6 +801,8 @@ impl Glean { .pings_enabled .extend(cfg.pings_enabled); + remote_settings_config.event_threshold = cfg.event_threshold; + // Update remote_settings epoch self.remote_settings_epoch.fetch_add(1, Ordering::SeqCst); } diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index 95778d8f7a..20cb0f151e 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::fs; @@ -10,9 +10,9 @@ use std::io; use std::num::NonZeroU64; use std::path::Path; use std::str; -#[cfg(target_os = "android")] use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::RwLock; +use std::time::{Duration, Instant}; use crate::ErrorKind; @@ -183,13 +183,6 @@ use crate::Glean; use crate::Lifetime; use crate::Result; -/// The number of writes we accept writes to the ping-lifetime in-memory map -/// before data is flushed to disk. -/// -/// Only considered if `delay_ping_lifetime_io` is set to `true`. -#[cfg(target_os = "android")] -const PING_LIFETIME_THRESHOLD: usize = 1000; - pub struct Database { /// Handle to the database environment. rkv: Rkv, @@ -209,12 +202,25 @@ pub struct Database { /// A count of how many database writes have been done since the last ping-lifetime flush. /// - /// A ping-lifetime flush is automatically done after `PING_LIFETIME_THRESHOLD` writes. + /// A ping-lifetime flush is automatically done after `ping_lifetime_threshold` writes. /// /// Only relevant if `delay_ping_lifetime_io` is set to `true`, - #[cfg(target_os = "android")] ping_lifetime_count: AtomicUsize, + /// Write-count threshold when to auto-flush. `0` disables it. + ping_lifetime_threshold: usize, + + /// The last time the `lifetime=ping` data was flushed to disk. + /// + /// Data is flushed to disk automatically when the last flush was more than + /// `ping_lifetime_max_time` ago. + /// + /// Only relevant if `delay_ping_lifetime_io` is set to `true`, + ping_lifetime_store_ts: Cell, + + /// After what time to auto-flush. 0 disables it. + ping_lifetime_max_time: Duration, + /// Initial file size when opening the database. file_size: Option, @@ -276,7 +282,12 @@ impl Database { /// /// It also loads any Lifetime::Ping data that might be /// persisted, in case `delay_ping_lifetime_io` is set. - pub fn new(data_path: &Path, delay_ping_lifetime_io: bool) -> Result { + pub fn new( + data_path: &Path, + delay_ping_lifetime_io: bool, + ping_lifetime_threshold: usize, + ping_lifetime_max_time: Duration, + ) -> Result { let path = data_path.join("db"); log::debug!("Database path: {:?}", path.display()); let file_size = database_size(&path); @@ -296,14 +307,18 @@ impl Database { // The value was chosen at random. let write_timings = RefCell::new(Vec::with_capacity(64)); + let now = Instant::now(); + let db = Self { rkv, user_store, ping_store, application_store, ping_lifetime_data, - #[cfg(target_os = "android")] ping_lifetime_count: AtomicUsize::new(0), + ping_lifetime_threshold, + ping_lifetime_store_ts: Cell::new(now), + ping_lifetime_max_time, file_size, rkv_load_state, write_timings, @@ -851,8 +866,8 @@ impl Database { .expect("Can't read ping lifetime data"); // We can reset the write-counter. Current data has been persisted. - #[cfg(target_os = "android")] self.ping_lifetime_count.store(0, Ordering::Release); + self.ping_lifetime_store_ts.replace(Instant::now()); self.write_with_store(Lifetime::Ping, |mut writer, store| { for (key, value) in data.iter() { @@ -874,36 +889,54 @@ impl Database { &self, data: &BTreeMap, ) -> Result<()> { - #[cfg(target_os = "android")] - { - self.ping_lifetime_count.fetch_add(1, Ordering::Release); + if self.ping_lifetime_threshold == 0 && self.ping_lifetime_max_time.is_zero() { + log::trace!("Auto-flush disabled."); + return Ok(()); + } - let write_count = self.ping_lifetime_count.load(Ordering::Relaxed); - if write_count < PING_LIFETIME_THRESHOLD { - return Ok(()); - } + let write_count = self.ping_lifetime_count.fetch_add(1, Ordering::Release) + 1; + let last_write = self.ping_lifetime_store_ts.get(); + let elapsed = last_write.elapsed(); - self.ping_lifetime_count.store(0, Ordering::Release); - let write_result = self.write_with_store(Lifetime::Ping, |mut writer, store| { - for (key, value) in data.iter() { - let encoded = - bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed"); - // There is no need for `get_storage_key` here because - // the key is already formatted from when it was saved - // to ping_lifetime_data. - store.put(&mut writer, key, &rkv::Value::Blob(&encoded))?; - } - writer.commit()?; - Ok(()) - }); + if (self.ping_lifetime_threshold == 0 || write_count < self.ping_lifetime_threshold) + && (self.ping_lifetime_max_time.is_zero() || elapsed < self.ping_lifetime_max_time) + { + log::trace!( + "Not flushing. write_count={} (threshold={}), elapsed={:?} (max={:?})", + write_count, + self.ping_lifetime_threshold, + elapsed, + self.ping_lifetime_max_time + ); + return Ok(()); + } - return write_result; + if self.ping_lifetime_threshold > 0 && write_count >= self.ping_lifetime_threshold { + log::debug!( + "Flushing database due to threshold of {} reached.", + self.ping_lifetime_threshold + ) + } else if !self.ping_lifetime_max_time.is_zero() && elapsed >= self.ping_lifetime_max_time { + log::debug!( + "Flushing database due to last write more than {:?} ago", + self.ping_lifetime_max_time + ); } - #[cfg(not(target_os = "android"))] - { - _ = data; // suppress unused_variables warning. + + self.ping_lifetime_count.store(0, Ordering::Release); + self.ping_lifetime_store_ts.replace(Instant::now()); + self.write_with_store(Lifetime::Ping, |mut writer, store| { + for (key, value) in data.iter() { + let encoded = + bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed"); + // There is no need for `get_storage_key` here because + // the key is already formatted from when it was saved + // to ping_lifetime_data. + store.put(&mut writer, key, &rkv::Value::Blob(&encoded))?; + } + writer.commit()?; Ok(()) - } + }) } } @@ -917,7 +950,7 @@ mod test { #[test] fn test_panicks_if_fails_dir_creation() { let path = Path::new("/!#\"'@#°ç"); - assert!(Database::new(path, false).is_err()); + assert!(Database::new(path, false, 0, Duration::ZERO).is_err()); } #[test] @@ -935,7 +968,7 @@ mod test { let dir = tempdir().unwrap(); let path = dir.path().join(os_str); - let res = Database::new(&path, false); + let res = Database::new(&path, false, 0, Duration::ZERO); assert!( res.is_ok(), @@ -959,7 +992,7 @@ mod test { let dir = tempdir().unwrap(); let path = dir.path().join(os_str); - let res = Database::new(&path, false); + let res = Database::new(&path, false, 0, Duration::ZERO); assert!( res.is_ok(), "Database should not fail at {}: {:?}", @@ -982,7 +1015,7 @@ mod test { let dir = tempdir().unwrap(); let path = dir.path().join(os_str); - let res = Database::new(&path, false); + let res = Database::new(&path, false, 0, Duration::ZERO); assert!( res.is_err(), "Database should not fail at {}: {:?}", @@ -994,7 +1027,7 @@ mod test { #[test] fn test_data_dir_rkv_inits() { let dir = tempdir().unwrap(); - Database::new(dir.path(), false).unwrap(); + Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(dir.path().exists()); } @@ -1003,7 +1036,7 @@ mod test { fn test_ping_lifetime_metric_recorded() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(db.ping_lifetime_data.is_none()); @@ -1039,7 +1072,7 @@ mod test { fn test_application_lifetime_metric_recorded() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); // Attempt to record a known value. let test_value = "test-value"; @@ -1076,7 +1109,7 @@ mod test { fn test_user_lifetime_metric_recorded() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); // Attempt to record a known value. let test_value = "test-value"; @@ -1110,7 +1143,7 @@ mod test { fn test_clear_ping_storage() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); // Attempt to record a known value for every single lifetime. let test_storage = "test-storage"; @@ -1185,7 +1218,7 @@ mod test { fn test_remove_single_metric() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let test_storage = "test-storage-single-lifetime"; let metric_id_pattern = "telemetry_test.single_metric"; @@ -1241,7 +1274,7 @@ mod test { fn test_delayed_ping_lifetime_persistence() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); let test_storage = "test-storage"; assert!(db.ping_lifetime_data.is_some()); @@ -1357,7 +1390,7 @@ mod test { let test_metric_id = "telemetry_test.test_name"; { - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); // Attempt to record a known value. db.record_per_lifetime( @@ -1396,7 +1429,7 @@ mod test { // Now create a new instace of the db and check if data was // correctly loaded from rkv to memory. { - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); // Verify that test_value is in memory. let data = match &db.ping_lifetime_data { @@ -1425,7 +1458,7 @@ mod test { fn test_delayed_ping_lifetime_clear() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); let test_storage = "test-storage"; assert!(db.ping_lifetime_data.is_some()); @@ -1498,7 +1531,7 @@ mod test { // Attempt to record metric with the record and record_with functions, // this should work since upload is enabled. - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); db.record(&glean, &test_data, &Metric::String("record".to_owned())); db.iter_store_from( Lifetime::Ping, @@ -1595,7 +1628,7 @@ mod test { let f = File::create(safebin).expect("create database file"); drop(f); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(dir.path().exists()); assert!( @@ -1616,7 +1649,7 @@ mod test { let safebin = database_dir.join("data.safe.bin"); fs::write(safebin, "").expect("write to database file"); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(dir.path().exists()); assert!( @@ -1665,7 +1698,7 @@ mod test { // First open should migrate the data. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); @@ -1685,7 +1718,7 @@ mod test { // Next open should not re-create the LMDB files. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file exists"); assert!(!datamdb.exists(), "LMDB data should not be recreated"); @@ -1764,7 +1797,7 @@ mod test { // First open should try migration and ignore it, because destination is not empty. // It also deletes the leftover LMDB database. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); @@ -1829,7 +1862,7 @@ mod test { // First open should try migration and ignore it, because destination is not empty. // It also deletes the leftover LMDB database. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); @@ -1874,7 +1907,7 @@ mod test { // safe-mode does not write an empty database to disk. // It also deletes the leftover LMDB database. { - let _db = Database::new(dir.path(), false).unwrap(); + let _db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(!safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); diff --git a/glean-core/src/dispatcher/global.rs b/glean-core/src/dispatcher/global.rs index f90a681a5e..0c1f2a6fa0 100644 --- a/glean-core/src/dispatcher/global.rs +++ b/glean-core/src/dispatcher/global.rs @@ -11,10 +11,7 @@ use std::time::Duration; use super::{DispatchError, DispatchGuard, Dispatcher}; use crossbeam_channel::RecvTimeoutError; -#[cfg(feature = "preinit_million_queue")] pub const GLOBAL_DISPATCHER_LIMIT: usize = 1000000; -#[cfg(not(feature = "preinit_million_queue"))] -pub const GLOBAL_DISPATCHER_LIMIT: usize = 1000; static GLOBAL_DISPATCHER: Lazy>> = Lazy::new(|| RwLock::new(Some(Dispatcher::new(GLOBAL_DISPATCHER_LIMIT)))); diff --git a/glean-core/src/glean.udl b/glean-core/src/glean.udl index a94e0d54dc..bc8ca9ec36 100644 --- a/glean-core/src/glean.udl +++ b/glean-core/src/glean.udl @@ -94,6 +94,8 @@ dictionary InternalConfiguration { string? experimentation_id; boolean enable_internal_pings; record> ping_schedule; + u64 ping_lifetime_threshold; + u64 ping_lifetime_max_time; // in millis }; // How to specify the rate pings may be uploaded before they are throttled. diff --git a/glean-core/src/lib.rs b/glean-core/src/lib.rs index ad38e83ddc..5558f490b7 100644 --- a/glean-core/src/lib.rs +++ b/glean-core/src/lib.rs @@ -143,6 +143,11 @@ pub struct InternalConfiguration { /// Maps a ping name to a list of pings to schedule along with it. /// Only used if the ping's own ping schedule list is empty. pub ping_schedule: HashMap>, + + /// Write count threshold when to auto-flush. `0` disables it. + pub ping_lifetime_threshold: u64, + /// After what time to auto-flush. 0 disables it. + pub ping_lifetime_max_time: u64, } /// How to specify the rate at which pings may be uploaded before they are throttled. diff --git a/glean-core/src/lib_unit_tests.rs b/glean-core/src/lib_unit_tests.rs index 635ba39736..7ea320858a 100644 --- a/glean-core/src/lib_unit_tests.rs +++ b/glean-core/src/lib_unit_tests.rs @@ -199,6 +199,8 @@ fn experimentation_id_is_set_correctly() { experimentation_id: Some(experimentation_id.to_string()), enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }) .unwrap(); diff --git a/glean-core/src/metrics/remote_settings_config.rs b/glean-core/src/metrics/remote_settings_config.rs index e7a560e81c..ad89e27606 100644 --- a/glean-core/src/metrics/remote_settings_config.rs +++ b/glean-core/src/metrics/remote_settings_config.rs @@ -31,6 +31,12 @@ pub struct RemoteSettingsConfig { /// enabled state of the ping of the same name. #[serde(default)] pub pings_enabled: HashMap, + + /// The threshold of events that will be buffered before an events ping is + /// collected and submitted. + /// It overrides the value configured at initialization time. + #[serde(default)] + pub event_threshold: Option, } impl RemoteSettingsConfig { diff --git a/glean-core/src/traits/mod.rs b/glean-core/src/traits/mod.rs index 4115609fdd..d292862613 100644 --- a/glean-core/src/traits/mod.rs +++ b/glean-core/src/traits/mod.rs @@ -54,3 +54,16 @@ pub use self::timing_distribution::TimingDistribution; pub use self::url::Url; pub use self::uuid::Uuid; pub use crate::histogram::HistogramType; + +#[doc(hidden)] +pub mod __serde_helper { + /// Deserialize value to a `Vec`. + /// + /// If the value was `null` return an empty vector. + pub fn vec_null<'de, D: serde::Deserializer<'de>, T: serde::Deserialize<'de>>( + d: D, + ) -> Result, D::Error> { + let res: Option> = serde::Deserialize::deserialize(d)?; + Ok(res.unwrap_or_default()) + } +} diff --git a/glean-core/tests/common/mod.rs b/glean-core/tests/common/mod.rs index 5891169374..7bc08436df 100644 --- a/glean-core/tests/common/mod.rs +++ b/glean-core/tests/common/mod.rs @@ -65,6 +65,8 @@ pub fn new_glean(tempdir: Option) -> (Glean, tempfile::TempDi experimentation_id: None, enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }; let glean = Glean::new(cfg).unwrap(); diff --git a/glean-core/tests/event.rs b/glean-core/tests/event.rs index 5495087871..dcc9668a7f 100644 --- a/glean-core/tests/event.rs +++ b/glean-core/tests/event.rs @@ -8,6 +8,8 @@ use crate::common::*; use std::collections::HashMap; use std::fs; +use serde_json::json; + use glean_core::metrics::*; use glean_core::{ get_timestamp_ms, test_get_num_recorded_errors, CommonMetricData, ErrorType, Lifetime, @@ -221,6 +223,82 @@ fn test_sending_of_event_ping_when_it_fills_up() { } } +#[test] +fn test_server_knobs_config_changing_max_events() { + let (mut glean, _t) = new_glean(None); + + let store_names: Vec = vec!["events".into()]; + + for store_name in &store_names { + glean.register_ping_type(&PingType::new( + store_name.clone(), + true, + false, + true, + true, + true, + vec![], + vec!["max_capacity".to_string()], + )); + } + + // 1. Set up an event to record + let click = EventMetric::new( + CommonMetricData { + name: "click".into(), + category: "ui".into(), + send_in_pings: store_names, + disabled: false, + lifetime: Lifetime::Ping, + ..Default::default() + }, + vec!["test_event_number".into()], + ); + + // 2. Set a Server Knobs configuration to disable the metrics + let remote_settings_config = json!( + { + "event_threshold": 50 + } + ) + .to_string(); + glean + .apply_server_knobs_config(RemoteSettingsConfig::try_from(remote_settings_config).unwrap()); + + // 3. Record 51 events. We expect to get the first 50 in the first ping and 1 + // remaining afterward + for i in 0..51 { + let mut extra = HashMap::new(); + extra.insert("test_event_number".to_string(), i.to_string()); + click.record_sync(&glean, i, extra, 0); + } + + assert_eq!(1, click.get_value(&glean, "events").unwrap().len()); + + let (url, json, _) = &get_queued_pings(glean.get_data_path()).unwrap()[0]; + assert!(url.starts_with(format!("/submit/{}/events/", glean.get_application_id()).as_str())); + assert_eq!(50, json["events"].as_array().unwrap().len()); + assert_eq!( + "max_capacity", + json["ping_info"].as_object().unwrap()["reason"] + .as_str() + .unwrap() + ); + + for i in 0..50 { + let event = &json["events"].as_array().unwrap()[i]; + assert_eq!(i.to_string(), event["extra"]["test_event_number"]); + } + + let snapshot = glean + .event_storage() + .snapshot_as_json(&glean, "events", false) + .unwrap(); + assert_eq!(1, snapshot.as_array().unwrap().len()); + let event = &snapshot.as_array().unwrap()[0]; + assert_eq!(50.to_string(), event["extra"]["test_event_number"]); +} + #[test] fn extra_keys_must_be_recorded_and_truncated_if_needed() { let (glean, _t) = new_glean(None); @@ -487,6 +565,8 @@ fn with_event_timestamps() { experimentation_id: None, // Enabling event timestamps enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }; let glean = Glean::new(cfg).unwrap(); diff --git a/glean-core/tests/ping_maker.rs b/glean-core/tests/ping_maker.rs index 242029fec3..32d5fc777a 100644 --- a/glean-core/tests/ping_maker.rs +++ b/glean-core/tests/ping_maker.rs @@ -93,6 +93,8 @@ fn test_metrics_must_report_experimentation_id() { experimentation_id: Some("test-experimentation-id".to_string()), enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }) .unwrap(); let ping_maker = PingMaker::new(); @@ -147,6 +149,8 @@ fn experimentation_id_is_removed_if_send_if_empty_is_false() { experimentation_id: Some("test-experimentation-id".to_string()), enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }) .unwrap(); let ping_maker = PingMaker::new(); diff --git a/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy b/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy index d43f7c6a69..34f6013f71 100644 --- a/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy +++ b/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy @@ -50,7 +50,7 @@ abstract class GleanMetricsYamlTransform implements TransformAction { // The version of glean_parser to install from PyPI. - private String GLEAN_PARSER_VERSION = "14.3" + private String GLEAN_PARSER_VERSION = "14.5" // The version of Miniconda is explicitly specified. // Miniconda3-4.5.12 is known to not work on Windows. private String MINICONDA_VERSION = "24.3.0-0" @@ -552,7 +552,7 @@ except: void apply(Project project) { isOffline = project.gradle.startParameter.offline - project.ext.glean_version = "60.4.0" + project.ext.glean_version = "60.5.0" def parserVersion = gleanParserVersion(project) // Print the required glean_parser version to the console. This is diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 21db8eaa07..5ca9dd3c6b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,10 +16,10 @@ kotlinx-serialization = "1.6.3" rust-android-gradle = "0.9.4" # AndroidX -androidx-annotation = "1.8.0" +androidx-annotation = "1.8.1" androidx-appcompat = "1.6.1" androidx-browser = "1.8.0" -androidx-lifecycle = "2.7.0" +androidx-lifecycle = "2.8.4" androidx-work = "2.9.0" # JNA @@ -40,10 +40,10 @@ androidx-test-uiautomator = "2.3.0" junit = "4.13.2" mockito = "5.12.0" mockwebserver = "4.12.0" -robolectric = "4.12.1" +robolectric = "4.13" # Miscellaneous Gradle plugins -jacoco = "0.8.11" +jacoco = "0.8.12" python-envs = "0.0.31" [libraries] diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e6441136f3..2c3521197d 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4413138c9..09523c0e54 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index b740cf1339..f5feea6d6b 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 7101f8e467..9b42019c79 100755 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## diff --git a/pyproject.toml b/pyproject.toml index 48c0414f29..bcb807d75d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "glean-sdk" -version = "60.4.0" +version = "60.5.0" requires-python = ">=3.8" classifiers = [ "Intended Audience :: Developers", @@ -22,7 +22,7 @@ maintainers = [ dependencies = [ "semver>=2.13.0", - "glean_parser~=14.3", + "glean_parser~=14.5", ] [project.urls] diff --git a/samples/android/app/metrics.yaml b/samples/android/app/metrics.yaml index 6597c8ef96..aa6c8e9128 100644 --- a/samples/android/app/metrics.yaml +++ b/samples/android/app/metrics.yaml @@ -28,6 +28,9 @@ browser.engagement: key2: type: string description: "This is key two" + And1WithUnusualCASING: + type: boolean + description: "This is key three" expires: 2 metadata: tags: @@ -198,6 +201,10 @@ party: type: string diameter: type: number + labels: + type: array + items: + type: string animals: type: object description: > diff --git a/samples/android/app/src/main/java/org/mozilla/samples/gleancore/MainActivity.kt b/samples/android/app/src/main/java/org/mozilla/samples/gleancore/MainActivity.kt index 33530d617b..36b6bfe023 100644 --- a/samples/android/app/src/main/java/org/mozilla/samples/gleancore/MainActivity.kt +++ b/samples/android/app/src/main/java/org/mozilla/samples/gleancore/MainActivity.kt @@ -48,7 +48,9 @@ open class MainActivity : AppCompatActivity() { // Record an object val balloons = Party.BalloonsObject() - balloons.add(Party.BalloonsObjectItem(colour = "red", diameter = 5)) + val labels = Party.BalloonsObjectItemLabels() + labels.add("round") + balloons.add(Party.BalloonsObjectItem(colour = "red", diameter = 5, labels = labels)) balloons.add(Party.BalloonsObjectItem(colour = "green")) Party.balloons.set(balloons) diff --git a/samples/ios/app/glean-sample-app/ViewController.swift b/samples/ios/app/glean-sample-app/ViewController.swift index 82a6ed4d6c..72da0d1d94 100644 --- a/samples/ios/app/glean-sample-app/ViewController.swift +++ b/samples/ios/app/glean-sample-app/ViewController.swift @@ -59,7 +59,9 @@ class ViewController: UIViewController { // Record an object var balloons: Party.BalloonsObject = [] - balloons.append(Party.BalloonsObjectItem(colour: "red", diameter: 5)) + var labels: Party.BalloonsObjectItemLabels = [] + labels.append("round") + balloons.append(Party.BalloonsObjectItem(colour: "red", diameter: 5, labels: labels)) balloons.append(Party.BalloonsObjectItem(colour: "green")) Party.balloons.set(balloons) diff --git a/samples/ios/app/glean-sample-appUITests/ViewControllerTest.swift b/samples/ios/app/glean-sample-appUITests/ViewControllerTest.swift index d8d04e10fd..1c370815b4 100644 --- a/samples/ios/app/glean-sample-appUITests/ViewControllerTest.swift +++ b/samples/ios/app/glean-sample-appUITests/ViewControllerTest.swift @@ -44,6 +44,13 @@ class ViewControllerTest: XCTestCase { XCTAssertEqual(value, expectedValue) } + func checkObjectData() { + let metrics = lastPingJson!["metrics"] as! [String: Any] + let objects = metrics["object"] as! [String: Any] + let balloons = objects["party.balloons"] as! [Any] + XCTAssertEqual(balloons.count, 2) + } + func testViewControllerInteraction() { let server = setupServer(expectPingType: "sample") @@ -77,6 +84,7 @@ class ViewControllerTest: XCTestCase { } checkCustomCounterData(expectedValue: 4) + checkObjectData() server.stop() } diff --git a/samples/ios/app/metrics.yaml b/samples/ios/app/metrics.yaml index 6302504feb..ad68e51e1f 100644 --- a/samples/ios/app/metrics.yaml +++ b/samples/ios/app/metrics.yaml @@ -28,6 +28,9 @@ browser.engagement: key2: type: string description: "This is key two" + And1WithUnusualCASING: + type: boolean + description: "This is key three" expires: 2100-01-01 no_lint: - EXPIRATION_DATE_TOO_FAR @@ -178,6 +181,8 @@ party: notification_emails: - CHANGE-ME@example.com expires: never + send_in_pings: + - sample structure: type: array items: @@ -187,7 +192,10 @@ party: type: string diameter: type: number - + labels: + type: array + items: + type: string animals: type: object description: > @@ -201,6 +209,8 @@ party: data_sensitivity: - technical expires: never + send_in_pings: + - sample structure: type: array items: diff --git a/samples/rust/Cargo.toml b/samples/rust/Cargo.toml index 6c397056db..a377535724 100644 --- a/samples/rust/Cargo.toml +++ b/samples/rust/Cargo.toml @@ -9,6 +9,7 @@ license = "MIT" env_logger = { version = "0.10.0", default-features = false, features = ["humantime"] } flate2 = "1.0.19" glean = { path = "../../glean-core/rlb" } +serde_json = "1.0.44" tempfile = "3.3.0" [build-dependencies] diff --git a/samples/rust/metrics.yaml b/samples/rust/metrics.yaml index 367a437c84..fe9bc86e47 100644 --- a/samples/rust/metrics.yaml +++ b/samples/rust/metrics.yaml @@ -68,6 +68,17 @@ test.metrics: - aLabel - 2label + sample_event: + <<: *defaults + type: event + extra_keys: + key_one: + type: quantity + description: "This is key one" + And1WithUnusualCASING: + type: boolean + description: "This is key two" + party: balloons: type: object @@ -91,3 +102,28 @@ party: type: string diameter: type: number + + drinks: + type: object + description: | + Just testing objects + bugs: + - https://bugzilla.mozilla.org/1910809 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com + expires: never + send_in_pings: + - prototype + structure: + type: array + items: + type: object + properties: + name: + type: string + ingredients: + type: array + items: + type: string diff --git a/samples/rust/src/main.rs b/samples/rust/src/main.rs index a221cb31a9..54a546bf0e 100644 --- a/samples/rust/src/main.rs +++ b/samples/rust/src/main.rs @@ -12,7 +12,7 @@ use std::time::Duration; use tempfile::Builder; use flate2::read::GzDecoder; -use glean::{net, ClientInfoMetrics, ConfigurationBuilder}; +use glean::{net, ClientInfoMetrics, ConfigurationBuilder, ErrorType}; pub mod glean_metrics { include!(concat!(env!("OUT_DIR"), "/glean_metrics.rs")); @@ -100,6 +100,19 @@ fn main() { ]); glean_metrics::party::balloons.set(balloons); + // Testing with empty and null values. + let drinks = serde_json::json!([ + { "name": "lemonade", "ingredients": ["lemon", "water", "sugar"] }, + { "name": "sparkling-water", "ingredients": [] }, + { "name": "still-water", "ingredients": null }, + ]); + glean_metrics::party::drinks.set_string(drinks.to_string()); + + assert_eq!( + 0, + glean_metrics::party::drinks.test_get_num_recorded_errors(ErrorType::InvalidValue) + ); + glean_metrics::prototype.submit(None); // Need to wait a short time for Glean to actually act. thread::sleep(Duration::from_millis(100)); diff --git a/taskcluster/scripts/cross-compile-setup.sh b/taskcluster/scripts/cross-compile-setup.sh index 06ae95c586..3a1fadd59f 100755 --- a/taskcluster/scripts/cross-compile-setup.sh +++ b/taskcluster/scripts/cross-compile-setup.sh @@ -10,9 +10,9 @@ export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_TOOLCHA export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_AR=/builds/worker/cctools/bin/x86_64-apple-darwin-ar export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_RANLIB=/builds/worker/cctools/bin/x86_64-apple-darwin-ranlib export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_LD_LIBRARY_PATH=/builds/worker/clang/lib -export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C linker=/builds/worker/clang/bin/clang -C link-arg=-fuse-ld=/builds/worker/cctools/bin/x86_64-apple-darwin-ld -C link-arg=-B -C link-arg=/builds/worker/cctools/bin -C link-arg=-target -C link-arg=x86_64-apple-darwin -C link-arg=-isysroot -C link-arg=/tmp/MacOSX10.12.sdk -C link-arg=-Wl,-syslibroot,/tmp/MacOSX10.12.sdk -C link-arg=-Wl,-dead_strip" -export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_CFLAGS_x86_64_apple_darwin="-B /builds/worker/cctools/bin -target x86_64-apple-darwin -isysroot /tmp/MacOSX10.12.sdk -Wl,-syslibroot,/tmp/MacOSX10.12.sdk -Wl,-dead_strip" - +export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C linker=/builds/worker/clang/bin/clang -C link-arg=-fuse-ld=/builds/worker/cctools/bin/x86_64-apple-darwin-ld -C link-arg=-B -C link-arg=/builds/worker/cctools/bin -C link-arg=-target -C link-arg=x86_64-apple-darwin -C link-arg=-isysroot -C link-arg=/tmp/MacOSX11.0.sdk -C link-arg=-Wl,-syslibroot,/tmp/MacOSX11.0.sdk -C link-arg=-Wl,-dead_strip" +# For ring's use of `cc`. +export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_CFLAGS_x86_64_apple_darwin="-B /builds/worker/cctools/bin -target x86_64-apple-darwin -isysroot /tmp/MacOSX11.0.sdk -Wl,-syslibroot,/tmp/MacOSX11.0.sdk -Wl,-dead_strip" # aarch64 Darwin (M1/Silicon) export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_AARCH64_APPLE_DARWIN_CC=/builds/worker/clang/bin/clang export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_AARCH64_APPLE_DARWIN_TOOLCHAIN_PREFIX=/builds/worker/cctools/bin @@ -45,17 +45,22 @@ export TARGET_CFLAGS="-DNDEBUG" # (drop the "index." prefix, ensure the "public/build" path matches the artifacts of the TC task) pushd /builds/worker curl -sfSL --retry 5 --retry-delay 10 \ - https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.cache.level-3.toolchains.v3.linux64-cctools-port.hash.bec815dd1e64cd75ba803c1f002c646e151c784d804791bffbc23b2ae41f545f/artifacts/public/build/cctools.tar.zst > cctools.tar.zst + https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.cache.level-3.toolchains.v3.linux64-cctools-port.pushdate.2024.07.23.20240723071212/artifacts/public%2Fbuild%2Fcctools.tar.zst > cctools.tar.zst tar -I zstd -xf cctools.tar.zst rm cctools.tar.zst curl -sfSL --retry 5 --retry-delay 10 \ - https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.cache.level-3.toolchains.v3.clang-dist-toolchain.hash.9e5dd392eefd9eb1ef3ff7aa40d2823f60b280cb54d716e0f85f0b50573e0d69/artifacts/public/build/clang-dist-toolchain.tar.xz > clang-dist-toolchain.tar.xz + https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.cache.level-3.toolchains.v3.clang-dist-toolchain.pushdate.2024.07.30.20240730145721/artifacts/public%2Fbuild%2Fclang-dist-toolchain.tar.xz > clang-dist-toolchain.tar.xz tar -xf clang-dist-toolchain.tar.xz mv builds/worker/toolchains/clang clang rm clang-dist-toolchain.tar.xz + +# Fixup symlink +rm /builds/worker/clang/bin/clang +ln -s /builds/worker/clang/bin/clang-18 /builds/worker/clang/bin/clang + popd -pushd /tmp +pushd /tmp || exit tooltool.py \ --url=http://taskcluster/tooltool.mozilla-releng.net/ \ @@ -65,10 +70,16 @@ tooltool.py \ # so we do it manually. tar -I zstd -xf "MacOSX11.0.sdk.tar.zst" +popd || exit + rustup target add x86_64-apple-darwin rustup target add aarch64-apple-darwin rustup target add x86_64-pc-windows-gnu -popd +# Verify paths after extraction and permission changes +echo "Verifying paths after extraction" +ls -la /builds/worker/clang/bin +file /builds/worker/clang/bin/clang +file /builds/worker/clang/bin/clang-18 set +eu