diff --git a/src/core/serializers.ts b/src/core/serializers.ts index 50f91001..4da2cb09 100644 --- a/src/core/serializers.ts +++ b/src/core/serializers.ts @@ -2,7 +2,7 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; +import { Study_Channel, Study_Platform } from '../proto/generated/study'; export function getPlatformNameFromString(protoPlatfrom: string): string { const PREFIX = 'PLATFORM_'; @@ -32,42 +32,3 @@ export function getChannelName( ): string { return getChannelNameFromString(Study_Channel[protoChannel], isBraveSpecific); } - -function unixSecondToUTCString(unixTimeSeconds: number): string { - return new Date(unixTimeSeconds * 1000).toUTCString(); -} - -export function serializePlatforms(platforms?: string[]): string | undefined { - if (platforms === undefined) return undefined; - return platforms.map((v) => getPlatformNameFromString(v)).join(', '); -} - -export function serializeChannels(channels?: string[]): string | undefined { - if (channels === undefined) return undefined; - return channels.join(', '); -} - -// Converts a study to JSON that is ready to be serialized. Some field are -// removed, some are converted to a human readable format. -export function studyToJSON(study: Study): Record { - const json = Study.toJson(study, { useProtoFieldName: true }) as Record< - string, - any - > | null; - if (json === null) { - throw new Error('Failed to convert study to JSON'); - } - const filter = json.filter; - delete json.consistency; - delete json.activation_type; - if (filter !== undefined) { - if (filter.end_date !== undefined) - filter.end_date = unixSecondToUTCString(filter.end_date); - if (filter.start_date !== undefined) { - filter.start_date = unixSecondToUTCString(filter.start_date); - } - filter.platform = serializePlatforms(filter.platform); - filter.channel = serializeChannels(filter.channel); - } - return json; -} diff --git a/src/finch_tracker/main.ts b/src/finch_tracker/main.ts index 27bd543c..b3b9647b 100644 --- a/src/finch_tracker/main.ts +++ b/src/finch_tracker/main.ts @@ -98,7 +98,7 @@ async function main(): Promise { } if (updateData) { - storeDataToDirectory(seedData, storageDir, options); + await storeDataToDirectory(seedData, storageDir, options); if (commitData) { newGitSha1 = commitAllChanges(storageDir); } diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 8f57c577..7967fdb6 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -4,32 +4,42 @@ // You can obtain one at https://mozilla.org/MPL/2.0/. import * as fs from 'fs'; +import * as os from 'os'; import { describe, expect, test } from '@jest/globals'; +import path from 'path'; import { StudyPriority } from '../core/study_processor'; import { ItemAction, makeSummary, summaryToJson } from '../core/summary'; import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; -import { serializeStudies } from './tracker_lib'; - -function serialize(json: Record) { - const ordered = Object.keys(json) - .sort() - .reduce((res: Record, key) => { - res[key] = json[key]; - return res; - }, {}); - return JSON.stringify(ordered, undefined, 2); +import { storeDataToDirectory } from './tracker_lib'; + +function readDirectory(dir: string): string { + const files = fs + .readdirSync(dir, { recursive: true, encoding: 'utf-8' }) + .sort(); + let result = ''; + + for (const file of files) { + const filePath = path.join(dir, file); + if (!file.endsWith('.json5')) { + continue; + } + const content = fs.readFileSync(filePath, 'utf-8'); + result += file + '\n' + content + '\n'; + } + return result; } -test('seed serialization', () => { +test('seed serialization', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracker-')); const data = fs.readFileSync('src/test/data/seed1.bin'); - const map = serializeStudies(data, { + await storeDataToDirectory(data, tempDir, { minMajorVersion: 116, isBraveSeed: true, }); - const serializedOutput = serialize(map); + const serializedOutput = readDirectory(path.join(tempDir)); const serializedExpectations = fs .readFileSync('src/test/data/seed1.bin.processing_expectations') .toString(); diff --git a/src/finch_tracker/tracker_lib.ts b/src/finch_tracker/tracker_lib.ts index 7c1149a6..6a752ebb 100644 --- a/src/finch_tracker/tracker_lib.ts +++ b/src/finch_tracker/tracker_lib.ts @@ -8,7 +8,6 @@ import * as fs from 'fs'; import * as path from 'path'; import { type ProcessingOptions } from '../core/base_types'; -import { studyToJSON } from '../core/serializers'; import { ProcessedStudy, StudyPriority, @@ -16,6 +15,7 @@ import { } from '../core/study_processor'; import { Study } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; +import { writeStudyFile } from '../seed_tools/utils/study_json_utils'; import { downloadUrl, getSeedPath, getStudyPath } from './node_utils'; export async function fetchChromeSeedData(): Promise { @@ -24,18 +24,17 @@ export async function fetchChromeSeedData(): Promise { return await downloadUrl(kChromeSeedUrl); } -// Processes, groups by name and converts to JSON a list of studies. -export function serializeStudies( +// Groups studies by name and priority. +export function groupStudies( seedData: Buffer, options: ProcessingOptions, -): Record { - const map: Record = {}; +): Record { + const map: Record = {}; const seed = VariationsSeed.fromBinary(seedData); const addStudy = (path: string, study: Study) => { - const json = studyToJSON(study); const list = map[path]; - if (list !== undefined) list.push(json); - else map[path] = [json]; + if (list !== undefined) list.push(study); + else map[path] = [study]; }; for (const study of seed.study) { @@ -74,20 +73,20 @@ export function commitAllChanges(directory: string): string | undefined { // Processes and serializes a given seed to disk (including grouping to // subdirectories/files). -export function storeDataToDirectory( +export async function storeDataToDirectory( seedData: Buffer, directory: string, options: ProcessingOptions, -): void { +): Promise { const studyDirectory = getStudyPath(directory); fs.rmSync(studyDirectory, { recursive: true, force: true }); - const map = serializeStudies(seedData, options); + const map = groupStudies(seedData, options); - for (const [name, json] of Object.entries(map)) { - const fileName = `${studyDirectory}/${name}`; + for (const [name, study] of Object.entries(map)) { + const fileName = `${studyDirectory}/${name}.json5`; const dirname = path.dirname(fileName); fs.mkdirSync(dirname, { recursive: true }); - fs.writeFileSync(fileName, JSON.stringify(json, null, 2) + '\n'); + await writeStudyFile(study, fileName, { isChromium: !options.isBraveSeed }); } // TODO: maybe start to use s3 instead of git one day? diff --git a/src/test/data/seed1.bin.processing_expectations b/src/test/data/seed1.bin.processing_expectations index 4a861f33..136a186b 100644 --- a/src/test/data/seed1.bin.processing_expectations +++ b/src/test/data/seed1.bin.processing_expectations @@ -1,298 +1,438 @@ -{ - "all-by-name/BetaStudy": [ - { - "name": "BetaStudy", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } +study/all-by-name/BetaStudy.json5 +[ + { + name: 'BetaStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + ], + filter: { + channel: [ + 'BETA', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "BETA", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/BlocklistedStudy": [ - { - "name": "BlocklistedStudy", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "Ukm" - ] - } - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/BlocklistedStudy.json5 +[ + { + name: 'BlocklistedStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'Ukm', + ], + }, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/EndedByDate": [ - { - "name": "EndedByDate", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/EndedByDate.json5 +[ + { + name: 'EndedByDate', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + ], + filter: { + end_date: '2022-09-14T17:10:24.000Z', + channel: [ + 'RELEASE', ], - "filter": { - "end_date": "Wed, 14 Sep 2022 17:10:24 GMT", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/EndedMaxVersion": [ - { - "name": "EndedMaxVersion", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "max_version": "99.1.49.83", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/Stable-100": [ - { - "name": "Stable-100", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 99, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/EndedMaxVersion.json5 +[ + { + name: 'EndedMaxVersion', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 1 - } + }, + ], + filter: { + max_version: '99.1.49.83', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/Stable-100.json5 +[ + { + name: 'Stable-100', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 99, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Default', + probability_weight: 1, + }, + ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/Stable-50": [ - { - "name": "Stable-50", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 50, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/Stable-50.json5 +[ + { + name: 'Stable-50', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 50, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/Stable-min": [ - { - "name": "Stable-min", - "experiment": [ - { - "name": "Enabled_1", - "probability_weight": 5, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/Stable-min.json5 +[ + { + name: 'Stable-min', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled_1', + probability_weight: 5, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Control_1", - "probability_weight": 45 + }, + { + name: 'Control_1', + probability_weight: 45, + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/StudyKillSwitch.json5 +[ + { + name: 'StudyKillSwitch', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + ], + filter: { + max_version: '299.0.0.0', + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/StudyKillSwitch": [ - { - "name": "StudyKillSwitch", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/blocklisted/BlocklistedStudy.json5 +[ + { + name: 'BlocklistedStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'Ukm', + ], + }, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "max_version": "299.0.0.0", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "blocklisted/BlocklistedStudy": [ - { - "name": "BlocklistedStudy", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "Ukm" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-100%/Stable-100": [ - { - "name": "Stable-100", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 99, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-100%/Stable-100.json5 +[ + { + name: 'Stable-100', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 99, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 1 - } + }, + { + name: 'Default', + probability_weight: 1, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-50%/Stable-50": [ - { - "name": "Stable-50", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 50, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-50%/Stable-50.json5 +[ + { + name: 'Stable-50', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 50, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-emergency-kill-switch/StudyKillSwitch": [ - { - "name": "StudyKillSwitch", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "max_version": "299.0.0.0", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-min/Stable-min": [ - { - "name": "Stable-min", - "experiment": [ - { - "name": "Enabled_1", - "probability_weight": 5, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-emergency-kill-switch/StudyKillSwitch.json5 +[ + { + name: 'StudyKillSwitch', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Control_1", - "probability_weight": 45 + }, + ], + filter: { + max_version: '299.0.0.0', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-min/Stable-min.json5 +[ + { + name: 'Stable-min', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled_1', + probability_weight: 5, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + { + name: 'Control_1', + probability_weight: 45, + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ] -} \ No newline at end of file + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +]