Use seed_tool serialization

src/core/serializers.ts

@@ -2,7 +2,7 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
-import { Study, Study_Channel, Study_Platform } from '../proto/generated/study';
+import { Study_Channel, Study_Platform } from '../proto/generated/study';
 
 export function getPlatformNameFromString(protoPlatfrom: string): string {
   const PREFIX = 'PLATFORM_';
@@ -32,42 +32,3 @@ export function getChannelName(
 ): string {
   return getChannelNameFromString(Study_Channel[protoChannel], isBraveSpecific);
 }
-
-function unixSecondToUTCString(unixTimeSeconds: number): string {
-  return new Date(unixTimeSeconds * 1000).toUTCString();
-}
-
-export function serializePlatforms(platforms?: string[]): string | undefined {
-  if (platforms === undefined) return undefined;
-  return platforms.map((v) => getPlatformNameFromString(v)).join(', ');
-}
-
-export function serializeChannels(channels?: string[]): string | undefined {
-  if (channels === undefined) return undefined;
-  return channels.join(', ');
-}
-
-// Converts a study to JSON that is ready to be serialized. Some field are
-// removed, some are converted to a human readable format.
-export function studyToJSON(study: Study): Record<string, any> {
-  const json = Study.toJson(study, { useProtoFieldName: true }) as Record<
-    string,
-    any
-  > | null;
-  if (json === null) {
-    throw new Error('Failed to convert study to JSON');
-  }
-  const filter = json.filter;
-  delete json.consistency;
-  delete json.activation_type;
-  if (filter !== undefined) {
-    if (filter.end_date !== undefined)
-      filter.end_date = unixSecondToUTCString(filter.end_date);
-    if (filter.start_date !== undefined) {
-      filter.start_date = unixSecondToUTCString(filter.start_date);
-    }
-    filter.platform = serializePlatforms(filter.platform);
-    filter.channel = serializeChannels(filter.channel);
-  }
-  return json;
-}

src/finch_tracker/main.ts

@@ -98,7 +98,7 @@ async function main(): Promise<void> {
   }
 
   if (updateData) {
-    storeDataToDirectory(seedData, storageDir, options);
+    await storeDataToDirectory(seedData, storageDir, options);
     if (commitData) {
       newGitSha1 = commitAllChanges(storageDir);
     }

src/finch_tracker/tracker_lib.test.ts

@@ -4,32 +4,42 @@
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import * as fs from 'fs';
+import * as os from 'os';
 
 import { describe, expect, test } from '@jest/globals';
+import path from 'path';
 import { StudyPriority } from '../core/study_processor';
 import { ItemAction, makeSummary, summaryToJson } from '../core/summary';
 import { Study, Study_Channel, Study_Platform } from '../proto/generated/study';
 import { VariationsSeed } from '../proto/generated/variations_seed';
-import { serializeStudies } from './tracker_lib';
-
-function serialize(json: Record<string, any>) {
-  const ordered = Object.keys(json)
-    .sort()
-    .reduce((res: Record<string, any>, key) => {
-      res[key] = json[key];
-      return res;
-    }, {});
-  return JSON.stringify(ordered, undefined, 2);
+import { storeDataToDirectory } from './tracker_lib';
+
+function readDirectory(dir: string): string {
+  const files = fs
+    .readdirSync(dir, { recursive: true, encoding: 'utf-8' })
+    .sort();
+  let result = '';
+
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    if (!file.endsWith('.json5')) {
+      continue;
+    }
+    const content = fs.readFileSync(filePath, 'utf-8');
+    result += file + '\n' + content + '\n';
+  }
+  return result;
 }
 
-test('seed serialization', () => {
+test('seed serialization', async () => {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracker-'));
   const data = fs.readFileSync('src/test/data/seed1.bin');
-  const map = serializeStudies(data, {
+  await storeDataToDirectory(data, tempDir, {
     minMajorVersion: 116,
     isBraveSeed: true,
   });
-  const serializedOutput = serialize(map);
 
+  const serializedOutput = readDirectory(path.join(tempDir));
   const serializedExpectations = fs
     .readFileSync('src/test/data/seed1.bin.processing_expectations')
     .toString();

src/finch_tracker/tracker_lib.ts

@@ -8,14 +8,14 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import { type ProcessingOptions } from '../core/base_types';
-import { studyToJSON } from '../core/serializers';
 import {
   ProcessedStudy,
   StudyPriority,
   priorityToText,
 } from '../core/study_processor';
 import { Study } from '../proto/generated/study';
 import { VariationsSeed } from '../proto/generated/variations_seed';
+import { writeStudyFile } from '../seed_tools/utils/study_json_utils';
 import { downloadUrl, getSeedPath, getStudyPath } from './node_utils';
 
 export async function fetchChromeSeedData(): Promise<Buffer> {
@@ -24,18 +24,17 @@ export async function fetchChromeSeedData(): Promise<Buffer> {
   return await downloadUrl(kChromeSeedUrl);
 }
 
-// Processes, groups by name and converts to JSON a list of studies.
-export function serializeStudies(
+// Groups studies by name and priority.
+export function groupStudies(
   seedData: Buffer,
   options: ProcessingOptions,
-): Record<string, any[]> {
-  const map: Record<string, any[]> = {};
+): Record<string, Study[]> {
+  const map: Record<string, Study[]> = {};
   const seed = VariationsSeed.fromBinary(seedData);
   const addStudy = (path: string, study: Study) => {
-    const json = studyToJSON(study);
     const list = map[path];
-    if (list !== undefined) list.push(json);
-    else map[path] = [json];
+    if (list !== undefined) list.push(study);
+    else map[path] = [study];
   };
 
   for (const study of seed.study) {
@@ -74,20 +73,20 @@ export function commitAllChanges(directory: string): string | undefined {
 
 // Processes and serializes a given seed to disk (including grouping to
 // subdirectories/files).
-export function storeDataToDirectory(
+export async function storeDataToDirectory(
   seedData: Buffer,
   directory: string,
   options: ProcessingOptions,
-): void {
+): Promise<void> {
   const studyDirectory = getStudyPath(directory);
   fs.rmSync(studyDirectory, { recursive: true, force: true });
-  const map = serializeStudies(seedData, options);
+  const map = groupStudies(seedData, options);
 
-  for (const [name, json] of Object.entries(map)) {
-    const fileName = `${studyDirectory}/${name}`;
+  for (const [name, study] of Object.entries(map)) {
+    const fileName = `${studyDirectory}/${name}.json5`;
     const dirname = path.dirname(fileName);
     fs.mkdirSync(dirname, { recursive: true });
-    fs.writeFileSync(fileName, JSON.stringify(json, null, 2) + '\n');
+    await writeStudyFile(study, fileName, { isChromium: !options.isBraveSeed });
   }
 
   // TODO: maybe start to use s3 instead of git one day?