feat!: decode with coreDiscoveryId rather than coreDiscoveryKey

BREAKING CHANGE: this changes the expected
parameter for decode() to be coreDiscoveryId, to
match an upstream change in the multi-core-indexer
to use coreDiscoveryId rather than
coreDiscoveryKey. This reduces a round-trip
conversion from buffer to string, and will also
facilitate sending entries between threads when we
run indexing in a separate worker thread.

src/decode.ts

@@ -25,7 +25,7 @@ import cenc from 'compact-encoding'
 import { DATA_TYPE_ID_BYTES, SCHEMA_VERSION_BYTES } from './constants.js'
 import {
   ExhaustivenessError,
-  VersionIdObject,
+  VERSION_ID_SEPARATOR,
   getOwn,
   getProtoTypeName,
 } from './lib/utils.js'
@@ -46,9 +46,10 @@ for (const [schemaName, dataTypeId] of Object.entries(dataTypeIds) as Array<
  * */
 export function decode(
   buf: Buffer,
-  versionObj: VersionIdObject
+  { index, coreDiscoveryId }: { index: number; coreDiscoveryId: string }
 ): MapeoDocDecode {
   const schemaDef = decodeBlockPrefix(buf)
+  const versionId = coreDiscoveryId + VERSION_ID_SEPARATOR + index
 
   const encodedMsg = buf.subarray(
     DATA_TYPE_ID_BYTES + SCHEMA_VERSION_BYTES,
@@ -61,25 +62,25 @@ export function decode(
 
   switch (message.schemaName) {
     case 'projectSettings':
-      return convertProjectSettings(message, versionObj)
+      return convertProjectSettings(message, versionId)
     case 'observation':
-      return convertObservation(message, versionObj)
+      return convertObservation(message, versionId)
     case 'field':
-      return convertField(message, versionObj)
+      return convertField(message, versionId)
     case 'preset':
-      return convertPreset(message, versionObj)
+      return convertPreset(message, versionId)
     case 'role':
-      return convertRole(message, versionObj)
+      return convertRole(message, versionId)
     case 'deviceInfo':
-      return convertDeviceInfo(message, versionObj)
+      return convertDeviceInfo(message, versionId)
     case 'coreOwnership':
-      return convertCoreOwnership(message, versionObj)
+      return convertCoreOwnership(message, versionId)
     case 'icon':
-      return convertIcon(message, versionObj)
+      return convertIcon(message, versionId)
     case 'translation':
-      return convertTranslation(message, versionObj)
+      return convertTranslation(message, versionId)
     case 'track':
-      return convertTrack(message, versionObj)
+      return convertTrack(message, versionId)
     default:
       throw new ExhaustivenessError(message)
   }

src/lib/decode-conversions.ts

@@ -35,18 +35,13 @@ import type { Track_1_Position } from '../proto/track/v1.js'
 import { ProjectSettings_1_ConfigMetadata } from '../proto/projectSettings/v1.js'
 import { ProjectSettings } from '../schema/projectSettings.js'
 import type { Position } from '../schema/observation.js'
-import {
-  assert,
-  ExhaustivenessError,
-  getVersionId,
-  VersionIdObject,
-} from './utils.js'
+import { assert, ExhaustivenessError, getVersionId } from './utils.js'
 
 /** Function type for converting a protobuf type of any version for a particular
  * schema name, and returning the most recent JSONSchema type */
 type ConvertFunction<TSchemaName extends SchemaName> = (
   message: Extract<ProtoTypesWithSchemaInfo, { schemaName: TSchemaName }>,
-  versionObj: VersionIdObject
+  versionId: string
 ) => FilterBySchemaName<MapeoDocDecode, TSchemaName>
 
 function ensure(
@@ -59,10 +54,10 @@ function ensure(
 
 export const convertProjectSettings: ConvertFunction<'projectSettings'> = (
   message,
-  versionObj
+  versionId
 ) => {
   const { common, schemaVersion, defaultPresets, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
 
   let configMetadata: undefined | ProjectSettings['configMetadata']
   if (rest.configMetadata) {
@@ -101,10 +96,10 @@ function convertConfigMetadata(
 
 export const convertObservation: ConvertFunction<'observation'> = (
   message,
-  versionObj
+  versionId
 ) => {
   const { common, metadata, schemaVersion, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   let presetRef
 
   if (rest.presetRef) {
@@ -138,9 +133,9 @@ export const convertObservation: ConvertFunction<'observation'> = (
 
 type FieldOptions = FilterBySchemaName<MapeoDoc, 'field'>['options']
 
-export const convertField: ConvertFunction<'field'> = (message, versionObj) => {
+export const convertField: ConvertFunction<'field'> = (message, versionId) => {
   const { common, schemaVersion, tagKey, type, label, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   ensure(tagKey, 'field', 'tagKey')
   return {
     ...jsonSchemaCommon,
@@ -172,10 +167,10 @@ type JsonSchemaPresetGeomItem = FilterBySchemaName<
 
 export const convertPreset: ConvertFunction<'preset'> = (
   message,
-  versionObj
+  versionId
 ) => {
   const { common, schemaVersion, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
 
   const geometry = rest.geometry.filter(
     (geomType): geomType is JsonSchemaPresetGeomItem =>
@@ -209,11 +204,11 @@ export const convertPreset: ConvertFunction<'preset'> = (
   }
 }
 
-export const convertRole: ConvertFunction<'role'> = (message, versionObj) => {
+export const convertRole: ConvertFunction<'role'> = (message, versionId) => {
   const { common, schemaVersion, fromIndex, roleId, ...rest } = message
   ensure(roleId.length, 'role', 'roleId')
   ensure(typeof fromIndex === 'number', 'role', 'fromIndex')
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   return {
     ...jsonSchemaCommon,
     ...rest,
@@ -224,10 +219,10 @@ export const convertRole: ConvertFunction<'role'> = (message, versionObj) => {
 
 export const convertDeviceInfo: ConvertFunction<'deviceInfo'> = (
   message,
-  versionObj
+  versionId
 ) => {
   const { common, schemaVersion, name, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   return {
     ...jsonSchemaCommon,
     ...rest,
@@ -237,7 +232,7 @@ export const convertDeviceInfo: ConvertFunction<'deviceInfo'> = (
 
 export const convertCoreOwnership: ConvertFunction<'coreOwnership'> = (
   message,
-  versionObj
+  versionId
 ) => {
   const {
     common,
@@ -256,7 +251,7 @@ export const convertCoreOwnership: ConvertFunction<'coreOwnership'> = (
   ensure(dataCoreId.byteLength, 'coreOwnership', 'dataCoreId')
   ensure(blobCoreId.byteLength, 'coreOwnership', 'blobCoreId')
   ensure(blobIndexCoreId.byteLength, 'coreOwnership', 'blobIndexCoreId')
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   return {
     ...jsonSchemaCommon,
     ...rest,
@@ -269,9 +264,9 @@ export const convertCoreOwnership: ConvertFunction<'coreOwnership'> = (
   }
 }
 
-export const convertIcon: ConvertFunction<'icon'> = (message, versionObj) => {
+export const convertIcon: ConvertFunction<'icon'> = (message, versionId) => {
   const { common, schemaVersion, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
 
   const variants: Icon['variants'] = []
   for (const variant of message.variants) {
@@ -288,7 +283,7 @@ export const convertIcon: ConvertFunction<'icon'> = (message, versionObj) => {
 
 export const convertTranslation: ConvertFunction<'translation'> = (
   message,
-  versionObj
+  versionId
 ) => {
   const {
     common,
@@ -298,7 +293,7 @@ export const convertTranslation: ConvertFunction<'translation'> = (
     regionCode,
     ...rest
   } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   ensure(message.docRef, 'translation', 'docRef')
   ensure(message.docRef.versionId, 'translation.docRef', 'versionId')
   ensure(propertyRef, 'translation', 'propertyRef')
@@ -316,9 +311,9 @@ export const convertTranslation: ConvertFunction<'translation'> = (
   }
 }
 
-export const convertTrack: ConvertFunction<'track'> = (message, versionObj) => {
+export const convertTrack: ConvertFunction<'track'> = (message, versionId) => {
   const { common, schemaVersion, ...rest } = message
-  const jsonSchemaCommon = convertCommon(common, versionObj)
+  const jsonSchemaCommon = convertCommon(common, versionId)
   const locations = message.locations.map(convertTrackPosition)
 
   const observationRefs: Track['observationRefs'] = []
@@ -465,7 +460,7 @@ function convertTagPrimitive({
 
 function convertCommon(
   common: ProtoTypesWithSchemaInfo['common'],
-  versionObj: VersionIdObject
+  versionId: string
 ): Omit<MapeoCommon, 'schemaName'> {
   if (
     !common ||
@@ -476,8 +471,6 @@ function convertCommon(
     throw new Error('Missing required common properties')
   }
 
-  const versionId = getVersionId(versionObj)
-
   /** @type {string} */ let originalVersionId
   if (common.originalVersionId) {
     originalVersionId = getVersionId(common.originalVersionId)

src/lib/utils.ts

@@ -42,6 +42,8 @@ export type VersionIdObject = {
   index: number
 }
 
+export const VERSION_ID_SEPARATOR = '/'
+
 /**
  * Get a string versionId from a core key and index in that core. A versionId
  * uniquely identifies a record in the underlying Hypercore storage used by
@@ -58,7 +60,7 @@ export function getVersionId({ coreDiscoveryKey, index }: VersionIdObject) {
     Number.isSafeInteger(index) && index >= 0,
     'version ID index must be a non-negative integer'
   )
-  return coreDiscoveryKey.toString('hex') + '/' + index
+  return coreDiscoveryKey.toString('hex') + VERSION_ID_SEPARATOR + index
 }
 
 /**
@@ -69,7 +71,7 @@ export function getVersionId({ coreDiscoveryKey, index }: VersionIdObject) {
  * @returns coreKey as a Buffer and index in the core
  */
 export function parseVersionId(versionId: string): VersionIdObject {
-  const items = versionId.split('/')
+  const items = versionId.split(VERSION_ID_SEPARATOR)
   if (!items[0] || !items[1]) throw new Error('Invalid versionId')
   const coreDiscoveryKey = Buffer.from(items[0], 'hex')
   if (coreDiscoveryKey.length !== 32) throw new Error('Invalid versionId')

test/index.test.js

@@ -6,7 +6,6 @@ import {
   encode,
   decode,
   decodeBlockPrefix,
-  parseVersionId,
   validate,
   valueOf,
 } from '../dist/index.js'
@@ -20,6 +19,7 @@ import {
   badDocs,
 } from './fixtures/index.js'
 import { cachedValues } from './fixtures/cached.js'
+import { VERSION_ID_SEPARATOR } from '../dist/lib/utils.js'
 
 /** @import { SchemaName } from '../dist/types.js' */
 
@@ -341,3 +341,8 @@ function removeBlockPrefix(buf) {
   const blockPrefix = encodeBlockPrefix(decodeBlockPrefix(buf))
   return buf.slice(blockPrefix.byteLength)
 }
+
+function parseVersionId(versionId) {
+  const [coreDiscoveryId, index] = versionId.split(VERSION_ID_SEPARATOR)
+  return { coreDiscoveryId, index: Number.parseInt(index) }
+}