From d699ad2b0be45da98921ba0c3a4602962c0e45d3 Mon Sep 17 00:00:00 2001 From: cdm-publisher Date: Thu, 19 Oct 2023 18:33:12 +0000 Subject: [PATCH] Microsoft CDM Release Version Update: --- objectModel/CSharp/Directory.Build.props | 4 +- .../DataPartitionPatternTest.cs | 14 +++ .../Cdm/CdmDataPartitionPatternDefinition.cs | 47 +++++++- .../CdmLocalEntityDeclarationDefinition.cs | 14 ++- .../Cdm/CdmManifestDefinition.cs | 9 +- .../Enums/CdmLogCode.cs | 2 + .../Resx/LogMessages.Designer.cs | 18 +++ .../Resx/LogMessages.resx | 6 + .../Utilities/FileStatusCheckOptions.cs | 2 + objectModel/Java/objectmodel/pom.xml | 4 +- objectModel/Java/pom.xml | 2 +- objectModel/Python/cdm/enums/cdm_log_code.py | 1 + .../cdm_data_partition_pattern_def.py | 32 +++-- .../cdm_local_entity_declaration_def.py | 11 +- .../cdm/objectmodel/cdm_manifest_def.py | 5 +- objectModel/Python/cdm/resx/logmessages.txt | 1 + .../utilities/file_status_check_options.py | 2 + objectModel/Python/setup.py | 2 +- .../test_data_partition_pattern.py | 9 ++ .../Cdm/CdmDataPartitionPatternDefinition.ts | 112 +++++++++++++++--- .../TypeScript/Cdm/CdmDocumentDefinition.ts | 3 + .../CdmLocalEntityDeclarationDefinition.ts | 19 ++- .../TypeScript/Cdm/CdmManifestDefinition.ts | 6 +- .../TypeScript/CdmStandards/package-lock.json | 4 +- .../TypeScript/CdmStandards/package.json | 2 +- objectModel/TypeScript/Enums/cdmLogCode.ts | 1 + .../LocalEntityDeclarationPersistence.ts | 2 +- .../ModelJson/ManifestPersistence.ts | 3 - .../Utilities/fileStatusCheckOptions.ts | 4 +- .../DataPartitionPattern.test.ts | 12 ++ .../Persistence/ModelJson/modelJson.test.ts | 17 +++ .../ModelJson/modelJsonExtensibility.test.ts | 4 +- .../Persistence/PersistenceLayer.test.ts | 3 +- objectModel/TypeScript/package-lock.json | 21 +++- objectModel/TypeScript/package.json | 2 +- objectModel/TypeScript/resx/logMessages.json | 1 + objectModel/TypeScript/tsconfig.json | 4 +- 37 files changed, 346 insertions(+), 59 deletions(-) diff --git a/objectModel/CSharp/Directory.Build.props b/objectModel/CSharp/Directory.Build.props index f89a696a15..212c196023 100644 --- a/objectModel/CSharp/Directory.Build.props +++ b/objectModel/CSharp/Directory.Build.props @@ -1,9 +1,9 @@ - 1.7.3 + 1.7.4 2.8.0 2.8.0 - 1.7.3-preview1 + 1.7.4-preview1 $(MSBuildThisFileDirectory)\CodeCoverage.runsettings \ No newline at end of file diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs index 2fb3f68b89..4d5a09acf5 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs @@ -919,5 +919,19 @@ public async Task TestFetchAllFilesMetadata() var fetchNullManifest = await corpus.FetchObjectAsync("fetchNull:/manifest.manifest.cdm.json"); await fetchNullManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions); } + + /// + /// Test RegexTimeout is handled correctly + /// + [TestMethod] + public async Task TestRegexTimeout() + { + var outOfRangeExpectedLogCodes = new HashSet { CdmLogCode.ErrRegexTimeoutOutOfRange }; + CdmCorpusDefinition outOfRangeCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestFetchAllFilesMetadata), expectedCodes: outOfRangeExpectedLogCodes); + var fileStatusCheckOptions = new FileStatusCheckOptions() { RegexTimeoutSeconds = 0 }; + + var outOfRangeManifest = await outOfRangeCorpus.FetchObjectAsync("manifest.manifest.cdm.json"); + await outOfRangeManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions); + } } } diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs index 923bd7230f..7b62186c4d 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs @@ -20,6 +20,11 @@ public class CdmDataPartitionPatternDefinition : CdmObjectDefinitionBase, CdmFil { private static readonly string Tag = nameof(CdmDataPartitionPatternDefinition); + /// + /// Regex timeout value + /// + private static TimeSpan DefaultRegexTimeoutValue = TimeSpan.FromSeconds(1); + private TraitToPropertyMap TraitToPropertyMap { get; } /// @@ -182,6 +187,11 @@ public override bool IsDerivedFrom(string baseName, ResolveOptions resOpt) /// public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions) + { + await this.FileStatusCheckAsyncInternal(fileStatusCheckOptions); + } + + internal async Task FileStatusCheckAsyncInternal(FileStatusCheckOptions fileStatusCheckOptions) { using (Logger.EnterScope(nameof(CdmDataPartitionPatternDefinition), Ctx, nameof(FileStatusCheckAsync))) { @@ -204,7 +214,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt if (pathTuple == null) { Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrStorageNullCorpusPath); - return; + return true; } nameSpace = pathTuple.Item1; @@ -213,7 +223,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt if (adapter == null) { Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrDocAdapterNotFound, this.InDocument.Name); - return; + return true; } // get a list of all corpusPaths under the root @@ -230,7 +240,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt if (fileInfoList == null) { Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrFetchingFileMetadataNull, nameSpace); - return; + return true; } if (nameSpace != null) @@ -261,9 +271,20 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt string regularExpression = !String.IsNullOrWhiteSpace(this.GlobPattern) ? this.GlobPatternToRegex(this.GlobPattern) : this.RegularExpression; Regex regexPattern = null; + TimeSpan regexTimeoutValue = fileStatusCheckOptions?.RegexTimeoutSeconds != null ? TimeSpan.FromSeconds(fileStatusCheckOptions.RegexTimeoutSeconds.Value) : DefaultRegexTimeoutValue; + try { - regexPattern = new Regex(regularExpression); + regexPattern = new Regex(regularExpression, RegexOptions.None, regexTimeoutValue); + } + catch (ArgumentOutOfRangeException rangeEx) + { + Logger.Error( + this.Ctx, + Tag, + nameof(FileStatusCheckAsync), + this.AtCorpusPath, + CdmLogCode.ErrRegexTimeoutOutOfRange, regexTimeoutValue.ToString(), rangeEx.Message); } catch (Exception e) { @@ -302,7 +323,20 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt string fileName = fi.Key; CdmFileMetadata partitionMetadata = fi.Value; - Match m = regexPattern.Match(fileName); + Match m; + + try + { + m = regexPattern.Match(fileName); + } + catch (RegexMatchTimeoutException e) + { + Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrRegexTimeout, e.Message); + + // do not continue processing the manifest/entity/partition pattern if timeout + return false; + } + if (m.Success && m.Length > 1 && m.Value == fileName) { // create a map of arguments out of capture groups @@ -337,7 +371,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt if (pathTuple == null) { Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrStorageNullCorpusPath, this.AtCorpusPath); - return; + return true; } CdmTraitCollection exhibitsTraits = this.ExhibitsTraits; @@ -374,6 +408,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt } } } + return true; } } diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmLocalEntityDeclarationDefinition.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmLocalEntityDeclarationDefinition.cs index 11b10c8cfc..a7c3967df2 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmLocalEntityDeclarationDefinition.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmLocalEntityDeclarationDefinition.cs @@ -215,6 +215,11 @@ public async Task FileStatusCheckAsync() } public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null) + { + await FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions); + } + + internal async Task FileStatusCheckAsyncInternal(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null) { using ((this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace) as StorageAdapterBase)?.CreateFileQueryCacheContext()) { @@ -236,7 +241,12 @@ public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFil } else { - await pattern.FileStatusCheckAsync(fileStatusCheckOptions); + bool shouldContinue = await pattern.FileStatusCheckAsyncInternal(fileStatusCheckOptions); + + if (!shouldContinue) + { + return false; + } } } @@ -276,6 +286,8 @@ public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFil this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime); await this.ReportMostRecentTimeAsync(this.LastFileModifiedTime); + + return true; } } diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmManifestDefinition.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmManifestDefinition.cs index 30697a5214..ead90b1078 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmManifestDefinition.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmManifestDefinition.cs @@ -561,9 +561,14 @@ public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFil { await entity.FileStatusCheckAsync(); } - else if (entity is CdmLocalEntityDeclarationDefinition) + else if (entity is CdmLocalEntityDeclarationDefinition localEntity) { - await (entity as CdmLocalEntityDeclarationDefinition).FileStatusCheckAsync(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions); + bool shouldContinue = await localEntity.FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions); + + if (!shouldContinue) + { + return; + } } } diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Enums/CdmLogCode.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Enums/CdmLogCode.cs index 01b50c68c0..3c375e1046 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Enums/CdmLogCode.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Enums/CdmLogCode.cs @@ -107,6 +107,8 @@ public enum CdmLogCode ErrProjStringError, ErrProjUnsupportedAttrGroups, ErrProjUnsupportedSource, + ErrRegexTimeout, + ErrRegexTimeoutOutOfRange, ErrRelMaxResolvedAttrReached, ErrResolutionFailure, ErrResolveEntityFailure, diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.Designer.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.Designer.cs index 5fe41ba527..67b0293556 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.Designer.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.Designer.cs @@ -1005,6 +1005,24 @@ internal static string ErrProjUnsupportedSource { } } + /// + /// Looks up a localized string similar to The regex calculation timed out.. + /// + internal static string ErrRegexTimeout { + get { + return ResourceManager.GetString("ErrRegexTimeout", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Regex timeout '{0}' is not valid. Reason '{1}'. + /// + internal static string ErrRegexTimeoutOutOfRange { + get { + return ResourceManager.GetString("ErrRegexTimeoutOutOfRange", resourceCulture); + } + } + /// /// Looks up a localized string similar to Maximum number of resolved attributes reached for the entity: {0}.. /// diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx index 131b7890c6..64beee50f3 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx @@ -687,4 +687,10 @@ Unmounting the cdm namespace in order to use offline mode has been deprecated. The default behavior will now be using the CdmStandards package instead of the online schema store. Please remove this Unmount API call. + + The regex calculation timed out. + + + Regex timeout '{0}' is not valid. Reason '{1}' + \ No newline at end of file diff --git a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/FileStatusCheckOptions.cs b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/FileStatusCheckOptions.cs index 15b2b19764..5d796bb74c 100644 --- a/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/FileStatusCheckOptions.cs +++ b/objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/FileStatusCheckOptions.cs @@ -6,5 +6,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Utilities public class FileStatusCheckOptions { public bool IncludeDataPartitionSize { get; set; } + + public double? RegexTimeoutSeconds { get; set; } } } diff --git a/objectModel/Java/objectmodel/pom.xml b/objectModel/Java/objectmodel/pom.xml index 06ed0eb600..83a14f3f88 100644 --- a/objectModel/Java/objectmodel/pom.xml +++ b/objectModel/Java/objectmodel/pom.xml @@ -21,7 +21,7 @@ 1.15 2.13.2 2.13.4.2 - 8 + 9 1.11.0 UTF-8 1.5.0 @@ -179,8 +179,6 @@ ${java.version} - 1.8 - 1.8 org.apache.maven.plugins maven-compiler-plugin diff --git a/objectModel/Java/pom.xml b/objectModel/Java/pom.xml index 878ce32f20..7bf75a1f01 100644 --- a/objectModel/Java/pom.xml +++ b/objectModel/Java/pom.xml @@ -5,7 +5,7 @@ - 1.7.3 + 1.7.4 2.8.0 diff --git a/objectModel/Python/cdm/enums/cdm_log_code.py b/objectModel/Python/cdm/enums/cdm_log_code.py index b44c200c20..dc140f4fae 100644 --- a/objectModel/Python/cdm/enums/cdm_log_code.py +++ b/objectModel/Python/cdm/enums/cdm_log_code.py @@ -107,6 +107,7 @@ class CdmLogCode(AutoNumber): ERR_PROJ_STRING_ERROR = () ERR_PROJ_UNSUPPORTED_ATTR_GROUPS = () ERR_PROJ_UNSUPPORTED_SOURCE = () + ERR_REGEX_TIMEOUT = () ERR_REL_MAX_RESOLVED_ATTR_REACHED = () ERR_RESOLUTION_FAILURE = () ERR_RESOLVE_ENTITY_FAILURE = () diff --git a/objectModel/Python/cdm/objectmodel/cdm_data_partition_pattern_def.py b/objectModel/Python/cdm/objectmodel/cdm_data_partition_pattern_def.py index 04838e641a..d3a7cc7f23 100644 --- a/objectModel/Python/cdm/objectmodel/cdm_data_partition_pattern_def.py +++ b/objectModel/Python/cdm/objectmodel/cdm_data_partition_pattern_def.py @@ -6,9 +6,8 @@ from typing import cast, Dict, List, Optional, TYPE_CHECKING import regex -from cdm.enums import CdmLogCode, PartitionFileStatusCheckType, CdmIncrementalPartitionType -from cdm.enums import CdmObjectType -from cdm.utilities import logger, ResolveOptions, StorageUtils, TraitToPropertyMap +from cdm.enums import CdmLogCode, CdmObjectType +from cdm.utilities import FileStatusCheckOptions, logger, ResolveOptions, StorageUtils, TraitToPropertyMap from .cdm_file_status import CdmFileStatus from .cdm_local_entity_declaration_def import CdmLocalEntityDeclarationDefinition @@ -19,6 +18,7 @@ from cdm.objectmodel import CdmCorpusContext from cdm.utilities import VisitCallback +regex_timeout = 1 class CdmDataPartitionPatternDefinition(CdmObjectDefinition, CdmFileStatus): def __init__(self, ctx: 'CdmCorpusContext', name: str) -> None: @@ -94,6 +94,9 @@ def copy(self, res_opt: Optional['ResolveOptions'] = None, host: Optional['CdmDa return copy async def file_status_check_async(self, file_status_check_options = None) -> None: + await self._file_status_check_async_internal(file_status_check_options) + + async def _file_status_check_async_internal(self, file_status_check_options: Optional[FileStatusCheckOptions] = None) -> None: """Check the modified time for this object and any children.""" with logger._enter_scope(self._TAG, self.ctx, self.file_status_check_async.__name__): namespace = None @@ -108,7 +111,7 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non path_tuple = StorageUtils.split_namespace_path(root_corpus) if not path_tuple: logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH) - return + return True namespace = path_tuple[0] adapter = self.ctx.corpus.storage.fetch_adapter(namespace) @@ -129,7 +132,7 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non if file_info_list is None: logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_FETCHING_FILE_METADATA_NULL, namespace) - return + return True if namespace is not None: # remove root of the search from the beginning of all paths so anything in the root is not found by regex. @@ -167,8 +170,20 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non incremental_partition_location_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(incremental_partition.location, self.in_document) incremental_partition_path_hash_set.add(incremental_partition_location_full_path) + if file_status_check_options is not None and 'regex_timeout_seconds' in file_status_check_options: + configured_regex_timeout = file_status_check_options['regex_timeout_seconds'] + else: + configured_regex_timeout = regex_timeout + for file_name,partition_metadata in cleaned_file_list.items(): - m = reg.fullmatch(file_name) + try: + m = reg.fullmatch(file_name, timeout=configured_regex_timeout) + except TimeoutError as e: + logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_REGEX_TIMEOUT) + + # do not continue processing the manifest/entity/partition pattern if timeout + return False + if m: # create a map of arguments out of capture groups. args = defaultdict(list) # type: Dict[str, List[str]] @@ -193,10 +208,10 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non path_tuple = StorageUtils.split_namespace_path(full_path) if not path_tuple: logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH) - return + return True exhibits_traits = self.exhibits_traits - if file_status_check_options is not None and file_status_check_options['include_data_partition_size'] and partition_metadata is not None and partition_metadata['file_size_bytes'] is not None: + if file_status_check_options is not None and 'include_data_partition_size' in file_status_check_options and partition_metadata is not None and 'file_size_bytes' in partition_metadata is not None: exhibits_traits = CdmTraitCollection(self.ctx, self) for trait in self.exhibits_traits: exhibits_traits.append(trait) @@ -213,6 +228,7 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non local_ent_dec_def_owner._create_partition_from_pattern( location_corpus_path, exhibits_traits, args, self.specialized_schema, last_modified_time) data_partition_path_set.add(full_path) + return True def glob_pattern_to_regex(self, pattern: str) -> str: diff --git a/objectModel/Python/cdm/objectmodel/cdm_local_entity_declaration_def.py b/objectModel/Python/cdm/objectmodel/cdm_local_entity_declaration_def.py index df601affa1..e3fec9dc58 100644 --- a/objectModel/Python/cdm/objectmodel/cdm_local_entity_declaration_def.py +++ b/objectModel/Python/cdm/objectmodel/cdm_local_entity_declaration_def.py @@ -101,6 +101,12 @@ async def file_status_check_async(self, partition_file_status_check_type: Option 'CdmIncrementalPartitionType'] = CdmIncrementalPartitionType.NONE, file_status_check_options: Optional[FileStatusCheckOptions] = False) -> None: """Check the modified time for this object and any children.""" + await self._file_status_check_async_internal(partition_file_status_check_type, incremental_type, file_status_check_options) + + async def _file_status_check_async_internal(self, partition_file_status_check_type: Optional[ + 'PartitionFileStatusCheckType'] = PartitionFileStatusCheckType.FULL, incremental_type: Optional[ + 'CdmIncrementalPartitionType'] = CdmIncrementalPartitionType.NONE, file_status_check_options: Optional[FileStatusCheckOptions] = False) -> bool: + context = self.ctx.corpus.storage.fetch_adapter(self.in_document._namespace).create_file_query_cache_context() try: full_path = self.ctx.corpus.storage.create_absolute_corpus_path(self.entity_path, self.in_document) @@ -120,7 +126,9 @@ async def file_status_check_async(self, partition_file_status_check_type: Option Constants._INCREMENTAL_TRAIT_NAME, CdmLocalEntityDeclarationDefinition.data_partition_patterns.fget.__name__) else: - await pattern.file_status_check_async(file_status_check_options) + should_continue = await pattern._file_status_check_async_internal(file_status_check_options) + if not should_continue: + return False for partition in self.data_partitions: if partition.is_incremental: @@ -148,6 +156,7 @@ async def file_status_check_async(self, partition_file_status_check_type: Option await self.report_most_recent_time_async(self.last_file_modified_time) finally: context.dispose() + return True def _should_call_file_status_check(self, incremental_type: 'CdmIncrementalPartitionType', is_pattern: bool, pattern_or_partition_obj: 'CdmObjectDefinition') -> bool: diff --git a/objectModel/Python/cdm/objectmodel/cdm_manifest_def.py b/objectModel/Python/cdm/objectmodel/cdm_manifest_def.py index 4beffb2e63..f50e9dacaf 100644 --- a/objectModel/Python/cdm/objectmodel/cdm_manifest_def.py +++ b/objectModel/Python/cdm/objectmodel/cdm_manifest_def.py @@ -284,7 +284,10 @@ async def file_status_check_async(self, partition_file_status_check_type: Option if isinstance(entity, CdmReferencedEntityDeclarationDefinition): await entity.file_status_check_async() elif isinstance(entity, CdmLocalEntityDeclarationDefinition): - await cast(CdmLocalEntityDeclarationDefinition, entity).file_status_check_async(partition_file_status_check_type, incremental_type, file_status_check_options) + should_continue = await cast(CdmLocalEntityDeclarationDefinition, entity)._file_status_check_async_internal(partition_file_status_check_type, incremental_type, file_status_check_options) + + if not should_continue: + return for sub_manifest in self.sub_manifests: await sub_manifest.file_status_check_async() diff --git a/objectModel/Python/cdm/resx/logmessages.txt b/objectModel/Python/cdm/resx/logmessages.txt index 5d0c93224c..0777c057d0 100644 --- a/objectModel/Python/cdm/resx/logmessages.txt +++ b/objectModel/Python/cdm/resx/logmessages.txt @@ -98,6 +98,7 @@ ERR_PROJ_RENAME_FORMAT_IS_NOT_SET: RenameFormat should be set for this operation ERR_PROJ_SOURCE_ERROR: Source can only be another projection in a type attribute. ERR_PROJ_STRING_ERROR: Unable to get number for string '{0}'. Using default value '{1}'. ERR_PROJ_UNSUPPORTED_ATTR_GROUPS: Array expansion operation does not support attribute groups. +ERR_REGEX_TIMEOUT: The regex calculation timed out. ERR_REL_MAX_RESOLVED_ATTR_REACHED: Maximum number of resolved attributes reached for the entity: {0}. ERR_RESOLUTION_FAILURE: Parameter '{0}' has the dataType of '{1}' but the value '{2}' doesn't resolve to a known {3} reference ERR_RESOLVE_ENTITY_FAILURE: Failed to resolve entity {0} diff --git a/objectModel/Python/cdm/utilities/file_status_check_options.py b/objectModel/Python/cdm/utilities/file_status_check_options.py index 7a42e97aa3..2a86b8cc21 100644 --- a/objectModel/Python/cdm/utilities/file_status_check_options.py +++ b/objectModel/Python/cdm/utilities/file_status_check_options.py @@ -7,3 +7,5 @@ class FileStatusCheckOptions: def __init__(self) -> None: self.include_data_partition_size = None # type: Optional[bool] + + self.regex_timeout_seconds = None # type: Optional[float] diff --git a/objectModel/Python/setup.py b/objectModel/Python/setup.py index f3880cd36e..ef3b898d5c 100644 --- a/objectModel/Python/setup.py +++ b/objectModel/Python/setup.py @@ -77,7 +77,7 @@ def run(self): setuptools.setup( name='commondatamodel-objectmodel', - version='1.7.3', + version='1.7.4', author='Microsoft', description='Common Data Model Object Model library for Python', url='https://github.com/microsoft/CDM/tree/master/objectModel/Python', diff --git a/objectModel/Python/tests/cdm/data_partition_pattern/test_data_partition_pattern.py b/objectModel/Python/tests/cdm/data_partition_pattern/test_data_partition_pattern.py index 8bdcbd51b2..2d4d4f61d3 100644 --- a/objectModel/Python/tests/cdm/data_partition_pattern/test_data_partition_pattern.py +++ b/objectModel/Python/tests/cdm/data_partition_pattern/test_data_partition_pattern.py @@ -762,3 +762,12 @@ async def test_fetch_all_files_metadata(self): corpus.storage.mount('fetchNull', FetchAllMetadataNullAdapter(test_local_adapter)) fetch_null_manifest = await corpus.fetch_object_async('fetchNull:/manifest.manifest.cdm.json') await fetch_null_manifest.file_status_check_async(PartitionFileStatusCheckType.FULL, CdmIncrementalPartitionType.NONE, file_status_check_options) + + @async_test + async def test_regex_timeout(self): + expected_log_codes = {CdmLogCode.ERR_REGEX_TIMEOUT} + corpus = TestHelper.get_local_corpus(self.test_subpath, 'TestFetchAllFilesMetadata', expected_codes=expected_log_codes) + file_status_check__options = {'regex_timeout_seconds': 0} + + manifest = await corpus.fetch_object_async('manifest.manifest.cdm.json') + await manifest.file_status_check_async(file_status_check_options=file_status_check__options) \ No newline at end of file diff --git a/objectModel/TypeScript/Cdm/CdmDataPartitionPatternDefinition.ts b/objectModel/TypeScript/Cdm/CdmDataPartitionPatternDefinition.ts index 241c6debce..8925a4433c 100644 --- a/objectModel/TypeScript/Cdm/CdmDataPartitionPatternDefinition.ts +++ b/objectModel/TypeScript/Cdm/CdmDataPartitionPatternDefinition.ts @@ -21,7 +21,68 @@ import { isLocalEntityDeclarationDefinition } from '../Utilities/cdmObjectTypeGu import { Logger, enterScope } from '../Utilities/Logging/Logger'; import { StorageUtils } from '../Utilities/StorageUtils'; import { using } from "using-statement"; -import path = require('node:path'); +import * as util from 'util'; +import * as path from 'path'; + +const isInBrowser = typeof window !== 'undefined' && typeof window.document !== 'undefined'; + +let execFile; +if (!isInBrowser) { + execFile = util.promisify(require('child_process').execFile); +} + +const evaluateRegexp = async (regularExpression: string, configuredRegexTimeout: number, cleanedFileNames: string[]): Promise<[Map, Error]> => { + if (isInBrowser) { + let regexPattern: RegExp; + try { + regexPattern = new RegExp(regularExpression); + } catch (e) { + throw new Error('Regex init error'); + } + + const browserResult: Map = new Map(); + for (const name of cleanedFileNames) { + browserResult.set(name, regexPattern.exec(name)); + } + + return [browserResult, undefined]; + } + + let error: Error = undefined; + const defaultRegexTimeout: number = Math.max(3000, 1000 * cleanedFileNames.length); + const regexPromise = execFile('node', [path.resolve(__dirname, '../Utilities/evaluateRegexp.js'), regularExpression, ...cleanedFileNames]); + + const timeout = setTimeout(() => { + var spawn = require('child_process').spawn; + spawn("taskkill", ["/pid", regexPromise.child.pid, '/f', '/t']); + + error = new Error('Regex timeout.'); + }, configuredRegexTimeout != undefined ? configuredRegexTimeout : defaultRegexTimeout); + + const { stdout, stderr } = await regexPromise; + + clearTimeout(timeout); + + if (error) { + return [undefined, error]; + } + + if (stdout) { + let jsonOutput; + try { + jsonOutput = JSON.parse(stdout); + } catch (e) { + return [undefined, e]; + } + return [new Map(jsonOutput), undefined]; + } + + if (stderr) { + return [undefined, new Error(stderr)]; + } + + return undefined; +}; /** * The object model implementation for Data Partition Pattern. @@ -203,6 +264,13 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i * @inheritdoc */ public async fileStatusCheckAsync(fileStatusCheckOptions?: fileStatusCheckOptions): Promise { + await this.fileStatusCheckAsyncInternal(fileStatusCheckOptions); + } + + /** + * @internal + */ + public async fileStatusCheckAsyncInternal(fileStatusCheckOptions?: fileStatusCheckOptions): Promise { return await using(enterScope(CdmDataPartitionPatternDefinition.name, this.ctx, this.fileStatusCheckAsync.name), async _ => { let namespace: string = undefined; let adapter: StorageAdapterBase = undefined; @@ -220,7 +288,7 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i const pathTuple: [string, string] = StorageUtils.splitNamespacePath(rootCorpus); if (!pathTuple) { Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrStorageNullCorpusPath); - return; + return true; } namespace = pathTuple[0]; @@ -228,7 +296,7 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i if (adapter === undefined) { Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrDocAdapterNotFound, this.inDocument.name); - return; + return true; } // get a list of all corpusPaths under the root @@ -242,7 +310,7 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i if (fileInfoList === undefined) { Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrFetchingFileMetadataNull, namespace); - return; + return true; } if (namespace !== undefined) { @@ -263,15 +331,21 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i } const regularExpression: string = this.globPattern && this.globPattern.trim() !== '' ? this.globPatternToRegex(this.globPattern) : this.regularExpression; - let regexPattern: RegExp; - try { - regexPattern = new RegExp(regularExpression); - } catch (e) { - Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrValdnInvalidExpression, this.globPattern, this.regularExpression, e.message); + const [regexMatchMap, err]: [Map, Error] = await evaluateRegexp(regularExpression, fileStatusCheckOptions?.regexTimeoutSeconds, Array.from(cleanedFileList.keys())); + + if (err) { + if (err.message && err.message.indexOf('Regex init error') !== -1) { + Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrValdnInvalidExpression, this.globPattern, this.regularExpression, err.message); + } else { + Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrRegexTimeout); + + // do not continue processing the manifest/entity/partition pattern if timeout + return false; + } } - if (regexPattern !== undefined) { + if (regexMatchMap !== undefined) { const dataPartitionPathSet: Set = new Set(); if (localEntDecDefOwner.dataPartitions) { for (const dataPartition of localEntDecDefOwner.dataPartitions) { @@ -288,11 +362,16 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i } } - for (const fi of cleanedFileList) { - const fileName: string = fi[0]; - const partitionMetadata: CdmFileMetadata = fi[1]; + for (const regexMatch of regexMatchMap.entries()) { + const fileName: string = regexMatch[0]; + + if (!cleanedFileList.has(fileName)) { + continue; + } + + const partitionMetadata: CdmFileMetadata = cleanedFileList.get(fileName); + const m: RegExpExecArray = regexMatch[1]; - const m: RegExpExecArray = regexPattern.exec(fileName); if (m && m.length > 0 && m[0] === fileName) { // create a map of arguments out of capture groups const args: Map = new Map(); @@ -309,14 +388,14 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i } } - // put the origial but cleaned up root back onto the matched doc as the location stored in the partition + // put the original but cleaned up root back onto the matched doc as the location stored in the partition const locationCorpusPath: string = `${rootCleaned}${fileName}`; const fullPath: string = `${rootCorpus}${fileName}`; // Remove namespace from path const pathTuple: [string, string] = StorageUtils.splitNamespacePath(fullPath); if (!pathTuple) { Logger.error(this.ctx, this.TAG, this.fileStatusCheckAsync.name, this.atCorpusPath, cdmLogCode.ErrStorageNullCorpusPath); - return; + return true; } let exhibitsTraits: CdmTraitCollection = this.exhibitsTraits; @@ -345,6 +424,7 @@ export class CdmDataPartitionPatternDefinition extends CdmObjectDefinitionBase i } } } + return true; }); } diff --git a/objectModel/TypeScript/Cdm/CdmDocumentDefinition.ts b/objectModel/TypeScript/Cdm/CdmDocumentDefinition.ts index 5c55a33c98..975611bbc3 100644 --- a/objectModel/TypeScript/Cdm/CdmDocumentDefinition.ts +++ b/objectModel/TypeScript/Cdm/CdmDocumentDefinition.ts @@ -614,6 +614,9 @@ export class CdmDocumentDefinition extends cdmObjectSimple implements CdmDocumen // return p.measure(bodyCode); } + /** + * @deprecated Access 'name' member directly. + */ public getName(): string { // let bodyCode = () => { diff --git a/objectModel/TypeScript/Cdm/CdmLocalEntityDeclarationDefinition.ts b/objectModel/TypeScript/Cdm/CdmLocalEntityDeclarationDefinition.ts index d3fd8fa168..dd1fe314f7 100644 --- a/objectModel/TypeScript/Cdm/CdmLocalEntityDeclarationDefinition.ts +++ b/objectModel/TypeScript/Cdm/CdmLocalEntityDeclarationDefinition.ts @@ -236,6 +236,17 @@ export class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase incrementalType: cdmIncrementalPartitionType = cdmIncrementalPartitionType.None, fileStatusCheckOptions: fileStatusCheckOptions = undefined ): Promise { + await this.fileStatusCheckAsyncInternal(partitionCheckType, incrementalType, fileStatusCheckOptions); + } + + /** + * @internal + */ + public async fileStatusCheckAsyncInternal( + partitionCheckType: partitionFileStatusCheckType = partitionFileStatusCheckType.Full, + incrementalType: cdmIncrementalPartitionType = cdmIncrementalPartitionType.None, + fileStatusCheckOptions: fileStatusCheckOptions = undefined + ): Promise { let adapter: StorageAdapterBase = this.ctx.corpus.storage.fetchAdapter(this.inDocument.namespace); let cacheContext: StorageAdapterCacheContext = (adapter !== undefined) ? adapter.createFileQueryCacheContext() : undefined; @@ -252,7 +263,11 @@ export class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase Logger.error(pattern.ctx, this.TAG, this.fileStatusCheckAsync.name, pattern.atCorpusPath, cdmLogCode.ErrUnexpectedIncrementalPartitionTrait, CdmDataPartitionPatternDefinition.name, pattern.fetchObjectDefinitionName(), constants.INCREMENTAL_TRAIT_NAME, 'dataPartitionPatterns'); } else { - await pattern.fileStatusCheckAsync(fileStatusCheckOptions); + const shouldContinue: boolean = await pattern.fileStatusCheckAsyncInternal(fileStatusCheckOptions); + + if (!shouldContinue) { + return false; + } } } @@ -290,6 +305,8 @@ export class CdmLocalEntityDeclarationDefinition extends CdmObjectDefinitionBase cacheContext.dispose() } } + + return true; } /** diff --git a/objectModel/TypeScript/Cdm/CdmManifestDefinition.ts b/objectModel/TypeScript/Cdm/CdmManifestDefinition.ts index 468bc361b2..1a20525965 100644 --- a/objectModel/TypeScript/Cdm/CdmManifestDefinition.ts +++ b/objectModel/TypeScript/Cdm/CdmManifestDefinition.ts @@ -476,7 +476,11 @@ export class CdmManifestDefinition extends CdmDocumentDefinition implements CdmO if (isReferencedEntityDeclarationDefinition(entity)) { await entity.fileStatusCheckAsync(); } else if (isLocalEntityDeclarationDefinition(entity)) { - await entity.fileStatusCheckAsync(partitionCheckType, incrementalType, fileStatusCheckOptions); + const shouldContinue: boolean = await entity.fileStatusCheckAsyncInternal(partitionCheckType, incrementalType, fileStatusCheckOptions); + + if (!shouldContinue) { + return; + } } } diff --git a/objectModel/TypeScript/CdmStandards/package-lock.json b/objectModel/TypeScript/CdmStandards/package-lock.json index b423d82966..9c0040b407 100644 --- a/objectModel/TypeScript/CdmStandards/package-lock.json +++ b/objectModel/TypeScript/CdmStandards/package-lock.json @@ -1,12 +1,12 @@ { "name": "cdm.objectmodel.cdmstandards", - "version": "2.8.0", + "version": "2.8.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cdm.objectmodel.cdmstandards", - "version": "2.8.0", + "version": "2.8.3", "license": "MIT", "dependencies": { "@types/node": "^14.14.31" diff --git a/objectModel/TypeScript/CdmStandards/package.json b/objectModel/TypeScript/CdmStandards/package.json index 6c2bedc16c..77e21ce58a 100644 --- a/objectModel/TypeScript/CdmStandards/package.json +++ b/objectModel/TypeScript/CdmStandards/package.json @@ -1,6 +1,6 @@ { "name": "cdm.objectmodel.cdmstandards", - "version": "2.8.0", + "version": "2.8.3", "description": "The Common Data Model standard schema and entities.", "main": "index.js", "scripts": { diff --git a/objectModel/TypeScript/Enums/cdmLogCode.ts b/objectModel/TypeScript/Enums/cdmLogCode.ts index c872e60d75..76466b2e21 100644 --- a/objectModel/TypeScript/Enums/cdmLogCode.ts +++ b/objectModel/TypeScript/Enums/cdmLogCode.ts @@ -81,6 +81,7 @@ export enum cdmLogCode { ErrProjStringError, ErrProjUnsupportedAttrGroups, ErrProjUnsupportedSource, + ErrRegexTimeout, ErrRelMaxResolvedAttrReached, ErrResolutionFailure, ErrResolveEntityFailure, diff --git a/objectModel/TypeScript/Persistence/ModelJson/LocalEntityDeclarationPersistence.ts b/objectModel/TypeScript/Persistence/ModelJson/LocalEntityDeclarationPersistence.ts index 03b3ba87c5..fbd46b44ba 100644 --- a/objectModel/TypeScript/Persistence/ModelJson/LocalEntityDeclarationPersistence.ts +++ b/objectModel/TypeScript/Persistence/ModelJson/LocalEntityDeclarationPersistence.ts @@ -49,7 +49,7 @@ export class LocalEntityDeclarationPersistence { documentFolder.documents.push(entityDoc); // Entity schema path is the path to the doc containing the entity definition. - localEntityDec.entityPath = ctx.corpus.storage.createRelativeCorpusPath(`${entityDoc.atCorpusPath}/${dataObj.name}`, manifest); + localEntityDec.entityPath = ctx.corpus.storage.createRelativeCorpusPath(`${entityDoc.atCorpusPath}/${dataObj.name}`, documentFolder); localEntityDec.explanation = dataObj.description; diff --git a/objectModel/TypeScript/Persistence/ModelJson/ManifestPersistence.ts b/objectModel/TypeScript/Persistence/ModelJson/ManifestPersistence.ts index 104aa87078..e83683859e 100644 --- a/objectModel/TypeScript/Persistence/ModelJson/ManifestPersistence.ts +++ b/objectModel/TypeScript/Persistence/ModelJson/ManifestPersistence.ts @@ -50,9 +50,6 @@ export class ManifestPersistence { const manifest: CdmManifestDefinition = ctx.corpus.MakeObject(cdmObjectType.manifestDef, obj.name); manifest.virtualLocation = folder.folderPath + CdmConstants.modelJsonExtension; - // We need to set up folder path and namespace of a manifest to be able to retrieve that object. - folder.documents.push(manifest); - if (obj['cdm:imports']) { obj['cdm:imports'].forEach((impElement: object) => { const importObj: CdmImport = CdmFolder.ImportPersistence.fromData(ctx, impElement as Import); diff --git a/objectModel/TypeScript/Utilities/fileStatusCheckOptions.ts b/objectModel/TypeScript/Utilities/fileStatusCheckOptions.ts index 092c63e195..e928f6a6e5 100644 --- a/objectModel/TypeScript/Utilities/fileStatusCheckOptions.ts +++ b/objectModel/TypeScript/Utilities/fileStatusCheckOptions.ts @@ -2,5 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. export class fileStatusCheckOptions { - public includeDataPartitionSize: boolean; + public includeDataPartitionSize?: boolean; + + public regexTimeoutSeconds?: number; } diff --git a/objectModel/TypeScript/__test__/Cdm/DataPartitionPattern/DataPartitionPattern.test.ts b/objectModel/TypeScript/__test__/Cdm/DataPartitionPattern/DataPartitionPattern.test.ts index 8506bcd197..cae6016e0f 100644 --- a/objectModel/TypeScript/__test__/Cdm/DataPartitionPattern/DataPartitionPattern.test.ts +++ b/objectModel/TypeScript/__test__/Cdm/DataPartitionPattern/DataPartitionPattern.test.ts @@ -1103,4 +1103,16 @@ describe('Cdm/DataPartitionPattern/DataPartitionPattern', () => { var fetchNullManifest = await corpus.fetchObjectAsync('fetchNull:/manifest.manifest.cdm.json'); await fetchNullManifest.fileStatusCheckAsync(partitionFileStatusCheckType.Full, cdmIncrementalPartitionType.None, fileStatusCheckOptions); }); + + /** + * Test Regex Timeout handled correctly + */ + it('TestRegexTimeout', async () => { + const expectedLogCodes = new Set([cdmLogCode.ErrRegexTimeout]); + const corpus: CdmCorpusDefinition = testHelper.getLocalCorpus(testsSubpath, 'TestFetchAllFilesMetadata', undefined, false, expectedLogCodes); + const fileStatusCheckOptions: fileStatusCheckOptions = { regexTimeoutSeconds: 0 }; + + const manifest: CdmManifestDefinition = await corpus.fetchObjectAsync('manifest.manifest.cdm.json'); + await manifest.fileStatusCheckAsync(partitionFileStatusCheckType.Full, cdmIncrementalPartitionType.None, fileStatusCheckOptions); + }); }); diff --git a/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJson.test.ts b/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJson.test.ts index 4bc314b2cd..f5890f6170 100644 --- a/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJson.test.ts +++ b/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJson.test.ts @@ -405,6 +405,23 @@ describe('Persistence.ModelJson.ModelJson', () => { testHelper.expectCdmLogCodeEquality(corpus, cdmLogCode.ErrPersistModelJsonRefEntityInvalidLocation, true); }); + /** + * Test resulting manifest file is only added once and manifest name is correctly named + */ + it('TestNameOnModelLoad', async () => { + const corpus: CdmCorpusDefinition = testHelper.getLocalCorpus(testsSubpath, 'TestNameOnModelLoad'); + const manifest: CdmManifestDefinition = await corpus.fetchObjectAsync('model.json'); + + const folder: CdmFolderDefinition = corpus.storage.fetchRootFolder('local'); + + // folder should contain one manifest, one entity file, and an extensions file + expect(folder.documents.length) + .toBe(3); + + expect(manifest.getName()) + .toBe(modelJsonExtension); + }); + /** * Handles the obtained output. * If needed, writes the output to a test debugging file. diff --git a/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJsonExtensibility.test.ts b/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJsonExtensibility.test.ts index 9ea5931074..ae7de354ca 100644 --- a/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJsonExtensibility.test.ts +++ b/objectModel/TypeScript/__test__/Persistence/ModelJson/modelJsonExtensibility.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -import { CdmCorpusDefinition, CdmDocumentDefinition, CdmFolderDefinition, CdmManifestDefinition } from '../../../internal'; +import { CdmConstants, CdmCorpusDefinition, CdmDocumentDefinition, CdmFolderDefinition, CdmManifestDefinition } from '../../../internal'; import { CdmFolder, ModelJson } from '../../../Persistence'; import { DocumentPersistence as cdmDocument } from '../../../Persistence/CdmFolder/DocumentPersistence'; import { DocumentContent } from '../../../Persistence/CdmFolder/types'; @@ -99,7 +99,7 @@ describe('Persistence.ModelJson.ModelJsonExtensibility', () => { } const expectedOutput: string = - testHelper.getExpectedOutputFileContent(testsSubpath, 'TestModelJsonExtensibilityManifestDocs', manifest.name); + testHelper.getExpectedOutputFileContent(testsSubpath, 'TestModelJsonExtensibilityManifestDocs', `${manifest.manifestName}${CdmConstants.manifestExtension}`); testHelper.assertSameObjectWasSerialized(expectedOutput, serializedManifest); }); diff --git a/objectModel/TypeScript/__test__/Persistence/PersistenceLayer.test.ts b/objectModel/TypeScript/__test__/Persistence/PersistenceLayer.test.ts index d5615bbf8b..ffcd07d87b 100644 --- a/objectModel/TypeScript/__test__/Persistence/PersistenceLayer.test.ts +++ b/objectModel/TypeScript/__test__/Persistence/PersistenceLayer.test.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. import { + CdmConstants, CdmCorpusDefinition, CdmDocumentDefinition, CdmEntityDefinition, @@ -114,7 +115,7 @@ describe('Persistence.PersistenceLayerTest', () => { await manifestFromModelJson.saveAsAsync(newManifestFromModelJsonName, true); // Verify that model.json persistence was called by comparing the saved document to the original model.json. serializedManifest = allDocs.get(`/${newManifestFromModelJsonName}`); - expectedOutputManifest = testHelper.getExpectedOutputFileContent(testsSubpath, testName, manifestFromModelJson.name); + expectedOutputManifest = testHelper.getExpectedOutputFileContent(testsSubpath, testName, `${manifestFromModelJson.manifestName}${CdmConstants.manifestExtension}`); testHelper.assertSameObjectWasSerialized(expectedOutputManifest, serializedManifest); }); diff --git a/objectModel/TypeScript/package-lock.json b/objectModel/TypeScript/package-lock.json index a530f09960..b4b190ad49 100644 --- a/objectModel/TypeScript/package-lock.json +++ b/objectModel/TypeScript/package-lock.json @@ -1,16 +1,17 @@ { "name": "cdm.objectmodel", - "version": "1.7.2", + "version": "1.7.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cdm.objectmodel", - "version": "1.7.2", + "version": "1.7.4", "license": "MIT", "dependencies": { "@azure/msal-node": "^1.14.6", "@types/node": "^14.14.31", + "cdm.objectmodel.cdmstandards": "^2.8.0", "guid-typescript": "^1.0.9", "using-statement": "^0.3.1" }, @@ -2647,6 +2648,14 @@ "url": "https://opencollective.com/browserslist" } }, + "node_modules/cdm.objectmodel.cdmstandards": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/cdm.objectmodel.cdmstandards/-/cdm.objectmodel.cdmstandards-2.8.3.tgz", + "integrity": "sha512-o+p8C6Gw2i/jFpHECJrIj3vggykHo5bSXogQ5PPClwQLccrYIgP2U+gi/NXI6TY4q2Gr2L+O9rrlCZBoFxKIEA==", + "dependencies": { + "@types/node": "^14.14.31" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -8946,6 +8955,14 @@ "integrity": "sha512-QQmLOGJ3DEgokHbMSA8cj2a+geXqmnpyOFT0lhQV6P3/YOJvGDEwoedcwxEQ30gJIwIIunHIicunJ2rzK5gB2A==", "dev": true }, + "cdm.objectmodel.cdmstandards": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/cdm.objectmodel.cdmstandards/-/cdm.objectmodel.cdmstandards-2.8.3.tgz", + "integrity": "sha512-o+p8C6Gw2i/jFpHECJrIj3vggykHo5bSXogQ5PPClwQLccrYIgP2U+gi/NXI6TY4q2Gr2L+O9rrlCZBoFxKIEA==", + "requires": { + "@types/node": "^14.14.31" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", diff --git a/objectModel/TypeScript/package.json b/objectModel/TypeScript/package.json index 1f48a0a135..6e316b0da4 100644 --- a/objectModel/TypeScript/package.json +++ b/objectModel/TypeScript/package.json @@ -1,6 +1,6 @@ { "name": "cdm.objectmodel", - "version": "1.7.2", + "version": "1.7.4", "description": "The typescript object model for CDM.", "main": "lib/index.js", "types": "lib/index.d.ts", diff --git a/objectModel/TypeScript/resx/logMessages.json b/objectModel/TypeScript/resx/logMessages.json index b8362ca076..3b9822591e 100644 --- a/objectModel/TypeScript/resx/logMessages.json +++ b/objectModel/TypeScript/resx/logMessages.json @@ -78,6 +78,7 @@ "ErrProjStringError": "Unable to get number for string '{0}'. Using default value '{1}'.", "ErrProjUnsupportedAttrGroups": "Array expansion operation does not support attribute groups.", "ErrProjUnsupportedSource": "Unsupported source type '{0}' this operation '{1}'.", + "ErrRegexTimeout": "The regex calculation timed out.", "ErrRelMaxResolvedAttrReached": "Maximum number of resolved attributes reached for the entity: {0}.", "ErrResolutionFailure": "Parameter '{0}' has the dataType of '{1}' but the value '{2}' doesn't resolve to a known {3} reference", "ErrResolveEntityFailure": "Failed to resolve entity {0}", diff --git a/objectModel/TypeScript/tsconfig.json b/objectModel/TypeScript/tsconfig.json index 1620bd5345..1b8297717c 100644 --- a/objectModel/TypeScript/tsconfig.json +++ b/objectModel/TypeScript/tsconfig.json @@ -8,10 +8,12 @@ "sourceMap": true, "baseUrl": ".", "outDir": "./lib", - "resolveJsonModule": true + "resolveJsonModule": true, + "allowJs": true }, "include": [ "./index.ts", + "./Utilities/evaluateRegexp.js", "./__test__/**/*.ts" ] } \ No newline at end of file