Merge pull request #368 from microsoft/features/publish-20231019-2

Microsoft CDM Release Version Update: 1.7.4

objectModel/CSharp/Directory.Build.props

@@ -1,9 +1,9 @@
 <Project>
   <PropertyGroup>
-    <CdmObjectModelVersion>1.7.3</CdmObjectModelVersion>
+    <CdmObjectModelVersion>1.7.4</CdmObjectModelVersion>
     <CdmStandardsVersion>2.8.0</CdmStandardsVersion>
     <CdmPQModelsVersion>2.8.0</CdmPQModelsVersion>
-    <CdmTelemetryVersion>1.7.3-preview1</CdmTelemetryVersion>
+    <CdmTelemetryVersion>1.7.4-preview1</CdmTelemetryVersion>
     <RunSettingsFilePath>$(MSBuildThisFileDirectory)\CodeCoverage.runsettings</RunSettingsFilePath>
   </PropertyGroup>
 </Project>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs

@@ -919,5 +919,19 @@ public async Task TestFetchAllFilesMetadata()
             var fetchNullManifest = await corpus.FetchObjectAsync<CdmManifestDefinition>("fetchNull:/manifest.manifest.cdm.json");
             await fetchNullManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
         }
+
+        /// <summary>
+        /// Test RegexTimeout is handled correctly
+        /// </summary>
+        [TestMethod]
+        public async Task TestRegexTimeout()
+        {
+            var outOfRangeExpectedLogCodes = new HashSet<CdmLogCode> { CdmLogCode.ErrRegexTimeoutOutOfRange };
+            CdmCorpusDefinition outOfRangeCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestFetchAllFilesMetadata), expectedCodes: outOfRangeExpectedLogCodes);
+            var fileStatusCheckOptions = new FileStatusCheckOptions() { RegexTimeoutSeconds = 0 };
+
+            var outOfRangeManifest = await outOfRangeCorpus.FetchObjectAsync<CdmManifestDefinition>("manifest.manifest.cdm.json");
+            await outOfRangeManifest.FileStatusCheckAsync(fileStatusCheckOptions: fileStatusCheckOptions);
+        }
     }
 }

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmDataPartitionPatternDefinition.cs

@@ -20,6 +20,11 @@ public class CdmDataPartitionPatternDefinition : CdmObjectDefinitionBase, CdmFil
     {
         private static readonly string Tag = nameof(CdmDataPartitionPatternDefinition);
 
+        /// <summary>
+        /// Regex timeout value
+        /// </summary>
+        private static TimeSpan DefaultRegexTimeoutValue = TimeSpan.FromSeconds(1);
+
         private TraitToPropertyMap TraitToPropertyMap { get; }
 
         /// <summary>
@@ -182,6 +187,11 @@ public override bool IsDerivedFrom(string baseName, ResolveOptions resOpt)
 
         /// <inheritdoc />
         public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOptions)
+        {
+            await this.FileStatusCheckAsyncInternal(fileStatusCheckOptions);
+        }
+
+        internal async Task<bool> FileStatusCheckAsyncInternal(FileStatusCheckOptions fileStatusCheckOptions)
         {
             using (Logger.EnterScope(nameof(CdmDataPartitionPatternDefinition), Ctx, nameof(FileStatusCheckAsync)))
             {
@@ -204,7 +214,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                     if (pathTuple == null)
                     {
                         Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrStorageNullCorpusPath);
-                        return;
+                        return true;
                     }
 
                     nameSpace = pathTuple.Item1;
@@ -213,7 +223,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                     if (adapter == null)
                     {
                         Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrDocAdapterNotFound, this.InDocument.Name);
-                        return;
+                        return true;
                     }
 
                     // get a list of all corpusPaths under the root
@@ -230,7 +240,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                 if (fileInfoList == null)
                 {
                     Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrFetchingFileMetadataNull, nameSpace);
-                    return;
+                    return true;
                 }
 
                 if (nameSpace != null)
@@ -261,9 +271,20 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                         string regularExpression = !String.IsNullOrWhiteSpace(this.GlobPattern) ? this.GlobPatternToRegex(this.GlobPattern) : this.RegularExpression;
                         Regex regexPattern = null;
 
+                        TimeSpan regexTimeoutValue = fileStatusCheckOptions?.RegexTimeoutSeconds != null ? TimeSpan.FromSeconds(fileStatusCheckOptions.RegexTimeoutSeconds.Value) : DefaultRegexTimeoutValue;
+
                         try
                         {
-                            regexPattern = new Regex(regularExpression);
+                            regexPattern = new Regex(regularExpression, RegexOptions.None, regexTimeoutValue);
+                        }
+                        catch (ArgumentOutOfRangeException rangeEx)
+                        {
+                            Logger.Error(
+                                this.Ctx,
+                                Tag,
+                                nameof(FileStatusCheckAsync),
+                                this.AtCorpusPath,
+                                CdmLogCode.ErrRegexTimeoutOutOfRange, regexTimeoutValue.ToString(), rangeEx.Message);
                         }
                         catch (Exception e)
                         {
@@ -302,7 +323,20 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                                 string fileName = fi.Key;
                                 CdmFileMetadata partitionMetadata = fi.Value;
 
-                                Match m = regexPattern.Match(fileName);
+                                Match m;
+
+                                try
+                                {
+                                    m = regexPattern.Match(fileName);
+                                }
+                                catch (RegexMatchTimeoutException e)
+                                {
+                                    Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrRegexTimeout, e.Message);
+
+                                    // do not continue processing the manifest/entity/partition pattern if timeout
+                                    return false;
+                                }
+
                                 if (m.Success && m.Length > 1 && m.Value == fileName)
                                 {
                                     // create a map of arguments out of capture groups
@@ -337,7 +371,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                                     if (pathTuple == null)
                                     {
                                         Logger.Error(this.Ctx, Tag, nameof(FileStatusCheckAsync), this.AtCorpusPath, CdmLogCode.ErrStorageNullCorpusPath, this.AtCorpusPath);
-                                        return;
+                                        return true;
                                     }
 
                                     CdmTraitCollection exhibitsTraits = this.ExhibitsTraits;
@@ -374,6 +408,7 @@ public async Task FileStatusCheckAsync(FileStatusCheckOptions fileStatusCheckOpt
                         }
                     }
                 }
+                return true;
             }
         }
 

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmLocalEntityDeclarationDefinition.cs

@@ -215,6 +215,11 @@ public async Task FileStatusCheckAsync()
         }
 
         public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null)
+        {
+            await FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions);
+        }
+
+        internal async Task<bool> FileStatusCheckAsyncInternal(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None, FileStatusCheckOptions fileStatusCheckOptions = null)
         {
             using ((this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace) as StorageAdapterBase)?.CreateFileQueryCacheContext())
             {
@@ -236,7 +241,12 @@ public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFil
                         }
                         else
                         {
-                            await pattern.FileStatusCheckAsync(fileStatusCheckOptions);
+                            bool shouldContinue = await pattern.FileStatusCheckAsyncInternal(fileStatusCheckOptions);
+
+                            if (!shouldContinue)
+                            {
+                                return false;
+                            }
                         }
                     }
 
@@ -276,6 +286,8 @@ public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFil
                 this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime);
 
                 await this.ReportMostRecentTimeAsync(this.LastFileModifiedTime);
+
+                return true;
             }
         }
 

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Cdm/CdmManifestDefinition.cs

@@ -561,9 +561,14 @@ public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFil
                         {
                             await entity.FileStatusCheckAsync();
                         }
-                        else if (entity is CdmLocalEntityDeclarationDefinition)
+                        else if (entity is CdmLocalEntityDeclarationDefinition localEntity)
                         {
-                            await (entity as CdmLocalEntityDeclarationDefinition).FileStatusCheckAsync(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions);
+                            bool shouldContinue = await localEntity.FileStatusCheckAsyncInternal(partitionFileStatusCheckType, incrementalType, fileStatusCheckOptions);
+
+                            if (!shouldContinue)
+                            {
+                                return;
+                            }
                         }
                     }
 

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Enums/CdmLogCode.cs

@@ -107,6 +107,8 @@ public enum CdmLogCode
         ErrProjStringError,
         ErrProjUnsupportedAttrGroups,
         ErrProjUnsupportedSource,
+        ErrRegexTimeout,
+        ErrRegexTimeoutOutOfRange,
         ErrRelMaxResolvedAttrReached,
         ErrResolutionFailure,
         ErrResolveEntityFailure,

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Resx/LogMessages.resx

@@ -687,4 +687,10 @@
   <data name="WarnUnmountCdmNamespace" xml:space="preserve">
     <value>Unmounting the cdm namespace in order to use offline mode has been deprecated. The default behavior will now be using the CdmStandards package instead of the online schema store. Please remove this Unmount API call.</value>
   </data>
+  <data name="ErrRegexTimeout" xml:space="preserve">
+    <value>The regex calculation timed out.</value>
+  </data>
+  <data name="ErrRegexTimeoutOutOfRange" xml:space="preserve">
+    <value>Regex timeout '{0}' is not valid. Reason '{1}'</value>
+  </data>
 </root>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel/Utilities/FileStatusCheckOptions.cs

@@ -6,5 +6,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Utilities
     public class FileStatusCheckOptions
     {
         public bool IncludeDataPartitionSize { get; set; }
+
+        public double? RegexTimeoutSeconds { get; set; }
     }
 }

objectModel/Java/objectmodel/pom.xml

@@ -21,7 +21,7 @@
     <apache.commons-codec.version>1.15</apache.commons-codec.version>
     <fasterxml.jackson.version>2.13.2</fasterxml.jackson.version>
     <fasterxml.databind.version>2.13.4.2</fasterxml.databind.version>
-    <java.version>8</java.version>
+    <java.version>9</java.version>
     <microsoft.msal4j.version>1.11.0</microsoft.msal4j.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <skycreamer.jsonassert.version>1.5.0</skycreamer.jsonassert.version>
@@ -179,8 +179,6 @@
       <plugin>
         <configuration>
           <release>${java.version}</release>
-          <source>1.8</source>
-          <target>1.8</target>
         </configuration>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>

objectModel/Java/pom.xml

@@ -5,7 +5,7 @@
 
     <properties>
         <!-- Increment version for each official release of the SDK -->
-        <revision>1.7.3</revision>
+        <revision>1.7.4</revision>
         <cdmstandards-revision>2.8.0</cdmstandards-revision>
     </properties>
 

objectModel/Python/cdm/enums/cdm_log_code.py

@@ -107,6 +107,7 @@ class CdmLogCode(AutoNumber):
     ERR_PROJ_STRING_ERROR = ()
     ERR_PROJ_UNSUPPORTED_ATTR_GROUPS = ()
     ERR_PROJ_UNSUPPORTED_SOURCE = ()
+    ERR_REGEX_TIMEOUT = ()
     ERR_REL_MAX_RESOLVED_ATTR_REACHED = ()
     ERR_RESOLUTION_FAILURE = ()
     ERR_RESOLVE_ENTITY_FAILURE = ()

objectModel/Python/cdm/objectmodel/cdm_data_partition_pattern_def.py

@@ -6,9 +6,8 @@
 from typing import cast, Dict, List, Optional, TYPE_CHECKING
 import regex
 
-from cdm.enums import CdmLogCode, PartitionFileStatusCheckType, CdmIncrementalPartitionType
-from cdm.enums import CdmObjectType
-from cdm.utilities import logger, ResolveOptions, StorageUtils, TraitToPropertyMap
+from cdm.enums import CdmLogCode, CdmObjectType
+from cdm.utilities import FileStatusCheckOptions, logger, ResolveOptions, StorageUtils, TraitToPropertyMap
 
 from .cdm_file_status import CdmFileStatus
 from .cdm_local_entity_declaration_def import CdmLocalEntityDeclarationDefinition
@@ -19,6 +18,7 @@
     from cdm.objectmodel import CdmCorpusContext
     from cdm.utilities import VisitCallback
 
+regex_timeout = 1
 
 class CdmDataPartitionPatternDefinition(CdmObjectDefinition, CdmFileStatus):
     def __init__(self, ctx: 'CdmCorpusContext', name: str) -> None:
@@ -94,6 +94,9 @@ def copy(self, res_opt: Optional['ResolveOptions'] = None, host: Optional['CdmDa
         return copy
 
     async def file_status_check_async(self, file_status_check_options = None) -> None:
+        await self._file_status_check_async_internal(file_status_check_options)
+
+    async def _file_status_check_async_internal(self, file_status_check_options: Optional[FileStatusCheckOptions] = None) -> None:
         """Check the modified time for this object and any children."""
         with logger._enter_scope(self._TAG, self.ctx, self.file_status_check_async.__name__):
             namespace = None
@@ -108,7 +111,7 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non
                 path_tuple = StorageUtils.split_namespace_path(root_corpus)
                 if not path_tuple:
                     logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH)
-                    return
+                    return True
 
                 namespace = path_tuple[0]
                 adapter = self.ctx.corpus.storage.fetch_adapter(namespace)
@@ -129,7 +132,7 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non
             if file_info_list is None:
                 logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path,
                 CdmLogCode.ERR_FETCHING_FILE_METADATA_NULL, namespace)
-                return
+                return True
 
             if namespace is not None:
                 # remove root of the search from the beginning of all paths so anything in the root is not found by regex.
@@ -167,8 +170,20 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non
                                 incremental_partition_location_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(incremental_partition.location, self.in_document)
                                 incremental_partition_path_hash_set.add(incremental_partition_location_full_path)
 
+                        if file_status_check_options is not None and 'regex_timeout_seconds' in file_status_check_options:
+                            configured_regex_timeout = file_status_check_options['regex_timeout_seconds']
+                        else:
+                            configured_regex_timeout = regex_timeout
+
                         for file_name,partition_metadata in cleaned_file_list.items():
-                            m = reg.fullmatch(file_name)
+                            try:
+                                m = reg.fullmatch(file_name, timeout=configured_regex_timeout)
+                            except TimeoutError as e:
+                                logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_REGEX_TIMEOUT)
+
+                                # do not continue processing the manifest/entity/partition pattern if timeout
+                                return False
+
                             if m:
                                 # create a map of arguments out of capture groups.
                                 args = defaultdict(list)  # type: Dict[str, List[str]]
@@ -193,10 +208,10 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non
                                 path_tuple = StorageUtils.split_namespace_path(full_path)
                                 if not path_tuple:
                                     logger.error(self.ctx, self._TAG, CdmDataPartitionPatternDefinition.file_status_check_async.__name__, self.at_corpus_path, CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH)
-                                    return
+                                    return True
 
                                 exhibits_traits = self.exhibits_traits
-                                if file_status_check_options is not None and file_status_check_options['include_data_partition_size'] and partition_metadata is not None and partition_metadata['file_size_bytes'] is not None:
+                                if file_status_check_options is not None and 'include_data_partition_size' in file_status_check_options and partition_metadata is not None and 'file_size_bytes' in partition_metadata is not None:
                                     exhibits_traits = CdmTraitCollection(self.ctx, self)
                                     for trait in self.exhibits_traits:
                                         exhibits_traits.append(trait)
@@ -213,6 +228,7 @@ async def file_status_check_async(self, file_status_check_options = None) -> Non
                                     local_ent_dec_def_owner._create_partition_from_pattern(
                                         location_corpus_path, exhibits_traits, args, self.specialized_schema, last_modified_time)
                                     data_partition_path_set.add(full_path)
+            return True
 
 
     def glob_pattern_to_regex(self, pattern: str) -> str: