diff --git a/api/swagger.yml b/api/swagger.yml index e6f8cd51e07..d7e06729ea3 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -1027,6 +1027,7 @@ components: - pre_sign_support - pre_sign_support_ui - import_support + - import_validity_regex properties: blockstore_type: type: string @@ -1042,6 +1043,8 @@ components: type: boolean import_support: type: boolean + import_validity_regex: + type: string VersionConfig: type: object diff --git a/clients/java/api/openapi.yaml b/clients/java/api/openapi.yaml index cc623284d48..b64bda191d5 100644 --- a/clients/java/api/openapi.yaml +++ b/clients/java/api/openapi.yaml @@ -7036,6 +7036,7 @@ components: blockstore_type: blockstore_type pre_sign_support_ui: true import_support: true + import_validity_regex: import_validity_regex default_namespace_prefix: default_namespace_prefix pre_sign_support: true properties: @@ -7053,11 +7054,14 @@ components: type: boolean import_support: type: boolean + import_validity_regex: + type: string required: - blockstore_namespace_ValidityRegex - blockstore_namespace_example - blockstore_type - import_support + - import_validity_regex - pre_sign_support - pre_sign_support_ui type: object diff --git a/clients/java/docs/StorageConfig.md b/clients/java/docs/StorageConfig.md index 44d2024214e..6f176a63b5e 100644 --- a/clients/java/docs/StorageConfig.md +++ b/clients/java/docs/StorageConfig.md @@ -14,6 +14,7 @@ Name | Type | Description | Notes **preSignSupport** | **Boolean** | | **preSignSupportUi** | **Boolean** | | **importSupport** | **Boolean** | | +**importValidityRegex** | **String** | | diff --git a/clients/java/src/main/java/io/lakefs/clients/api/model/StorageConfig.java b/clients/java/src/main/java/io/lakefs/clients/api/model/StorageConfig.java index a389f4e9a0a..c0a589f661b 100644 --- a/clients/java/src/main/java/io/lakefs/clients/api/model/StorageConfig.java +++ b/clients/java/src/main/java/io/lakefs/clients/api/model/StorageConfig.java @@ -57,6 +57,10 @@ public class StorageConfig { @SerializedName(SERIALIZED_NAME_IMPORT_SUPPORT) private Boolean importSupport; + public static final String SERIALIZED_NAME_IMPORT_VALIDITY_REGEX = "import_validity_regex"; + @SerializedName(SERIALIZED_NAME_IMPORT_VALIDITY_REGEX) + private String importValidityRegex; + public StorageConfig blockstoreType(String blockstoreType) { @@ -219,6 +223,29 @@ public void setImportSupport(Boolean importSupport) { } + public StorageConfig importValidityRegex(String importValidityRegex) { + + this.importValidityRegex = importValidityRegex; + return this; + } + + /** + * Get importValidityRegex + * @return importValidityRegex + **/ + @javax.annotation.Nonnull + @ApiModelProperty(required = true, value = "") + + public String getImportValidityRegex() { + return importValidityRegex; + } + + + public void setImportValidityRegex(String importValidityRegex) { + this.importValidityRegex = importValidityRegex; + } + + @Override public boolean equals(Object o) { if (this == o) { @@ -234,12 +261,13 @@ public boolean equals(Object o) { Objects.equals(this.defaultNamespacePrefix, storageConfig.defaultNamespacePrefix) && Objects.equals(this.preSignSupport, storageConfig.preSignSupport) && Objects.equals(this.preSignSupportUi, storageConfig.preSignSupportUi) && - Objects.equals(this.importSupport, storageConfig.importSupport); + Objects.equals(this.importSupport, storageConfig.importSupport) && + Objects.equals(this.importValidityRegex, storageConfig.importValidityRegex); } @Override public int hashCode() { - return Objects.hash(blockstoreType, blockstoreNamespaceExample, blockstoreNamespaceValidityRegex, defaultNamespacePrefix, preSignSupport, preSignSupportUi, importSupport); + return Objects.hash(blockstoreType, blockstoreNamespaceExample, blockstoreNamespaceValidityRegex, defaultNamespacePrefix, preSignSupport, preSignSupportUi, importSupport, importValidityRegex); } @Override @@ -253,6 +281,7 @@ public String toString() { sb.append(" preSignSupport: ").append(toIndentedString(preSignSupport)).append("\n"); sb.append(" preSignSupportUi: ").append(toIndentedString(preSignSupportUi)).append("\n"); sb.append(" importSupport: ").append(toIndentedString(importSupport)).append("\n"); + sb.append(" importValidityRegex: ").append(toIndentedString(importValidityRegex)).append("\n"); sb.append("}"); return sb.toString(); } diff --git a/clients/java/src/test/java/io/lakefs/clients/api/model/StorageConfigTest.java b/clients/java/src/test/java/io/lakefs/clients/api/model/StorageConfigTest.java index b0e969c1774..0a46368e0c9 100644 --- a/clients/java/src/test/java/io/lakefs/clients/api/model/StorageConfigTest.java +++ b/clients/java/src/test/java/io/lakefs/clients/api/model/StorageConfigTest.java @@ -96,4 +96,12 @@ public void importSupportTest() { // TODO: test importSupport } + /** + * Test the property 'importValidityRegex' + */ + @Test + public void importValidityRegexTest() { + // TODO: test importValidityRegex + } + } diff --git a/clients/python/docs/StorageConfig.md b/clients/python/docs/StorageConfig.md index 85f44afccdf..c9f469c1ddb 100644 --- a/clients/python/docs/StorageConfig.md +++ b/clients/python/docs/StorageConfig.md @@ -10,6 +10,7 @@ Name | Type | Description | Notes **pre_sign_support** | **bool** | | **pre_sign_support_ui** | **bool** | | **import_support** | **bool** | | +**import_validity_regex** | **str** | | **default_namespace_prefix** | **str** | | [optional] **any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional] diff --git a/clients/python/lakefs_client/model/storage_config.py b/clients/python/lakefs_client/model/storage_config.py index 4bdd63d1121..d8c452d7a1a 100644 --- a/clients/python/lakefs_client/model/storage_config.py +++ b/clients/python/lakefs_client/model/storage_config.py @@ -88,6 +88,7 @@ def openapi_types(): 'pre_sign_support': (bool,), # noqa: E501 'pre_sign_support_ui': (bool,), # noqa: E501 'import_support': (bool,), # noqa: E501 + 'import_validity_regex': (str,), # noqa: E501 'default_namespace_prefix': (str,), # noqa: E501 } @@ -103,6 +104,7 @@ def discriminator(): 'pre_sign_support': 'pre_sign_support', # noqa: E501 'pre_sign_support_ui': 'pre_sign_support_ui', # noqa: E501 'import_support': 'import_support', # noqa: E501 + 'import_validity_regex': 'import_validity_regex', # noqa: E501 'default_namespace_prefix': 'default_namespace_prefix', # noqa: E501 } @@ -113,7 +115,7 @@ def discriminator(): @classmethod @convert_js_args_to_python_args - def _from_openapi_data(cls, blockstore_type, blockstore_namespace_example, blockstore_namespace_validity_regex, pre_sign_support, pre_sign_support_ui, import_support, *args, **kwargs): # noqa: E501 + def _from_openapi_data(cls, blockstore_type, blockstore_namespace_example, blockstore_namespace_validity_regex, pre_sign_support, pre_sign_support_ui, import_support, import_validity_regex, *args, **kwargs): # noqa: E501 """StorageConfig - a model defined in OpenAPI Args: @@ -123,6 +125,7 @@ def _from_openapi_data(cls, blockstore_type, blockstore_namespace_example, block pre_sign_support (bool): pre_sign_support_ui (bool): import_support (bool): + import_validity_regex (str): Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -189,6 +192,7 @@ def _from_openapi_data(cls, blockstore_type, blockstore_namespace_example, block self.pre_sign_support = pre_sign_support self.pre_sign_support_ui = pre_sign_support_ui self.import_support = import_support + self.import_validity_regex = import_validity_regex for var_name, var_value in kwargs.items(): if var_name not in self.attribute_map and \ self._configuration is not None and \ @@ -209,7 +213,7 @@ def _from_openapi_data(cls, blockstore_type, blockstore_namespace_example, block ]) @convert_js_args_to_python_args - def __init__(self, blockstore_type, blockstore_namespace_example, blockstore_namespace_validity_regex, pre_sign_support, pre_sign_support_ui, import_support, *args, **kwargs): # noqa: E501 + def __init__(self, blockstore_type, blockstore_namespace_example, blockstore_namespace_validity_regex, pre_sign_support, pre_sign_support_ui, import_support, import_validity_regex, *args, **kwargs): # noqa: E501 """StorageConfig - a model defined in OpenAPI Args: @@ -219,6 +223,7 @@ def __init__(self, blockstore_type, blockstore_namespace_example, blockstore_nam pre_sign_support (bool): pre_sign_support_ui (bool): import_support (bool): + import_validity_regex (str): Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -283,6 +288,7 @@ def __init__(self, blockstore_type, blockstore_namespace_example, blockstore_nam self.pre_sign_support = pre_sign_support self.pre_sign_support_ui = pre_sign_support_ui self.import_support = import_support + self.import_validity_regex = import_validity_regex for var_name, var_value in kwargs.items(): if var_name not in self.attribute_map and \ self._configuration is not None and \ diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index e6f8cd51e07..d7e06729ea3 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -1027,6 +1027,7 @@ components: - pre_sign_support - pre_sign_support_ui - import_support + - import_validity_regex properties: blockstore_type: type: string @@ -1042,6 +1043,8 @@ components: type: boolean import_support: type: boolean + import_validity_regex: + type: string VersionConfig: type: object diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 89ea45905ae..492541e81c4 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -1452,6 +1452,7 @@ func (c *Controller) GetStorageConfig(w http.ResponseWriter, r *http.Request) { PreSignSupport: info.PreSignSupport, PreSignSupportUi: info.PreSignSupportUI, ImportSupport: info.ImportSupport, + ImportValidityRegex: info.ImportValidityRegex, } writeResponse(w, r, http.StatusOK, response) } diff --git a/pkg/block/azure/adapter.go b/pkg/block/azure/adapter.go index cecc9398fab..eb5cdd041cf 100644 --- a/pkg/block/azure/adapter.go +++ b/pkg/block/azure/adapter.go @@ -570,7 +570,8 @@ func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectP func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo { info := block.DefaultStorageNamespaceInfo(block.BlockstoreTypeAzure) - info.ValidityRegex = `^https?://[a-z,0-9]+\.blob\.core\.windows\.net` + info.ImportValidityRegex = `^https?://[a-z0-9_-]+\.(blob|adls)\.core\.windows\.net` // added adls for import hint validation in UI + info.ValidityRegex = `^https?://[a-z0-9_-]+\.blob\.core\.windows\.net` info.Example = "https://mystorageaccount.blob.core.windows.net/mycontainer/" if a.disablePreSigned { info.PreSignSupport = false diff --git a/pkg/block/namespace.go b/pkg/block/namespace.go index ddcd571d55a..65f01ba546b 100644 --- a/pkg/block/namespace.go +++ b/pkg/block/namespace.go @@ -51,6 +51,7 @@ type StorageNamespaceInfo struct { PreSignSupport bool PreSignSupportUI bool ImportSupport bool + ImportValidityRegex string } type QualifiedKey interface { @@ -189,9 +190,10 @@ func DefaultValidationRegex(scheme string) string { func DefaultStorageNamespaceInfo(scheme string) StorageNamespaceInfo { return StorageNamespaceInfo{ - ValidityRegex: DefaultValidationRegex(scheme), - Example: DefaultExample(scheme), - PreSignSupport: true, - ImportSupport: true, + ValidityRegex: DefaultValidationRegex(scheme), + Example: DefaultExample(scheme), + PreSignSupport: true, + ImportSupport: true, + ImportValidityRegex: DefaultValidationRegex(scheme), } } diff --git a/webui/src/lib/hooks/storageConfig.tsx b/webui/src/lib/hooks/storageConfig.tsx index 429eb4ba80a..dfc1cd8af53 100644 --- a/webui/src/lib/hooks/storageConfig.tsx +++ b/webui/src/lib/hooks/storageConfig.tsx @@ -16,6 +16,7 @@ type StorageConfigContextType = { blockstore_type: string | null; default_namespace_prefix: string | null; import_support: boolean; + import_validity_regex: string | null; pre_sign_support: boolean; pre_sign_support_ui: boolean; }; @@ -28,6 +29,7 @@ const storageConfigInitialState: StorageConfigContextType = { blockstore_type: null, default_namespace_prefix: null, import_support: false, + import_validity_regex: null, pre_sign_support: false, pre_sign_support_ui: false, }; diff --git a/webui/src/pages/repositories/services/import_data.jsx b/webui/src/pages/repositories/services/import_data.jsx index fc978e0e90c..4cf6075e89d 100644 --- a/webui/src/pages/repositories/services/import_data.jsx +++ b/webui/src/pages/repositories/services/import_data.jsx @@ -97,8 +97,8 @@ const ImportForm = ({ }) => { const [isSourceValid, setIsSourceValid] = useState(true); - const storageNamespaceValidityRegexStr = config.blockstore_namespace_ValidityRegex; - const storageNamespaceValidityRegex = RegExp(storageNamespaceValidityRegexStr); + const importValidityRegexStr = config.import_validity_regex; + const storageNamespaceValidityRegex = RegExp(importValidityRegexStr); const updateSourceURLValidity = () => { if (!sourceRef.current.value) { updateSrcValidity(true); @@ -124,7 +124,7 @@ const ImportForm = ({ onChange={updateSourceURLValidity}/> {isSourceValid === false && - {`Import source should match the following pattern: "${storageNamespaceValidityRegexStr}"`} + {`Import source should match the following pattern: "${importValidityRegexStr}"`} } {isSourceValid &&