diff --git a/api/swagger.yml b/api/swagger.yml index bda8a0d83b8..9ebc23a52eb 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -4556,43 +4556,6 @@ paths: $ref: "#/components/schemas/GarbageCollectionConfig" 401: $ref: "#/components/responses/Unauthorized" - /templates/{template_location}: - get: - tags: - - templates - operationId: expandTemplate - description: fetch and expand template - parameters: - - in: path - name: template_location - required: true - schema: - type: string - description: URL of the template; must be relative (to a URL configured on the server). - example: spark.submit.conf.tt - - in: query - name: params - schema: - type: object - additionalProperties: - type: string - example: - lakefs_url: https://lakefs.example.com - style: form - explode: true - responses: - 200: - description: expanded template - content: - "*/*": - schema: - format: binary - 401: - $ref: "#/components/responses/Unauthorized" - 404: - $ref: "#/components/responses/NotFound" - default: - $ref: "#/components/responses/ServerError" /statistics: post: diff --git a/clients/java/README.md b/clients/java/README.md index 9b9f76b05c0..e16dc594fc8 100644 --- a/clients/java/README.md +++ b/clients/java/README.md @@ -240,7 +240,6 @@ Class | Method | HTTP request | Description *TagsApi* | [**deleteTag**](docs/TagsApi.md#deleteTag) | **DELETE** /repositories/{repository}/tags/{tag} | delete tag *TagsApi* | [**getTag**](docs/TagsApi.md#getTag) | **GET** /repositories/{repository}/tags/{tag} | get tag *TagsApi* | [**listTags**](docs/TagsApi.md#listTags) | **GET** /repositories/{repository}/tags | list tags -*TemplatesApi* | [**expandTemplate**](docs/TemplatesApi.md#expandTemplate) | **GET** /templates/{template_location} | ## Documentation for Models diff --git a/clients/java/api/openapi.yaml b/clients/java/api/openapi.yaml index ad6bd5ebed9..e4085851a66 100644 --- a/clients/java/api/openapi.yaml +++ b/clients/java/api/openapi.yaml @@ -5495,60 +5495,6 @@ paths: tags: - config x-accepts: application/json - /templates/{template_location}: - get: - description: fetch and expand template - operationId: expandTemplate - parameters: - - description: URL of the template; must be relative (to a URL configured on - the server). - example: spark.submit.conf.tt - explode: false - in: path - name: template_location - required: true - schema: - type: string - style: simple - - explode: true - in: query - name: params - required: false - schema: - additionalProperties: - type: string - example: - lakefs_url: https://lakefs.example.com - type: object - style: form - responses: - "200": - content: - '*/*': - schema: - format: binary - description: expanded template - "401": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Unauthorized - "404": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Resource Not Found - default: - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Internal Server Error - tags: - - templates - x-accepts: application/json /statistics: post: operationId: postStatsEvents diff --git a/clients/java/docs/TemplatesApi.md b/clients/java/docs/TemplatesApi.md deleted file mode 100644 index cee49474a4a..00000000000 --- a/clients/java/docs/TemplatesApi.md +++ /dev/null @@ -1,104 +0,0 @@ -# TemplatesApi - -All URIs are relative to *http://localhost/api/v1* - -Method | HTTP request | Description -------------- | ------------- | ------------- -[**expandTemplate**](TemplatesApi.md#expandTemplate) | **GET** /templates/{template_location} | - - - -# **expandTemplate** -> Object expandTemplate(templateLocation, params) - - - -fetch and expand template - -### Example -```java -// Import classes: -import io.lakefs.clients.api.ApiClient; -import io.lakefs.clients.api.ApiException; -import io.lakefs.clients.api.Configuration; -import io.lakefs.clients.api.auth.*; -import io.lakefs.clients.api.models.*; -import io.lakefs.clients.api.TemplatesApi; - -public class Example { - public static void main(String[] args) { - ApiClient defaultClient = Configuration.getDefaultApiClient(); - defaultClient.setBasePath("http://localhost/api/v1"); - - // Configure HTTP basic authorization: basic_auth - HttpBasicAuth basic_auth = (HttpBasicAuth) defaultClient.getAuthentication("basic_auth"); - basic_auth.setUsername("YOUR USERNAME"); - basic_auth.setPassword("YOUR PASSWORD"); - - // Configure API key authorization: cookie_auth - ApiKeyAuth cookie_auth = (ApiKeyAuth) defaultClient.getAuthentication("cookie_auth"); - cookie_auth.setApiKey("YOUR API KEY"); - // Uncomment the following line to set a prefix for the API key, e.g. "Token" (defaults to null) - //cookie_auth.setApiKeyPrefix("Token"); - - // Configure HTTP bearer authorization: jwt_token - HttpBearerAuth jwt_token = (HttpBearerAuth) defaultClient.getAuthentication("jwt_token"); - jwt_token.setBearerToken("BEARER TOKEN"); - - // Configure API key authorization: oidc_auth - ApiKeyAuth oidc_auth = (ApiKeyAuth) defaultClient.getAuthentication("oidc_auth"); - oidc_auth.setApiKey("YOUR API KEY"); - // Uncomment the following line to set a prefix for the API key, e.g. "Token" (defaults to null) - //oidc_auth.setApiKeyPrefix("Token"); - - // Configure API key authorization: saml_auth - ApiKeyAuth saml_auth = (ApiKeyAuth) defaultClient.getAuthentication("saml_auth"); - saml_auth.setApiKey("YOUR API KEY"); - // Uncomment the following line to set a prefix for the API key, e.g. "Token" (defaults to null) - //saml_auth.setApiKeyPrefix("Token"); - - TemplatesApi apiInstance = new TemplatesApi(defaultClient); - String templateLocation = "spark.submit.conf.tt"; // String | URL of the template; must be relative (to a URL configured on the server). - Map params = new HashMap(); // Map | - try { - Object result = apiInstance.expandTemplate(templateLocation, params); - System.out.println(result); - } catch (ApiException e) { - System.err.println("Exception when calling TemplatesApi#expandTemplate"); - System.err.println("Status code: " + e.getCode()); - System.err.println("Reason: " + e.getResponseBody()); - System.err.println("Response headers: " + e.getResponseHeaders()); - e.printStackTrace(); - } - } -} -``` - -### Parameters - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - **templateLocation** | **String**| URL of the template; must be relative (to a URL configured on the server). | - **params** | [**Map<String, String>**](String.md)| | [optional] - -### Return type - -**Object** - -### Authorization - -[basic_auth](../README.md#basic_auth), [cookie_auth](../README.md#cookie_auth), [jwt_token](../README.md#jwt_token), [oidc_auth](../README.md#oidc_auth), [saml_auth](../README.md#saml_auth) - -### HTTP request headers - - - **Content-Type**: Not defined - - **Accept**: */*, application/json - -### HTTP response details -| Status code | Description | Response headers | -|-------------|-------------|------------------| -**200** | expanded template | - | -**401** | Unauthorized | - | -**404** | Resource Not Found | - | -**0** | Internal Server Error | - | - diff --git a/clients/java/src/main/java/io/lakefs/clients/api/TemplatesApi.java b/clients/java/src/main/java/io/lakefs/clients/api/TemplatesApi.java deleted file mode 100644 index c4a22d2c00a..00000000000 --- a/clients/java/src/main/java/io/lakefs/clients/api/TemplatesApi.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * lakeFS API - * lakeFS HTTP API - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -package io.lakefs.clients.api; - -import io.lakefs.clients.api.ApiCallback; -import io.lakefs.clients.api.ApiClient; -import io.lakefs.clients.api.ApiException; -import io.lakefs.clients.api.ApiResponse; -import io.lakefs.clients.api.Configuration; -import io.lakefs.clients.api.Pair; -import io.lakefs.clients.api.ProgressRequestBody; -import io.lakefs.clients.api.ProgressResponseBody; - -import com.google.gson.reflect.TypeToken; - -import java.io.IOException; - - -import io.lakefs.clients.api.model.Error; - -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class TemplatesApi { - private ApiClient localVarApiClient; - - public TemplatesApi() { - this(Configuration.getDefaultApiClient()); - } - - public TemplatesApi(ApiClient apiClient) { - this.localVarApiClient = apiClient; - } - - public ApiClient getApiClient() { - return localVarApiClient; - } - - public void setApiClient(ApiClient apiClient) { - this.localVarApiClient = apiClient; - } - - /** - * Build call for expandTemplate - * @param templateLocation URL of the template; must be relative (to a URL configured on the server). (required) - * @param params (optional) - * @param _callback Callback for upload/download progress - * @return Call to execute - * @throws ApiException If fail to serialize the request body object - * @http.response.details - - - - - - -
Status Code Description Response Headers
200 expanded template -
401 Unauthorized -
404 Resource Not Found -
0 Internal Server Error -
- */ - public okhttp3.Call expandTemplateCall(String templateLocation, Map params, final ApiCallback _callback) throws ApiException { - Object localVarPostBody = null; - - // create path and map variables - String localVarPath = "/templates/{template_location}" - .replaceAll("\\{" + "template_location" + "\\}", localVarApiClient.escapeString(templateLocation.toString())); - - List localVarQueryParams = new ArrayList(); - List localVarCollectionQueryParams = new ArrayList(); - Map localVarHeaderParams = new HashMap(); - Map localVarCookieParams = new HashMap(); - Map localVarFormParams = new HashMap(); - - if (params != null) { - localVarQueryParams.addAll(localVarApiClient.parameterToPair("params", params)); - } - - final String[] localVarAccepts = { - "*/*", "application/json" - }; - final String localVarAccept = localVarApiClient.selectHeaderAccept(localVarAccepts); - if (localVarAccept != null) { - localVarHeaderParams.put("Accept", localVarAccept); - } - - final String[] localVarContentTypes = { - - }; - final String localVarContentType = localVarApiClient.selectHeaderContentType(localVarContentTypes); - localVarHeaderParams.put("Content-Type", localVarContentType); - - String[] localVarAuthNames = new String[] { "basic_auth", "cookie_auth", "jwt_token", "oidc_auth", "saml_auth" }; - return localVarApiClient.buildCall(localVarPath, "GET", localVarQueryParams, localVarCollectionQueryParams, localVarPostBody, localVarHeaderParams, localVarCookieParams, localVarFormParams, localVarAuthNames, _callback); - } - - @SuppressWarnings("rawtypes") - private okhttp3.Call expandTemplateValidateBeforeCall(String templateLocation, Map params, final ApiCallback _callback) throws ApiException { - - // verify the required parameter 'templateLocation' is set - if (templateLocation == null) { - throw new ApiException("Missing the required parameter 'templateLocation' when calling expandTemplate(Async)"); - } - - - okhttp3.Call localVarCall = expandTemplateCall(templateLocation, params, _callback); - return localVarCall; - - } - - /** - * - * fetch and expand template - * @param templateLocation URL of the template; must be relative (to a URL configured on the server). (required) - * @param params (optional) - * @return Object - * @throws ApiException If fail to call the API, e.g. server error or cannot deserialize the response body - * @http.response.details - - - - - - -
Status Code Description Response Headers
200 expanded template -
401 Unauthorized -
404 Resource Not Found -
0 Internal Server Error -
- */ - public Object expandTemplate(String templateLocation, Map params) throws ApiException { - ApiResponse localVarResp = expandTemplateWithHttpInfo(templateLocation, params); - return localVarResp.getData(); - } - - /** - * - * fetch and expand template - * @param templateLocation URL of the template; must be relative (to a URL configured on the server). (required) - * @param params (optional) - * @return ApiResponse<Object> - * @throws ApiException If fail to call the API, e.g. server error or cannot deserialize the response body - * @http.response.details - - - - - - -
Status Code Description Response Headers
200 expanded template -
401 Unauthorized -
404 Resource Not Found -
0 Internal Server Error -
- */ - public ApiResponse expandTemplateWithHttpInfo(String templateLocation, Map params) throws ApiException { - okhttp3.Call localVarCall = expandTemplateValidateBeforeCall(templateLocation, params, null); - Type localVarReturnType = new TypeToken(){}.getType(); - return localVarApiClient.execute(localVarCall, localVarReturnType); - } - - /** - * (asynchronously) - * fetch and expand template - * @param templateLocation URL of the template; must be relative (to a URL configured on the server). (required) - * @param params (optional) - * @param _callback The callback to be executed when the API call finishes - * @return The request call - * @throws ApiException If fail to process the API call, e.g. serializing the request body object - * @http.response.details - - - - - - -
Status Code Description Response Headers
200 expanded template -
401 Unauthorized -
404 Resource Not Found -
0 Internal Server Error -
- */ - public okhttp3.Call expandTemplateAsync(String templateLocation, Map params, final ApiCallback _callback) throws ApiException { - - okhttp3.Call localVarCall = expandTemplateValidateBeforeCall(templateLocation, params, _callback); - Type localVarReturnType = new TypeToken(){}.getType(); - localVarApiClient.executeAsync(localVarCall, localVarReturnType, _callback); - return localVarCall; - } -} diff --git a/clients/java/src/test/java/io/lakefs/clients/api/TemplatesApiTest.java b/clients/java/src/test/java/io/lakefs/clients/api/TemplatesApiTest.java deleted file mode 100644 index 0f1d4ea5309..00000000000 --- a/clients/java/src/test/java/io/lakefs/clients/api/TemplatesApiTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * lakeFS API - * lakeFS HTTP API - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -package io.lakefs.clients.api; - -import io.lakefs.clients.api.ApiException; -import io.lakefs.clients.api.model.Error; -import org.junit.Test; -import org.junit.Ignore; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * API tests for TemplatesApi - */ -@Ignore -public class TemplatesApiTest { - - private final TemplatesApi api = new TemplatesApi(); - - - /** - * - * - * fetch and expand template - * - * @throws ApiException - * if the Api call fails - */ - @Test - public void expandTemplateTest() throws ApiException { - String templateLocation = null; - Map params = null; - Object response = api.expandTemplate(templateLocation, params); - // TODO: test validations - } - -} diff --git a/clients/python/.openapi-generator/FILES b/clients/python/.openapi-generator/FILES index fa67a29681a..e77e0047b14 100644 --- a/clients/python/.openapi-generator/FILES +++ b/clients/python/.openapi-generator/FILES @@ -102,7 +102,6 @@ docs/StorageConfig.md docs/StorageURI.md docs/TagCreation.md docs/TagsApi.md -docs/TemplatesApi.md docs/UnderlyingObjectProperties.md docs/UpdatePasswordByToken.md docs/UpdateToken.md @@ -128,7 +127,6 @@ lakefs_client/api/repositories_api.py lakefs_client/api/retention_api.py lakefs_client/api/staging_api.py lakefs_client/api/tags_api.py -lakefs_client/api/templates_api.py lakefs_client/api_client.py lakefs_client/apis/__init__.py lakefs_client/client.py @@ -336,7 +334,6 @@ test/test_storage_config.py test/test_storage_uri.py test/test_tag_creation.py test/test_tags_api.py -test/test_templates_api.py test/test_underlying_object_properties.py test/test_update_password_by_token.py test/test_update_token.py diff --git a/clients/python/README.md b/clients/python/README.md index c673f9a91ed..d40a5a0406d 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -221,7 +221,6 @@ Class | Method | HTTP request | Description *TagsApi* | [**delete_tag**](docs/TagsApi.md#delete_tag) | **DELETE** /repositories/{repository}/tags/{tag} | delete tag *TagsApi* | [**get_tag**](docs/TagsApi.md#get_tag) | **GET** /repositories/{repository}/tags/{tag} | get tag *TagsApi* | [**list_tags**](docs/TagsApi.md#list_tags) | **GET** /repositories/{repository}/tags | list tags -*TemplatesApi* | [**expand_template**](docs/TemplatesApi.md#expand_template) | **GET** /templates/{template_location} | ## Documentation For Models diff --git a/clients/python/docs/TemplatesApi.md b/clients/python/docs/TemplatesApi.md deleted file mode 100644 index 498fb3e7b1b..00000000000 --- a/clients/python/docs/TemplatesApi.md +++ /dev/null @@ -1,128 +0,0 @@ -# lakefs_client.TemplatesApi - -All URIs are relative to *http://localhost/api/v1* - -Method | HTTP request | Description -------------- | ------------- | ------------- -[**expand_template**](TemplatesApi.md#expand_template) | **GET** /templates/{template_location} | - - -# **expand_template** -> bool, date, datetime, dict, float, int, list, str, none_type expand_template(template_location) - - - -fetch and expand template - -### Example - -* Basic Authentication (basic_auth): -* Api Key Authentication (cookie_auth): -* Bearer (JWT) Authentication (jwt_token): -* Api Key Authentication (oidc_auth): -* Api Key Authentication (saml_auth): - -```python -import time -import lakefs_client -from lakefs_client.api import templates_api -from lakefs_client.model.error import Error -from pprint import pprint -# Defining the host is optional and defaults to http://localhost/api/v1 -# See configuration.py for a list of all supported configuration parameters. -configuration = lakefs_client.Configuration( - host = "http://localhost/api/v1" -) - -# The client must configure the authentication and authorization parameters -# in accordance with the API server security policy. -# Examples for each auth method are provided below, use the example that -# satisfies your auth use case. - -# Configure HTTP basic authorization: basic_auth -configuration = lakefs_client.Configuration( - username = 'YOUR_USERNAME', - password = 'YOUR_PASSWORD' -) - -# Configure API key authorization: cookie_auth -configuration.api_key['cookie_auth'] = 'YOUR_API_KEY' - -# Uncomment below to setup prefix (e.g. Bearer) for API key, if needed -# configuration.api_key_prefix['cookie_auth'] = 'Bearer' - -# Configure Bearer authorization (JWT): jwt_token -configuration = lakefs_client.Configuration( - access_token = 'YOUR_BEARER_TOKEN' -) - -# Configure API key authorization: oidc_auth -configuration.api_key['oidc_auth'] = 'YOUR_API_KEY' - -# Uncomment below to setup prefix (e.g. Bearer) for API key, if needed -# configuration.api_key_prefix['oidc_auth'] = 'Bearer' - -# Configure API key authorization: saml_auth -configuration.api_key['saml_auth'] = 'YOUR_API_KEY' - -# Uncomment below to setup prefix (e.g. Bearer) for API key, if needed -# configuration.api_key_prefix['saml_auth'] = 'Bearer' - -# Enter a context with an instance of the API client -with lakefs_client.ApiClient(configuration) as api_client: - # Create an instance of the API class - api_instance = templates_api.TemplatesApi(api_client) - template_location = "spark.submit.conf.tt" # str | URL of the template; must be relative (to a URL configured on the server). - params = { - "key": "key_example", - } # {str: (str,)} | (optional) - - # example passing only required values which don't have defaults set - try: - api_response = api_instance.expand_template(template_location) - pprint(api_response) - except lakefs_client.ApiException as e: - print("Exception when calling TemplatesApi->expand_template: %s\n" % e) - - # example passing only required values which don't have defaults set - # and optional values - try: - api_response = api_instance.expand_template(template_location, params=params) - pprint(api_response) - except lakefs_client.ApiException as e: - print("Exception when calling TemplatesApi->expand_template: %s\n" % e) -``` - - -### Parameters - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - **template_location** | **str**| URL of the template; must be relative (to a URL configured on the server). | - **params** | **{str: (str,)}**| | [optional] - -### Return type - -**bool, date, datetime, dict, float, int, list, str, none_type** - -### Authorization - -[basic_auth](../README.md#basic_auth), [cookie_auth](../README.md#cookie_auth), [jwt_token](../README.md#jwt_token), [oidc_auth](../README.md#oidc_auth), [saml_auth](../README.md#saml_auth) - -### HTTP request headers - - - **Content-Type**: Not defined - - **Accept**: */*, application/json - - -### HTTP response details - -| Status code | Description | Response headers | -|-------------|-------------|------------------| -**200** | expanded template | - | -**401** | Unauthorized | - | -**404** | Resource Not Found | - | -**0** | Internal Server Error | - | - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) - diff --git a/clients/python/lakefs_client/api/templates_api.py b/clients/python/lakefs_client/api/templates_api.py deleted file mode 100644 index e951c85ddbc..00000000000 --- a/clients/python/lakefs_client/api/templates_api.py +++ /dev/null @@ -1,167 +0,0 @@ -""" - lakeFS API - - lakeFS HTTP API # noqa: E501 - - The version of the OpenAPI document: 0.1.0 - Contact: services@treeverse.io - Generated by: https://openapi-generator.tech -""" - - -import re # noqa: F401 -import sys # noqa: F401 - -from lakefs_client.api_client import ApiClient, Endpoint as _Endpoint -from lakefs_client.model_utils import ( # noqa: F401 - check_allowed_values, - check_validations, - date, - datetime, - file_type, - none_type, - validate_and_convert_types -) -from lakefs_client.model.error import Error - - -class TemplatesApi(object): - """NOTE: This class is auto generated by OpenAPI Generator - Ref: https://openapi-generator.tech - - Do not edit the class manually. - """ - - def __init__(self, api_client=None): - if api_client is None: - api_client = ApiClient() - self.api_client = api_client - self.expand_template_endpoint = _Endpoint( - settings={ - 'response_type': (bool, date, datetime, dict, float, int, list, str, none_type,), - 'auth': [ - 'basic_auth', - 'cookie_auth', - 'jwt_token', - 'oidc_auth', - 'saml_auth' - ], - 'endpoint_path': '/templates/{template_location}', - 'operation_id': 'expand_template', - 'http_method': 'GET', - 'servers': None, - }, - params_map={ - 'all': [ - 'template_location', - 'params', - ], - 'required': [ - 'template_location', - ], - 'nullable': [ - ], - 'enum': [ - ], - 'validation': [ - ] - }, - root_map={ - 'validations': { - }, - 'allowed_values': { - }, - 'openapi_types': { - 'template_location': - (str,), - 'params': - ({str: (str,)},), - }, - 'attribute_map': { - 'template_location': 'template_location', - 'params': 'params', - }, - 'location_map': { - 'template_location': 'path', - 'params': 'query', - }, - 'collection_format_map': { - } - }, - headers_map={ - 'accept': [ - '*/*', - 'application/json' - ], - 'content_type': [], - }, - api_client=api_client - ) - - def expand_template( - self, - template_location, - **kwargs - ): - """expand_template # noqa: E501 - - fetch and expand template # noqa: E501 - This method makes a synchronous HTTP request by default. To make an - asynchronous HTTP request, please pass async_req=True - - >>> thread = api.expand_template(template_location, async_req=True) - >>> result = thread.get() - - Args: - template_location (str): URL of the template; must be relative (to a URL configured on the server). - - Keyword Args: - params ({str: (str,)}): [optional] - _return_http_data_only (bool): response data without head status - code and headers. Default is True. - _preload_content (bool): if False, the urllib3.HTTPResponse object - will be returned without reading/decoding response data. - Default is True. - _request_timeout (int/float/tuple): timeout setting for this request. If - one number provided, it will be total request timeout. It can also - be a pair (tuple) of (connection, read) timeouts. - Default is None. - _check_input_type (bool): specifies if type checking - should be done one the data sent to the server. - Default is True. - _check_return_type (bool): specifies if type checking - should be done one the data received from the server. - Default is True. - _host_index (int/None): specifies the index of the server - that we want to use. - Default is read from the configuration. - async_req (bool): execute request asynchronously - - Returns: - bool, date, datetime, dict, float, int, list, str, none_type - If the method is called asynchronously, returns the request - thread. - """ - kwargs['async_req'] = kwargs.get( - 'async_req', False - ) - kwargs['_return_http_data_only'] = kwargs.get( - '_return_http_data_only', True - ) - kwargs['_preload_content'] = kwargs.get( - '_preload_content', True - ) - kwargs['_request_timeout'] = kwargs.get( - '_request_timeout', None - ) - kwargs['_check_input_type'] = kwargs.get( - '_check_input_type', True - ) - kwargs['_check_return_type'] = kwargs.get( - '_check_return_type', True - ) - kwargs['_host_index'] = kwargs.get('_host_index') - kwargs['template_location'] = \ - template_location - return self.expand_template_endpoint.call_with_http_info(**kwargs) - diff --git a/clients/python/lakefs_client/apis/__init__.py b/clients/python/lakefs_client/apis/__init__.py index 0d92819de42..9b0888a23a3 100644 --- a/clients/python/lakefs_client/apis/__init__.py +++ b/clients/python/lakefs_client/apis/__init__.py @@ -30,4 +30,3 @@ from lakefs_client.api.retention_api import RetentionApi from lakefs_client.api.staging_api import StagingApi from lakefs_client.api.tags_api import TagsApi -from lakefs_client.api.templates_api import TemplatesApi diff --git a/clients/python/lakefs_client/client.py b/clients/python/lakefs_client/client.py index 1a941ff7e7f..2589e24bd63 100644 --- a/clients/python/lakefs_client/client.py +++ b/clients/python/lakefs_client/client.py @@ -19,7 +19,6 @@ from lakefs_client.api import retention_api from lakefs_client.api import staging_api from lakefs_client.api import tags_api -from lakefs_client.api import templates_api class _WrappedApiClient(lakefs_client.ApiClient): @@ -61,7 +60,6 @@ def __init__(self, configuration=None, header_name=None, header_value=None, cook self.retention_api = retention_api.RetentionApi(self._api) self.staging_api = staging_api.StagingApi(self._api) self.tags_api = tags_api.TagsApi(self._api) - self.templates_api = templates_api.TemplatesApi(self._api) @staticmethod def _ensure_endpoint(configuration): diff --git a/clients/python/test/test_templates_api.py b/clients/python/test/test_templates_api.py deleted file mode 100644 index 5e8ccbc2c4d..00000000000 --- a/clients/python/test/test_templates_api.py +++ /dev/null @@ -1,35 +0,0 @@ -""" - lakeFS API - - lakeFS HTTP API # noqa: E501 - - The version of the OpenAPI document: 0.1.0 - Contact: services@treeverse.io - Generated by: https://openapi-generator.tech -""" - - -import unittest - -import lakefs_client -from lakefs_client.api.templates_api import TemplatesApi # noqa: E501 - - -class TestTemplatesApi(unittest.TestCase): - """TemplatesApi unit test stubs""" - - def setUp(self): - self.api = TemplatesApi() # noqa: E501 - - def tearDown(self): - pass - - def test_expand_template(self): - """Test case for expand_template - - """ - pass - - -if __name__ == '__main__': - unittest.main() diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go index c327629ebe7..966079d80c5 100644 --- a/cmd/lakefs/cmd/run.go +++ b/cmd/lakefs/cmd/run.go @@ -44,10 +44,8 @@ import ( "github.com/treeverse/lakefs/pkg/logging" tablediff "github.com/treeverse/lakefs/pkg/plugins/diff" "github.com/treeverse/lakefs/pkg/stats" - "github.com/treeverse/lakefs/pkg/templater" "github.com/treeverse/lakefs/pkg/upload" "github.com/treeverse/lakefs/pkg/version" - "github.com/treeverse/lakefs/templates" ) const ( @@ -204,8 +202,6 @@ var runCmd = &cobra.Command{ } } - templater := templater.NewService(templates.Content, cfg, authService) - actionsService := actions.NewService( ctx, actionsStore, @@ -268,7 +264,6 @@ var runCmd = &cobra.Command{ auditChecker, logger.WithField("service", "api_gateway"), emailer, - templater, cfg.Gateways.S3.DomainNames, cfg.UISnippets(), upload.DefaultPathProvider, diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index bda8a0d83b8..9ebc23a52eb 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -4556,43 +4556,6 @@ paths: $ref: "#/components/schemas/GarbageCollectionConfig" 401: $ref: "#/components/responses/Unauthorized" - /templates/{template_location}: - get: - tags: - - templates - operationId: expandTemplate - description: fetch and expand template - parameters: - - in: path - name: template_location - required: true - schema: - type: string - description: URL of the template; must be relative (to a URL configured on the server). - example: spark.submit.conf.tt - - in: query - name: params - schema: - type: object - additionalProperties: - type: string - example: - lakefs_url: https://lakefs.example.com - style: form - explode: true - responses: - 200: - description: expanded template - content: - "*/*": - schema: - format: binary - 401: - $ref: "#/components/responses/Unauthorized" - 404: - $ref: "#/components/responses/NotFound" - default: - $ref: "#/components/responses/ServerError" /statistics: post: diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 909894123b0..30eae107cc4 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -43,7 +43,6 @@ import ( tablediff "github.com/treeverse/lakefs/pkg/plugins/diff" "github.com/treeverse/lakefs/pkg/samplerepo" "github.com/treeverse/lakefs/pkg/stats" - "github.com/treeverse/lakefs/pkg/templater" "github.com/treeverse/lakefs/pkg/upload" "github.com/treeverse/lakefs/pkg/validator" "github.com/treeverse/lakefs/pkg/version" @@ -96,7 +95,6 @@ type Controller struct { AuditChecker AuditChecker Logger logging.Logger Emailer *email.Emailer - Templater templater.Service sessionStore sessions.Store PathProvider upload.PathProvider otfDiffService *tablediff.Service @@ -4302,52 +4300,6 @@ func (c *Controller) UpdatePassword(w http.ResponseWriter, r *http.Request, body writeResponse(w, r, http.StatusCreated, nil) } -func (c *Controller) ExpandTemplate(w http.ResponseWriter, r *http.Request, templateLocation string, p apigen.ExpandTemplateParams) { - if !c.authorize(w, r, permissions.Node{ - Permission: permissions.Permission{ - Action: permissions.ReadObjectAction, - Resource: permissions.TemplateArn(templateLocation), - }, - }) { - return - } - ctx := r.Context() - - u, err := auth.GetUser(ctx) - if err != nil { - writeError(w, r, http.StatusInternalServerError, "request performed with no user") - } - - // Override bug in OpenAPI generated code: parameters do not show up - // in p.Params.AdditionalProperties, so force them in there. - if len(p.Params.AdditionalProperties) == 0 && len(r.URL.Query()) > 0 { - p.Params.AdditionalProperties = make(map[string]string, len(r.Header)) - for k, v := range r.URL.Query() { - p.Params.AdditionalProperties[k] = v[0] - } - } - - c.LogAction(ctx, "expand_template", r, "", "", "") - err = c.Templater.Expand(ctx, w, u, templateLocation, p.Params.AdditionalProperties) - if err != nil { - c.Logger.WithError(err).WithField("location", templateLocation).Error("Template expansion failed") - } - - if errors.Is(err, templater.ErrNotAuthorized) { - writeError(w, r, http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)) - return - } - if errors.Is(err, templater.ErrNotFound) { - writeError(w, r, http.StatusNotFound, http.StatusText(http.StatusNotFound)) - return - } - if err != nil { - writeError(w, r, http.StatusInternalServerError, "expansion failed") - return - } - // Response already written during expansion. -} - func (c *Controller) GetLakeFSVersion(w http.ResponseWriter, r *http.Request) { ctx := r.Context() _, err := auth.GetUser(ctx) @@ -4655,25 +4607,7 @@ func resolvePathList(objects, prefixes *[]string) []catalog.PathRecord { return pathRecords } -func NewController( - cfg *config.Config, - catalog catalog.Interface, - authenticator auth.Authenticator, - authService auth.Service, - blockAdapter block.Adapter, - metadataManager auth.MetadataManager, - migrator Migrator, - collector stats.Collector, - cloudMetadataProvider cloud.MetadataProvider, - actions actionsHandler, - auditChecker AuditChecker, - logger logging.Logger, - emailer *email.Emailer, - templater templater.Service, - sessionStore sessions.Store, - pathProvider upload.PathProvider, - otfDiffService *tablediff.Service, -) *Controller { +func NewController(cfg *config.Config, catalog catalog.Interface, authenticator auth.Authenticator, authService auth.Service, blockAdapter block.Adapter, metadataManager auth.MetadataManager, migrator Migrator, collector stats.Collector, cloudMetadataProvider cloud.MetadataProvider, actions actionsHandler, auditChecker AuditChecker, logger logging.Logger, emailer *email.Emailer, sessionStore sessions.Store, pathProvider upload.PathProvider, otfDiffService *tablediff.Service) *Controller { return &Controller{ Config: cfg, Catalog: catalog, @@ -4688,7 +4622,6 @@ func NewController( AuditChecker: auditChecker, Logger: logger, Emailer: emailer, - Templater: templater, sessionStore: sessionStore, PathProvider: pathProvider, otfDiffService: otfDiffService, diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 8f2bd500b8e..af787ff2bd1 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -3,7 +3,6 @@ package api_test import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -14,7 +13,6 @@ import ( "net/http/httptest" "path/filepath" "reflect" - "regexp" "sort" "strconv" "strings" @@ -3694,79 +3692,6 @@ func TestController_CherryPick(t *testing.T) { }) } -func TestController_ExpandTemplate(t *testing.T) { - clt, _ := setupClientWithAdmin(t) - ctx := context.Background() - - t.Run("not-found", func(t *testing.T) { - resp, err := clt.ExpandTemplateWithResponse(ctx, "no/template/here", &apigen.ExpandTemplateParams{}) - testutil.Must(t, err) - if resp.HTTPResponse.StatusCode != http.StatusNotFound { - t.Errorf("Expanding a nonexistent template should fail with status %d got %d\n\t%s\n\t%+v", http.StatusNotFound, resp.HTTPResponse.StatusCode, string(resp.Body), resp) - } - }) - - t.Run("spark.conf", func(t *testing.T) { - const lfsURL = "https://lakefs.example.test" - expected := []struct { - name string - pattern string - }{ - {"impl", `spark\.hadoop\.fs\.s3a\.impl=org\.apache\.hadoop\.fs\.s3a\.S3AFileSystem`}, - {"access.key", `spark\.hadoop\.fs\.s3a\.access.key=AKIA.*`}, - {"secret.key", `spark\.hadoop\.fs\.s3a\.secret.key=`}, - {"s3a_endpoint", `spark\.hadoop\.fs\.s3a\.endpoint=` + lfsURL}, - } - - // OpenAPI places additional query params in the wrong - // place. Use a request editor to place them directly as a - // query string. - resp, err := clt.ExpandTemplateWithResponse(ctx, "spark.submit.conf.tt", &apigen.ExpandTemplateParams{}, - func(_ context.Context, req *http.Request) error { - values := req.URL.Query() - values.Add("lakefs_url", lfsURL) - req.URL.RawQuery = values.Encode() - return nil - }) - testutil.Must(t, err) - if resp.HTTPResponse.StatusCode != http.StatusOK { - t.Errorf("Expansion failed with status %d\n\t%s\n\t%+v", resp.HTTPResponse.StatusCode, string(resp.Body), resp) - } - - contentType := resp.HTTPResponse.Header.Values("Content-Type") - if len(contentType) != 1 { - t.Errorf("Expansion returned %d content types: %v", len(contentType), contentType) - } - if contentType[0] != "application/x-conf" { - t.Errorf("Expansion returned content type %s not application/x-conf", contentType[0]) - } - - for _, e := range expected { - re := regexp.MustCompile(e.pattern) - if !re.Match(resp.Body) { - t.Errorf("Expansion result has no %s: /%s/\n\t%s", e.name, e.pattern, string(resp.Body)) - } - } - }) - - t.Run("fail", func(t *testing.T) { - resp, err := clt.ExpandTemplateWithResponse(ctx, "fail.tt", &apigen.ExpandTemplateParams{}) - testutil.Must(t, err) - if resp.HTTPResponse.StatusCode != http.StatusInternalServerError { - t.Errorf("Expansion should fail with status %d got %d\n\t%s\n\t%+v", http.StatusInternalServerError, resp.HTTPResponse.StatusCode, string(resp.Body), resp) - } - - parsed := make(map[string]string, 0) - err = json.Unmarshal(resp.Body, &parsed) - if err != nil { - t.Errorf("Unmarshal body: %s", err) - } - if parsed["message"] != "expansion failed" { - t.Errorf("Expected \"expansion failed\" message, got %+v", parsed) - } - }) -} - func TestController_UpdatePolicy(t *testing.T) { clt, _ := setupClientWithAdmin(t) ctx := context.Background() diff --git a/pkg/api/serve.go b/pkg/api/serve.go index b50ff2bad13..cfc8b4dcf73 100644 --- a/pkg/api/serve.go +++ b/pkg/api/serve.go @@ -25,7 +25,6 @@ import ( "github.com/treeverse/lakefs/pkg/logging" tablediff "github.com/treeverse/lakefs/pkg/plugins/diff" "github.com/treeverse/lakefs/pkg/stats" - "github.com/treeverse/lakefs/pkg/templater" "github.com/treeverse/lakefs/pkg/upload" ) @@ -36,26 +35,7 @@ const ( extensionValidationExcludeBody = "x-validation-exclude-body" ) -func Serve( - cfg *config.Config, - catalog catalog.Interface, - middlewareAuthenticator auth.Authenticator, - authService auth.Service, - blockAdapter block.Adapter, - metadataManager auth.MetadataManager, - migrator Migrator, - collector stats.Collector, - cloudMetadataProvider cloud.MetadataProvider, - actions actionsHandler, - auditChecker AuditChecker, - logger logging.Logger, - emailer *email.Emailer, - templater templater.Service, - gatewayDomains []string, - snippets []params.CodeSnippet, - pathProvider upload.PathProvider, - otfService *tablediff.Service, -) http.Handler { +func Serve(cfg *config.Config, catalog catalog.Interface, middlewareAuthenticator auth.Authenticator, authService auth.Service, blockAdapter block.Adapter, metadataManager auth.MetadataManager, migrator Migrator, collector stats.Collector, cloudMetadataProvider cloud.MetadataProvider, actions actionsHandler, auditChecker AuditChecker, logger logging.Logger, emailer *email.Emailer, gatewayDomains []string, snippets []params.CodeSnippet, pathProvider upload.PathProvider, otfService *tablediff.Service) http.Handler { logger.Info("initialize OpenAPI server") swagger, err := apigen.GetSwagger() if err != nil { @@ -77,25 +57,7 @@ func Serve( AuthMiddleware(logger, swagger, middlewareAuthenticator, authService, sessionStore, &oidcConfig, &cookieAuthConfig), MetricsMiddleware(swagger), ) - controller := NewController( - cfg, - catalog, - middlewareAuthenticator, - authService, - blockAdapter, - metadataManager, - migrator, - collector, - cloudMetadataProvider, - actions, - auditChecker, - logger, - emailer, - templater, - sessionStore, - pathProvider, - otfService, - ) + controller := NewController(cfg, catalog, middlewareAuthenticator, authService, blockAdapter, metadataManager, migrator, collector, cloudMetadataProvider, actions, auditChecker, logger, emailer, sessionStore, pathProvider, otfService) apigen.HandlerFromMuxWithBaseURL(controller, apiRouter, apiutil.BaseURL) r.Mount("/_health", httputil.ServeHealth()) diff --git a/pkg/api/serve_test.go b/pkg/api/serve_test.go index 2dd24425756..b4d8da130ec 100644 --- a/pkg/api/serve_test.go +++ b/pkg/api/serve_test.go @@ -36,11 +36,9 @@ import ( "github.com/treeverse/lakefs/pkg/logging" tablediff "github.com/treeverse/lakefs/pkg/plugins/diff" "github.com/treeverse/lakefs/pkg/stats" - "github.com/treeverse/lakefs/pkg/templater" "github.com/treeverse/lakefs/pkg/testutil" "github.com/treeverse/lakefs/pkg/upload" "github.com/treeverse/lakefs/pkg/version" - "github.com/treeverse/lakefs/templates" ) const ( @@ -190,31 +188,11 @@ func setupHandlerWithWalkerFactory(t testing.TB, factory catalog.WalkerFactory) auditChecker := version.NewDefaultAuditChecker(cfg.Security.AuditCheckURL, "", nil) emailer, err := email.NewEmailer(email.Params(cfg.Email)) - tmpl := templater.NewService(templates.Content, cfg, authService) otfDiffService := tablediff.NewMockService() testutil.Must(t, err) - handler := api.Serve( - cfg, - c, - authenticator, - authService, - c.BlockAdapter, - meta, - migrator, - collector, - nil, - actionsService, - auditChecker, - logging.ContextUnavailable(), - emailer, - tmpl, - nil, - nil, - upload.DefaultPathProvider, - otfDiffService, - ) + handler := api.Serve(cfg, c, authenticator, authService, c.BlockAdapter, meta, migrator, collector, nil, actionsService, auditChecker, logging.ContextUnavailable(), emailer, nil, nil, upload.DefaultPathProvider, otfDiffService) return handler, &dependencies{ blocks: c.BlockAdapter, diff --git a/pkg/loadtest/local_load_test.go b/pkg/loadtest/local_load_test.go index dd239f70188..a1b9797105f 100644 --- a/pkg/loadtest/local_load_test.go +++ b/pkg/loadtest/local_load_test.go @@ -86,26 +86,7 @@ func TestLocalLoad(t *testing.T) { auditChecker := version.NewDefaultAuditChecker(conf.Security.AuditCheckURL, "", nil) emailer, err := email.NewEmailer(email.Params(conf.Email)) testutil.Must(t, err) - handler := api.Serve( - conf, - c, - authenticator, - authService, - blockAdapter, - meta, - migrator, - &stats.NullCollector{}, - nil, - actionsService, - auditChecker, - logging.ContextUnavailable(), - emailer, - nil, - nil, - nil, - upload.DefaultPathProvider, - nil, - ) + handler := api.Serve(conf, c, authenticator, authService, blockAdapter, meta, migrator, &stats.NullCollector{}, nil, actionsService, auditChecker, logging.ContextUnavailable(), emailer, nil, nil, upload.DefaultPathProvider, nil) ts := httptest.NewServer(handler) defer ts.Close() diff --git a/pkg/templater/expander.go b/pkg/templater/expander.go deleted file mode 100644 index 32797b7f050..00000000000 --- a/pkg/templater/expander.go +++ /dev/null @@ -1,177 +0,0 @@ -package templater - -import ( - "context" - "errors" - "fmt" - "io" - "io/fs" - "net/http" - "path" - "strings" - "text/template" - - "github.com/treeverse/lakefs/pkg/auth" - auth_model "github.com/treeverse/lakefs/pkg/auth/model" - "github.com/treeverse/lakefs/pkg/config" -) - -const ( - // Prefix inside templates FS to access actual contents. - embeddedContentPrefix = "content" -) - -var ( - ErrNotFound = errors.New("template not found") - ErrPathTraversalBlocked = errors.New("path traversal blocked") -) - -type AuthService interface { - auth.Authorizer - auth.CredentialsCreator -} - -type Phase string - -const ( - PhasePrepare Phase = "prepare" - PhaseExpand Phase = "expand" -) - -type ControlledParams struct { - Ctx context.Context - Phase Phase - Auth AuthService - // Headers are the HTTP response headers. They may only be modified - // during PhasePrepare. - Header http.Header - // User is the user expanding the template. - User *auth_model.User - // Store is a place for funcs to keep stuff around, mostly between - // phases. Each func should use its name as its index. - Store map[string]interface{} -} - -type UncontrolledData struct { - // UserName is the name of the executing user. - Username string - // Query is the (parsed) querystring of the HTTP access. - Query map[string]string -} - -// Params parametrizes a single template expansion. -type Params struct { - // Controlled is the data visible to functions to control expansion. - // It is _not_ directly visible to templates for expansion. - Controlled *ControlledParams - // Data is directly visible to templates for expansion, with no - // authorization required. - Data *UncontrolledData -} - -// Expander is a template that may be expanded as requested by users. -type Expander interface { - // Expand serves the template into w using the parameters specified - // in params. If during expansion a template function fails, - // returns an error without writing anything to w. (However if - // expansion fails for other reasons, Expand may write to w!) - Expand(w io.Writer, params *Params) error -} - -type expander struct { - template *template.Template - cfg *config.Config - auth AuthService -} - -// MakeExpander creates an expander for the text of tmpl. -func MakeExpander(name, tmpl string, cfg *config.Config, auth AuthService) (Expander, error) { - t := template.New(name).Funcs(templateFuncs).Option("missingkey=error") - t, err := t.Parse(tmpl) - if err != nil { - return nil, err - } - - return &expander{ - template: t, - cfg: cfg, - auth: auth, - }, nil -} - -func (e *expander) Expand(w io.Writer, params *Params) error { - // Expand with no output: verify that no template functions will - // fail. - params.Controlled.Phase = PhasePrepare - if err := e.expandTo(io.Discard, params); err != nil { - return fmt.Errorf("prepare: %w", err) - } - params.Controlled.Phase = PhaseExpand - if err := e.expandTo(w, params); err != nil { - return fmt.Errorf("execute: %w", err) - } - return nil -} - -func (e *expander) expandTo(w io.Writer, params *Params) error { - clone, err := e.template.Clone() - if err != nil { - return err - } - wrappedFuncs := WrapFuncMapWithData(templateFuncs, params.Controlled) - clone.Funcs(wrappedFuncs) - return clone.Execute(w, params.Data) -} - -// ExpanderMap reads and caches Expanders from a fs.FS. Currently, it -// provides no uncaching as it is only used with a prebuilt FS. -type ExpanderMap struct { - fs fs.FS - cfg *config.Config - auth AuthService - - expanders map[string]Expander -} - -func NewExpanderMap(fs fs.FS, cfg *config.Config, auth AuthService) *ExpanderMap { - return &ExpanderMap{ - fs: fs, - cfg: cfg, - auth: auth, - expanders: make(map[string]Expander, 0), - } -} - -func (em *ExpanderMap) Get(_ context.Context, _, name string) (Expander, error) { - if e, ok := em.expanders[name]; ok { - // Fast-path through the cache - if e == nil { - // Negative cache - return nil, ErrNotFound - } - return e, nil - } - - // Compute path - p := path.Join(embeddedContentPrefix, name) - if !strings.HasPrefix(p, embeddedContentPrefix+"/") { - // Path traversal, fail - return nil, fmt.Errorf("%s: %w", name, ErrPathTraversalBlocked) - } - - tmpl, err := fs.ReadFile(em.fs, p) - if errors.Is(err, fs.ErrNotExist) { - return nil, ErrNotFound - } - if err != nil { - return nil, err - } - - e, err := MakeExpander(name, string(tmpl), em.cfg, em.auth) - if err != nil { - // Store negative cache result - e = nil - } - em.expanders[name] = e - return e, err -} diff --git a/pkg/templater/funcmap.go b/pkg/templater/funcmap.go deleted file mode 100644 index e420594a3f4..00000000000 --- a/pkg/templater/funcmap.go +++ /dev/null @@ -1,42 +0,0 @@ -package templater - -import ( - "errors" - "fmt" - "reflect" - "text/template" -) - -var ErrBadFuncRet = errors.New("wrong number of return values") - -// WrapFuncMapWithData wraps a funcMap to a FuncMap that passes data as a -// first element when calling each element. -func WrapFuncMapWithData(funcMap template.FuncMap, data interface{}) template.FuncMap { - ret := make(template.FuncMap, len(funcMap)) - for k, v := range funcMap { - funcValue := reflect.ValueOf(v) - ret[k] = func(args ...interface{}) (interface{}, error) { - argVals := make([]reflect.Value, 0, len(args)+1) - argVals = append(argVals, reflect.ValueOf(data)) - for _, arg := range args { - argVals = append(argVals, reflect.ValueOf(arg)) - } - retVals := funcValue.Call(argVals) - switch len(retVals) { - case 1: - return retVals[0].Interface(), nil - case 2: //nolint:gomnd - errInterface := retVals[1].Interface() - var err error - if errInterface != nil { - err = errInterface.(error) - } - return retVals[0].Interface(), err - default: - // TODO(ariels): This is an implementation error; panic? - return nil, fmt.Errorf("call to %s returned %d values: %w", k, len(retVals), ErrBadFuncRet) - } - } - } - return ret -} diff --git a/pkg/templater/funcmap_test.go b/pkg/templater/funcmap_test.go deleted file mode 100644 index 6bd918139dd..00000000000 --- a/pkg/templater/funcmap_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package templater_test - -import ( - "github.com/treeverse/lakefs/pkg/templater" - - "errors" - "fmt" - "reflect" - "testing" - "text/template" -) - -func asError(i interface{}) error { - if i == nil { - return nil - } - return i.(error) -} - -func TestWrapFuncMapWithData(t *testing.T) { - errFailed := errors.New("(for testing)") - baseFuncs := template.FuncMap{ - "ok": func(data string) string { return data + ":" + "yes" }, - "maybe": func(data string) (string, error) { - if data != "" { - return "", fmt.Errorf("Fail on %s %w", data, errFailed) - } - return "no data", nil - }, - "join": func(dataSep string, a, b string) (string, error) { - return a + dataSep + b, nil - }, - } - - type Case struct { - Name string - FuncName string - Data string - Args []interface{} - Err error - Result string - } - cases := []Case{ - { - Name: "call ok", - FuncName: "ok", - Data: "data", - Result: "data:yes", - }, { - Name: "call maybe", - FuncName: "maybe", - Data: "", - Result: "no data", - }, { - Name: "fail maybe", - FuncName: "maybe", - Data: "fail", - Err: errFailed, - }, { - Name: "join", - FuncName: "join", - Data: "+", - Args: []interface{}{"foo", "bar"}, - Result: "foo+bar", - }, - } - - for _, c := range cases { - t.Run(c.Name, func(t *testing.T) { - funcs := templater.WrapFuncMapWithData(baseFuncs, c.Data) - f := reflect.ValueOf(funcs[c.FuncName]) - argsVal := make([]reflect.Value, 0, len(c.Args)) - for _, arg := range c.Args { - argsVal = append(argsVal, reflect.ValueOf(arg)) - } - r := f.Call(argsVal) - switch len(r) { - case 1: - res := r[0].Interface().(string) - if res != c.Result { - t.Errorf("Got result %q when expecting %q", res, c.Result) - } - case 2: - res := r[0].Interface().(string) - if res != c.Result { - t.Errorf("Got result %q when expecting %q", res, c.Result) - } - err := asError(r[1].Interface()) - if !errors.Is(err, c.Err) { - if c.Err == nil { - t.Errorf("Got unexpected error %v", err) - } else { - t.Errorf("Got error %v when expecting %v", err, c.Err) - } - } - default: - t.Errorf("Got %d return values: %+v", len(r), r) - } - }) - } -} diff --git a/pkg/templater/funcs.go b/pkg/templater/funcs.go deleted file mode 100644 index fcb1517ef17..00000000000 --- a/pkg/templater/funcs.go +++ /dev/null @@ -1,96 +0,0 @@ -package templater - -import ( - "errors" - "fmt" - "text/template" - - "github.com/treeverse/lakefs/pkg/auth" - "github.com/treeverse/lakefs/pkg/permissions" -) - -var ( - ErrNotAuthorized = errors.New("unauthorized") - // ErrTemplateFailed is an error generated by the "fail" template function. - ErrTemplateFailed = errors.New("") -) - -type Credentials struct { - Key, Secret string -} - -var templateFuncs = template.FuncMap{ - "new_credentials": newCredentials, - "contenttype": contentType, - "fail": fail, -} - -// newCredentials creates new credentials for the user and returns them. -// name is used to cache the generated new credentials during a single call -// (between prepare and expand phases, but also between different parts of -// the template). -func newCredentials(params *ControlledParams, name string) (*Credentials, error) { - resp, err := params.Auth.Authorize(params.Ctx, &auth.AuthorizationRequest{ - Username: params.User.Username, - RequiredPermissions: permissions.Node{ - Permission: permissions.Permission{ - Action: permissions.CreateCredentialsAction, - Resource: permissions.UserArn(params.User.Username), - }, - }, - }) - if err == nil && resp.Error != nil { - err = resp.Error - } - if err != nil { - return nil, fmt.Errorf("create credentials for %+v: %w", params.User, err) - } - if !resp.Allowed { - return nil, fmt.Errorf("create credentials for %+v: %w", params.User, ErrNotAuthorized) - } - - var ( - generatedCredsI interface{} - ok bool - generatedCreds map[string]*Credentials - ) - - generatedCredsI, ok = params.Store["new_credentials"] - if !ok { - generatedCreds = make(map[string]*Credentials) - params.Store["new_credentials"] = generatedCreds - } else { - generatedCreds = generatedCredsI.(map[string]*Credentials) - } - - if retCreds, ok := generatedCreds[name]; ok { - return retCreds, nil - } - - // TODO(ariels): monitor! - - // TODO(ariels): Name these credentials (once auth allows named credentials...). - credentials, err := params.Auth.CreateCredentials(params.Ctx, params.User.Username) - if err != nil { - return nil, fmt.Errorf("create credentials for %+v: %w", params.User, err) - } - - retCreds := &Credentials{Key: credentials.AccessKeyID, Secret: credentials.SecretAccessKey} - generatedCreds[name] = retCreds - return retCreds, nil -} - -// fail - fails template expansion with the message passed in. -func fail(_ *ControlledParams, msg string) (string, error) { - return msg, fmt.Errorf("%s%w", msg, ErrTemplateFailed) -} - -// contentType sets the content type of the response. -func contentType(params *ControlledParams, contentType string) (string, error) { - if params.Phase == PhasePrepare { - params.Header.Add("Content-Type", contentType) - } - return "", nil -} - -// TODO(ariels): config, object diff --git a/pkg/templater/service.go b/pkg/templater/service.go deleted file mode 100644 index 5ac234dc6e6..00000000000 --- a/pkg/templater/service.go +++ /dev/null @@ -1,45 +0,0 @@ -package templater - -import ( - "github.com/treeverse/lakefs/pkg/auth/model" - "github.com/treeverse/lakefs/pkg/config" - - "context" - "io/fs" - "net/http" -) - -type Service interface { - Expand(ctx context.Context, w http.ResponseWriter, user *model.User, templateName string, query map[string]string) error -} - -type service struct { - auth AuthService - expanders *ExpanderMap -} - -func NewService(fs fs.FS, cfg *config.Config, auth AuthService) Service { - return &service{auth: auth, expanders: NewExpanderMap(fs, cfg, auth)} -} - -func (s *service) Expand(ctx context.Context, w http.ResponseWriter, user *model.User, templateName string, query map[string]string) error { - e, err := s.expanders.Get(ctx, user.Username, templateName) - if err != nil { - return err - } - - params := &Params{ - Controlled: &ControlledParams{ - Ctx: ctx, - Auth: s.auth, - Header: w.Header(), - Store: make(map[string]interface{}, 0), - User: user, - }, - Data: &UncontrolledData{ - Username: user.Username, - Query: query, - }, - } - return e.Expand(w, params) -} diff --git a/templates/content.go b/templates/content.go deleted file mode 100644 index 50d4519c066..00000000000 --- a/templates/content.go +++ /dev/null @@ -1,8 +0,0 @@ -package templates - -import "embed" - -// Content embeds templates content folder -// -//go:embed content -var Content embed.FS diff --git a/templates/content/fail.tt b/templates/content/fail.tt deleted file mode 100644 index 5e35dffcfb9..00000000000 --- a/templates/content/fail.tt +++ /dev/null @@ -1,2 +0,0 @@ -This is a template. -{{ fail "failure message" }} diff --git a/templates/content/python.notebook.ipynb.tt b/templates/content/python.notebook.ipynb.tt deleted file mode 100644 index e1a5a48efe3..00000000000 --- a/templates/content/python.notebook.ipynb.tt +++ /dev/null @@ -1,442 +0,0 @@ -{{contenttype "application/x-ipynb+json" -}} - -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9b2c8fa0-1702-411a-b11c-3190679bf31c", - "metadata": {}, - "source": [ - "# Integration of lakeFS with Spark and Python\n", - "\n", - "## Use Case: Isolated Testing Environment\n", - "\n", - "## Access lakeFS using the S3A gateway" - ] - }, - { - "cell_type": "markdown", - "id": "387f4469-9708-435d-bec9-c943fd87a0bf", - "metadata": {}, - "source": [ - "## Using your lakeFS credentials" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dfa46799-5f86-4104-9447-d91328edbff4", - "metadata": {}, - "outputs": [], - "source": [ -{{- with $creds := new_credentials "spark"}} - "lakefsAccessKey = '{{$creds.Key}}'\n", - "lakefsSecretKey = '{{$creds.Secret}}'\n", -{{- end}} - "lakefsEndPoint = '{{ .Query.lakefs_url }}'" - ] - }, - { - "cell_type": "markdown", - "id": "13d1c562-9899-4343-b887-c5c018ea79b0", - "metadata": {}, - "source": [ - "## Storage Information\n", - "#### Change the Storage Namespace to a location in the bucket you’ve configured. The storage namespace is a location in the underlying storage where data for this repository will be stored." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25b151c2-743d-43e1-9f2f-25482967c207", - "metadata": {}, - "outputs": [], - "source": [ - "storageNamespace = 's3:///' # e.g. \"s3://username-lakefs-cloud/\"" - ] - }, - { - "cell_type": "markdown", - "id": "c093db8e-68d9-409f-bde7-73ee4ceca5a5", - "metadata": {}, - "source": [ - "## Versioning Information" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3b839850-5954-4631-8e7e-d0dee6d17dde", - "metadata": {}, - "outputs": [], - "source": [ - "sourceBranch = \"main\"\n", - "newBranch = \"experiment1\"\n", - "newPath = \"partitioned_data\"\n", - "fileName = \"lakefs_test.csv\"" - ] - }, - { - "cell_type": "markdown", - "id": "a7ede096-1b84-4b59-bdd1-2608ced6be51", - "metadata": {}, - "source": [ - "## Working with the lakeFS Python client API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "468e98e4-7e06-4373-b36b-f1763cc7755d", - "metadata": {}, - "outputs": [], - "source": [ - "import lakefs_client\n", - "from lakefs_client import models\n", - "from lakefs_client.client import LakeFSClient\n", - "\n", - "# lakeFS credentials and endpoint\n", - "configuration = lakefs_client.Configuration()\n", - "configuration.username = lakefsAccessKey\n", - "configuration.password = lakefsSecretKey\n", - "configuration.host = lakefsEndPoint\n", - "\n", - "client = LakeFSClient(configuration)" - ] - }, - { - "cell_type": "markdown", - "id": "8724cfd8-bda5-4c16-bff6-e05d0ad7f74a", - "metadata": {}, - "source": [ - "## You can change lakeFS repo name (it can be an existing repo or provide another repo name)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0da678dc-5a25-4b90-9dea-30052f45e5d5", - "metadata": {}, - "outputs": [], - "source": [ - "repo = \"my-repo\"" - ] - }, - { - "cell_type": "markdown", - "id": "2422d8af-c714-4d32-aa7b-eddfacbbddb4", - "metadata": {}, - "source": [ - "## If above mentioned repo already exists on your lakeFS server then you can skip following step otherwise create a new repo:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19a4d613-f76d-497d-844d-21aadf1fc50e", - "metadata": {}, - "outputs": [], - "source": [ - "client.repositories.create_repository(\n", - " repository_creation=models.RepositoryCreation(\n", - " name=repo,\n", - " storage_namespace=storageNamespace,\n", - " default_branch=sourceBranch))" - ] - }, - { - "cell_type": "markdown", - "id": "70239947-a305-4f3a-a275-c9a528bafce6", - "metadata": {}, - "source": [ - "## Upload a file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d2c1d011-b1ae-444b-88da-bf79f5eb00b8", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "contentToUpload = open(os.path.expanduser('~')+'/'+fileName, 'rb') # Only a single file per upload which must be named \\\\\\\"content\\\\\\\"\n", - "client.objects.upload_object(\n", - " repository=repo,\n", - " branch=sourceBranch,\n", - " path=fileName, content=contentToUpload)" - ] - }, - { - "cell_type": "markdown", - "id": "ed1629fe-2b8d-442b-bb42-9a9e1779875e", - "metadata": {}, - "source": [ - "## Commit changes and attach some metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "87b1708e-7d0a-413c-85c9-d5266cde2e20", - "metadata": {}, - "outputs": [], - "source": [ - "client.commits.commit(\n", - " repository=repo,\n", - " branch=sourceBranch,\n", - " commit_creation=models.CommitCreation(\n", - " message='Added my first file!',\n", - " metadata={'using': 'python_api'}))" - ] - }, - { - "cell_type": "markdown", - "id": "64b53725-95e6-4c78-9e17-fa271424a3d4", - "metadata": {}, - "source": [ - "## S3A Gateway configuration\n", - "\n", - "##### Note: lakeFS can be configured to work with Spark in two ways:\n", - "###### * Access lakeFS using the S3A gateway https://docs.lakefs.io/integrations/spark.html#access-lakefs-using-the-s3a-gateway.\n", - "###### * Access lakeFS using the lakeFS-specific Hadoop FileSystem https://docs.lakefs.io/integrations/spark.html#access-lakefs-using-the-lakefs-specific-hadoop-filesystem." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b5886ad4-d97d-4ab8-a158-9c034fa0e293", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.context import SparkContext\n", - "from pyspark.sql.session import SparkSession\n", - "sc = SparkContext.getOrCreate()\n", - "spark = SparkSession(sc)\n", - "\n", - "sc._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\", lakefsAccessKey)\n", - "sc._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\", lakefsSecretKey)\n", - "sc._jsc.hadoopConfiguration().set(\"fs.s3a.endpoint\", lakefsEndPoint)\n", - "sc._jsc.hadoopConfiguration().set(\"fs.s3a.path.style.access\", \"true\")" - ] - }, - { - "cell_type": "markdown", - "id": "ddd6f1a5-16f3-4954-b18e-294df65a0cc0", - "metadata": {}, - "source": [ - "## Reading data by using S3A Gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "562dce4b-59b4-460d-9414-753980fbd944", - "metadata": {}, - "outputs": [], - "source": [ - "dataPath = f\"s3a://{repo}/{sourceBranch}/{fileName}\"\n", - "\n", - "df = spark.read.csv(dataPath)\n", - "df.show()" - ] - }, - { - "cell_type": "markdown", - "id": "a2ca865d-0aa0-46bb-b6f2-54d7205dda64", - "metadata": {}, - "source": [ - "# Experimentation Starts" - ] - }, - { - "cell_type": "markdown", - "id": "53e7ed20-7896-47c7-b526-816f4464569a", - "metadata": {}, - "source": [ - "## List the repository branches by using lakeFS Python client API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31099b5c-d59e-4ce3-8da0-4050382052ff", - "metadata": {}, - "outputs": [], - "source": [ - "results = map(\n", - " lambda n:[n.id,n.commit_id],\n", - " client.branches.list_branches(\n", - " repository=repo).results)\n", - "\n", - "from tabulate import tabulate\n", - "print(tabulate(\n", - " results,\n", - " headers=['id','commit_id']))" - ] - }, - { - "cell_type": "markdown", - "id": "49c7cd60-8a5a-4f90-9f2b-527093844411", - "metadata": {}, - "source": [ - "## Create a new branch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cce638f4-f63e-45d4-a3b5-1dcf85ddf1bc", - "metadata": {}, - "outputs": [], - "source": [ - "client.branches.create_branch(\n", - " repository=repo,\n", - " branch_creation=models.BranchCreation(\n", - " name=newBranch,\n", - " source=sourceBranch))" - ] - }, - { - "cell_type": "markdown", - "id": "4cff360f-69c6-4abc-9454-addae4a07a44", - "metadata": {}, - "source": [ - "## Partition the data and write to new branch by using S3A Gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "53e7a3be-a572-4512-b219-227b9501c24b", - "metadata": {}, - "outputs": [], - "source": [ - "newDataPath = f\"s3a://{repo}/{newBranch}/{newPath}\"\n", - "\n", - "df.write.partitionBy(\"_c0\").csv(newDataPath)" - ] - }, - { - "cell_type": "markdown", - "id": "03d0e30a-9638-465d-9ee5-9dd41ba99fd8", - "metadata": {}, - "source": [ - "## Diffing a single branch will show all the uncommitted changes on that branch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2bd88694-f32a-49b2-a6df-af73e6e2bb67", - "metadata": {}, - "outputs": [], - "source": [ - "results = map(\n", - " lambda n:[n.path,n.path_type,n.size_bytes,n.type],\n", - " client.branches.diff_branch(\n", - " repository=repo,\n", - " branch=newBranch).results)\n", - "\n", - "from tabulate import tabulate\n", - "print(tabulate(\n", - " results,\n", - " headers=['Path','Path Type','Size(Bytes)','Type']))" - ] - }, - { - "cell_type": "markdown", - "id": "b2faa4ae-b2e4-47ab-b51f-b61d9ed4d846", - "metadata": {}, - "source": [ - "## Commit changes and attach some metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cfc00b7e-5f9a-46f2-b6fd-0d64ee440943", - "metadata": {}, - "outputs": [], - "source": [ - "client.commits.commit(\n", - " repository=repo,\n", - " branch=newBranch,\n", - " commit_creation=models.CommitCreation(\n", - " message='Partitioned CSV file!',\n", - " metadata={'using': 'python_api'}))" - ] - }, - { - "cell_type": "markdown", - "id": "6fc7c991-05e9-4adb-a071-c8e43f6181d7", - "metadata": {}, - "source": [ - "## Diff between the new branch and the source branch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "08216aa6-c39e-46a8-aaac-d1e2a0f89d79", - "metadata": {}, - "outputs": [], - "source": [ - "results = map(\n", - " lambda n:[n.path,n.path_type,n.size_bytes,n.type],\n", - " client.refs.diff_refs(\n", - " repository=repo,\n", - " left_ref=sourceBranch,\n", - " right_ref=newBranch).results)\n", - "\n", - "from tabulate import tabulate\n", - "print(tabulate(\n", - " results,\n", - " headers=['Path','Path Type','Size(Bytes)','Type']))" - ] - }, - { - "cell_type": "markdown", - "id": "5762d1a0-276f-4a20-8ca0-18f4a463a622", - "metadata": {}, - "source": [ - "# Experimentation Completes\n", - "\n", - "## Delete new branch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ac4765b-5a15-4296-a2a3-73d4fe54ef37", - "metadata": {}, - "outputs": [], - "source": [ - "client.branches.delete_branch(\n", - " repository=repo,\n", - " branch=newBranch)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/templates/content/spark.core.site.conf.tt b/templates/content/spark.core.site.conf.tt deleted file mode 100644 index f30eb3f20a6..00000000000 --- a/templates/content/spark.core.site.conf.tt +++ /dev/null @@ -1,25 +0,0 @@ -{{contenttype "application/xml" -}} - - - fs.s3a.path.style.access - true - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.endpoint - {{ .Query.lakefs_url }} - -{{- with $creds := new_credentials "spark"}} - - fs.s3a.access.key - {{$creds.Key}} - - - fs.s3a.secret.key - {{$creds.Secret}} - -{{- end}} - diff --git a/templates/content/spark.databricks.conf.tt b/templates/content/spark.databricks.conf.tt deleted file mode 100644 index 4a64a0da837..00000000000 --- a/templates/content/spark.databricks.conf.tt +++ /dev/null @@ -1,8 +0,0 @@ -{{contenttype "application/x-conf" -}} -{{- with $creds := new_credentials "spark" -}} -spark.hadoop.fs.s3a.access.key {{$creds.Key}} -spark.hadoop.fs.s3a.secret.key {{$creds.Secret}} -{{- end}} -spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem -spark.hadoop.fs.s3a.endpoint {{ .Query.lakefs_url }} -spark.hadoop.fs.s3a.path.style.access true diff --git a/templates/content/spark.metastore.readme.tt b/templates/content/spark.metastore.readme.tt deleted file mode 100644 index 395325a3612..00000000000 --- a/templates/content/spark.metastore.readme.tt +++ /dev/null @@ -1,96 +0,0 @@ -{{contenttype "text/markdown" -}} -## {{ .Query.repo }} - -This repository was created with the Spark Quickstart wizard. - -### What we did - -1. Created the repository `{{ .Query.repo }}` and the branch `{{ .Query.branch }}`. -{{if .Query.import_location -}} -1. Imported your data from `{{ .Query.import_location }}`. -{{- end}} -1. Generated configuration snippets to get started with Spark and lakeFS. - -### How to use Spark with lakeFS - -Use the configuration we generated to point your cluster to lakeFS. -Now your Spark code can interact with lakeFS: - -1. Read a dataframe from lakeFS: - - ``` - val df = spark.read.parquet("s3a://{{ .Query.repo }}/{{ .Query.branch }}/sensor_data") - ``` - -1. Write a dataframe to lakeFS: - - ``` - df.write.partitionBy("example-column").parquet(s"s3a://{{ .Query.repo }}/{{ .Query.branch }}/output_table/") - ``` - -### Working with Hive Metastore - -In Hive Metastore (or AWS Glue), a table location can only be under a single lakeFS branch. -For example, consider a table pointing to the `{{ .Query.branch }}` branch: - -``` -CREATE EXTERNAL TABLE `sensor_data` ( - `ts` timestamp, - `value` double - PARTITIONED BY (`dt` string) - LOCATION 's3a://{{ .Query.repo }}/{{ .Query.branch }}/sensor_data'; -``` - -Suppose you create a new branch, `new_feature`. You will not be able to access the table from the new branch, -since your catalog doesn't point to a location in this branch. - -To solve this, use the [lakectl metastore commands](https://docs.lakefs.io/integrations/glue_hive_metastore.html#commands): - -1. [Download and configure lakectl](https://www.youtube.com/watch?v=8nO7RT411nA). - -1. Add the following to the lakectl configuration file (by default ~/.lakectl.yaml`): - - #### Hive - - ```conf - metastore: - type: hive - hive: - uri: : - ``` - - #### Glue - - ```conf - metastore: - type: glue - glue: - catalog-id: - region: - profile: default # optional, implies using a credentials file - credentials: - access_key_id: - secret_access_key: - ``` - -1. Use the `lakectl metastore copy` command to create a table pointing to the same path under the new branch: - - ``` - lakectl metastore copy --from-schema default --from-table sensor_data --to-schema default --to-table sensor_new_feature --to-branch new_feature - ``` - -1. You can now query the data from the new branch: - - ``` - SELECT * FROM sensor_new_feature; - ``` - -1. Read the [docs](https://docs.lakefs.io/integrations/glue_hive_metastore.html#commands) to learn more about the `lakectl metastore` commands. - -### Suggested model - -For simplicity, we recommend creating a schema for each branch. That way, you can use the same table name across different schemas: - -``` -CREATE SCHEMA {{ .Query.branch }}_schema with (location = 's3://{{ .Query.repo }}/{{ .Query.branch }}/'); -``` diff --git a/templates/content/spark.submit.conf.tt b/templates/content/spark.submit.conf.tt deleted file mode 100644 index 4480eab680d..00000000000 --- a/templates/content/spark.submit.conf.tt +++ /dev/null @@ -1,14 +0,0 @@ -{{contenttype "application/x-conf" -}} - -spark-submit --class \ - --master \ - --deploy-mode \ - -c spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \ -{{- with $creds := new_credentials "spark"}} - -c spark.hadoop.fs.s3a.access.key={{$creds.Key}} \ - -c spark.hadoop.fs.s3a.secret.key={{$creds.Secret}} \ -{{- end}} - -c spark.hadoop.fs.s3a.endpoint={{ .Query.lakefs_url }} \ - -c spark.hadoop.fs.s3a.path.style.access=true \ - \ - [application-arguments]