From e4e5bf6d849a8702900596007335bfd01924dee7 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 12 Nov 2024 17:49:08 +0100 Subject: [PATCH] Azure short-lived token support initial draft Signed-off-by: Ben --- src/agent/agent.js | 3 ++- src/api/account_api.js | 16 ++++++++++++++++ src/api/common_api.js | 2 +- src/hosted_agents/hosted_agents.js | 13 ++++++++++++- src/server/system_services/account_server.js | 14 ++++++++++++++ src/server/system_services/bucket_server.js | 1 + src/server/system_services/pool_server.js | 5 +++++ .../system_services/schemas/account_schema.js | 6 +++++- .../system_services/schemas/pool_schema.js | 6 +++++- src/util/cloud_utils.js | 10 ++++++++++ 10 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 581b2ffae8..c3e9e6067a 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -115,7 +115,8 @@ class Agent { params.cloud_info.endpoint_type === 'IBM_COS') { this.node_type = 'BLOCK_STORE_S3'; this.block_store = new BlockStoreS3(block_store_options); - } else if (params.cloud_info.endpoint_type === 'AZURE') { + } else if (params.cloud_info.endpoint_type === 'AZURE' || + params.cloud_info.endpoint_type === 'AZURESTS') { const connection_string = cloud_utils.get_azure_new_connection_string({ endpoint: params.cloud_info.endpoint, access_key: params.cloud_info.access_keys.access_key, diff --git a/src/api/account_api.js b/src/api/account_api.js index 6e21f0a6e6..f72953bc37 100644 --- a/src/api/account_api.js +++ b/src/api/account_api.js @@ -447,6 +447,10 @@ module.exports = { identity: { $ref: 'common_api#/definitions/access_key' }, secret: { $ref: 'common_api#/definitions/secret_key' }, azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' }, + azure_subscription_id: { type: 'string' }, + azure_tenant_id: { type: 'string' }, + azure_client_id: { type: 'string' }, + azure_region: { type: 'string' }, aws_sts_arn: { type: 'string' }, @@ -508,6 +512,18 @@ module.exports = { identity: { $ref: 'common_api#/definitions/access_key' }, secret: { $ref: 'common_api#/definitions/secret_key' }, azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' }, + azure_subscription_id: { + type: 'string' + }, + azure_tenant_id: { + type: 'string' + }, + azure_client_id: { + type: 'string' + }, + azure_region: { + type: 'string' + }, aws_sts_arn: { type: 'string' }, diff --git a/src/api/common_api.js b/src/api/common_api.js index e9f4c948a2..efc38564cd 100644 --- a/src/api/common_api.js +++ b/src/api/common_api.js @@ -751,7 +751,7 @@ module.exports = { endpoint_type: { type: 'string', - enum: ['AWSSTS', 'AWS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS'] + enum: ['AWSSTS', 'AWS', 'AZURE', 'AZURESTS', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS'] }, block_md: { diff --git a/src/hosted_agents/hosted_agents.js b/src/hosted_agents/hosted_agents.js index 9ce6105915..5fab98b4da 100644 --- a/src/hosted_agents/hosted_agents.js +++ b/src/hosted_agents/hosted_agents.js @@ -164,6 +164,10 @@ class HostedAgents { secret_key: pool.cloud_pool_info.access_keys.secret_key }, aws_sts_arn: pool.cloud_pool_info.aws_sts_arn, + azure_client_id: pool.cloud_pool_info.azure_client_id, + azure_tenant_id: pool.cloud_pool_info.azure_tenant_id, + azure_region: pool.cloud_pool_info.azure_region, + azure_subscription_id: pool.cloud_pool_info.azure_subscription_id, region: pool.cloud_pool_info.region, pool_name: pool.name } : { @@ -174,7 +178,7 @@ class HostedAgents { node_name, host_id, storage_path, - token_wrapper, + token_wrapper, create_node_token_wrapper, routing_hint: 'LOOPBACK' }; @@ -182,6 +186,13 @@ class HostedAgents { agent_params[pool_info_property] = pool_info; if (pool.cloud_pool_info && pool.cloud_pool_info.storage_limit) agent_params.storage_limit = pool.cloud_pool_info.storage_limit; if (pool.cloud_pool_info && pool.cloud_pool_info.aws_sts_arn) agent_params.aws_sts_arn = pool.cloud_pool_info.aws_sts_arn; + if (pool.cloud_pool_info) { + const { azure_client_id, azure_tenant_id, azure_region, azure_subscription_id } = pool.cloud_pool_info; + if (azure_client_id) agent_params.azure_client_id = azure_client_id; + if (azure_tenant_id) agent_params.azure_tenant_id = azure_tenant_id; + if (azure_region) agent_params.azure_region = azure_region; + if (azure_subscription_id) agent_params.azure_subscription_id = azure_subscription_id; + } dbg.log0(`running agent with params ${util.inspect(agent_params)}`); const agent = new Agent(agent_params); this._started_agents[node_name] = { diff --git a/src/server/system_services/account_server.js b/src/server/system_services/account_server.js index 98a5d6be4d..dca70c773a 100644 --- a/src/server/system_services/account_server.js +++ b/src/server/system_services/account_server.js @@ -784,6 +784,16 @@ async function add_external_connection(req) { }; } + if (req.rpc_params.azure_subscription_id && + req.rpc_params.azure_tenant_id && + req.rpc_params.azure_client_id && + req.rpc_params.azure_client_secret) { + info.azure_subscription_id = req.rpc_params.azure_subscription_id; + info.azure_tenant_id = req.rpc_params.azure_tenant_id; + info.azure_client_id = req.rpc_params.azure_client_id; + info.azure_region = req.rpc_params.azure_region; + } + info.cp_code = req.rpc_params.cp_code || undefined; info.auth_method = req.rpc_params.auth_method || config.DEFAULT_S3_AUTH_METHOD[info.endpoint_type] || undefined; info = _.omitBy(info, _.isUndefined); @@ -958,6 +968,10 @@ async function _check_external_connection_internal(connection) { case 'AZURE': { return check_azure_connection(connection); } + case 'AZURESTS': { + // TODO - Requires example secret to implement + return check_azure_sts_connection(connection); + } case 'AWSSTS': { return check_aws_sts_connection(connection); } diff --git a/src/server/system_services/bucket_server.js b/src/server/system_services/bucket_server.js index 0dbaf92bd2..a32b867273 100644 --- a/src/server/system_services/bucket_server.js +++ b/src/server/system_services/bucket_server.js @@ -1196,6 +1196,7 @@ async function get_cloud_buckets(req) { req.rpc_params.connection ); if (connection.endpoint_type === 'AZURE') { + // TODO - utilize CCO secret for authentication const blob_svc = azure_storage.BlobServiceClient.fromConnectionString( cloud_utils.get_azure_new_connection_string(connection)); const used_cloud_buckets = cloud_utils.get_used_cloud_targets(['AZURE'], diff --git a/src/server/system_services/pool_server.js b/src/server/system_services/pool_server.js index 48e602d55a..6a298cece8 100644 --- a/src/server/system_services/pool_server.js +++ b/src/server/system_services/pool_server.js @@ -369,6 +369,10 @@ async function create_cloud_pool(req) { account_id: req.account._id }, aws_sts_arn: connection.aws_sts_arn, + azure_subscription_id: connection.azure_subscription_id, + azure_tenant_id: connection.azure_tenant_id, + azure_client_id: connection.azure_client_id, + azure_region: connection.azure_region, region: connection.region, endpoint_type: connection.endpoint_type || 'AWS', backingstore: req.rpc_params.backingstore, @@ -416,6 +420,7 @@ async function create_cloud_pool(req) { FLASHBLADE: 'BLOCK_STORE_S3', IBM_COS: 'BLOCK_STORE_S3', AZURE: 'BLOCK_STORE_AZURE', + AZURESTS: 'BLOCK_STORE_AZURE', GOOGLE: 'BLOCK_STORE_GOOGLE' }; diff --git a/src/server/system_services/schemas/account_schema.js b/src/server/system_services/schemas/account_schema.js index 3f3a7c94d3..3689ffcf22 100644 --- a/src/server/system_services/schemas/account_schema.js +++ b/src/server/system_services/schemas/account_schema.js @@ -70,6 +70,10 @@ module.exports = { access_key: { $ref: 'common_api#/definitions/access_key' }, secret_key: { $ref: 'common_api#/definitions/secret_key' }, azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' }, + azure_subscription_id: { type: 'string' }, + azure_tenant_id: { type: 'string' }, + azure_client_id: { type: 'string' }, + azure_region: { type: 'string' }, aws_sts_arn: { type: 'string' }, @@ -82,7 +86,7 @@ module.exports = { cp_code: { type: 'string' }, endpoint_type: { type: 'string', - enum: ['AWSSTS', 'AWS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS'] + enum: ['AWSSTS', 'AWS', 'AZURE', 'AZURESTS', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS'] }, } } diff --git a/src/server/system_services/schemas/pool_schema.js b/src/server/system_services/schemas/pool_schema.js index 71965c1925..62ced4a612 100644 --- a/src/server/system_services/schemas/pool_schema.js +++ b/src/server/system_services/schemas/pool_schema.js @@ -108,6 +108,10 @@ module.exports = { aws_sts_arn: { type: 'string' }, +azure_subscription_id: { type: 'string' }, + azure_tenant_id: { type: 'string' }, + azure_client_id: { type: 'string' }, + azure_region: { type: 'string' }, backingstore: { type: 'object', properties: { @@ -138,7 +142,7 @@ module.exports = { }, endpoint_type: { type: 'string', - enum: ['AWSSTS', 'AWS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS'] + enum: ['AWSSTS', 'AWS', 'AZURE', 'AZURESTS', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS'] }, agent_info: { type: 'object', diff --git a/src/util/cloud_utils.js b/src/util/cloud_utils.js index c963d05ceb..01c70d6e3b 100644 --- a/src/util/cloud_utils.js +++ b/src/util/cloud_utils.js @@ -58,6 +58,14 @@ async function generate_aws_sts_creds(params, roleSessionName) { ); } +async function createAzureSTSClient(params, additionalParams) { + //TODO - Requires example secret to implement +} + +async function generate_azure_sts_creds(params, roleSessionName) { + //TODO - Requires example secret to implement +} + function get_signed_url(params, expiry = 604800) { const s3 = new AWS.S3({ endpoint: params.endpoint, @@ -205,3 +213,5 @@ exports.set_noobaa_s3_connection = set_noobaa_s3_connection; exports.createSTSS3Client = createSTSS3Client; exports.generate_aws_sts_creds = generate_aws_sts_creds; exports.generate_access_keys = generate_access_keys; +exports.createSTSS3Client = createSTSS3Client; +exports.createAzureSTSClient = createAzureSTSClient;