From c7d6003c4fff9acb2bb2dfdd9e297f5ee7450ba0 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Thu, 24 Oct 2024 16:06:07 -0500 Subject: [PATCH] added participation file to S3 and updated the scripts --- tools/scripts/format-participation-file.py | 6 ++-- tools/scripts/retrieve-fms-file.py | 40 +++++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/tools/scripts/format-participation-file.py b/tools/scripts/format-participation-file.py index fb8b969cd..a04b8a26b 100755 --- a/tools/scripts/format-participation-file.py +++ b/tools/scripts/format-participation-file.py @@ -7,12 +7,12 @@ def main(argv): # Open the workbook and select the second sheet wb = load_workbook(filename = argv[0]) - sh = wb['2023_Providers'] + sh = wb['2024_Providers'] data_list = [] for row in sh.iter_rows(sh.min_row+1, sh.max_row): data = OrderedDict() - data['npi'] = row[12].value - data['tin'] = row[14].value + data['npi'] = row[10].value + data['tin'] = row[11].value data['apm_entity_id'] = row[0].value data_list.append(data) j = json.dumps(data_list) diff --git a/tools/scripts/retrieve-fms-file.py b/tools/scripts/retrieve-fms-file.py index 734592419..793e54866 100755 --- a/tools/scripts/retrieve-fms-file.py +++ b/tools/scripts/retrieve-fms-file.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 +import os import sys import boto3 import argparse import requests +import urllib.request import simplejson as json from io import BytesIO from dotenv import dotenv_values from openpyxl import load_workbook -config = dotenv_values("local.env") +config = dotenv_values("../../local.env") s3_client = boto3.client('s3') pcf_filename = "pcf_apm_entity_ids.json" @@ -29,7 +31,7 @@ def get_user_inputs(): return args -def download_from_fms(auth_url, fms_url, fms_token, fms_path): +def download_from_fms(auth_url, fms_url, fms_token, fms_path, filename): d = {'client_assertion': fms_token, 'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer', 'grant_type': 'client_credentials', @@ -45,10 +47,10 @@ def download_from_fms(auth_url, fms_url, fms_token, fms_path): } ) s2s_token = get_s2s_token.json()["data"]["token"] - # print('starting download from fms for file - ' + fms_path) + # print('starting download from fms for file - ' + filename) get_download_url = requests.post( url=fms_url + '/get-file', - json={"path": fms_path}, + json={"path": fms_path + filename}, verify=False, headers={ 'Accept': 'application/vnd.qpp.cms.gov.v2+json', @@ -56,15 +58,13 @@ def download_from_fms(auth_url, fms_url, fms_token, fms_path): } ) download_url = get_download_url.json()['presigned_url'] - download_result = requests.get(url=download_url) - return download_result + urllib.request.urlretrieve(download_url, filename) -def process_file(download_result): +def process_file(filename): print('processing file') - file_object = BytesIO(download_result.content) - wb = load_workbook(file_object) - sh = wb['2023_Practices'] + wb = load_workbook(filename) + sh = wb['2024_Practices'] data_list = [] for row in sh.iter_rows(sh.min_row + 1, sh.max_row): data_list.append(row[0].value) @@ -72,8 +72,8 @@ def process_file(download_result): return str(json_data).replace(" ", "") def update_local_repo(data): - # print('writing ' + pcf_filename + ' to local repository') - with open('./converter/src/main/resources/' + pcf_filename, 'w') as f: + # print('writing ' + pcf_filename + ' to local repository') + with open('../../converter/src/main/resources/' + pcf_filename, 'w') as f: f.write(data) @@ -88,16 +88,24 @@ def upload_to_s3(data): ) print(upload_status) +def delete_file(filename): + if os.path.exists("./" + filename): + os.remove("./" + filename) + print("File " + filename + " has been processed and removed successfully!") + else: + print("Can not process or delete file " + filename + ", as it doesn't exists") def main(): try: # args = get_user_inputs() # s3_url = download_from_fms(args.auth_url, args.fms_url, args.fms_token, args.fms_path) - download_result = download_from_fms(config.get('auth_url'), config.get('fms_url'), config.get('fms_token'), - config.get('fms_path')) - processed_data = process_file(download_result) + filename = config.get('filename') + download_from_fms(config.get('auth_url'), config.get('fms_url'), config.get('fms_token'), + config.get('fms_path'), filename) + processed_data = process_file(filename) update_local_repo(processed_data) - upload_to_s3(processed_data) + # upload_to_s3(processed_data) + delete_file(filename) except Exception as err: print(f"Unexpected Error. {err = }, {type(err) = }") sys.exit(1)