diff --git a/scripts/cpi/dvc.lock b/scripts/cpi/dvc.lock index f68475f8..e1a0684a 100644 --- a/scripts/cpi/dvc.lock +++ b/scripts/cpi/dvc.lock @@ -48,8 +48,12 @@ stages: size: 129729 - path: scripts/cpi/prepare.py hash: md5 - md5: eae0eadf2582f5c6398273d08614a390 - size: 8631 + md5: baa430f643d94e5596d409b15e943900 + size: 8605 + - path: working/upstream/metadata.csv + hash: md5 + md5: e8f918d67dad983a618a80d2185527dc + size: 134 - path: working/upstream/mm23-codes.csv hash: md5 md5: 83bd378b2ba90ee84ca2546ff6f51dc9 diff --git a/scripts/cpi/dvc.yaml b/scripts/cpi/dvc.yaml index 9b9ce570..9f1db36f 100644 --- a/scripts/cpi/dvc.yaml +++ b/scripts/cpi/dvc.yaml @@ -17,6 +17,7 @@ stages: - scripts/cpi/prepare.py - data/cpi/transformed_cpi.csv - working/upstream/mm23-codes.csv + - working/upstream/metadata.csv outs: - data/cpi/indicator.csv: cache: false diff --git a/scripts/cpi/prepare.py b/scripts/cpi/prepare.py index 993e430b..fafd1dfd 100644 --- a/scripts/cpi/prepare.py +++ b/scripts/cpi/prepare.py @@ -4,7 +4,7 @@ from transform import DATA_DIR as INPUTS_DIR from transform import n from scripts.util.util import iso_to_named_date -from scripts.util.metadata import read_meta, filter_for_dataset +from scripts.util.metadata import read_meta, extract_dates OUTPUTS_DIR = os.path.realpath(os.path.join('src', '_data', 'sources', 'cpi')) CPI_METADATA = os.path.join(OUTPUTS_DIR, 'metadata.json') @@ -12,9 +12,9 @@ def get_dates(): # read csv and make a new dataframe - metadata = read_meta().pipe(filter_for_dataset, 'MM23') - next_update = iso_to_named_date(metadata['next_update'].iloc[0]) - published = iso_to_named_date(metadata['last_update'].iloc[0]) + metadata = read_meta().pipe(extract_dates, 'MM23') + next_update = iso_to_named_date(metadata['next_update']) + published = iso_to_named_date(metadata['last_update']) dates = pd.Series( data={ 'published': published, diff --git a/scripts/labour-market/dvc.lock b/scripts/labour-market/dvc.lock index 69df2d34..82259bda 100644 --- a/scripts/labour-market/dvc.lock +++ b/scripts/labour-market/dvc.lock @@ -29,17 +29,21 @@ stages: size: 359985 - path: ../../scripts/util/ hash: md5 - md5: c7bb69be54b1e9f9ce79d49e29ead857.dir - size: 9655 + md5: 3727a82e6b3317df98198097a7646dc0.dir + size: 9441 nfiles: 6 + - path: ../../working/upstream/metadata.csv + hash: md5 + md5: e8f918d67dad983a618a80d2185527dc + size: 134 - path: config.py hash: md5 md5: c9187b99fb9c48a24be707561d15eaa2 size: 5131 - path: prepare.ipynb hash: md5 - md5: 8e26694a5328022c4ef105c1eb274952 - size: 55354 + md5: a169583bbac0a6b21b631a4f25058bf1 + size: 55309 - path: prepare.py hash: md5 md5: 09def7c536dd2fc8644c8100908aaebc diff --git a/scripts/labour-market/dvc.yaml b/scripts/labour-market/dvc.yaml index 0bafb244..c46588af 100644 --- a/scripts/labour-market/dvc.yaml +++ b/scripts/labour-market/dvc.yaml @@ -22,6 +22,7 @@ stages: - config.py - ${top}/scripts/util/ - ${top}/data/labour-market/monthly-rolling.csv + - ${top}/working/upstream/metadata.csv outs: - ${top}/src/_data/sources/labour-market/long_term_unemployed_last_3_years.csv: cache: false diff --git a/scripts/labour-market/prepare.ipynb b/scripts/labour-market/prepare.ipynb index bb647859..5d9b7444 100644 --- a/scripts/labour-market/prepare.ipynb +++ b/scripts/labour-market/prepare.ipynb @@ -21,7 +21,7 @@ "outputs": [], "source": [ "from prepare import LMS_EXTRACT, DASHBOARD_DIR, create_table, save_files, summarise, labour_market_status_variables, long_term_unemployed_variables\n", - "from scripts.util.metadata import read_meta, filter_for_dataset, extract_dates" + "from scripts.util.metadata import read_meta, extract_dates" ] }, { @@ -1166,7 +1166,7 @@ " long_term_unemployed=long_term_unemployed,\n", " labour_market_status=labour_market_status,\n", " ),\n", - " read_meta().pipe(filter_for_dataset, 'LMS').pipe(extract_dates),\n", + " read_meta().pipe(extract_dates, 'LMS'),\n", "])\n", "\n", "summary.to_json(os.path.join(DASHBOARD_DIR, 'latest.json'), indent=2, date_format='iso')" diff --git a/scripts/neet/dvc.lock b/scripts/neet/dvc.lock index 69d6a1f3..13922eef 100644 --- a/scripts/neet/dvc.lock +++ b/scripts/neet/dvc.lock @@ -14,6 +14,10 @@ stages: transform: cmd: papermill transform.ipynb ../../working/output/neet-transform.ipynb deps: + - path: ../../working/upstream/metadata.csv + hash: md5 + md5: e8f918d67dad983a618a80d2185527dc + size: 134 - path: ../../working/upstream/neet.csv hash: md5 md5: a6d894c2cdc326e9b9d803c8149457c8 diff --git a/scripts/neet/dvc.yaml b/scripts/neet/dvc.yaml index 621dea77..bda9fc85 100644 --- a/scripts/neet/dvc.yaml +++ b/scripts/neet/dvc.yaml @@ -6,6 +6,7 @@ stages: deps: - transform.ipynb - ${top}/working/upstream/neet.csv + - ${top}/working/upstream/metadata.csv outs: - ${top}/data/neet/neet.csv: cache: false diff --git a/scripts/util/metadata.py b/scripts/util/metadata.py index 599816bb..e39cc67b 100644 --- a/scripts/util/metadata.py +++ b/scripts/util/metadata.py @@ -7,19 +7,12 @@ def read_meta(): # read csv and make a new dataframe - metadata = pd.read_csv(METADATA_FILE) - metadata.last_update = pd.to_datetime(metadata.last_update, format='ISO8601') - metadata.next_update = pd.to_datetime(metadata.next_update, format='ISO8601') + metadata = pd.read_csv(METADATA_FILE, parse_dates=['last_update', 'next_update'], index_col=['id']) return metadata -def filter_for_dataset(metadata, id): - metadata = metadata[metadata.id == id].reset_index() - return metadata - - -def extract_dates(metadata): - return metadata.loc[0, ['last_update', 'next_update']] +def extract_dates(metadata, id): + return metadata.loc[id, ['last_update', 'next_update']] # next_update = metadata['next_update'].iloc[0] # published = metadata['last_update'].iloc[0] # dates = pd.Series(data={'published': published, 'next_update': next_update}, index=['published', 'next_update']) diff --git a/scripts/vacancies/dvc.lock b/scripts/vacancies/dvc.lock index 24597044..35e0c402 100644 --- a/scripts/vacancies/dvc.lock +++ b/scripts/vacancies/dvc.lock @@ -9,12 +9,16 @@ stages: size: 10506 - path: scripts/util/metadata.py hash: md5 - md5: 4c10007d68898289db269e5da494a0f5 - size: 911 + md5: cfc702d76a3920af615dbbf1a9408cfd + size: 697 - path: scripts/vacancies/prepare.py hash: md5 - md5: 714220570380a26716ec2053a008c262 - size: 5849 + md5: 3cf43cb424b9c3af8c5b362f36b36325 + size: 5823 + - path: working/upstream/metadata.csv + hash: md5 + md5: e8f918d67dad983a618a80d2185527dc + size: 134 - path: working/upstream/vacancies-growth-by-sector.csv hash: md5 md5: 8696accdcce4286973199b89da179271 diff --git a/scripts/vacancies/dvc.yaml b/scripts/vacancies/dvc.yaml index d7999720..b0486162 100644 --- a/scripts/vacancies/dvc.yaml +++ b/scripts/vacancies/dvc.yaml @@ -18,6 +18,7 @@ stages: - scripts/util/metadata.py - data/vacancies/vacancies_by_date.csv - working/upstream/vacancies-growth-by-sector.csv + - working/upstream/metadata.csv outs: - src/_data/sources/vacancies/: cache: false diff --git a/scripts/vacancies/prepare.py b/scripts/vacancies/prepare.py index 221a2e9e..6c640a8d 100644 --- a/scripts/vacancies/prepare.py +++ b/scripts/vacancies/prepare.py @@ -3,7 +3,7 @@ from scripts.util.date import lms_period_to_quarter_label from scripts.util.util import iso_to_named_date -from scripts.util.metadata import read_meta, filter_for_dataset +from scripts.util.metadata import read_meta, extract_dates DATA_DIR = os.path.join('src', '_data', 'sources', 'vacancies') RAW_DATA_DIR = os.path.realpath(os.path.join('data', 'vacancies')) @@ -132,9 +132,9 @@ def summarise(): def get_dates(): # read csv and make a new dataframe - metadata = read_meta().pipe(filter_for_dataset, 'LMS') - next_update = iso_to_named_date(metadata['next_update'].iloc[0]) - published = iso_to_named_date(metadata['last_update'].iloc[0]) + metadata = read_meta().pipe(extract_dates, 'LMS') + next_update = iso_to_named_date(metadata['next_update']) + published = iso_to_named_date(metadata['last_update']) dates = pd.Series(data={'published': published, 'next_update': next_update}, index=[ 'published', 'next_update']) dates.to_json(os.path.join(DATA_DIR, 'metadata.json'), date_format='iso') diff --git a/working/output/labour-market-prepare.ipynb b/working/output/labour-market-prepare.ipynb index ecba2207..b551c56f 100644 --- a/working/output/labour-market-prepare.ipynb +++ b/working/output/labour-market-prepare.ipynb @@ -3,19 +3,19 @@ { "cell_type": "code", "execution_count": 1, - "id": "1dcb703e", + "id": "1b3db7b5", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:05.753616Z", - "iopub.status.busy": "2023-10-12T16:54:05.752818Z", - "iopub.status.idle": "2023-10-12T16:54:05.915273Z", - "shell.execute_reply": "2023-10-12T16:54:05.914812Z" + "iopub.execute_input": "2023-10-16T16:00:35.891036Z", + "iopub.status.busy": "2023-10-16T16:00:35.890415Z", + "iopub.status.idle": "2023-10-16T16:00:36.041764Z", + "shell.execute_reply": "2023-10-16T16:00:36.041381Z" }, "papermill": { - "duration": 0.169313, - "end_time": "2023-10-12T16:54:05.916332", + "duration": 0.161006, + "end_time": "2023-10-16T16:00:36.042859", "exception": false, - "start_time": "2023-10-12T16:54:05.747019", + "start_time": "2023-10-16T16:00:35.881853", "status": "completed" }, "tags": [] @@ -33,19 +33,19 @@ { "cell_type": "code", "execution_count": 2, - "id": "aad17674", + "id": "e1ab0bf7", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:05.919327Z", - "iopub.status.busy": "2023-10-12T16:54:05.919169Z", - "iopub.status.idle": "2023-10-12T16:54:05.923186Z", - "shell.execute_reply": "2023-10-12T16:54:05.922808Z" + "iopub.execute_input": "2023-10-16T16:00:36.046027Z", + "iopub.status.busy": "2023-10-16T16:00:36.045833Z", + "iopub.status.idle": "2023-10-16T16:00:36.049378Z", + "shell.execute_reply": "2023-10-16T16:00:36.049067Z" }, "papermill": { - "duration": 0.006235, - "end_time": "2023-10-12T16:54:05.923856", + "duration": 0.006046, + "end_time": "2023-10-16T16:00:36.050170", "exception": false, - "start_time": "2023-10-12T16:54:05.917621", + "start_time": "2023-10-16T16:00:36.044124", "status": "completed" }, "tags": [] @@ -53,18 +53,18 @@ "outputs": [], "source": [ "from prepare import LMS_EXTRACT, DASHBOARD_DIR, create_table, save_files, summarise, labour_market_status_variables, long_term_unemployed_variables\n", - "from scripts.util.metadata import read_meta, filter_for_dataset, extract_dates" + "from scripts.util.metadata import read_meta, extract_dates" ] }, { "cell_type": "markdown", - "id": "a6b4437e", + "id": "9f16ce87", "metadata": { "papermill": { - "duration": 0.000975, - "end_time": "2023-10-12T16:54:05.925959", + "duration": 0.000996, + "end_time": "2023-10-16T16:00:36.052286", "exception": false, - "start_time": "2023-10-12T16:54:05.924984", + "start_time": "2023-10-16T16:00:36.051290", "status": "completed" }, "tags": [] @@ -76,19 +76,19 @@ { "cell_type": "code", "execution_count": 3, - "id": "881244de", + "id": "d37a4c40", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:05.928529Z", - "iopub.status.busy": "2023-10-12T16:54:05.928369Z", - "iopub.status.idle": "2023-10-12T16:54:05.937200Z", - "shell.execute_reply": "2023-10-12T16:54:05.936889Z" + "iopub.execute_input": "2023-10-16T16:00:36.054942Z", + "iopub.status.busy": "2023-10-16T16:00:36.054706Z", + "iopub.status.idle": "2023-10-16T16:00:36.063303Z", + "shell.execute_reply": "2023-10-16T16:00:36.062980Z" }, "papermill": { - "duration": 0.010962, - "end_time": "2023-10-12T16:54:05.937924", + "duration": 0.010961, + "end_time": "2023-10-16T16:00:36.064238", "exception": false, - "start_time": "2023-10-12T16:54:05.926962", + "start_time": "2023-10-16T16:00:36.053277", "status": "completed" }, "tags": [] @@ -103,13 +103,13 @@ }, { "cell_type": "markdown", - "id": "a05a90ef", + "id": "75f423a8", "metadata": { "papermill": { - "duration": 0.001044, - "end_time": "2023-10-12T16:54:05.940094", + "duration": 0.001047, + "end_time": "2023-10-16T16:00:36.066409", "exception": false, - "start_time": "2023-10-12T16:54:05.939050", + "start_time": "2023-10-16T16:00:36.065362", "status": "completed" }, "tags": [] @@ -121,19 +121,19 @@ { "cell_type": "code", "execution_count": 4, - "id": "642dd6ef", + "id": "ed4bfaa2", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:05.942857Z", - "iopub.status.busy": "2023-10-12T16:54:05.942635Z", - "iopub.status.idle": "2023-10-12T16:54:05.970567Z", - "shell.execute_reply": "2023-10-12T16:54:05.970276Z" + "iopub.execute_input": "2023-10-16T16:00:36.069055Z", + "iopub.status.busy": "2023-10-16T16:00:36.068918Z", + "iopub.status.idle": "2023-10-16T16:00:36.099127Z", + "shell.execute_reply": "2023-10-16T16:00:36.098795Z" }, "papermill": { - "duration": 0.030139, - "end_time": "2023-10-12T16:54:05.971347", + "duration": 0.032867, + "end_time": "2023-10-16T16:00:36.100323", "exception": false, - "start_time": "2023-10-12T16:54:05.941208", + "start_time": "2023-10-16T16:00:36.067456", "status": "completed" }, "tags": [] @@ -721,13 +721,13 @@ }, { "cell_type": "markdown", - "id": "c84d1e07", + "id": "8f585e27", "metadata": { "papermill": { - "duration": 0.001316, - "end_time": "2023-10-12T16:54:05.974135", + "duration": 0.001908, + "end_time": "2023-10-16T16:00:36.104541", "exception": false, - "start_time": "2023-10-12T16:54:05.972819", + "start_time": "2023-10-16T16:00:36.102633", "status": "completed" }, "tags": [] @@ -739,19 +739,19 @@ { "cell_type": "code", "execution_count": 5, - "id": "ea53e431", + "id": "2262ef99", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:05.977450Z", - "iopub.status.busy": "2023-10-12T16:54:05.977199Z", - "iopub.status.idle": "2023-10-12T16:54:05.980233Z", - "shell.execute_reply": "2023-10-12T16:54:05.979950Z" + "iopub.execute_input": "2023-10-16T16:00:36.110143Z", + "iopub.status.busy": "2023-10-16T16:00:36.109958Z", + "iopub.status.idle": "2023-10-16T16:00:36.114204Z", + "shell.execute_reply": "2023-10-16T16:00:36.113661Z" }, "papermill": { - "duration": 0.005426, - "end_time": "2023-10-12T16:54:05.980868", + "duration": 0.008554, + "end_time": "2023-10-16T16:00:36.115370", "exception": false, - "start_time": "2023-10-12T16:54:05.975442", + "start_time": "2023-10-16T16:00:36.106816", "status": "completed" }, "tags": [] @@ -787,19 +787,19 @@ { "cell_type": "code", "execution_count": 6, - "id": "de461eb6", + "id": "8f314782", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:05.984034Z", - "iopub.status.busy": "2023-10-12T16:54:05.983860Z", - "iopub.status.idle": "2023-10-12T16:54:06.006422Z", - "shell.execute_reply": "2023-10-12T16:54:06.006136Z" + "iopub.execute_input": "2023-10-16T16:00:36.121210Z", + "iopub.status.busy": "2023-10-16T16:00:36.121003Z", + "iopub.status.idle": "2023-10-16T16:00:36.145522Z", + "shell.execute_reply": "2023-10-16T16:00:36.145196Z" }, "papermill": { - "duration": 0.025, - "end_time": "2023-10-12T16:54:06.007205", + "duration": 0.028616, + "end_time": "2023-10-16T16:00:36.146531", "exception": false, - "start_time": "2023-10-12T16:54:05.982205", + "start_time": "2023-10-16T16:00:36.117915", "status": "completed" }, "tags": [] @@ -1276,13 +1276,13 @@ }, { "cell_type": "markdown", - "id": "96126bf7", + "id": "142aab26", "metadata": { "papermill": { - "duration": 0.001569, - "end_time": "2023-10-12T16:54:06.010469", + "duration": 0.002755, + "end_time": "2023-10-16T16:00:36.152085", "exception": false, - "start_time": "2023-10-12T16:54:06.008900", + "start_time": "2023-10-16T16:00:36.149330", "status": "completed" }, "tags": [] @@ -1294,19 +1294,19 @@ { "cell_type": "code", "execution_count": 7, - "id": "f95cba3f", + "id": "eddb4d4c", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:06.014235Z", - "iopub.status.busy": "2023-10-12T16:54:06.014019Z", - "iopub.status.idle": "2023-10-12T16:54:06.020890Z", - "shell.execute_reply": "2023-10-12T16:54:06.020577Z" + "iopub.execute_input": "2023-10-16T16:00:36.158121Z", + "iopub.status.busy": "2023-10-16T16:00:36.157950Z", + "iopub.status.idle": "2023-10-16T16:00:36.165693Z", + "shell.execute_reply": "2023-10-16T16:00:36.165324Z" }, "papermill": { - "duration": 0.009609, - "end_time": "2023-10-12T16:54:06.021662", + "duration": 0.011559, + "end_time": "2023-10-16T16:00:36.166521", "exception": false, - "start_time": "2023-10-12T16:54:06.012053", + "start_time": "2023-10-16T16:00:36.154962", "status": "completed" }, "tags": [] @@ -1318,7 +1318,7 @@ " long_term_unemployed=long_term_unemployed,\n", " labour_market_status=labour_market_status,\n", " ),\n", - " read_meta().pipe(filter_for_dataset, 'LMS').pipe(extract_dates),\n", + " read_meta().pipe(extract_dates, 'LMS'),\n", "])\n", "\n", "summary.to_json(os.path.join(DASHBOARD_DIR, 'latest.json'), indent=2, date_format='iso')" @@ -1327,19 +1327,19 @@ { "cell_type": "code", "execution_count": 8, - "id": "63d64901", + "id": "3c2f8be8", "metadata": { "execution": { - "iopub.execute_input": "2023-10-12T16:54:06.025617Z", - "iopub.status.busy": "2023-10-12T16:54:06.025401Z", - "iopub.status.idle": "2023-10-12T16:54:06.028009Z", - "shell.execute_reply": "2023-10-12T16:54:06.027751Z" + "iopub.execute_input": "2023-10-16T16:00:36.171807Z", + "iopub.status.busy": "2023-10-16T16:00:36.171533Z", + "iopub.status.idle": "2023-10-16T16:00:36.175354Z", + "shell.execute_reply": "2023-10-16T16:00:36.174926Z" }, "papermill": { - "duration": 0.005338, - "end_time": "2023-10-12T16:54:06.028752", + "duration": 0.008008, + "end_time": "2023-10-16T16:00:36.176434", "exception": false, - "start_time": "2023-10-12T16:54:06.023414", + "start_time": "2023-10-16T16:00:36.168426", "status": "completed" }, "tags": [] @@ -1374,13 +1374,13 @@ { "cell_type": "code", "execution_count": null, - "id": "511d2597", + "id": "53b97f2c", "metadata": { "papermill": { - "duration": 0.001696, - "end_time": "2023-10-12T16:54:06.032190", + "duration": 0.0025, + "end_time": "2023-10-16T16:00:36.181908", "exception": false, - "start_time": "2023-10-12T16:54:06.030494", + "start_time": "2023-10-16T16:00:36.179408", "status": "completed" }, "tags": [] @@ -1409,14 +1409,14 @@ }, "papermill": { "default_parameters": {}, - "duration": 1.201943, - "end_time": "2023-10-12T16:54:06.250602", + "duration": 1.212355, + "end_time": "2023-10-16T16:00:36.401112", "environment_variables": {}, "exception": null, "input_path": "prepare.ipynb", "output_path": "../../working/output/labour-market-prepare.ipynb", "parameters": {}, - "start_time": "2023-10-12T16:54:05.048659", + "start_time": "2023-10-16T16:00:35.188757", "version": "2.4.0" } }, diff --git a/working/output/neet-transform.ipynb b/working/output/neet-transform.ipynb index 90781788..3ba2446b 100644 --- a/working/output/neet-transform.ipynb +++ b/working/output/neet-transform.ipynb @@ -3,19 +3,19 @@ { "cell_type": "code", "execution_count": 1, - "id": "ac90490b", + "id": "18a65e8a", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.080378Z", - "iopub.status.busy": "2023-10-16T15:38:22.080036Z", - "iopub.status.idle": "2023-10-16T15:38:22.248000Z", - "shell.execute_reply": "2023-10-16T15:38:22.247627Z" + "iopub.execute_input": "2023-10-16T16:00:37.558018Z", + "iopub.status.busy": "2023-10-16T16:00:37.557842Z", + "iopub.status.idle": "2023-10-16T16:00:37.721661Z", + "shell.execute_reply": "2023-10-16T16:00:37.721307Z" }, "papermill": { - "duration": 0.174421, - "end_time": "2023-10-16T15:38:22.248967", + "duration": 0.170152, + "end_time": "2023-10-16T16:00:37.722666", "exception": false, - "start_time": "2023-10-16T15:38:22.074546", + "start_time": "2023-10-16T16:00:37.552514", "status": "completed" }, "tags": [] @@ -34,19 +34,19 @@ { "cell_type": "code", "execution_count": 2, - "id": "f0d30a39", + "id": "3522143e", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.252149Z", - "iopub.status.busy": "2023-10-16T15:38:22.251965Z", - "iopub.status.idle": "2023-10-16T15:38:22.254345Z", - "shell.execute_reply": "2023-10-16T15:38:22.254063Z" + "iopub.execute_input": "2023-10-16T16:00:37.725833Z", + "iopub.status.busy": "2023-10-16T16:00:37.725624Z", + "iopub.status.idle": "2023-10-16T16:00:37.727950Z", + "shell.execute_reply": "2023-10-16T16:00:37.727700Z" }, "papermill": { - "duration": 0.00477, - "end_time": "2023-10-16T15:38:22.255095", + "duration": 0.00455, + "end_time": "2023-10-16T16:00:37.728573", "exception": false, - "start_time": "2023-10-16T15:38:22.250325", + "start_time": "2023-10-16T16:00:37.724023", "status": "completed" }, "tags": [] @@ -59,19 +59,19 @@ { "cell_type": "code", "execution_count": 3, - "id": "26d52c53", + "id": "1ced03aa", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.257924Z", - "iopub.status.busy": "2023-10-16T15:38:22.257772Z", - "iopub.status.idle": "2023-10-16T15:38:22.259859Z", - "shell.execute_reply": "2023-10-16T15:38:22.259579Z" + "iopub.execute_input": "2023-10-16T16:00:37.731314Z", + "iopub.status.busy": "2023-10-16T16:00:37.731160Z", + "iopub.status.idle": "2023-10-16T16:00:37.733197Z", + "shell.execute_reply": "2023-10-16T16:00:37.732934Z" }, "papermill": { - "duration": 0.004297, - "end_time": "2023-10-16T15:38:22.260529", + "duration": 0.004143, + "end_time": "2023-10-16T16:00:37.733840", "exception": false, - "start_time": "2023-10-16T15:38:22.256232", + "start_time": "2023-10-16T16:00:37.729697", "status": "completed" }, "tags": [] @@ -89,19 +89,19 @@ { "cell_type": "code", "execution_count": 4, - "id": "5a55135f", + "id": "174fe4ac", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.263335Z", - "iopub.status.busy": "2023-10-16T15:38:22.263193Z", - "iopub.status.idle": "2023-10-16T15:38:22.268845Z", - "shell.execute_reply": "2023-10-16T15:38:22.268519Z" + "iopub.execute_input": "2023-10-16T16:00:37.736529Z", + "iopub.status.busy": "2023-10-16T16:00:37.736401Z", + "iopub.status.idle": "2023-10-16T16:00:37.741829Z", + "shell.execute_reply": "2023-10-16T16:00:37.741509Z" }, "papermill": { - "duration": 0.007891, - "end_time": "2023-10-16T15:38:22.269572", + "duration": 0.007519, + "end_time": "2023-10-16T16:00:37.742460", "exception": false, - "start_time": "2023-10-16T15:38:22.261681", + "start_time": "2023-10-16T16:00:37.734941", "status": "completed" }, "tags": [] @@ -114,19 +114,19 @@ { "cell_type": "code", "execution_count": 5, - "id": "830863d8", + "id": "9ff7a84c", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.272480Z", - "iopub.status.busy": "2023-10-16T15:38:22.272332Z", - "iopub.status.idle": "2023-10-16T15:38:22.274239Z", - "shell.execute_reply": "2023-10-16T15:38:22.273960Z" + "iopub.execute_input": "2023-10-16T16:00:37.745259Z", + "iopub.status.busy": "2023-10-16T16:00:37.745125Z", + "iopub.status.idle": "2023-10-16T16:00:37.747025Z", + "shell.execute_reply": "2023-10-16T16:00:37.746758Z" }, "papermill": { - "duration": 0.004193, - "end_time": "2023-10-16T15:38:22.274914", + "duration": 0.004038, + "end_time": "2023-10-16T16:00:37.747641", "exception": false, - "start_time": "2023-10-16T15:38:22.270721", + "start_time": "2023-10-16T16:00:37.743603", "status": "completed" }, "tags": [] @@ -145,19 +145,19 @@ { "cell_type": "code", "execution_count": 6, - "id": "a056dd1b", + "id": "4462fc37", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.277719Z", - "iopub.status.busy": "2023-10-16T15:38:22.277576Z", - "iopub.status.idle": "2023-10-16T15:38:22.280186Z", - "shell.execute_reply": "2023-10-16T15:38:22.279884Z" + "iopub.execute_input": "2023-10-16T16:00:37.750965Z", + "iopub.status.busy": "2023-10-16T16:00:37.750835Z", + "iopub.status.idle": "2023-10-16T16:00:37.753230Z", + "shell.execute_reply": "2023-10-16T16:00:37.752953Z" }, "papermill": { - "duration": 0.004784, - "end_time": "2023-10-16T15:38:22.280856", + "duration": 0.005056, + "end_time": "2023-10-16T16:00:37.753862", "exception": false, - "start_time": "2023-10-16T15:38:22.276072", + "start_time": "2023-10-16T16:00:37.748806", "status": "completed" }, "tags": [] @@ -170,19 +170,19 @@ { "cell_type": "code", "execution_count": 7, - "id": "7e1c79d6", + "id": "0caf04a4", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.283633Z", - "iopub.status.busy": "2023-10-16T15:38:22.283495Z", - "iopub.status.idle": "2023-10-16T15:38:22.286143Z", - "shell.execute_reply": "2023-10-16T15:38:22.285825Z" + "iopub.execute_input": "2023-10-16T16:00:37.839625Z", + "iopub.status.busy": "2023-10-16T16:00:37.839407Z", + "iopub.status.idle": "2023-10-16T16:00:37.842951Z", + "shell.execute_reply": "2023-10-16T16:00:37.842546Z" }, "papermill": { - "duration": 0.004785, - "end_time": "2023-10-16T15:38:22.286774", + "duration": 0.006681, + "end_time": "2023-10-16T16:00:37.844067", "exception": false, - "start_time": "2023-10-16T15:38:22.281989", + "start_time": "2023-10-16T16:00:37.837386", "status": "completed" }, "tags": [] @@ -197,19 +197,19 @@ { "cell_type": "code", "execution_count": 8, - "id": "620d9a32", + "id": "f7df7191", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.289643Z", - "iopub.status.busy": "2023-10-16T15:38:22.289520Z", - "iopub.status.idle": "2023-10-16T15:38:22.294950Z", - "shell.execute_reply": "2023-10-16T15:38:22.294667Z" + "iopub.execute_input": "2023-10-16T16:00:37.861598Z", + "iopub.status.busy": "2023-10-16T16:00:37.860994Z", + "iopub.status.idle": "2023-10-16T16:00:37.878179Z", + "shell.execute_reply": "2023-10-16T16:00:37.876928Z" }, "papermill": { - "duration": 0.007615, - "end_time": "2023-10-16T15:38:22.295555", + "duration": 0.025453, + "end_time": "2023-10-16T16:00:37.880640", "exception": false, - "start_time": "2023-10-16T15:38:22.287940", + "start_time": "2023-10-16T16:00:37.855187", "status": "completed" }, "tags": [] @@ -223,19 +223,19 @@ { "cell_type": "code", "execution_count": 9, - "id": "4733c4f0", + "id": "2751967b", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.298440Z", - "iopub.status.busy": "2023-10-16T15:38:22.298303Z", - "iopub.status.idle": "2023-10-16T15:38:22.303582Z", - "shell.execute_reply": "2023-10-16T15:38:22.303261Z" + "iopub.execute_input": "2023-10-16T16:00:37.888228Z", + "iopub.status.busy": "2023-10-16T16:00:37.887947Z", + "iopub.status.idle": "2023-10-16T16:00:37.898987Z", + "shell.execute_reply": "2023-10-16T16:00:37.897625Z" }, "papermill": { - "duration": 0.007518, - "end_time": "2023-10-16T15:38:22.304258", + "duration": 0.01674, + "end_time": "2023-10-16T16:00:37.900409", "exception": false, - "start_time": "2023-10-16T15:38:22.296740", + "start_time": "2023-10-16T16:00:37.883669", "status": "completed" }, "tags": [] @@ -248,19 +248,19 @@ { "cell_type": "code", "execution_count": 10, - "id": "54e4d075", + "id": "2d3d9193", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.307206Z", - "iopub.status.busy": "2023-10-16T15:38:22.307071Z", - "iopub.status.idle": "2023-10-16T15:38:22.310866Z", - "shell.execute_reply": "2023-10-16T15:38:22.310578Z" + "iopub.execute_input": "2023-10-16T16:00:37.903955Z", + "iopub.status.busy": "2023-10-16T16:00:37.903732Z", + "iopub.status.idle": "2023-10-16T16:00:37.908536Z", + "shell.execute_reply": "2023-10-16T16:00:37.908233Z" }, "papermill": { - "duration": 0.006024, - "end_time": "2023-10-16T15:38:22.311503", + "duration": 0.007504, + "end_time": "2023-10-16T16:00:37.909254", "exception": false, - "start_time": "2023-10-16T15:38:22.305479", + "start_time": "2023-10-16T16:00:37.901750", "status": "completed" }, "tags": [] @@ -284,19 +284,19 @@ { "cell_type": "code", "execution_count": 11, - "id": "9b583db1", + "id": "d1726e87", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.314446Z", - "iopub.status.busy": "2023-10-16T15:38:22.314320Z", - "iopub.status.idle": "2023-10-16T15:38:22.318579Z", - "shell.execute_reply": "2023-10-16T15:38:22.318260Z" + "iopub.execute_input": "2023-10-16T16:00:37.913025Z", + "iopub.status.busy": "2023-10-16T16:00:37.912843Z", + "iopub.status.idle": "2023-10-16T16:00:37.917852Z", + "shell.execute_reply": "2023-10-16T16:00:37.917536Z" }, "papermill": { - "duration": 0.006502, - "end_time": "2023-10-16T15:38:22.319232", + "duration": 0.007876, + "end_time": "2023-10-16T16:00:37.918605", "exception": false, - "start_time": "2023-10-16T15:38:22.312730", + "start_time": "2023-10-16T16:00:37.910729", "status": "completed" }, "tags": [] @@ -333,19 +333,19 @@ { "cell_type": "code", "execution_count": 12, - "id": "ab813426", + "id": "6b9a8619", "metadata": { "execution": { - "iopub.execute_input": "2023-10-16T15:38:22.322155Z", - "iopub.status.busy": "2023-10-16T15:38:22.322029Z", - "iopub.status.idle": "2023-10-16T15:38:22.328649Z", - "shell.execute_reply": "2023-10-16T15:38:22.328365Z" + "iopub.execute_input": "2023-10-16T16:00:37.921919Z", + "iopub.status.busy": "2023-10-16T16:00:37.921774Z", + "iopub.status.idle": "2023-10-16T16:00:37.928606Z", + "shell.execute_reply": "2023-10-16T16:00:37.928340Z" }, "papermill": { - "duration": 0.00888, - "end_time": "2023-10-16T15:38:22.329334", + "duration": 0.009357, + "end_time": "2023-10-16T16:00:37.929279", "exception": false, - "start_time": "2023-10-16T15:38:22.320454", + "start_time": "2023-10-16T16:00:37.919922", "status": "completed" }, "tags": [] @@ -381,13 +381,13 @@ { "cell_type": "code", "execution_count": null, - "id": "afff2e7c", + "id": "5ccfc9b1", "metadata": { "papermill": { - "duration": 0.001277, - "end_time": "2023-10-16T15:38:22.332043", + "duration": 0.001261, + "end_time": "2023-10-16T16:00:37.931928", "exception": false, - "start_time": "2023-10-16T15:38:22.330766", + "start_time": "2023-10-16T16:00:37.930667", "status": "completed" }, "tags": [] @@ -398,13 +398,13 @@ { "cell_type": "code", "execution_count": null, - "id": "35ad762f", + "id": "a2d547c2", "metadata": { "papermill": { - "duration": 0.001219, - "end_time": "2023-10-16T15:38:22.334542", + "duration": 0.001234, + "end_time": "2023-10-16T16:00:37.934449", "exception": false, - "start_time": "2023-10-16T15:38:22.333323", + "start_time": "2023-10-16T16:00:37.933215", "status": "completed" }, "tags": [] @@ -433,14 +433,14 @@ }, "papermill": { "default_parameters": {}, - "duration": 1.319783, - "end_time": "2023-10-16T15:38:22.552473", + "duration": 1.444809, + "end_time": "2023-10-16T16:00:38.150033", "environment_variables": {}, "exception": null, "input_path": "transform.ipynb", "output_path": "../../working/output/neet-transform.ipynb", "parameters": {}, - "start_time": "2023-10-16T15:38:21.232690", + "start_time": "2023-10-16T16:00:36.705224", "version": "2.4.0" } },