Remove dependency on EDD metadata.csv

open-innovations · Oct 12, 2023 · 33b4d73 · 33b4d73
1 parent 61b8f55
commit 33b4d73
Show file tree

Hide file tree

Showing 14 changed files with 120 additions and 126 deletions.
diff --git a/scripts/labour-market/dvc.lock b/scripts/labour-market/dvc.lock
@@ -29,13 +29,9 @@ stages:
       size: 359985
     - path: ../../scripts/util/
       hash: md5
-      md5: 8b6cedb3e68500af4d07c0b1deed75d5.dir
-      size: 9646
+      md5: c7bb69be54b1e9f9ce79d49e29ead857.dir
+      size: 9655
       nfiles: 6
-    - path: ../../working/metadata.csv
-      hash: md5
-      md5: 1cba7127d3fd375c3a10ef95b256b2b1
-      size: 7650
     - path: config.py
       hash: md5
       md5: c9187b99fb9c48a24be707561d15eaa2

diff --git a/scripts/labour-market/dvc.yaml b/scripts/labour-market/dvc.yaml
@@ -22,7 +22,6 @@ stages:
       - config.py
       - ${top}/scripts/util/
       - ${top}/data/labour-market/monthly-rolling.csv
-      - ${top}/working/metadata.csv
     outs:
       - ${top}/src/_data/sources/labour-market/long_term_unemployed_last_3_years.csv:
           cache: false

diff --git a/scripts/util/metadata.py b/scripts/util/metadata.py
@@ -2,7 +2,7 @@
 import pandas as pd
 
 TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
-METADATA_FILE = os.path.join(TOP_DIR, 'working/metadata.csv')
+METADATA_FILE = os.path.join(TOP_DIR, 'working/upstream/metadata.csv')
 
 
 def read_meta():

diff --git a/scripts/vacancies/dvc.lock b/scripts/vacancies/dvc.lock
@@ -9,8 +9,8 @@ stages:
       size: 10506
     - path: scripts/util/metadata.py
       hash: md5
-      md5: 0b8a89232411298d711fb437327810bc
-      size: 902
+      md5: 4c10007d68898289db269e5da494a0f5
+      size: 911
     - path: scripts/vacancies/prepare.py
       hash: md5
       md5: 612ecb35280882729350a4668a49e8c0

diff --git a/working/metadata.csv.dvc b/working/metadata.csv.dvc
diff --git a/working/output/labour-market-prepare.ipynb b/working/output/labour-market-prepare.ipynb
@@ -3,19 +3,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "id": "6024ee17",
+   "id": "1dcb703e",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.034001Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.033209Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.229538Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.229154Z"
+     "iopub.execute_input": "2023-10-12T16:54:05.753616Z",
+     "iopub.status.busy": "2023-10-12T16:54:05.752818Z",
+     "iopub.status.idle": "2023-10-12T16:54:05.915273Z",
+     "shell.execute_reply": "2023-10-12T16:54:05.914812Z"
     },
     "papermill": {
-     "duration": 0.203141,
-     "end_time": "2023-10-12T12:38:52.230683",
+     "duration": 0.169313,
+     "end_time": "2023-10-12T16:54:05.916332",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.027542",
+     "start_time": "2023-10-12T16:54:05.747019",
      "status": "completed"
     },
     "tags": []
@@ -33,19 +33,19 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "id": "323b3b83",
+   "id": "aad17674",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.233837Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.233664Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.237758Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.237437Z"
+     "iopub.execute_input": "2023-10-12T16:54:05.919327Z",
+     "iopub.status.busy": "2023-10-12T16:54:05.919169Z",
+     "iopub.status.idle": "2023-10-12T16:54:05.923186Z",
+     "shell.execute_reply": "2023-10-12T16:54:05.922808Z"
     },
     "papermill": {
-     "duration": 0.006602,
-     "end_time": "2023-10-12T12:38:52.238604",
+     "duration": 0.006235,
+     "end_time": "2023-10-12T16:54:05.923856",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.232002",
+     "start_time": "2023-10-12T16:54:05.917621",
      "status": "completed"
     },
     "tags": []
@@ -58,13 +58,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "26e6f8cb",
+   "id": "a6b4437e",
    "metadata": {
     "papermill": {
-     "duration": 0.001002,
-     "end_time": "2023-10-12T12:38:52.240701",
+     "duration": 0.000975,
+     "end_time": "2023-10-12T16:54:05.925959",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.239699",
+     "start_time": "2023-10-12T16:54:05.924984",
      "status": "completed"
     },
     "tags": []
@@ -76,19 +76,19 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "aec0850e",
+   "id": "881244de",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.243270Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.243115Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.252054Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.251714Z"
+     "iopub.execute_input": "2023-10-12T16:54:05.928529Z",
+     "iopub.status.busy": "2023-10-12T16:54:05.928369Z",
+     "iopub.status.idle": "2023-10-12T16:54:05.937200Z",
+     "shell.execute_reply": "2023-10-12T16:54:05.936889Z"
     },
     "papermill": {
-     "duration": 0.011193,
-     "end_time": "2023-10-12T12:38:52.252899",
+     "duration": 0.010962,
+     "end_time": "2023-10-12T16:54:05.937924",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.241706",
+     "start_time": "2023-10-12T16:54:05.926962",
      "status": "completed"
     },
     "tags": []
@@ -103,13 +103,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c12349f1",
+   "id": "a05a90ef",
    "metadata": {
     "papermill": {
-     "duration": 0.001055,
-     "end_time": "2023-10-12T12:38:52.255060",
+     "duration": 0.001044,
+     "end_time": "2023-10-12T16:54:05.940094",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.254005",
+     "start_time": "2023-10-12T16:54:05.939050",
      "status": "completed"
     },
     "tags": []
@@ -121,19 +121,19 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "ead5cade",
+   "id": "642dd6ef",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.257705Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.257513Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.285094Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.284764Z"
+     "iopub.execute_input": "2023-10-12T16:54:05.942857Z",
+     "iopub.status.busy": "2023-10-12T16:54:05.942635Z",
+     "iopub.status.idle": "2023-10-12T16:54:05.970567Z",
+     "shell.execute_reply": "2023-10-12T16:54:05.970276Z"
     },
     "papermill": {
-     "duration": 0.029831,
-     "end_time": "2023-10-12T12:38:52.285946",
+     "duration": 0.030139,
+     "end_time": "2023-10-12T16:54:05.971347",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.256115",
+     "start_time": "2023-10-12T16:54:05.941208",
      "status": "completed"
     },
     "tags": []
@@ -721,13 +721,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "a82da65f",
+   "id": "c84d1e07",
    "metadata": {
     "papermill": {
-     "duration": 0.001294,
-     "end_time": "2023-10-12T12:38:52.288716",
+     "duration": 0.001316,
+     "end_time": "2023-10-12T16:54:05.974135",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.287422",
+     "start_time": "2023-10-12T16:54:05.972819",
      "status": "completed"
     },
     "tags": []
@@ -739,19 +739,19 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "dc01ca5f",
+   "id": "ea53e431",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.291970Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.291826Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.294898Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.294615Z"
+     "iopub.execute_input": "2023-10-12T16:54:05.977450Z",
+     "iopub.status.busy": "2023-10-12T16:54:05.977199Z",
+     "iopub.status.idle": "2023-10-12T16:54:05.980233Z",
+     "shell.execute_reply": "2023-10-12T16:54:05.979950Z"
     },
     "papermill": {
-     "duration": 0.005586,
-     "end_time": "2023-10-12T12:38:52.295571",
+     "duration": 0.005426,
+     "end_time": "2023-10-12T16:54:05.980868",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.289985",
+     "start_time": "2023-10-12T16:54:05.975442",
      "status": "completed"
     },
     "tags": []
@@ -787,19 +787,19 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "71d7cb23",
+   "id": "de461eb6",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.298809Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.298674Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.321040Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.320788Z"
+     "iopub.execute_input": "2023-10-12T16:54:05.984034Z",
+     "iopub.status.busy": "2023-10-12T16:54:05.983860Z",
+     "iopub.status.idle": "2023-10-12T16:54:06.006422Z",
+     "shell.execute_reply": "2023-10-12T16:54:06.006136Z"
     },
     "papermill": {
-     "duration": 0.024863,
-     "end_time": "2023-10-12T12:38:52.321780",
+     "duration": 0.025,
+     "end_time": "2023-10-12T16:54:06.007205",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.296917",
+     "start_time": "2023-10-12T16:54:05.982205",
      "status": "completed"
     },
     "tags": []
@@ -1276,13 +1276,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "be2bcaf8",
+   "id": "96126bf7",
    "metadata": {
     "papermill": {
-     "duration": 0.001548,
-     "end_time": "2023-10-12T12:38:52.325008",
+     "duration": 0.001569,
+     "end_time": "2023-10-12T16:54:06.010469",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.323460",
+     "start_time": "2023-10-12T16:54:06.008900",
      "status": "completed"
     },
     "tags": []
@@ -1294,19 +1294,19 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "4a796335",
+   "id": "f95cba3f",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.328744Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.328597Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.336123Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.335798Z"
+     "iopub.execute_input": "2023-10-12T16:54:06.014235Z",
+     "iopub.status.busy": "2023-10-12T16:54:06.014019Z",
+     "iopub.status.idle": "2023-10-12T16:54:06.020890Z",
+     "shell.execute_reply": "2023-10-12T16:54:06.020577Z"
     },
     "papermill": {
-     "duration": 0.010282,
-     "end_time": "2023-10-12T12:38:52.336854",
+     "duration": 0.009609,
+     "end_time": "2023-10-12T16:54:06.021662",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.326572",
+     "start_time": "2023-10-12T16:54:06.012053",
      "status": "completed"
     },
     "tags": []
@@ -1327,19 +1327,19 @@
   {
    "cell_type": "code",
    "execution_count": 8,
-   "id": "a8682fd5",
+   "id": "63d64901",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-12T12:38:52.340997Z",
-     "iopub.status.busy": "2023-10-12T12:38:52.340833Z",
-     "iopub.status.idle": "2023-10-12T12:38:52.343926Z",
-     "shell.execute_reply": "2023-10-12T12:38:52.343577Z"
+     "iopub.execute_input": "2023-10-12T16:54:06.025617Z",
+     "iopub.status.busy": "2023-10-12T16:54:06.025401Z",
+     "iopub.status.idle": "2023-10-12T16:54:06.028009Z",
+     "shell.execute_reply": "2023-10-12T16:54:06.027751Z"
     },
     "papermill": {
-     "duration": 0.006269,
-     "end_time": "2023-10-12T12:38:52.344884",
+     "duration": 0.005338,
+     "end_time": "2023-10-12T16:54:06.028752",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.338615",
+     "start_time": "2023-10-12T16:54:06.023414",
      "status": "completed"
     },
     "tags": []
@@ -1374,13 +1374,13 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "3a858c79",
+   "id": "511d2597",
    "metadata": {
     "papermill": {
-     "duration": 0.003022,
-     "end_time": "2023-10-12T12:38:52.350480",
+     "duration": 0.001696,
+     "end_time": "2023-10-12T16:54:06.032190",
      "exception": false,
-     "start_time": "2023-10-12T12:38:52.347458",
+     "start_time": "2023-10-12T16:54:06.030494",
      "status": "completed"
     },
     "tags": []
@@ -1409,14 +1409,14 @@
   },
   "papermill": {
    "default_parameters": {},
-   "duration": 1.270241,
-   "end_time": "2023-10-12T12:38:52.572617",
+   "duration": 1.201943,
+   "end_time": "2023-10-12T16:54:06.250602",
    "environment_variables": {},
    "exception": null,
    "input_path": "prepare.ipynb",
    "output_path": "../../working/output/labour-market-prepare.ipynb",
    "parameters": {},
-   "start_time": "2023-10-12T12:38:51.302376",
+   "start_time": "2023-10-12T16:54:05.048659",
    "version": "2.4.0"
   }
  },