201 update dependencies

ojd_daps_skills/app/requirements_app.txt

@@ -1,4 +1,2 @@
-numpy==1.21.1
 ojd-daps-skills
-awscli==1.27.25
-streamlit== 1.16.0
+streamlit==1.16.0

ojd_daps_skills/getters/download_public_data.py

@@ -4,7 +4,6 @@
 
 
 def download():
-
     public_data_dir = os.path.join(PROJECT_DIR, PUBLIC_DATA_FOLDER_NAME)
 
     os.system(
@@ -14,5 +13,43 @@ def download():
     os.system(f"rm {public_data_dir}.zip")
 
 
+if __name__ == "__main__":
+    download()
+
+from ojd_daps_skills import PUBLIC_DATA_FOLDER_NAME, PROJECT_DIR
+
+import os
+import boto3
+from botocore.exceptions import ClientError
+from botocore import UNSIGNED
+from botocore.config import Config
+from zipfile import ZipFile
+
+
+def download():
+    """Download public data. Expected to run once on first use."""
+    s3 = boto3.client(
+        "s3", region_name="eu-west-1", config=Config(signature_version=UNSIGNED)
+    )
+
+    bucket_name = "open-jobs-indicators"
+    key = f"escoe_extension/{PUBLIC_DATA_FOLDER_NAME}.zip"
+
+    public_data_dir = os.path.join(PROJECT_DIR, PUBLIC_DATA_FOLDER_NAME)
+
+    try:
+        s3.download_file(bucket_name, key, f"{public_data_dir}.zip")
+
+        with ZipFile(f"{public_data_dir}.zip", "r") as zip_ref:
+            zip_ref.extractall(PROJECT_DIR)
+
+        os.remove(f"{public_data_dir}.zip")
+
+    except ClientError as ce:
+        print(f"Error: {ce}")
+    except FileNotFoundError as fnfe:
+        print(f"Error: {fnfe}")
+
+
 if __name__ == "__main__":
     download()

requirements.txt

@@ -1,13 +1,13 @@
-numpy==1.22.4
-scipy==1.8.1
+numpy==1.24.4
+scipy==1.10.1
 pandas==1.3.5
 tqdm==4.64.0
 filelock==3.7.1
 typer==0.4.1
 sh==1.14.2
-transformers==4.20.1
+transformers==4.33.3
 sentence-transformers==2.2.2
-scikit-learn==0.23.2
+scikit-learn==1.3.1
 spacy==3.4.0
 nervaluate==0.1.8
 s3fs==2022.5.0

requirements_dev.txt

@@ -9,7 +9,6 @@ black
 Sphinx
 sphinxcontrib-napoleon
 sphinx-rtd-theme
-awscli==1.27.32
 pre-commit
 pre-commit-hooks
 spacy==3.4.0