chore: add script to find python dependencies

.github/workflows/ci.yml

@@ -26,8 +26,13 @@ jobs:
     - name: Install pip
       run: pip install -r requirements/pip.txt
 
+    - name: upgrade setuptool
+      run: pip install --upgrade setuptools
+
     - name: Install Dependencies
-      run: make dev-install
+      run: |
+        python -m pip install --upgrade pip
+        make dev-install
 
     - name: Run Tests
       run: make test

Makefile

@@ -34,9 +34,9 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)  ## update the requirements/*.txt files with
 	pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
 	pip install -qr requirements/pip.txt
 	pip install -qr requirements/pip-tools.txt
-	pip-compile --upgrade -o requirements/base.txt requirements/base.in
+	pip-compile --upgrade --allow-unsafe -o requirements/base.txt requirements/base.in
 	pip-compile --upgrade -o requirements/development.txt requirements/development.in
-	for fextra in edx_repo_tools/*/extra.in; do pip-compile --upgrade -o $${fextra%.in}.txt $$fextra; done
+	for fextra in edx_repo_tools/*/extra*.in; do pip-compile --upgrade -o $${fextra%.in}.txt $$fextra; done
 
 lint:				## run pylint
 	pylint *.py edx_repo_tools tests

edx_repo_tools/audit_gh_users/extra.txt

@@ -6,11 +6,11 @@
 #
 click==8.1.7
     # via -r edx_repo_tools/audit_gh_users/extra.in
-fastcore==1.5.29
+fastcore==1.5.33
     # via ghapi
-ghapi==1.0.4
+ghapi==1.0.5
     # via -r edx_repo_tools/audit_gh_users/extra.in
-packaging==23.2
+packaging==24.0
     # via
     #   fastcore
     #   ghapi

edx_repo_tools/conventional_commits/extra-py312.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
 #    make upgrade
@@ -8,36 +8,40 @@ alembic==1.13.1
     # via dataset
 banal==1.0.6
     # via dataset
-contourpy==1.2.0
+contourpy==1.1.1
     # via matplotlib
 cycler==0.12.1
     # via matplotlib
 dataset==1.6.2
     # via -r edx_repo_tools/conventional_commits/extra-py312.in
-fonttools==4.49.0
+fonttools==4.51.0
     # via matplotlib
-greenlet==3.0.3
+importlib-metadata==6.11.0
     # via
-    #   -c edx_repo_tools/conventional_commits/../../requirements/constraints.txt
-    #   sqlalchemy
+    #   -c edx_repo_tools/conventional_commits/../../requirements/common_constraints.txt
+    #   alembic
+importlib-resources==6.4.0
+    # via
+    #   alembic
+    #   matplotlib
 kiwisolver==1.4.5
     # via matplotlib
-mako==1.3.2
+mako==1.3.3
     # via alembic
 markupsafe==2.1.5
     # via mako
-matplotlib==3.8.3
+matplotlib==3.7.5
     # via -r edx_repo_tools/conventional_commits/extra-py312.in
-numpy==1.26.4
+numpy==1.24.4
     # via
     #   contourpy
     #   matplotlib
     #   pandas
-packaging==23.2
+packaging==24.0
     # via matplotlib
-pandas==2.2.1
+pandas==2.0.3
     # via -r edx_repo_tools/conventional_commits/extra-py312.in
-pillow==10.2.0
+pillow==10.3.0
     # via matplotlib
 pyparsing==3.1.2
     # via matplotlib
@@ -53,7 +57,11 @@ sqlalchemy==1.4.52
     # via
     #   alembic
     #   dataset
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via alembic
 tzdata==2024.1
     # via pandas
+zipp==3.18.1
+    # via
+    #   importlib-metadata
+    #   importlib-resources

edx_repo_tools/conventional_commits/extra.txt

@@ -14,21 +14,19 @@ cycler==0.12.1
     # via matplotlib
 dataset==1.6.2
     # via -r edx_repo_tools/conventional_commits/extra.in
-fonttools==4.49.0
+fonttools==4.51.0
     # via matplotlib
-greenlet==3.0.3
+importlib-metadata==6.11.0
     # via
-    #   -c edx_repo_tools/conventional_commits/../../requirements/constraints.txt
-    #   sqlalchemy
-importlib-metadata==7.0.1
-    # via alembic
-importlib-resources==6.1.2
+    #   -c edx_repo_tools/conventional_commits/../../requirements/common_constraints.txt
+    #   alembic
+importlib-resources==6.4.0
     # via
     #   alembic
     #   matplotlib
 kiwisolver==1.4.5
     # via matplotlib
-mako==1.3.2
+mako==1.3.3
     # via alembic
 markupsafe==2.1.5
     # via mako
@@ -39,11 +37,11 @@ numpy==1.24.4
     #   contourpy
     #   matplotlib
     #   pandas
-packaging==23.2
+packaging==24.0
     # via matplotlib
 pandas==2.0.3
     # via -r edx_repo_tools/conventional_commits/extra.in
-pillow==10.2.0
+pillow==10.3.0
     # via matplotlib
 pyparsing==3.1.2
     # via matplotlib
@@ -59,11 +57,11 @@ sqlalchemy==1.4.52
     # via
     #   alembic
     #   dataset
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via alembic
 tzdata==2024.1
     # via pandas
-zipp==3.17.0
+zipp==3.18.1
     # via
     #   importlib-metadata
     #   importlib-resources

edx_repo_tools/find_dependencies/extra.txt

@@ -8,19 +8,19 @@ certifi==2024.2.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
-idna==3.6
+idna==3.7
     # via requests
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
     # via markdown-it-py
-pygments==2.17.2
+pygments==2.18.0
     # via rich
 requests==2.31.0
     # via -r edx_repo_tools/find_dependencies/extra.in
 rich==13.7.1
     # via -r edx_repo_tools/find_dependencies/extra.in
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via rich
 urllib3==2.2.1
     # via requests

edx_repo_tools/find_dependencies/find_python_dependencies.py

@@ -0,0 +1,75 @@
+"""
+Spider and catalog dependencies.
+$ python find_python_dependencies.py $FILE_PATH
+"""
+
+import json
+import os
+import requirements
+import sys
+from pathlib import Path
+import requests
+
+
+# The first of these we find is the requirements file we'll examine:
+PY_REQS = [
+    "requirements/edx/base.txt",
+    "requirements/base.txt",
+    "requirements.txt",
+]
+
+def request_package_info_url(package):
+        base_url = "https://pypi.org/pypi/"
+        url = f"{base_url}{package}/json"
+        response = requests.get(url)
+        if response.status_code == 200:
+            data_dict = response.json()
+            info = data_dict["info"]
+            return info["home_page"]            
+        else:
+            print(f"Failed to retrieve data for package {package}. Status code:", response.status_code)    
+
+FIRST_PARTY_ORGS = ["openedx"]
+
+SECOND_PARTY_ORGS = [
+    "edx", "edx-unsupported", "edx-solutions",
+    "mitodl",
+    "overhangio",
+    "open-craft", "eduNEXT", "raccoongang",
+]
+
+def urls_in_orgs(urls, orgs):
+    """
+    Find urls that are in any of the `orgs`.
+    """
+    return sorted(
+        url for url in urls
+        if any(f"/{org}/" in url for org in orgs)
+    )
+
+def main(dirs=None, org=None):
+    """
+    Analyze the requirements in input directory mentioned on the command line.    
+    """
+    packages_url = [] 
+    if dirs is None:
+         dirs = sys.argv[1:]
+
+    for i_dir, repo_dir in enumerate(dirs, start=1):
+        with open(repo_dir) as fbase:
+            # Read each line (package name) in the file
+            for req in requirements.parse(fbase):
+                home_page = request_package_info_url(req.name)
+                if home_page is not None:
+                    if match := urls_in_orgs([home_page], SECOND_PARTY_ORGS):
+                        packages_url.append(home_page)
+
+    print("== DONE ==============")
+    print("Second party packages:")
+    print("\n".join(packages_url))
+
+    if packages_url:
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

edx_repo_tools/repo_access_scraper/extra.txt

@@ -8,9 +8,9 @@ greenlet==3.0.3
     # via
     #   -c edx_repo_tools/repo_access_scraper/../../requirements/constraints.txt
     #   playwright
-playwright==1.42.0
+playwright==1.43.0
     # via -r edx_repo_tools/repo_access_scraper/extra.in
-pyee==11.0.1
+pyee==11.1.0
     # via playwright
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via pyee

edx_repo_tools/repo_checks/extra.txt

@@ -12,13 +12,13 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via -r edx_repo_tools/repo_checks/extra.in
-fastcore==1.5.29
+fastcore==1.5.33
     # via ghapi
-ghapi==1.0.4
+ghapi==1.0.5
     # via -r edx_repo_tools/repo_checks/extra.in
-idna==3.6
+idna==3.7
     # via requests
-packaging==23.2
+packaging==24.0
     # via
     #   fastcore
     #   ghapi