From d398b64c132e971690c71faaddf2afb1339f5c19 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 12 Jun 2024 11:16:57 -0500 Subject: [PATCH 1/5] Add workflow to test HSDS with socket --- .github/workflows/python-package.yml | 88 +++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 703bdc84..883fd7da 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -168,4 +168,90 @@ jobs: - name: Shut down Docker if: ${{matrix.build-method == 'docker' && matrix.os != 'windows-latest'}} run: | - ./stopall.sh \ No newline at end of file + ./stopall.sh + + build-and-test-socket: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.12"] + build-method: ["manual"] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Lint with flake8 + shell: bash + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --max-complexity=10 --max-line-length=127 --statistics + + - name: Install package + shell: bash + run: | + pip install -e . + + - name: Run unit tests + shell: bash + run: | + pytest + + - name: Start HSDS + shell: bash + if: ${{matrix.build-method == 'manual'}} + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock + BUCKET_NAME: hsdstest + LOG_LEVEL: DEBUG + ROOT_DIR: ${{github.workspace}}/hsdsdata + run: | + cp admin/config/groups.default admin/config/groups.txt + cp admin/config/passwd.default admin/config/passwd.txt + ./runall.sh --no-docker 1 & + + - name: Wait for node startup + shell: bash + run: | + sleep 45 + + - name: Run HSDS tests + id: hsds-tests + shell: bash + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + BUCKET_NAME: hsdstest + LOG_LEVEL: DEBUG + run: | + python testall.py + + - name: Show HSDS Logs on Fail (Manual) + # Only run if the whole workflow failed due to HSDS tests + if: ${{failure() && steps.hsds-tests.outcome == 'failure' && (matrix.build-method == 'manual')}} + run: | + cat hs.log From a707bac8f0c75dbd7f57cf383d87a724f1981362 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 12 Jun 2024 12:29:09 -0500 Subject: [PATCH 2/5] Require urllib3 < 2.0.0 --- pyproject.toml | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index acc295b8..a04e137a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "requests-unixsocket", "simplejson", "s3fs", + "urllib3 < 2.0.0" ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index d16cb14b..e5b7fd71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ requests<=2.32.2 requests-unixsocket simplejson s3fs +urllib3<2.0.0 From afa052236964a089467369cb099bd16426be53ce Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 12 Jun 2024 12:43:07 -0500 Subject: [PATCH 3/5] requests <=2.31.0 --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a04e137a..969213ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "pyjwt", "pytz", "pyyaml", - "requests <= 2.32.2", + "requests <= 2.31.0", "requests-unixsocket", "simplejson", "s3fs", diff --git a/requirements.txt b/requirements.txt index e5b7fd71..59b2a4af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ psutil pyjwt pytz pyyaml -requests<=2.32.2 +requests<=2.31.0 requests-unixsocket simplejson s3fs From 7a58943d09acb1462f52cd417c00a15e4e814d98 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 12 Jun 2024 12:54:05 -0500 Subject: [PATCH 4/5] Use socket setup from VOL REST workflow --- .github/workflows/python-package.yml | 89 +++++++++++----------------- 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 883fd7da..34934776 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -171,14 +171,28 @@ jobs: ./stopall.sh build-and-test-socket: + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + HSDS_USERNAME: test_user1 + HSDS_PASSWORD: test + HSDS_PATH: /home/test_user1/ + HDF5_API_TEST_PATH_PREFIX: /home/test_user1/ + HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock + HDF5_VOL_CONNECTOR: REST + ROOT_DIR: ${{github.workspace}}/hsdsdata + BUCKET_NAME: hsdstest + + name: Test HSDS with socket + runs-on: ubuntu-latest strategy: - fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] python-version: ["3.12"] - build-method: ["manual"] - runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -186,72 +200,41 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install HSDS dependencies shell: bash run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest + python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - shell: bash - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --max-complexity=10 --max-line-length=127 --statistics - - - name: Install package + - name: Install HSDS package shell: bash run: | pip install -e . - - name: Run unit tests + # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections + - name: Fix requests version + run: | + pip install requests==2.31.0 + + - name: Run HSDS unit tests shell: bash run: | pytest - name: Start HSDS - shell: bash - if: ${{matrix.build-method == 'manual'}} - env: - ADMIN_PASSWORD: admin - ADMIN_USERNAME: admin - USER_NAME: test_user1 - USER_PASSWORD: test - USER2_NAME: test_user2 - USER2_PASSWORD: test - HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock - BUCKET_NAME: hsdstest - LOG_LEVEL: DEBUG - ROOT_DIR: ${{github.workspace}}/hsdsdata run: | + mkdir ${{github.workspace}}/hsdsdata + mkdir ${{github.workspace}}/hsdsdata/hsdstest cp admin/config/groups.default admin/config/groups.txt cp admin/config/passwd.default admin/config/passwd.txt - ./runall.sh --no-docker 1 & + ROOT_DIR=${{github.workspace}}/hsdsdata ./runall.sh --no-docker 1 & + sleep 10 - - name: Wait for node startup - shell: bash + - name: Test HSDS setup run: | - sleep 45 + python tests/integ/setup_test.py - - name: Run HSDS tests - id: hsds-tests - shell: bash - env: - ADMIN_PASSWORD: admin - ADMIN_USERNAME: admin - USER_NAME: test_user1 - USER_PASSWORD: test - USER2_NAME: test_user2 - USER2_PASSWORD: test - BUCKET_NAME: hsdstest - LOG_LEVEL: DEBUG - run: | - python testall.py - - - name: Show HSDS Logs on Fail (Manual) - # Only run if the whole workflow failed due to HSDS tests - if: ${{failure() && steps.hsds-tests.outcome == 'failure' && (matrix.build-method == 'manual')}} - run: | - cat hs.log + - name: Test HSDS + run : | + python testall.py \ No newline at end of file From 9c9ced9dfacc9f72040ef933f8abefd2ce752899 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 12 Jun 2024 13:05:14 -0500 Subject: [PATCH 5/5] Fix workflows --- .github/workflows/dependency-review.yml | 6 +++--- .github/workflows/python-package.yml | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 583dff93..bf0656fb 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -16,8 +16,8 @@ jobs: steps: - name: 'Checkout Repository' uses: actions/checkout@v3 + - name: 'Dependency Review' - uses: actions/dependency-review-action@v1 + uses: actions/dependency-review-action@v4 with: - # verify=False is never used in HSDS - allow-ghsas: ["GHSA-9wx4-h78v-vm56"] + allow-ghsas: GHSA-9wx4-h78v-vm56 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 34934776..4e59c269 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -186,13 +186,14 @@ jobs: HDF5_VOL_CONNECTOR: REST ROOT_DIR: ${{github.workspace}}/hsdsdata BUCKET_NAME: hsdstest - + name: Test HSDS with socket - runs-on: ubuntu-latest strategy: matrix: + os: [ubuntu-latest] python-version: ["3.12"] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -213,9 +214,9 @@ jobs: pip install -e . # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections - - name: Fix requests version - run: | - pip install requests==2.31.0 + # - name: Fix requests version + # run: | + # pip install requests==2.31.0 - name: Run HSDS unit tests shell: bash