Merge pull request #270 from EESSI/develop

release v0.5.0

.flake8

@@ -1,3 +1,11 @@
+# This file is part of the EESSI build-and-deploy bot,
+# see https://github.com/EESSI/eessi-bot-software-layer
+#
+# author: Kenneth Hoste (@boegel)
+#
+# license: GPLv2
+#
+
 [flake8]
 max-line-length = 120
 

.github/workflows/scorecards.yml

@@ -1,6 +1,11 @@
 # This workflow uses actions that are not certified by GitHub. They are provided
 # by a third-party and are governed by separate terms of service, privacy
 # policy, and support documentation.
+#
+# author: Alan O'Cais (@ocaisa)
+#
+# license: GPLv2
+#
 
 name: Scorecards supply-chain security
 on:

.github/workflows/tests.yaml

@@ -1,3 +1,16 @@
+# This file is part of the EESSI build-and-deploy bot,
+# see https://github.com/EESSI/eessi-bot-software-layer
+#
+# The bot helps with requests to add software installations to the
+# EESSI software layer, see https://github.com/EESSI/software-layer
+#
+# author: Kenneth Hoste (@boegel)
+# author: Alan O'Cais (@ocaisa)
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
 name: Run tests
 on: [push, pull_request]
 # Declare default permissions as read only.

.gitignore

@@ -1,3 +1,15 @@
+# This file is part of the EESSI build-and-deploy bot,
+# see https://github.com/EESSI/eessi-bot-software-layer
+#
+# The bot helps with requests to add software installations to the
+# EESSI software layer, see https://github.com/EESSI/software-layer
+#
+# author: Bob Droege (@bedroge)
+# author: Hafsa Naeem (@hafsa-naeem)
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
 __pycache__/
 *.py[cod]
 *.log

.hound.yml

@@ -1,3 +1,13 @@
+# This file is part of the EESSI build-and-deploy bot,
+# see https://github.com/EESSI/eessi-bot-software-layer
+#
+# The bot helps with requests to add software installations to the
+# EESSI software layer, see https://github.com/EESSI/software-layer
+#
+# author: Kenneth Hoste (@boegel)
+#
+# license: GPLv2
+#
 flake8:
   enabled: true
   config_file: .flake8

README.md

@@ -175,7 +175,9 @@ You can exit the virtual environment simply by running `deactivate`.
 
 ### <a name="step4.1"></a>Step 4.1: Installing tools to access S3 bucket
 
-The [`scripts/eessi-upload-to-staging`](https://github.com/EESSI/eessi-bot-software-layer/blob/main/scripts/eessi-upload-to-staging) script uploads a tarball and an associated metadata file to an S3 bucket.
+The
+[`scripts/eessi-upload-to-staging`](https://github.com/EESSI/eessi-bot-software-layer/blob/main/scripts/eessi-upload-to-staging)
+script uploads an artefact and an associated metadata file to an S3 bucket.
 
 It needs two tools for this:
 * the `aws` command to actually upload the files;
@@ -444,14 +446,17 @@ information about the result of the command that was run (can be empty).
 
 The `[deploycfg]` section defines settings for uploading built artefacts (tarballs).
 ```
-tarball_upload_script = PATH_TO_EESSI_BOT/scripts/eessi-upload-to-staging
+artefact_upload_script = PATH_TO_EESSI_BOT/scripts/eessi-upload-to-staging
 ```
-`tarball_upload_script` provides the location for the script used for uploading built software packages to an S3 bucket.
+`artefact_upload_script` provides the location for the script used for uploading built software packages to an S3 bucket.
 
 ```
 endpoint_url = URL_TO_S3_SERVER
 ```
-`endpoint_url` provides an endpoint (URL) to a server hosting an S3 bucket. The server could be hosted by a commercial cloud provider like AWS or Azure, or running in a private environment, for example, using Minio. The bot uploads tarballs to the bucket which will be periodically scanned by the ingestion procedure at the Stratum 0 server.
+`endpoint_url` provides an endpoint (URL) to a server hosting an S3 bucket. The
+server could be hosted by a commercial cloud provider like AWS or Azure, or
+running in a private environment, for example, using Minio. The bot uploads
+artefacts to the bucket which will be periodically scanned by the ingestion procedure at the Stratum 0 server.
 
 
 ```ini
@@ -466,7 +471,7 @@ bucket_name = {
 }
 ```
 
-`bucket_name` is the name of the bucket used for uploading of tarballs.
+`bucket_name` is the name of the bucket used for uploading of artefacts.
 The bucket must be available on the default server (`https://${bucket_name}.s3.amazonaws.com`), or the one provided via `endpoint_url`.
 
 `bucket_name` can be specified as a string value to use the same bucket for all target repos, or it can be mapping from target repo id to bucket name.
@@ -481,7 +486,7 @@ The `upload_policy` defines what policy is used for uploading built artefacts to
 |`upload_policy` value|Policy|
 |:--------|:--------------------------------|
 |`all`|Upload all artefacts (mulitple uploads of the same artefact possible).|
-|`latest`|For each build target (prefix in tarball name `eessi-VERSION-{software,init,compat}-OS-ARCH)` only upload the latest built artefact.|
+|`latest`|For each build target (prefix in artefact name `eessi-VERSION-{software,init,compat}-OS-ARCH)` only upload the latest built artefact.|
 |`once`|Only once upload any built artefact for the build target.|
 |`none`|Do not upload any built artefacts.|
 
@@ -496,30 +501,30 @@ deployment), or a space delimited list of GitHub accounts.
 no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but this person does not have permission to trigger deployments
 ```
 This defines a message that is added to the status table in a PR comment
-corresponding to a job whose tarball should have been uploaded (e.g., after
+corresponding to a job whose artefact should have been uploaded (e.g., after
 setting the `bot:deploy` label).
 
 
 ```
 metadata_prefix = LOCATION_WHERE_METADATA_FILE_GETS_DEPOSITED
-tarball_prefix = LOCATION_WHERE_TARBALL_GETS_DEPOSITED
+artefact_prefix = LOCATION_WHERE_TARBALL_GETS_DEPOSITED
 ```
 
 These two settings are used to define where (which directory) in the S3 bucket
-(see `bucket_name` above) the metadata file and the tarball will be stored. The
+(see `bucket_name` above) the metadata file and the artefact will be stored. The
 value `LOCATION...` can be a string value to always use the same 'prefix'
 regardless of the target CVMFS repository, or can be a mapping of a target
 repository id (see also `repo_target_map` below) to a prefix.
 
 The prefix itself can use some (environment) variables that are set within
-the upload script (see `tarball_upload_script` above). Currently those are:
+the upload script (see `artefact_upload_script` above). Currently those are:
  * `'${github_repository}'` (which would be expanded to the full name of the GitHub
    repository, e.g., `EESSI/software-layer`),
  * `'${legacy_aws_path}'` (which expands to the legacy/old prefix being used for
-   storing tarballs/metadata files, the old prefix is
+   storing artefacts/metadata files, the old prefix is
    `EESSI_VERSION/TARBALL_TYPE/OS_TYPE/CPU_ARCHITECTURE/TIMESTAMP/`), _and_
  * `'${pull_request_number}'` (which would be expanded to the number of the pull
-   request from which the tarball originates).
+   request from which the artefact originates).
 Note, it's important to single-quote (`'`) the variables as shown above, because
 they may likely not be defined when the bot calls the upload script.
 
@@ -529,7 +534,7 @@ The list of supported variables can be shown by running
 **Examples:**
 ```
 metadata_prefix = {"eessi.io-2023.06": "new/${github_repository}/${pull_request_number}"}
-tarball_prefix = {
+artefact_prefix = {
     "eessi-pilot-2023.06": "",
     "eessi.io-2023.06": "new/${github_repository}/${pull_request_number}"
     }
@@ -656,46 +661,6 @@ running_job = job `{job_id}` is running
 #### `[finished_job_comments]` section
 
 The `[finished_job_comments]` section sets templates for messages about finished jobs.
-```
-success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir
-```
-`success` specifies the message for a successful job that produced a tarball.
-
-```
-failure = :cry: FAILURE
-```
-`failure` specifies the message for a failed job.
-
-```
-no_slurm_out = No slurm output `{slurm_out}` in job dir
-```
-`no_slurm_out` specifies the message for missing Slurm output file.
-
-```
-slurm_out = Found slurm output `{slurm_out}` in job dir
-```
-`slurm_out` specifies the message for found Slurm output file.
-
-```
-missing_modules = Slurm output lacks message "No missing modules!".
-```
-`missing_modules` is used to signal the lack of a message that all modules were built.
-
-```
-no_tarball_message = Slurm output lacks message about created tarball.
-```
-`no_tarball_message` is used to signal the lack of a message about a created tarball.
-
-```
-no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir.
-```
-`no_matching_tarball` is used to signal a missing tarball.
-
-```
-multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected.
-```
-`multiple_tarballs` is used to report that multiple tarballs have been found.
-
 ```
 job_result_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for details)_</summary><ul><li>Job results file `{filename}` does not exist in job directory, or parsing it failed.</li><li>No artefacts were found/reported.</li></ul></details>
 ```

RELEASE_NOTES

@@ -1,6 +1,21 @@
 This file contains a description of the major changes to the EESSI
 build-and-deploy bot. For more detailed information, please see the git log.
 
+v0.5.0 (16 May 2024)
+--------------------------
+
+This is a minor release of the EESSI build-and-deploy bot.
+
+Improvements:
+* list authors in source code files where applicable (#261)
+* make display of build targets more concise (#262)
+* use _bot_jobJOBID.result files created by bot/check-build.sh when determining
+  which build artefacts should be deployed (#263)
+  * this makes the bot fully agnostic to what it builds
+* reorganise declaration, initialisation and use of string constants _and_ verify
+* that required configuration settings are defined in 'app.cfg' (#266)
+
+
 v0.4.0 (28 February 2024)
 --------------------------
 

app.cfg.example

@@ -1,3 +1,19 @@
+# This file is part of the EESSI build-and-deploy bot,
+# see https://github.com/EESSI/eessi-bot-software-layer
+#
+# The bot helps with requests to add software installations to the
+# EESSI software layer, see https://github.com/EESSI/software-layer
+#
+# author: Kenneth Hoste (@boegel)
+# author: Bob Droege (@bedroge)
+# author: Hafsa Naeem (@hafsa-naeem)
+# author: Jonas Qvigstad (@jonas-lq)
+# author: Pedro Santos Neves (@Neves-P)
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
 # Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5
 
 [github]
@@ -111,7 +127,7 @@ no_build_permission_comment = Label `bot:build` has been set by user `{build_lab
 
 [deploycfg]
 # script for uploading built software packages
-tarball_upload_script = PATH_TO_EESSI_BOT/scripts/eessi-upload-to-staging
+artefact_upload_script = PATH_TO_EESSI_BOT/scripts/eessi-upload-to-staging
 
 # URL to S3/minio bucket
 #   if attribute is set, bucket_base will be constructed as follows
@@ -144,11 +160,11 @@ upload_policy = once
 # value can be a space delimited list of GH accounts
 deploy_permission =
 
-# template for comment when user who set a label has no permission to trigger deploying tarballs
+# template for comment when user who set a label has no permission to trigger deploying artefacts
 no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but this person does not have permission to trigger deployments
 
 # settings for where (directory) in the S3 bucket to store the metadata file and
-# the tarball
+# the artefact
 # - Can be a string value to always use the same 'prefix' regardless of the target
 #   CVMFS repository, or can be a mapping of a target repository id (see also
 #   repo_target_map) to a prefix.
@@ -157,17 +173,17 @@ no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_
 #   * 'github_repository' (which would be expanded to the full name of the GitHub
 #     repository, e.g., 'EESSI/software-layer'),
 #   * 'legacy_aws_path' (which expands to the legacy/old prefix being used for
-#     storing tarballs/metadata files) and
+#     storing artefacts/metadata files) and
 #   * 'pull_request_number' (which would be expanded to the number of the pull
-#     request from which the tarball originates).
+#     request from which the artefact originates).
 # - The list of supported variables can be shown by running
 #   `scripts/eessi-upload-to-staging --list-variables`.
 # - Examples:
 #   metadata_prefix = {"eessi.io-2023.06": "new/${github_repository}/${pull_request_number}"}
-#   tarball_prefix = {"eessi-pilot-2023.06": "", "eessi.io-2023.06": "new/${github_repository}/${pull_request_number}"}
+#   artefact_prefix = {"eessi-pilot-2023.06": "", "eessi.io-2023.06": "new/${github_repository}/${pull_request_number}"}
 # If left empty, the old/legacy prefix is being used.
 metadata_prefix =
-tarball_prefix =
+artefact_prefix =
 
 
 [architecturetargets]
@@ -231,14 +247,6 @@ running_job = job `{job_id}` is running
 
 
 [finished_job_comments]
-success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir
-failure = :cry: FAILURE
-no_slurm_out = No slurm output `{slurm_out}` in job dir
-slurm_out = Found slurm output `{slurm_out}` in job dir
-missing_modules = Slurm output lacks message "No missing modules!".
-no_tarball_message = Slurm output lacks message about created tarball.
-no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir.
-multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected.
 job_result_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for detailed information)_</summary><ul><li>Job results file `{filename}` does not exist in job directory, or parsing it failed.</li><li>No artefacts were found/reported.</li></ul></details>
 job_test_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for detailed information)_</summary><ul><li>Job test file `{filename}` does not exist in job directory, or parsing it failed.</li></ul></details>
 

connections/__init__.py

@@ -0,0 +1,10 @@
+# This file is part of the EESSI build-and-deploy bot,
+# see https://github.com/EESSI/eessi-bot-software-layer
+#
+# The bot helps with requests to add software installations to the
+# EESSI software layer, see https://github.com/EESSI/software-layer
+#
+# author: Bob Droege (@bedroge)
+#
+# license: GPLv2
+#

connections/github.py

@@ -5,6 +5,9 @@
 # EESSI software layer, see https://github.com/EESSI/software-layer
 #
 # author: Bob Droege (@bedroge)
+# author: Hafsa Naeem (@hafsa-naeem)
+# author: Jacob Ziemke (@jacobz137)
+# author: Lara Ramona Peeters (@laraPPr)
 # author: Thomas Roeblitz (@trz42)
 #
 # license: GPLv2
@@ -43,10 +46,10 @@ def get_token():
 
     global _token
     cfg = config.read_config()
-    github_cfg = cfg['github']
-    app_id = github_cfg.get('app_id')
-    installation_id = github_cfg.get('installation_id')
-    private_key_path = github_cfg.get('private_key')
+    github_cfg = cfg[config.SECTION_GITHUB]
+    app_id = github_cfg.get(config.GITHUB_SETTING_APP_ID)
+    installation_id = github_cfg.get(config.GITHUB_SETTING_INSTALLATION_ID)
+    private_key_path = github_cfg.get(config.GITHUB_SETTING_PRIVATE_KEY)
     private_key = ''
 
     with open(private_key_path, 'r') as private_key_file: