Merge pull request #1 from tsalemink/main

Add GitHub action to refresh the database.

.github/workflows/update-database.yaml

@@ -0,0 +1,53 @@
+name: update-database
+
+on:
+  # Triggers the workflow on issue close.
+  issues:
+    types: [closed]
+  # Allows you to run this workflow manually from the Actions tab.
+  workflow_dispatch:
+  # If we make any changes to the repository manually, this will ensure that the database stays up-to-date with the timestamp on the repo.
+  push:
+
+jobs:
+  update-database:
+    # If the workflow was triggered by closing an issue, only run if the issue contains the 'add-plugin' label.
+    if: ${{ github.event_name == 'workflow_dispatch' }} or ${{ github.event.label.name == 'add-plugin' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v4
+      - name: Retrieve database timestamp
+        id: time
+        run: echo "time=${{ github.event.repository.updated_at }}" >> $GITHUB_OUTPUT
+      - name: Retrieve URL submission
+        id: url
+        run: |
+          if ${{ github.event_name == 'issues' }}; then
+            echo "url=${{ github.event.issue.body }}" >> $GITHUB_OUTPUT
+          else
+            echo "url=''" >> $GITHUB_OUTPUT
+          fi
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install Python packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r generation/requirements.txt
+      - name: Run Python script
+        run: python generation/update_database.py ${{ steps.time.outputs.time }} ${{ steps.url.outputs.url }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Commit files
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add -A
+          git diff-index --quiet HEAD || (git commit -a -m "Update database." --allow-empty)
+      - name: Push changes
+        uses: ad-m/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: main

README.rst

@@ -0,0 +1,19 @@
+===================
+MAP Plugin Database
+===================
+
+.. _MAP Client: https://github.com/MusculoskeletalAtlasProject/mapclient
+.. _MAP Plugin Database: https://github.com/MusculoskeletalAtlasProject/map-plugin-database
+
+The `MAP Client`_ is a cross-platform, plugin-based framework for managing workflows. Since the Plugin lies at the heart of the MAP
+framework, the ability to easily find and share MAP plugins is an important aspect of the MAP-Client's usability.
+
+The `MAP Plugin Database`_ contains basic information about all of the known, published MAP-Client plugins. The primary use of this
+database is to allow users of the MAP-Client to search for and install MAP-Client plugins directly from within the MAP-Client itself,
+rather than having to install plugins manually. Specifically, the *Plugin Finder* tool can be found in the MAP-Client application under
+the *Tools* menu dropdown.
+
+MAP-Client users are able to submit their own MAP plugins to the MAP Plugin Database. Currently the recommended process for doing this is
+to submit an issue in the `MAP Plugin Database`_ GitHub repository. The issue should be tagged with the *add-plugin* label. The body of the
+issue submission should contain only the GitHub repository path of the plugin to be added; this path should be in the format:
+*{organization}/{repository}*, for example: *mapclient-plugins/pointsourcestep*.

generation/requirements.txt

@@ -0,0 +1 @@
+PyGithub==1.55

generation/update_database.py

@@ -0,0 +1,137 @@
+import os
+import sys
+import json
+import datetime
+
+from github import Github
+from github.GithubException import UnknownObjectException, RateLimitExceededException
+
+
+def read_step_info(step_file):
+    def read_value(identifier):
+        value = read_line(line, identifier)
+        if not value:
+            extended_line = line + next(lines).strip(' \t\r\n')
+            value = read_line(extended_line, identifier)
+
+        return value
+
+    name = category = icon_path = None
+    lines = iter(step_file.splitlines())
+    for line in lines:
+        line = line.strip()
+
+        if line.startswith("super"):
+            name = read_value("__init__")
+        elif line.startswith("self._category"):
+            category = read_value("=")
+            if icon_path:
+                break
+        elif line.startswith("self._icon"):
+            icon_path = read_value("QImage")
+            if category:
+                break
+
+    return name, category, icon_path
+
+
+def read_line(line, identifier):
+
+    value = None
+    for quote in ["'", '"']:
+        start = line.find(quote, line.find(identifier) + len(identifier))
+        if start == -1:
+            continue
+        end = line.find(quote, start + 1)
+        value = line[start:end].strip(' "\'\t\r\n')
+
+    return value
+
+
+def read_file(file):
+    with open(file, "r") as file:
+        return json.load(file)
+
+
+def write_file(file, data):
+    with open(file, "w") as file:
+        json.dump(data, file)
+
+
+def check_plugins_for_updates():
+    def check_plugin_info():
+        name = repo.name
+        updated_at = repo.updated_at.timestamp()
+        if (name not in plugin_database.keys()) or (database_timestamp < updated_at):
+            step_paths = [
+                f'mapclientplugins/{name}/step.py',
+                f'mapclientplugins/{name}step/step.py',
+                f'mapclientplugins/{name[name.find(".") + 1:]}/step.py',
+                f'mapclientplugins/{name[name.find(".") + 1:]}step/step.py'
+            ]
+            step_file = None
+            for step_path in step_paths:
+                try:
+                    step_file = repo.get_contents(step_path).decoded_content.decode()
+                except UnknownObjectException:
+                    continue
+                else:
+                    formatted_name, category, icon_path = read_step_info(step_file)
+                    icon_name = ""
+                    url = repo.url
+                    version = get_latest_version(step_path)
+                    plugin_database[name] = {"_name": formatted_name, "_category": category, "_icon": icon_name, "_url": url,
+                                             "_version": version}
+                    break
+            if not step_file:
+                print(f"GitHub repository \"{repo.full_name}\" in not a valid MAP-Client plugin.")
+
+    # TODO: Once all MAP plugins have versioned-releases, update this with a version retrieval directly from the repo releases.
+    def get_latest_version(step_path):
+        init_file_path = os.path.dirname(step_path) + "/__init__.py"
+        init_file = repo.get_contents(init_file_path).decoded_content.decode()
+        version = ""
+        lines = iter(init_file.splitlines())
+        for line in lines:
+            line = line.strip()
+            if line.startswith("__version__"):
+                version = read_line(line, "=")
+
+        return version
+
+    plugin_sources = read_file("plugin_sources.json")
+    plugin_database = read_file("plugin_database.json")
+    database_timestamp = datetime.datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).timestamp()
+    url_submission = sys.argv[2]
+
+    plugin_orgs = plugin_sources["plugin_organizations"]
+    plugin_repos = plugin_sources["plugin_repositories"]
+    if url_submission:
+        plugin_repos.append(url_submission)
+        plugin_sources["plugin_repositories"] = plugin_repos
+
+    g = Github(os.environ["GITHUB_TOKEN"])
+    i = 0
+    while i < 2:
+        try:
+            for organisation in plugin_orgs:
+                org = g.get_organization(organisation)
+                for repo in org.get_repos():
+                    check_plugin_info()
+
+            for repository in plugin_repos:
+                repo = g.get_repo(repository)
+                check_plugin_info()
+
+            break
+        except RateLimitExceededException:
+            i += 1
+            if i < 2:
+                g = authenticate_github_user()
+
+    write_file("plugin_database.json", plugin_database)
+    write_file("plugin_sources.json", plugin_sources)
+
+
+if __name__ == '__main__':
+    check_plugins_for_updates()