From 9cbf86d651244d65e28ca9f393c283a0d42c8968 Mon Sep 17 00:00:00 2001
From: Yair Siman Tov <63305203+yairsimantov20@users.noreply.github.com>
Date: Mon, 3 Jun 2024 15:05:14 +0300
Subject: [PATCH] linter matrix (#682)

# Description

What - Lint takes a lot of time
Why - Linting all integrations every time
How - Running only the changed scopes lint with matrix

## Type of change

- [X] Non-breaking change (fix of existing functionality that will not
change current behavior)


<img width="641" alt="image"
src="https://github.com/port-labs/ocean/assets/63305203/79714f76-057c-41e1-a0b3-271a5969033a">
---
 .github/workflows/lint.yml | 45 ++++++++++++++++++++++++++++++++++----
 Makefile                   | 20 +----------------
 2 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 5a0d2e6c7d..711622e0c5 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -5,9 +5,44 @@ on:
   workflow_dispatch:
 
 jobs:
+  detect-changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Get list of changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v44.5.2
+      - name: Set matrix
+        id: set-matrix
+        run: |
+          folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md"
+          changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq)
+          if [ -z "$changed_folders" ]; then
+            changed_folders=""
+          fi
+
+          folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md"
+          other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l)
+          if [ "$other_changes" -ne 0 ]; then
+            # Add the root directory to the matrix if there are changes outside the integrations folder
+            changed_folders=$(echo -e "$changed_folders\n.")
+          fi
+
+          matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
   lint:
-    name: Lint
+    name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }}
+    needs: detect-changes
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }}
     steps:
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
@@ -15,9 +50,11 @@ jobs:
           python-version: '3.11'
       - name: Checkout Repo
         uses: actions/checkout@v4
-      - name: Install Dependencies
+      - name: Install dependencies
+        working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}
         run: |
-          make install/all
+          make install
       - name: Lint
+        working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}
         run: |
-          make lint/all
+          make lint
diff --git a/Makefile b/Makefile
index db8b867dfe..0d80f2c0c1 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ define deactivate_virtualenv
     fi
 endef
 
-.SILENT: install install/all lint lint/integrations lint/all build run new test clean
+.SILENT: install install/all lint build run new test clean
 
 
 # Install dependencies
@@ -69,24 +69,6 @@ lint:
 	$(ACTIVATE) && \
 	$(call run_checks,.)
 
-lint/integrations:
-	$(ACTIVATE) && \
-	exit_code=0; \
-	for dir in $(wildcard $(CURDIR)/integrations/*); do \
-		count=$$(find $$dir -type f -name '*.py' -not -path "*/venv/*" | wc -l); \
-		if [ $$count -ne 0 ]; then \
-			echo "Linting $$dir"; \
-		  	cd $$dir; \
-			$(MAKE) lint || exit_code=$$?; \
-			cd ../..; \
-		fi; \
-    done; \
-    if [ $$exit_code -ne 0 ]; then \
-        exit 1; \
-    fi
-
-lint/all: lint lint/integrations
-
 # Development commands
 build: 
 	$(ACTIVATE) && poetry build