Merge branch 'master' into tls_cert_refresh_period

.github/workflows/on_release.yml

@@ -86,13 +86,13 @@ jobs:
         run: sed 's/:latest/:test/g' docker/docker-compose-example.yml > docker/docker-compose-test.yml
 
       - name: Bring up stack
-        run: docker-compose -f docker/docker-compose-test.yml up -d
+        run: docker compose -f docker/docker-compose-test.yml up -d
 
       - name: Check if OPA is healthy
         run: ./scripts/wait-for.sh -t 60 http://localhost:8181/v1/data/users -- sleep 10 && curl -s "http://localhost:8181/v1/data/users" | jq '.result.bob.location.country == "US"'
 
       - name: Output container logs
-        run: docker-compose -f docker/docker-compose-test.yml logs
+        run: docker compose -f docker/docker-compose-test.yml logs
 
       # PUSH PHASE
       - name: Output local docker images

.github/workflows/pre-commit.yml

@@ -13,4 +13,13 @@ jobs:
     - uses: actions/setup-python@v4
       with:
         python-version: 3.x
-    - uses: pre-commit/[email protected]
+    - name: install pre-commit
+      run: python -m pip install 'pre-commit<4'
+    - name: show environment
+      run: python -m pip freeze --local
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+    - name: run pre-commit
+      run: pre-commit run --show-diff-on-failure --color=always --all-files

.github/workflows/tests.yml

@@ -25,6 +25,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -53,6 +54,7 @@ jobs:
 
   test-docker:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
       # BUILD PHASE
       - name: Checkout
@@ -115,7 +117,7 @@ jobs:
           OPAL_TESTS_POLICY_REPO_DEPLOY_KEY: ${{ secrets.OPAL_TESTS_POLICY_REPO_DEPLOY_KEY }}
         run: |
           # Prepare git for using tests policy repo
-          export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./opal-tests-policy-repo-key)
+          export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath .)/opal-tests-policy-repo-key
           echo "$OPAL_TESTS_POLICY_REPO_DEPLOY_KEY" > $OPAL_POLICY_REPO_SSH_KEY_PATH
           chmod 400 $OPAL_POLICY_REPO_SSH_KEY_PATH
 

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
@@ -18,8 +18,8 @@ repos:
     hooks:
       - id: codespell
         args: [--skip, "*.json"]
-  - repo: https://github.com/myint/docformatter
-    rev: v1.5.1
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.5
     hooks:
       - id: docformatter
         args: [--in-place]

README.md

@@ -36,25 +36,46 @@ Open Policy Administration Layer
 
 OPAL is an administration layer for Policy Engines such as <a target="_blank" href="https://www.openpolicyagent.org/">Open Policy Agent (OPA)</a>, and <a target="_blank" href="https://github.com/permitio/cedar-agent">AWS' Cedar Agent</a> detecting changes to both policy and policy data in realtime and pushing live updates to your agents. OPAL brings open-policy up to the speed needed by live applications.
 
-As your application state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), OPAL will make sure your services are always in sync with the authorization data and policy they need (and only those they need).
+As your app's data state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), OPAL will make sure your services are always in sync with the authorization data and policy they need (and only those they need).
 
-Check out our main site at <a target="_blank" href="https://opal.ac">OPAL.ac</a>, <a target="_blank" href="https://youtu.be/tG8jrdcc7Zo">this video</a> briefly explaining OPAL and how it works with OPA, and a deeper dive into it at [this OWASP DevSlop talk](https://www.youtube.com/watch?v=1_Iz0tRQCH4).
+Check out OPAL's main site at <a target="_blank" href="https://opal.ac">OPAL.ac</a>
 
-## Why use OPAL?
+## OPAL Use Cases
 
 OPAL is the easiest way to keep your solution's authorization layer up-to-date in realtime. It aggregates policy and data from across the field and integrates them seamlessly into the authorization layer, and is microservices and cloud-native.
 
-## OPA + OPAL == 💜
+Here are some of the main use cases for using OPAL:
+* **End-to-End [Fine-Grained Authorization](https://www.permit.io/blog/what-is-fine-grained-authorization-fga) service** that can be used with any policy language or data store
+* [Google-Zanzibar](https://www.permit.io/blog/what-is-google-zanzibar) support for Policy as Code engines such as OPA and AWS Cedar
+* Streamline permissions in microservice architectures using [centralized policy configuration with decentralized data](https://www.permit.io/blog/best-practices-for-implementing-hybrid-cloud-security) sources and policy engines
+* Manage and automate the deployment of multiple Open Policy Agent engines in a Cloud-Native environment
+
+<img src="https://github.com/permitio/opal/assets/4082578/99d3dd95-a7ff-45c2-805e-3d533f8b1e8c" alt="simplified" border="0">
+
+OPAL  uses a client-server stateless architecture. OPAL-Servers publish policy and data updates over a lightweight (websocket) PubSub Channel, which OPAL-clients subscribe to via topics. Upon updates, each client fetches data directly (from the source) to load it into its managed Policy Engine instance.
+
+
+### OPA + OPAL == 💜
 
 While OPA (Open Policy Agent) decouples policy from code in a highly-performant and elegant way, the challenge of keeping policy agents up-to-date remains.
 This is especially true in applications, where each user interaction or API call may affect access-control decisions.
-OPAL runs in the background, supercharging policy-agents, keeping them in sync with events in realtime.
+OPAL runs in the background, supercharging policy agents and keeping them in sync with events in real time.
 
-## AWS Cedar + OPAL == 💪
+### AWS Cedar + OPAL == 💪
 
 Cedar is a very powerful policy language, which powers AWS' AVP (Amazon Verified Permissions) - but what if you want to enjoy the power of Cedar on another cloud, locally, or on premise?
 This is where [Cedar-Agent](https://github.com/permitio/cedar-agent) and OPAL come in.
 
+This [video](https://youtu.be/tG8jrdcc7Zo) briefly explains OPAL and how it works with OPA, and a deeper dive into it at [this OWASP DevSlop talk](https://www.youtube.com/watch?v=1_Iz0tRQCH4).
+
+## Who's Using OPAL?
+OPAL is being used as the core engine of Permit.io Authorization Service and serves in production:
+* \> 10,000 policy engines deployment
+* \> 100,000 policy changes and data synchronizations every day
+* \> 10,000,000 authorization checks every day
+
+Besides Permit, OPAL is being used in Production in **Tesla**, **Walmart**, **The NBA**, **Intel**, **Cisco**, **Live-Oak Bank**, and thousands of other development teams and companies of all sizes.
+
 ## Documentation
 
 - 📃 &nbsp; [Full documentation is available here](https://docs.opal.ac)
@@ -79,7 +100,7 @@ This is where [Cedar-Agent](https://github.com/permitio/cedar-agent) and OPAL co
 
 ```
 curl -L https://raw.githubusercontent.com/permitio/opal/master/docker/docker-compose-example.yml \
-> docker-compose.yml && docker-compose up
+> docker-compose.yml && docker compose up
 ```
 
 <p>
@@ -104,22 +125,18 @@ curl -L https://raw.githubusercontent.com/permitio/opal/master/docker/docker-com
 
 - 🎨 &nbsp; [Key concepts and design](https://docs.opal.ac/overview/design)
 - 🏗️ &nbsp; [Architecture](https://docs.opal.ac/overview/architecture)
-<be>
-<br>
-OPAL  uses a client-server stateless architecture. OPAL-Servers publish policy and data updates over a lightweight (websocket) PubSub Channel, which OPAL-clients subscribe to via topics. Upon updates each client fetches data directly (from source) to load it in to its managed OPA instance.
-  <br>
-<img src="https://github.com/permitio/opal/assets/4082578/99d3dd95-a7ff-45c2-805e-3d533f8b1e8c" alt="simplified" border="0">
 <br>
-📖 &nbsp; For further reading check out our [Blog](https://bit.ly/opal_blog).
+
+📖 For further reading, check out our [Blog](https://io.permit.io/opal-readme-blog)
 
 ## Community
 
-Come talk to us about OPAL, or authorization in general - we would love to hear from you ❤️
+ We would love to chat with you about OPAL. [Join our Slack community](https://io.permit.io/opal-readme-slack) to chat about authorization, open-source, realtime communication, tech, or anything else!
 
-You can raise questions and ask for features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions), report issues in [**Github issues**](https://github.com/permitio/opal/issues), follow us on Twitter to get the latest OPAL updates, and join our Slack community to chat about authorization, open-source, realtime communication, tech, or anything else!
+You can raise questions and ask for features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions), report issues in [**Github issues**](https://github.com/permitio/opal/issues)
 </br>
 </br>
-If you are using our project, please consider giving us a ⭐️
+If you like our project, please consider giving us a ⭐️
 </br>
 
 [![Button][join-slack-link]][badge-slack-link] </br> [![Button][follow-twitter-link]][badge-twitter-link]

app-tests/run.sh

@@ -116,8 +116,10 @@ function test_data_publish {
 function test_statistics {
     echo "- Testing statistics feature"
     # Make sure 2 servers & 2 clients (repeat few times cause different workers might response)
-    for _ in {1..10}; do
-      curl -s 'http://localhost:7002/stats' --header "Authorization: Bearer $OPAL_DATA_SOURCE_TOKEN" | grep '"client_count":2,"server_count":2'
+    for port in {7002..7003}; do
+      for _ in {1..8}; do
+        curl -s "http://localhost:${port}/stats" --header "Authorization: Bearer $OPAL_DATA_SOURCE_TOKEN" | grep '"client_count":2,"server_count":2'
+      done
     done
 }
 
@@ -156,4 +158,20 @@ function main {
   # TODO: Test statistics feature again after broadcaster restart (should first fix statistics bug)
 }
 
-main
+# Retry test in case of failure to avoid flakiness
+MAX_RETRIES=5
+RETRY_COUNT=0
+
+while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  echo "Running test (attempt $((RETRY_COUNT+1)) of $MAX_RETRIES)..."
+  main && break
+  RETRY_COUNT=$((RETRY_COUNT + 1))
+  echo "Test failed, retrying..."
+done
+
+if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
+  echo "Tests failed after $MAX_RETRIES attempts."
+  exit 1
+fi
+
+echo "Tests passed successfully."

docker/Dockerfile

@@ -39,7 +39,7 @@ RUN chmod +x ./wait-for.sh
 RUN ln -s /opal/wait-for.sh /usr/wait-for.sh
 
 # netcat (nc) is used by the wait-for.sh script
-RUN apt-get update && apt-get install -y netcat-traditional jq && apt-get clean
+RUN apt-get update && apt-get install -y netcat-traditional jq wget && apt-get clean
 
 # copy startup script (create link at old path to maintain backward compatibility)
 COPY ./scripts/start.sh .
@@ -116,6 +116,7 @@ COPY --from=opa-extractor /opal/opa ./opa
 
 # enable inline OPA
 ENV OPAL_INLINE_OPA_ENABLED=true
+ENV OPAL_INLINE_OPA_EXEC_PATH=/opal/opa
 # expose opa port
 EXPOSE 8181
 USER opal
@@ -134,6 +135,7 @@ COPY --from=cedar-builder /tmp/cedar-agent/target/*/cedar-agent /bin/cedar-agent
 # enable inline Cedar agent
 ENV OPAL_POLICY_STORE_TYPE=CEDAR
 ENV OPAL_INLINE_CEDAR_ENABLED=true
+ENV OPAL_INLINE_CEDAR_EXEC_PATH=/bin/cedar-agent
 ENV OPAL_INLINE_CEDAR_CONFIG='{"addr": "0.0.0.0:8180"}'
 ENV OPAL_POLICY_STORE_URL=http://localhost:8180
 # expose cedar port

docker/docker-compose-api-policy-source-example.yml

@@ -1,3 +1,5 @@
+name: opal-api-policy-source-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

docker/docker-compose-example-cedar.yml

@@ -1,3 +1,5 @@
+name: opal-cedar-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

docker/docker-compose-example.yml

@@ -1,3 +1,5 @@
+name: opal-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

docker/docker-compose-git-webhook.yml

@@ -1,3 +1,5 @@
+name: opal-git-webhook-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

docker/docker-compose-scopes-example.yml

@@ -1,3 +1,5 @@
+name: opal-scopes-example
+
 services:
   redis:
     image: redis

docker/docker-compose-with-callbacks.yml

@@ -1,3 +1,5 @@
+name: opal-callbacks-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

docker/docker-compose-with-kafka-example.yml

@@ -1,3 +1,4 @@
+name: opal-kafka-example
 
 services:
   # Based on: https://developer.confluent.io/quickstart/kafka-docker/

docker/docker-compose-with-oauth-initial.yml

@@ -1,3 +1,5 @@
+name: opal-oauth-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

docker/docker-compose-with-rate-limiting.yml

@@ -1,3 +1,5 @@
+name: opal-rate-limiting-example
+
 # This docker compose example shows how to configure OPAL's rate limiting feature
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use

docker/docker-compose-with-security.yml

@@ -1,3 +1,5 @@
+name: opal-security-example
+
 # this docker compose file is relying on external environment variables!
 # run it by running the script: ./run-example-with-security.sh
 services:

docker/docker-compose-with-statistics.yml

@@ -1,3 +1,5 @@
+name: opal-statistics-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.