Merge remote-tracking branch 'origin/integrate-opensearch'

linagora · Jun 4, 2024 · d6d841f · d6d841f
2 parents e203f20 + 6f8f844
commit d6d841f
Show file tree

Hide file tree

Showing 74 changed files with 8,722 additions and 292 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -25,7 +25,7 @@ jobs:
     needs: build
     steps:
     - name: Add hosts for integration tests
-      run: sudo echo "127.0.0.1 localhost auth.example.com matrix.example.com matrix1.example.com matrix2.example.com matrix3.example.com federation.example.com" | sudo tee -a /etc/hosts
+      run: sudo echo "127.0.0.1 localhost auth.example.com matrix.example.com matrix1.example.com matrix2.example.com matrix3.example.com federation.example.com opensearch.example.com" | sudo tee -a /etc/hosts
     - uses: actions/checkout@v3
     - name: Set up Node LTS
       uses: actions/setup-node@v3

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -81,7 +81,7 @@
     "supertest": "^6.3.3",
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-dist": "^4.18.3",
-    "testcontainers": "^9.8.0",
+    "testcontainers": "^10.6.0",
     "toad-cache": "^3.3.0",
     "ts-jest": "^29.1.0",
     "typescript": "^4.9.5"

diff --git a/packages/federation-server/src/__testData__/docker-compose.yml b/packages/federation-server/src/__testData__/docker-compose.yml
@@ -3,6 +3,7 @@ version: '3.8'
 services:
   postgresql:
     image: postgres:13-bullseye
+    container_name: postgresql
     volumes:
       - ./synapse-data/matrix.example.com.log.config:/data/matrix.example.com.log.config
       - ./db/init-synapse-db.sh:/docker-entrypoint-initdb.d/init-synapse-db.sh
@@ -23,6 +24,7 @@ services:
 
   synapse-federation: &synapse_template
     image: matrixdotorg/synapse:v1.89.0
+    container_name: synapse-federation
     volumes:
       - ./synapse-data:/data
       - ./nginx/ssl/ca.pem:/etc/ssl/certs/ca.pem
@@ -44,6 +46,7 @@ services:
 
   synapse-1:
     <<: *synapse_template
+    container_name: synapse-1
     environment: 
       - UID=${MYUID}
       - VIRTUAL_PORT=8008
@@ -52,6 +55,7 @@ services:
 
   synapse-2:
     <<: *synapse_template
+    container_name: synapse-2
     environment: 
       - UID=${MYUID}
       - VIRTUAL_PORT=8008
@@ -60,6 +64,7 @@ services:
 
   synapse-3:
     <<: *synapse_template
+    container_name: synapse-3
     environment: 
       - UID=${MYUID}
       - VIRTUAL_PORT=8008
@@ -93,6 +98,7 @@ services:
 
   federation-server:
     image: federation-server
+    container_name: federation-server
     build:
       context: ../../../..
       dockerfile: ./packages/federation-server/Dockerfile
@@ -117,6 +123,7 @@ services:
 
   identity-server-1: &identity-server-template
     image: identity-server
+    container_name: identity-server-1
     build:
       context: ../../../..
       dockerfile: ./packages/federation-server/src/__testData__/identity-server/Dockerfile
@@ -139,6 +146,7 @@ services:
 
   identity-server-2:
     <<: *identity-server-template
+    container_name: identity-server-2
     depends_on:
       annuaire:
         condition: service_started
@@ -156,6 +164,7 @@ services:
 
   identity-server-3:
     <<: *identity-server-template
+    container_name: identity-server-3
     depends_on:
       annuaire:
         condition: service_started
@@ -173,6 +182,7 @@ services:
 
   nginx-proxy:
     image: nginxproxy/nginx-proxy
+    container_name: nginx-proxy
     ports:
       - 443:443
     volumes:

diff --git a/packages/federation-server/src/index.test.ts b/packages/federation-server/src/index.test.ts
@@ -1,5 +1,4 @@
 import { Hash } from '@twake/crypto'
-import dockerComposeV1, { v2 as dockerComposeV2 } from 'docker-compose'
 import express from 'express'
 import fs from 'fs'
 import type * as http from 'http'
@@ -57,7 +56,6 @@ describe('Federation server', () => {
   })
 
   describe('Integration tests', () => {
-    let containerNameSuffix: string
     let startedCompose: StartedDockerComposeEnvironment
     let identity1IPAddress: string
     let identity2IPAddress: string
@@ -172,43 +170,18 @@ describe('Federation server', () => {
       syswideCas.addCAs(
         path.join(pathToTestDataFolder, 'nginx', 'ssl', 'ca.pem')
       )
-      Promise.allSettled([dockerComposeV1.version(), dockerComposeV2.version()])
-        // eslint-disable-next-line @typescript-eslint/promise-function-async
-        .then((results) => {
-          const promiseSucceededIndex = results.findIndex(
-            (res) => res.status === 'fulfilled'
-          )
-          if (promiseSucceededIndex === -1) {
-            throw new Error('Docker compose is not installed')
-          }
-          containerNameSuffix = promiseSucceededIndex === 0 ? '_' : '-'
-          return new DockerComposeEnvironment(
-            path.join(pathToTestDataFolder),
-            'docker-compose.yml'
-          )
-            .withEnvironment({ MYUID: os.userInfo().uid.toString() })
-            .withWaitStrategy(
-              `postgresql${containerNameSuffix}1`,
-              Wait.forHealthCheck()
-            )
-            .withWaitStrategy(
-              `synapse-federation${containerNameSuffix}1`,
-              Wait.forHealthCheck()
-            )
-            .withWaitStrategy(
-              `synapse-1${containerNameSuffix}1`,
-              Wait.forHealthCheck()
-            )
-            .withWaitStrategy(
-              `synapse-2${containerNameSuffix}1`,
-              Wait.forHealthCheck()
-            )
-            .withWaitStrategy(
-              `synapse-3${containerNameSuffix}1`,
-              Wait.forHealthCheck()
-            )
-            .up()
-        })
+
+      new DockerComposeEnvironment(
+        path.join(pathToTestDataFolder),
+        'docker-compose.yml'
+      )
+        .withEnvironment({ MYUID: os.userInfo().uid.toString() })
+        .withWaitStrategy('postgresql', Wait.forHealthCheck())
+        .withWaitStrategy('synapse-federation', Wait.forHealthCheck())
+        .withWaitStrategy('synapse-1', Wait.forHealthCheck())
+        .withWaitStrategy('synapse-2', Wait.forHealthCheck())
+        .withWaitStrategy('synapse-3', Wait.forHealthCheck())
+        .up()
         // eslint-disable-next-line @typescript-eslint/promise-function-async
         .then((upResult) => {
           startedCompose = upResult
@@ -275,10 +248,10 @@ describe('Federation server', () => {
 
       beforeAll((done) => {
         identity1IPAddress = startedCompose
-          .getContainer(`identity-server-1${containerNameSuffix}1`)
+          .getContainer(`identity-server-1`)
           .getIpAddress('test')
         identity2IPAddress = startedCompose
-          .getContainer(`identity-server-2${containerNameSuffix}1`)
+          .getContainer(`identity-server-2`)
           .getIpAddress('test')
 
         confOriginalContent = fs.readFileSync(
@@ -295,9 +268,8 @@ describe('Federation server', () => {
           'utf-8'
         )
 
-        federationServerContainer = startedCompose.getContainer(
-          `federation-server${containerNameSuffix}1`
-        )
+        federationServerContainer =
+          startedCompose.getContainer('federation-server')
 
         federationServerContainer
           .restart()
@@ -746,25 +718,19 @@ describe('Federation server', () => {
                 'Certificates files for federation server has not been created'
               )
             return Promise.all([
-              startedCompose
-                .getContainer(`identity-server-1${containerNameSuffix}1`)
-                .restart(),
-              startedCompose
-                .getContainer(`identity-server-2${containerNameSuffix}1`)
-                .restart(),
-              startedCompose
-                .getContainer(`identity-server-3${containerNameSuffix}1`)
-                .restart()
+              startedCompose.getContainer('identity-server-1').restart(),
+              startedCompose.getContainer('identity-server-2').restart(),
+              startedCompose.getContainer('identity-server-3').restart()
             ])
           })
           // eslint-disable-next-line @typescript-eslint/promise-function-async
           .then(() => {
             identity1IPAddress = startedCompose
-              .getContainer(`identity-server-1${containerNameSuffix}1`)
+              .getContainer(`identity-server-1`)
               .getIpAddress('test')
 
             identity2IPAddress = startedCompose
-              .getContainer(`identity-server-2${containerNameSuffix}1`)
+              .getContainer(`identity-server-2`)
               .getIpAddress('test')
 
             const testConfig: Config = {
@@ -776,16 +742,16 @@ describe('Federation server', () => {
               database_user: 'twake',
               database_password: 'twake!1',
               database_host: `${startedCompose
-                .getContainer(`postgresql${containerNameSuffix}1`)
+                .getContainer(`postgresql`)
                 .getHost()}:5432`,
               database_name: 'federation',
               ldap_base: 'dc=example,dc=com',
               ldap_uri: `ldap://${startedCompose
-                .getContainer(`postgresql${containerNameSuffix}1`)
+                .getContainer(`postgresql`)
                 .getHost()}:389`,
               matrix_database_engine: 'pg',
               matrix_database_host: `${startedCompose
-                .getContainer(`postgresql${containerNameSuffix}1`)
+                .getContainer(`postgresql`)
                 .getHost()}:5432`,
               matrix_database_name: 'synapsefederation',
               matrix_database_user: 'synapse',

diff --git a/packages/matrix-application-server/src/index.ts b/packages/matrix-application-server/src/index.ts
@@ -26,7 +26,10 @@ export { EHttpMethod } from './routes'
 export {
   AppServerAPIError,
   validationErrorHandler,
-  type expressAppHandler
+  type expressAppHandler,
+  errorMiddleware,
+  allowCors,
+  methodNotAllowed
 } from './utils'
 
 export declare interface AppService {

diff --git a/packages/matrix-application-server/src/interfaces.ts b/packages/matrix-application-server/src/interfaces.ts
@@ -3,14 +3,15 @@ export interface TransactionRequestBody {
 }
 
 export interface ClientEvent {
-  content: Record<string, string | number | boolean>
+  content: Record<string, string | number | boolean | Record<string, string>>
   event_id: string
   origin_server_ts: number
   room_id: string
   sender: string
   state_key?: string
   type: string
   unsigned?: UnsignedData
+  redacts?: string
 }
 
 interface UnsignedData {

diff --git a/packages/matrix-identity-server/src/matrixDb/index.ts b/packages/matrix-identity-server/src/matrixDb/index.ts
@@ -9,6 +9,8 @@ type Collections =
   | 'room_stats_state'
   | 'local_media_repository'
   | 'room_aliases'
+  | 'room_stats_state'
+  | 'event_json'
 
 type Get = (
   table: Collections,

diff --git a/packages/tom-server/jest.config.js b/packages/tom-server/jest.config.js
@@ -2,7 +2,7 @@ import jestConfigBase from '../../jest-base.config.js'
 
 export default {
   ...jestConfigBase,
-  testTimeout: 120000,
+  testTimeout: 420000,
   setupFilesAfterEnv: ['<rootDir>/jest.setup.ts'],
   moduleNameMapper: {
     ...jestConfigBase.moduleNameMapper,

diff --git a/packages/tom-server/package.json b/packages/tom-server/package.json
@@ -40,6 +40,7 @@
     "test": "jest"
   },
   "dependencies": {
+    "@opensearch-project/opensearch": "^2.5.0",
     "@twake/matrix-application-server": "*",
     "@twake/matrix-identity-server": "*",
     "lodash": "^4.17.21",

diff --git a/packages/tom-server/src/application-server/__testData__/config.json b/packages/tom-server/src/application-server/__testData__/config.json
@@ -15,13 +15,14 @@
   "template_dir": "./templates",
   "ldap_base": "dc=example,dc=com",
   "ldap_uri": "ldap://localhost:21389/",
-  "matrix_server": "matrix.example.com",
+  "matrix_server": "matrix.example.com:444",
   "registration_file_path": "./src/application-server/__testData__/synapse-data/registration.yaml",
   "matrix_database_engine": "sqlite",
   "matrix_database_host": "./src/application-server/__testData__/synapse-data/homeserver.db",
   "namespaces": {
     "aliases": [{ "exclusive": false, "regex": "#_twake_.*" }],
     "users": [{ "exclusive": false, "regex": "@.*" }]
   },
+  "opensearch_is_activated": false,
   "push_ephemeral": true
 }
diff --git a/packages/tom-server/src/application-server/__testData__/docker-compose.yml b/packages/tom-server/src/application-server/__testData__/docker-compose.yml
@@ -3,6 +3,7 @@ version: '3.8'
 services:
   synapse:
     image: matrixdotorg/synapse:v1.89.0
+    container_name: synapse-tom-1
     volumes:
       - ./synapse-data:/data
       - ./nginx/ssl/auth.example.com.crt:/etc/ssl/certs/ca-certificates.crt
@@ -37,7 +38,9 @@ services:
   nginx-proxy:
     image: nginxproxy/nginx-proxy
     ports:
-      - 443:443
+      - 444:444
+    environment:
+      - HTTPS_PORT=444
     volumes:
       - /var/run/docker.sock:/tmp/docker.sock:ro
       - ./nginx/ssl:/etc/nginx/certs
diff --git a/packages/tom-server/src/application-server/__testData__/ldap/ldif/base_ldap_users.ldif b/packages/tom-server/src/application-server/__testData__/ldap/ldif/base_ldap_users.ldif
@@ -135,7 +135,7 @@ userPassword: jfett
 dn: uid=lskywalker,ou=users,dc=example,dc=com
 objectClass: inetOrgPerson
 uid: lskywalker
-cn: Luc Skywalker
+cn: Luke Skywalker
 sn: Lskywalker
 mail: [email protected]
 userPassword: lskywalker

diff --git a/packages/tom-server/src/application-server/__testData__/llng/lmConf-1.json b/packages/tom-server/src/application-server/__testData__/llng/lmConf-1.json
@@ -250,7 +250,7 @@
        "oidcRPMetaDataOptionsLogoutSessionRequired": 1,
        "oidcRPMetaDataOptionsLogoutType": "back",
        "oidcRPMetaDataOptionsPublic": 0,
-       "oidcRPMetaDataOptionsRedirectUris": "https://matrix.example.com/_synapse/client/oidc/callback",
+       "oidcRPMetaDataOptionsRedirectUris": "https://matrix.example.com:444/_synapse/client/oidc/callback",
        "oidcRPMetaDataOptionsRefreshToken": 0,
        "oidcRPMetaDataOptionsRequirePKCE": 0
      }

diff --git a/packages/tom-server/src/application-server/__testData__/synapse-data/homeserver.yaml b/packages/tom-server/src/application-server/__testData__/synapse-data/homeserver.yaml
@@ -10,7 +10,7 @@
 # each option, go to docs/usage/configuration/config_documentation.md or
 # https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html
 server_name: "example.com"
-public_baseurl: "https://matrix.example.com/"
+public_baseurl: "https://matrix.example.com:444/"
 pid_file: /data/homeserver.pid
 listeners:
   - port: 8008