Add Sharepoint Online connector functional tests (#1258)

.buildkite/nightly_steps.yml

@@ -46,6 +46,11 @@ steps:
       - ".buildkite/run_nigthly.sh sharepoint_server extra_small"
     artifact_paths:
       - "perf8-report-*/**/*"
+  - label: "🔨 Sharepoint Online"
+    command:
+      - ".buildkite/run_nigthly.sh sharepoint_online medium"
+    artifact_paths:
+      - "perf8-report-*/**/*"
   - label: "🔨 Microsoft SQL"
     command:
       - ".buildkite/run_nigthly.sh mssql"

tests/sources/fixtures/sharepoint_online/.env

@@ -1 +1 @@
-GRAPH_API_URL="http://localhost:10337"
+OVERRIDE_URL="http://localhost:10337"

tests/sources/fixtures/sharepoint_online/config.yml

@@ -0,0 +1,34 @@
+elasticsearch:
+  host: http://localhost:9200
+  username: elastic
+  password: changeme
+  ssl: true
+  bulk:
+    queue_max_size: 1024
+    queue_max_mem_size: 25
+    display_every: 100
+    chunk_size: 1000
+    max_concurrency: 5
+    chunk_max_mem_size: 5
+    concurrent_downloads: 10
+  request_timeout: 120
+  max_wait_duration: 120
+  initial_backoff_duration: 1
+  backoff_multiplier: 2
+  log_level: info
+
+service:
+  idling: 5
+  heartbeat: 300
+  max_errors: 20
+  max_errors_span: 600
+  max_concurrent_syncs: 1
+  job_cleanup_interval: 300
+  log_level: INFO
+
+connector_id: 'sharepoint_online'
+service_type: 'sharepoint_online'
+
+sources:
+  sharepoint_online: connectors.sources.sharepoint_online:SharepointOnlineDataSource
+

tests/sources/fixtures/sharepoint_online/connector.json

@@ -75,14 +75,64 @@
         "value": "*",
         "order": 5,
         "ui_restrictions": []
+      },
+      "use_text_extraction_service": {
+        "depends_on": [],
+        "display": "toggle",
+        "tooltip": "Requires a separate deployment of the Elastic Text Extraction Service. Requires that pipeline settings disable text extraction.",
+        "default_value": null,
+        "label": "Use text extraction service",
+        "sensitive": false,
+        "type": "bool",
+        "required": true,
+        "options": [],
+        "validations": [],
+        "value": false,
+        "order": 6,
+        "ui_restrictions": []
+      },
+      "use_document_level_security": {
+        "depends_on": [],
+        "display": "toggle",
+        "tooltip": "Document level security ensures identities and permissions set in Sharepoint Online are maintained in Elasticsearch. This enables you to restrict and personalize read-access users and groups have to documents in this index. Access control syncs ensure this metadata is kept up to date in your Elasticsearch documents.",
+        "default_value": null,
+        "label": "Enable document level security",
+        "sensitive": false,
+        "type": "bool",
+        "required": true,
+        "options": [],
+        "validations": [],
+        "value": false,
+        "order": 7,
+        "ui_restrictions": []
+      },
+      "fetch_users_by_site": {
+        "depends_on": [
+          {
+            "field": "use_document_level_security",
+            "value": true
+          }
+        ],
+        "display": "toggle",
+        "tooltip": "When only syncing a small subset of sites, it may be more efficient to fetch only the users who have access to those sites. However, this becomes inefficent the more sites (and the more users) that are intended to be synced, as there can be a lot of redundancy in membership between sites. This should not be used if all sites are being synced.",
+        "default_value": null,
+        "label": "Fetch ACLs for users based on site membership",
+        "sensitive": false,
+        "type": "bool",
+        "required": true,
+        "options": [],
+        "validations": [],
+        "value": false,
+        "order": 8,
+        "ui_restrictions": []
       }
     },
     "custom_scheduling": {},
     "description": null,
     "error": "Cannot connect to host enterprisesearchasd.sharepoint.com:443 ssl:default [nodename nor servname provided, or not known]",
     "features": {
-      "filtering_advanced_config": true,
-      "filtering_rules": true,
+      "filtering_advanced_config": false,
+      "filtering_rules": false,
       "sync_rules": {
         "advanced": {
           "enabled": true
@@ -122,9 +172,7 @@
           "advanced_snippet": {
             "updated_at": "2023-06-02T15:50:23.157Z",
             "created_at": "2023-06-02T15:50:23.157Z",
-            "value": {
-              "maxDataAge": 15
-            }
+            "value": {}
           },
           "rules": [
             {
@@ -146,12 +194,12 @@
       }
     ],
     "index_name": "search-sharepoint_online",
-    "is_native": true,
+    "is_native": false,
     "language": null,
     "last_seen": "2023-06-02T16:05:43.247794+00:00",
     "last_sync_error": "Cannot connect to host enterprisesearchasd.sharepoint.com:443 ssl:default [nodename nor servname provided, or not known]",
     "last_sync_scheduled_at": null,
-    "last_sync_status": "error",
+    "last_sync_status": null,
     "last_synced": "2023-06-02T16:08:30.745527+00:00",
     "name": "sharepoint-online-adv",
     "pipeline": {
@@ -163,12 +211,12 @@
     "scheduling": {
       "full": {
         "enabled": true,
-        "interval": "0 0 0 * * ?"
+        "interval": "0 * * * * ?"
       }
     },
     "service_type": "sharepoint_online",
-    "status": "error",
-    "id": "cVvIfIgBFjJwEZQVkrcy",
+    "status": "connected",
+    "id": "sharepoint_online",
     "last_indexed_document_count": 0,
     "last_deleted_document_count": 0
   }

tests/sources/fixtures/sharepoint_online/docker-compose.yml

@@ -77,7 +77,7 @@ services:
     build: .
     command: python fixture.py
     ports:
-      - "10337:10337"
+      - 10337:10337
     volumes:
       - .:/python-flask
     restart: always