From a6a0312019b3953cf7c54a32fa4a52e9aba940d4 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Tue, 6 Sep 2022 14:20:43 +1000
Subject: [PATCH 01/21] Test for searching with numbers in handle

---
 src/init-es.js                | 7 +++++++
 src/scripts/seed-data.js      | 8 ++++++++
 src/services/SearchService.js | 2 ++
 3 files changed, 17 insertions(+)

diff --git a/src/init-es.js b/src/init-es.js
index 7b37aee..7169d50 100644
--- a/src/init-es.js
+++ b/src/init-es.js
@@ -45,6 +45,13 @@ const initES = async () => {
       properties: {
         handleLower: { type: 'keyword' },
         handle: { type: 'keyword' },
+        handleSuggest: {
+          "type": "completion",
+          "analyzer": "standard",
+          "preserve_separators": true,
+          "preserve_position_increments": true,
+          "max_input_length": 200
+        },
         userId: { type: 'keyword' },
         status: { type: 'keyword' }
       }
diff --git a/src/scripts/seed-data.js b/src/scripts/seed-data.js
index 41de270..ff88409 100644
--- a/src/scripts/seed-data.js
+++ b/src/scripts/seed-data.js
@@ -24,6 +24,7 @@ const members = [{
   otherLangName: 'en',
   handle: 'denis',
   handleLower: 'denis',
+  handleSuggest: 'denis',
   status: 'ACTIVE',
   email: 'denis@topcoder.com',
   newEmail: 'denis2@topcoder.com',
@@ -66,6 +67,7 @@ const members = [{
   otherLangName: 'en',
   handle: 'testing',
   handleLower: 'testing',
+  handleSuggest: 'testing',
   status: 'ACTIVE',
   email: 'testing@topcoder.com',
   newEmail: 'testing2@topcoder.com',
@@ -127,6 +129,7 @@ const historyStats = {
   userId: 123,
   handle: 'denis',
   handleLower: 'denis',
+  handleSuggest: 'denis',
   DEVELOP: {
     subTracks: [
       {
@@ -180,6 +183,7 @@ const historyStatsPrivate = {
   groupId: 20000001,
   handle: 'denis',
   handleLower: 'denis',
+  handleSuggest: 'denis',
   DEVELOP: {
     subTracks: [
       {
@@ -232,6 +236,7 @@ const memberStats = {
   userId: 123,
   handle: 'denis',
   handleLower: 'denis',
+  handleSuggest: 'denis',
   maxRating: {
     rating: 1565,
     track: 'develop',
@@ -406,6 +411,7 @@ const memberPrivateStats = {
   groupId: 20000001,
   handle: 'denis',
   handleLower: 'denis',
+  handleSuggest: 'denis',
   maxRating: {
     rating: 1565,
     track: 'develop',
@@ -589,6 +595,7 @@ const memberAggregatedSkills = {
   userId: 123,
   handle: 'denis',
   handleLower: 'denis',
+  handleSuggest: 'denis',
   skills: {
     Java: {
       tagName: 'code',
@@ -613,6 +620,7 @@ const memberEnteredSkills = {
   userId: 123,
   userHandle: 'POSTMANE2E-denis',
   handleLower: 'postmane2e-denis',
+  handleSuggest: 'postmane2e-denis',
   skills: {
     286: {
       hidden: false,
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 18f512e..b08092b 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -136,6 +136,8 @@ async function autocomplete (currentUser, query) {
     var results = docsSuggestions.suggest['handle-suggestion'][0].options
     // custom filter & sort
     let regex = new RegExp(`^${query.term}`, `i`)
+    // sometimes .payload is not defined. so use _source instead
+    results = results.map(x => ({...x, payload: x.payload || x._source}))
     results = results
       .filter(x => regex.test(x.payload.handle))
       .sort((a, b) => a.payload.handle.localeCompare(b.payload.handle))

From ff586d5257bceb5e0f9a90303c9204da9315551b Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Wed, 14 Sep 2022 14:01:31 +1000
Subject: [PATCH 02/21] Update the index used for searching to the new, more
 complex one.

---
 config/default.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/default.js b/config/default.js
index d12849d..2c83165 100644
--- a/config/default.js
+++ b/config/default.js
@@ -45,7 +45,7 @@ module.exports = {
     HOST: process.env.ES_HOST || 'localhost:9200',
     API_VERSION: process.env.ES_API_VERSION || '6.8',
     // member index
-    MEMBER_PROFILE_ES_INDEX: process.env.MEMBER_PROFILE_ES_INDEX || 'members-2020-01',
+    MEMBER_PROFILE_ES_INDEX: process.env.MEMBER_PROFILE_ES_INDEX || 'members-2020-01-s3',
     // member type, ES 6.x accepts only 1 Type per index and it's mandatory to define it
     MEMBER_PROFILE_ES_TYPE: process.env.MEMBER_PROFILE_ES_TYPE || 'profiles',
     MEMBER_TRAIT_ES_INDEX: process.env.MEMBER_TRAIT_ES_INDEX || 'membertraits-2020-01',

From 3ddfa861d69025c5ebb5e5ce82c84c690c8d76f3 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Tue, 20 Sep 2022 07:25:00 +1000
Subject: [PATCH 03/21] Restrict auto-complete endpoint

---
 app-constants.js              |  2 ++
 src/common/helper.js          | 20 ++++++++++++++++++++
 src/routes.js                 |  2 --
 src/services/SearchService.js | 10 +++++-----
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/app-constants.js b/app-constants.js
index 1f302fe..5a817a8 100644
--- a/app-constants.js
+++ b/app-constants.js
@@ -2,6 +2,7 @@
  * App constants
  */
 const ADMIN_ROLES = ['administrator', 'admin']
+const AUTOCOMPLETE_ROLES = ['copilot', 'administrator', 'admin', 'Connect Copilot', 'Connect Account Manager', 'Connect Admin', 'Account Executive']
 
 const EVENT_ORIGINATOR = 'topcoder-member-api'
 
@@ -26,6 +27,7 @@ const MAMBO_GET_REWARDS_ALLOWED_FIELDS = [
 
 module.exports = {
   ADMIN_ROLES,
+  AUTOCOMPLETE_ROLES,
   EVENT_ORIGINATOR,
   EVENT_MIME_TYPE,
   TOPICS,
diff --git a/src/common/helper.js b/src/common/helper.js
index 1c1cd3f..123e7d0 100644
--- a/src/common/helper.js
+++ b/src/common/helper.js
@@ -111,6 +111,25 @@ function hasAdminRole (authUser) {
   return false
 }
 
+/**
+ * Check if the user has autocomplete role
+ * @param {Object} authUser the user
+ * @returns {Boolean} whether the user has autocomplete role
+ */
+function hasAutocompleteRole (authUser) {
+  if (!authUser.roles) {
+    return false
+  }
+  for (let i = 0; i < authUser.roles.length; i += 1) {
+    for (let j = 0; j < constants.AUTOCOMPLETE_ROLES.length; j += 1) {
+      if (authUser.roles[i].toLowerCase() === constants.AUTOCOMPLETE_ROLES[j].toLowerCase()) {
+        return true
+      }
+    }
+  }
+  return false
+}
+
 /**
  * Check if exists.
  *
@@ -751,6 +770,7 @@ module.exports = {
   autoWrapExpress,
   checkIfExists,
   hasAdminRole,
+  hasAutocompleteRole,
   getMemberByHandle,
   getEntityByHashKey,
   getEntityByHashRangeKey,
diff --git a/src/routes.js b/src/routes.js
index d713b5f..82f2264 100644
--- a/src/routes.js
+++ b/src/routes.js
@@ -28,7 +28,6 @@ module.exports = {
       controller: 'SearchController',
       method: 'autocomplete',
       auth: 'jwt',
-      allowNoToken: true,
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
   },
@@ -37,7 +36,6 @@ module.exports = {
       controller: 'SearchController',
       method: 'autocomplete',
       auth: 'jwt',
-      allowNoToken: true,
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
   },
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index b08092b..e75f1b8 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -124,11 +124,11 @@ searchMembers.schema = {
 async function autocomplete (currentUser, query) {
   // validate and parse fields param
   let fields = helper.parseCommaSeparatedString(query.fields, MEMBER_AUTOCOMPLETE_FIELDS) || MEMBER_AUTOCOMPLETE_FIELDS
-  // // if current user is not admin and not M2M, then exclude the admin/M2M only fields
-  // if (!currentUser || (!currentUser.isMachine && !helper.hasAdminRole(currentUser))) {
-  //   fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
-  //   // MEMBER_AUTOCOMPLETE_FIELDS = _.without(MEMBER_AUTOCOMPLETE_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
-  // }
+  // if current user is not autocomplete role and not M2M, then exclude the autocomplete/M2M only fields
+  if (!currentUser || (!currentUser.isMachine && !helper.hasAutocompleteRole(currentUser))) {
+    fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
+    // MEMBER_AUTOCOMPLETE_FIELDS = _.without(MEMBER_AUTOCOMPLETE_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
+  }
   // get suggestion based on querys term
   const docsSuggestions = await eshelper.getSuggestion(query, esClient, currentUser)
   if (docsSuggestions.hasOwnProperty('suggest')) {

From 47c26eec961cfd4ee0b20b7d338c2916a400bb7f Mon Sep 17 00:00:00 2001
From: Aditya Gopal <36183939+adityagopal@users.noreply.github.com>
Date: Tue, 14 Mar 2023 14:10:53 +0530
Subject: [PATCH 04/21] feat: query by email (#105)

* feat: query by email

* Added tgadmin to constants

* UnauthorizedError for No Token on query via Email
---
 .circleci/config.yml          |  3 +--
 app-constants.js              |  2 +-
 src/common/eshelper.js        |  3 +++
 src/services/SearchService.js | 11 +++++++++++
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 193672c..d7ebe52 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -69,7 +69,7 @@ workflows:
             branches:
               only:
                 - develop
-                - feature/gamification
+                - feature/email
       # Production builds are exectuted only on tagged commits to the
       # master branch.
       - "build-prod":
@@ -78,4 +78,3 @@ workflows:
             branches:
               only:
                 - master
-                - hotfix/inc-15
diff --git a/app-constants.js b/app-constants.js
index f1d8d4f..79d1409 100644
--- a/app-constants.js
+++ b/app-constants.js
@@ -1,7 +1,7 @@
 /**
  * App constants
  */
-const ADMIN_ROLES = ['administrator', 'admin']
+const ADMIN_ROLES = ['administrator', 'admin','tgadmin']
 const AUTOCOMPLETE_ROLES = ['copilot', 'administrator', 'admin', 'Connect Copilot', 'Connect Account Manager', 'Connect Admin', 'Account Executive']
 
 const EVENT_ORIGINATOR = 'topcoder-member-api'
diff --git a/src/common/eshelper.js b/src/common/eshelper.js
index 3fcd9ea..1980d66 100644
--- a/src/common/eshelper.js
+++ b/src/common/eshelper.js
@@ -33,6 +33,9 @@ async function getMembers (query, esClient, currentUser) {
   if (query.handle) {
     boolQueryMembers.push({ match_phrase: { handle: query.handle } })
   }
+  if (query.email) {
+    boolQueryMembers.push({ match_phrase: { email: query.email } })
+  }
   if (userIds.length > 0) {
     boolQueryMembers.push({ query: { terms: { userId: userIds } } })
   }
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 3289c89..25d975c 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -8,6 +8,7 @@ const config = require('config')
 const helper = require('../common/helper')
 const eshelper = require('../common/eshelper')
 const logger = require('../common/logger')
+const errors = require('../common/errors')
 
 const MEMBER_FIELDS = ['userId', 'handle', 'handleLower', 'firstName', 'lastName',
   'status', 'addresses', 'photoURL', 'homeCountryCode', 'competitionCountryCode',
@@ -37,6 +38,15 @@ async function searchMembers (currentUser, query) {
     MEMBER_STATS_FIELDS = _.without(MEMBER_STATS_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
   }
 
+  if (query.email != null && query.email.length > 0) {
+  if (currentUser == null) {
+    throw new errors.UnauthorizedError("Authentication token is required to query users by email");
+  }
+  if (!helper.hasAdminRole(currentUser)) {
+    throw new errors.BadRequestError("Admin role is required to query users by email");
+  }
+  }
+
   // search for the members based on query
   const docsMembers = await eshelper.getMembers(query, esClient, currentUser)
 
@@ -105,6 +115,7 @@ searchMembers.schema = {
     handlesLower: Joi.array(),
     handle: Joi.string(),
     handles: Joi.array(),
+    email: Joi.string(),
     userId: Joi.number(),
     userIds: Joi.array(),
     term: Joi.string(),

From fa097d05b288ba8780324805ff0555305071c905 Mon Sep 17 00:00:00 2001
From: Rakib Ansary <rakib.saikot@colorado.edu>
Date: Tue, 14 Mar 2023 14:49:18 +0600
Subject: [PATCH 05/21] fix: restrict search by email (#107)

* fix: restrict search by email

- restrict to users with roles: admin, administrator & tgadmin

* fix: typo
---
 app-constants.js              |  4 +++-
 src/common/helper.js          | 15 +++++++++++++++
 src/services/SearchService.js |  2 +-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/app-constants.js b/app-constants.js
index 79d1409..66c5df8 100644
--- a/app-constants.js
+++ b/app-constants.js
@@ -1,7 +1,8 @@
 /**
  * App constants
  */
-const ADMIN_ROLES = ['administrator', 'admin','tgadmin']
+const ADMIN_ROLES = ['administrator', 'admin']
+const SEARCH_BY_EMAIL_ROLES = ADMIN_ROLES.concat('tgadmin');
 const AUTOCOMPLETE_ROLES = ['copilot', 'administrator', 'admin', 'Connect Copilot', 'Connect Account Manager', 'Connect Admin', 'Account Executive']
 
 const EVENT_ORIGINATOR = 'topcoder-member-api'
@@ -29,6 +30,7 @@ const MAMBO_GET_REWARDS_ALLOWED_FIELDS = [
 
 module.exports = {
   ADMIN_ROLES,
+  SEARCH_BY_EMAIL_ROLES,
   AUTOCOMPLETE_ROLES,
   EVENT_ORIGINATOR,
   EVENT_MIME_TYPE,
diff --git a/src/common/helper.js b/src/common/helper.js
index 123e7d0..88a78f8 100644
--- a/src/common/helper.js
+++ b/src/common/helper.js
@@ -111,6 +111,20 @@ function hasAdminRole (authUser) {
   return false
 }
 
+function hasSearchByEmailRole (authUser) {
+  if (!authUser.roles) {
+    return false
+  }
+  for (let i = 0; i < authUser.roles.length; i += 1) {
+    for (let j = 0; j < constants.SEARCH_BY_EMAIL_ROLES.length; j += 1) {
+      if (authUser.roles[i].toLowerCase() === constants.SEARCH_BY_EMAIL_ROLES[j].toLowerCase()) {
+        return true
+      }
+    }
+  }
+  return false
+}
+
 /**
  * Check if the user has autocomplete role
  * @param {Object} authUser the user
@@ -771,6 +785,7 @@ module.exports = {
   checkIfExists,
   hasAdminRole,
   hasAutocompleteRole,
+  hasSearchByEmailRole,
   getMemberByHandle,
   getEntityByHashKey,
   getEntityByHashRangeKey,
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 25d975c..39e7745 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -42,7 +42,7 @@ async function searchMembers (currentUser, query) {
   if (currentUser == null) {
     throw new errors.UnauthorizedError("Authentication token is required to query users by email");
   }
-  if (!helper.hasAdminRole(currentUser)) {
+  if (!helper.hasSearchByEmailRole(currentUser)) {
     throw new errors.BadRequestError("Admin role is required to query users by email");
   }
   }

From 7b073e5899386304653e0edf85a9c514ef92d648 Mon Sep 17 00:00:00 2001
From: Thomas Kranitsas <thomas.kranitsas@topcoder.com>
Date: Fri, 7 Apr 2023 11:48:36 +0300
Subject: [PATCH 06/21] Update base image of deploy script

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d7ebe52..b5b509d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -13,7 +13,7 @@ install_dependency: &install_dependency
 install_deploysuite: &install_deploysuite
   name: Installation of install_deploysuite.
   command: |
-    git clone --branch v1.4.2 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+    git clone --branch v1.4.14 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
     cp ./../buildscript/master_deploy.sh .
     cp ./../buildscript/buildenv.sh .
     cp ./../buildscript/awsconfiguration.sh .

From 6569a5d5a4b408960f95c87150f2dd40537ee7f4 Mon Sep 17 00:00:00 2001
From: Gunasekar-K <Gunasekar.Kesavalu@wipro.com>
Date: Mon, 22 May 2023 20:48:06 +0530
Subject: [PATCH 07/21] read-only-root-file-system-fix

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b5b509d..20fa30e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -13,7 +13,7 @@ install_dependency: &install_dependency
 install_deploysuite: &install_deploysuite
   name: Installation of install_deploysuite.
   command: |
-    git clone --branch v1.4.14 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+    git clone --branch v1.4.15 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
     cp ./../buildscript/master_deploy.sh .
     cp ./../buildscript/buildenv.sh .
     cp ./../buildscript/awsconfiguration.sh .

From 4dbd523d13722c7ec8232aa34156155202249eaf Mon Sep 17 00:00:00 2001
From: Gunasekar-K <Gunasekar.Kesavalu@wipro.com>
Date: Mon, 22 May 2023 21:05:40 +0530
Subject: [PATCH 08/21] Update Dockerfile

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index e8b326b..17b5b67 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -10,4 +10,4 @@ WORKDIR /member-api-v5
 # Install the dependencies from package.json
 RUN yarn
 
-CMD yarn start
+CMD node app.js

From b0c34621376dbb496c8961b83ae91430161b7226 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 25 May 2023 11:12:48 +1000
Subject: [PATCH 09/21] Initial search by EMSI skills

---
 .nvmrc                              |   2 +-
 app-constants.js                    |   7 +-
 src/common/eshelper.js              |  93 +++++++++++++++++++++++-
 src/common/helper.js                |  23 +++++-
 src/controllers/SearchController.js |  11 +++
 src/routes.js                       |   6 +-
 src/scripts/view-es-data.js         |   3 +-
 src/services/SearchService.js       | 109 +++++++++++++++++++++++-----
 8 files changed, 228 insertions(+), 26 deletions(-)

diff --git a/.nvmrc b/.nvmrc
index 8066b68..dda41f4 100644
--- a/.nvmrc
+++ b/.nvmrc
@@ -1 +1 @@
-v8.9.2
\ No newline at end of file
+v12.22.12
\ No newline at end of file
diff --git a/app-constants.js b/app-constants.js
index 66c5df8..838c679 100644
--- a/app-constants.js
+++ b/app-constants.js
@@ -28,6 +28,10 @@ const MAMBO_GET_REWARDS_ALLOWED_FIELDS = [
   'awardedOn', 'expiryOn', 'isExpired', 'id'
 ]
 
+const BOOLEAN_OPERATOR = {
+  AND: 'AND',
+  OR: 'OR'
+}
 module.exports = {
   ADMIN_ROLES,
   SEARCH_BY_EMAIL_ROLES,
@@ -36,5 +40,6 @@ module.exports = {
   EVENT_MIME_TYPE,
   TOPICS,
   ES_SEARCH_MAX_SIZE,
-  MAMBO_GET_REWARDS_ALLOWED_FIELDS
+  MAMBO_GET_REWARDS_ALLOWED_FIELDS,
+  BOOLEAN_OPERATOR
 }
diff --git a/src/common/eshelper.js b/src/common/eshelper.js
index 1980d66..bc8bff9 100644
--- a/src/common/eshelper.js
+++ b/src/common/eshelper.js
@@ -3,6 +3,7 @@
  */
 const _ = require('lodash')
 const config = require('config')
+const { BOOLEAN_OPERATOR } = require('../../app-constants')
 
 /**
  * Fetch members profile form ES
@@ -58,6 +59,35 @@ async function getMembers (query, esClient, currentUser) {
   return docsMembers
 }
 
+/**
+ * Search members by skills
+ * @param {Object} query the HTTP request query
+ * @returns {Object} members skills
+ */
+async function searchBySkills (query, esClient) {
+  // construct ES query for skills
+  const esQuerySkills = {
+    index: config.get('ES.MEMBER_SKILLS_ES_INDEX'),
+    type: config.get('ES.MEMBER_SKILLS_ES_TYPE'),
+    body: {
+      sort: [{ userHandle: { order: query.sort } }]
+    }
+  }
+  const boolQuerySkills = []
+
+  if (query.handlesLower) {
+    boolQuerySkills.push({ query: { terms: { handleLower: query.handlesLower } } })
+  }
+  esQuerySkills.body.query = {
+    bool: {
+      filter: boolQuerySkills
+    }
+  }
+  // search with constructed query
+  const docsSkills = await esClient.search(esQuerySkills)
+  return docsSkills
+}
+
 /**
  * Fetch members skills form ES
  * @param {Object} query the HTTP request query
@@ -146,6 +176,66 @@ async function getSuggestion (query, esClient, currentUser) {
   return docsSuggestionMembers
 }
 
+/**
+ * Gets the members skills documents matching the provided criteria from Elasticsearch
+ * @param skillIds
+ * @param skillsBooleanOperator
+ * @param page
+ * @param perPage
+ * @param esClient
+ * @returns {Promise<*>}
+ */
+async function searchMembersSkills (skillIds, skillsBooleanOperator, page, perPage, esClient) {
+  // construct ES query for members skills
+  const esQuerySkills = {
+    index: config.get('ES.MEMBER_PROFILE_ES_INDEX'),
+    type: config.get('ES.MEMBER_PROFILE_ES_TYPE'),
+    from: 0,
+    size: 100,
+    body: {
+      sort: [{ createdAt: { order: 'desc' } }],
+      query: {
+        bool: {
+          filter: { bool: {} }
+        }
+      }
+    }
+  }
+
+  const mustMatchQuery = [] // will contain the filters with AND operator
+  const shouldFilter = [] // will contain the filters with OR operator
+
+  if (skillsBooleanOperator === BOOLEAN_OPERATOR.AND) {
+    for (const skillId of skillIds) {
+      const matchPhrase = {}
+      matchPhrase[`emsiSkills.emsiId`] = `${skillId}`
+      mustMatchQuery.push({
+        match_phrase: matchPhrase
+      })
+    }
+  } else {
+    for (const skillId of skillIds) {
+      const matchPhrase = {}
+      matchPhrase[`emsiSkills.emsiId`] = `${skillId}`
+      shouldFilter.push({
+        match_phrase: matchPhrase// eslint-disable-line
+      })
+    }
+  }
+
+  if (mustMatchQuery.length > 0) {
+    esQuerySkills.body.query.bool.filter.bool.must = mustMatchQuery
+  }
+
+  if (shouldFilter.length > 0) {
+    esQuerySkills.body.query.bool.filter.bool.should = shouldFilter
+  }
+  // search with constructed query
+  return esClient.search(esQuerySkills)
+}
+
+
+
 /**
  * Get total items
  * @param {Object} docs the HTTP request query
@@ -164,5 +254,6 @@ module.exports = {
   getMembersSkills,
   getMembersStats,
   getSuggestion,
-  getTotal
+  getTotal,
+  searchMembersSkills,
 }
diff --git a/src/common/helper.js b/src/common/helper.js
index 88a78f8..8925c51 100644
--- a/src/common/helper.js
+++ b/src/common/helper.js
@@ -779,6 +779,26 @@ const getM2MToken = () => {
   )
 }
 
+/**
+ * Gets the list of parameters from the query as an array
+ *
+ * @param query
+ * @param parameterName
+ * @returns {*[]}
+ */
+const getParamsFromQueryAsArray = async (query, parameterName) => {
+  const paramsArray = []
+  if (!_.isEmpty(query[parameterName])) {
+    if (!_.isArray(query[parameterName])) {
+      paramsArray.push(query[parameterName])
+    } else {
+      paramsArray.push(...query[parameterName])
+    }
+  }
+  return paramsArray
+}
+
+
 module.exports = {
   wrapExpress,
   autoWrapExpress,
@@ -812,5 +832,6 @@ module.exports = {
   getGroupId,
   getAllowedGroupIds,
   getMemberGroups,
-  getM2MToken
+  getM2MToken,
+  getParamsFromQueryAsArray
 }
diff --git a/src/controllers/SearchController.js b/src/controllers/SearchController.js
index e539694..dd39a45 100644
--- a/src/controllers/SearchController.js
+++ b/src/controllers/SearchController.js
@@ -26,7 +26,18 @@ async function autocomplete (req, res) {
   res.send(result.result)
 }
 
+/**
+ * Search members with additional parameters, like skills
+ * @param {Object} req the request
+ * @param {Object} res the response
+ */
+async function searchMembersBySkills (req, res) {
+  const result = await service.searchMembersBySkills(req.authUser, req.query)
+  helper.setResHeaders(req, res, result)
+  res.send(result.result)
+}
 module.exports = {
   searchMembers,
+  searchMembersBySkills,
   autocomplete
 }
diff --git a/src/routes.js b/src/routes.js
index 82f2264..7895e00 100644
--- a/src/routes.js
+++ b/src/routes.js
@@ -23,10 +23,10 @@ module.exports = {
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
   },
-  '/search/members/autocomplete': {
+  '/members/searchBySkills': {
     get: {
       controller: 'SearchController',
-      method: 'autocomplete',
+      method: 'searchMembersBySkills',
       auth: 'jwt',
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
@@ -160,5 +160,5 @@ module.exports = {
       allowNoToken: true,
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
-  },
+  }
 }
diff --git a/src/scripts/view-es-data.js b/src/scripts/view-es-data.js
index 68f85ca..c3bd4ce 100644
--- a/src/scripts/view-es-data.js
+++ b/src/scripts/view-es-data.js
@@ -12,13 +12,14 @@ if (process.argv.length <= 2) {
   process.exit(1)
 }
 const indexName = process.argv[2]
+const indexType = process.argv[3]
 
 const esClient = helper.getESClient()
 
 async function showESData () {
   const result = await esClient.search({
     index: indexName,
-    type: config.get('ES.MEMBER_PROFILE_ES_TYPE') // type name is same for all indices
+    type: indexType // type name is same for all indices
   })
   return result.hits.hits || []
 }
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index b0a417e..1def005 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -9,12 +9,17 @@ const helper = require('../common/helper')
 const eshelper = require('../common/eshelper')
 const logger = require('../common/logger')
 const errors = require('../common/errors')
+const { BOOLEAN_OPERATOR } = require('../../app-constants')
 
 const MEMBER_FIELDS = ['userId', 'handle', 'handleLower', 'firstName', 'lastName',
   'status', 'addresses', 'photoURL', 'homeCountryCode', 'competitionCountryCode',
   'description', 'email', 'tracks', 'maxRating', 'wins', 'createdAt', 'createdBy',
   'updatedAt', 'updatedBy', 'skills', 'stats', 'emsiSkills']
 
+const MEMBER_SORT_BY_FIELDS = ['userId', 'country', 'handle', 'firstName', 'lastName', 
+  'accountAge', 'numberOfChallengesWon', 
+  'numberOfChallengesPlaced']
+
 const MEMBER_AUTOCOMPLETE_FIELDS = ['userId', 'handle', 'handleLower',
   'status', 'email', 'createdAt', 'updatedAt']
 
@@ -39,19 +44,42 @@ async function searchMembers (currentUser, query) {
   }
 
   if (query.email != null && query.email.length > 0) {
-  if (currentUser == null) {
-    throw new errors.UnauthorizedError("Authentication token is required to query users by email");
-  }
-  if (!helper.hasSearchByEmailRole(currentUser)) {
-    throw new errors.BadRequestError("Admin role is required to query users by email");
-  }
+    if (currentUser == null) {
+      throw new errors.UnauthorizedError('Authentication token is required to query users by email')
+    }
+    if (!helper.hasSearchByEmailRole(currentUser)) {
+      throw new errors.BadRequestError('Admin role is required to query users by email')
+    }
   }
 
   // search for the members based on query
   const docsMembers = await eshelper.getMembers(query, esClient, currentUser)
 
+  return fillMembers(docsMembers, query, fields)
+}
+
+searchMembers.schema = {
+  currentUser: Joi.any(),
+  query: Joi.object().keys({
+    handleLower: Joi.string(),
+    handlesLower: Joi.array(),
+    handle: Joi.string(),
+    handles: Joi.array(),
+    email: Joi.string(),
+    userId: Joi.number(),
+    userIds: Joi.array(),
+    term: Joi.string(),
+    fields: Joi.string(),
+    page: Joi.page(),
+    perPage: Joi.perPage(),
+    sort: Joi.sort()
+  })
+}
+
+async function fillMembers(docsMembers, query, fields) {
   // get the total
   const total = eshelper.getTotal(docsMembers)
+
   let results = []
   if (total > 0) {
     // extract member profiles from hits
@@ -108,24 +136,68 @@ async function searchMembers (currentUser, query) {
   return { total: total, page: query.page, perPage: query.perPage, result: results }
 }
 
-searchMembers.schema = {
+// TODO - use some caching approach to replace these in-memory objects
+/**
+ * Search members by the given search query
+ *
+ * @param query The search query by which to search members
+ *
+ * @returns {Promise<[]>} The array of members matching the given query
+ */
+const searchMembersBySkills = async (currentUser, query) => {
+  const esClient = await helper.getESClient()
+  let skillIds = await helper.getParamsFromQueryAsArray(query, 'skillId')
+  const result = searchMembersBySkillsWithOptions(currentUser, query, skillIds, BOOLEAN_OPERATOR.AND, query.page, query.perPage, query.sortBy, query.sortOrder, esClient)
+  return result
+}
+
+searchMembersBySkills.schema = {
   currentUser: Joi.any(),
   query: Joi.object().keys({
-    handleLower: Joi.string(),
-    handlesLower: Joi.array(),
-    handle: Joi.string(),
-    handles: Joi.array(),
-    email: Joi.string(),
-    userId: Joi.number(),
-    userIds: Joi.array(),
-    term: Joi.string(),
-    fields: Joi.string(),
+    skillId: Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())),
     page: Joi.page(),
     perPage: Joi.perPage(),
-    sort: Joi.sort()
+    sortBy: Joi.string().valid(MEMBER_SORT_BY_FIELDS).default('numberOfChallengesWon'),
+    sortOrder: Joi.string().valid('asc', 'desc').default('desc')
   })
 }
 
+/**
+ * Search members matching the given skills
+ *
+ * @param currentUser
+ * @param skillsFilter
+ * @param skillsBooleanOperator
+ * @param page
+ * @param perPage
+ * @param sortBy
+ * @param sortOrder
+ * @param esClient
+ * @returns {Promise<*[]|{total, perPage, numberOfPages: number, data: *[], page}>}
+ */
+const searchMembersBySkillsWithOptions = async (currentUser, query, skillsFilter, skillsBooleanOperator, page, perPage, sortBy, sortOrder, esClient) => {
+  let fields = helper.parseCommaSeparatedString(query.fields, MEMBER_FIELDS) || MEMBER_FIELDS
+  // if current user is not admin and not M2M, then exclude the admin/M2M only fields
+  if (!currentUser || (!currentUser.isMachine && !helper.hasAdminRole(currentUser))) {
+    fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
+    MEMBER_STATS_FIELDS = _.without(MEMBER_STATS_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
+  }
+
+  const emptyResult = {
+    total: 0,
+    page,
+    perPage,
+    numberOfPages: 0,
+    data: []
+  }
+  if (_.isEmpty(skillsFilter)) {
+    return emptyResult
+  }
+
+  const membersSkillsDocs = await eshelper.searchMembersSkills(skillsFilter, skillsBooleanOperator, page, perPage, esClient)
+  
+  return fillMembers(membersSkillsDocs, query, fields)
+}
 /**
  * members autocomplete.
  * @param {Object} currentUser the user who performs operation
@@ -148,7 +220,7 @@ async function autocomplete (currentUser, query) {
     // custom filter & sort
     let regex = new RegExp(`^${query.term}`, `i`)
     // sometimes .payload is not defined. so use _source instead
-    results = results.map(x => ({...x, payload: x.payload || x._source}))
+    results = results.map(x => ({ ...x, payload: x.payload || x._source }))
     results = results
       .filter(x => regex.test(x.payload.handle))
       .sort((a, b) => a.payload.handle.localeCompare(b.payload.handle))
@@ -175,6 +247,7 @@ autocomplete.schema = {
 
 module.exports = {
   searchMembers,
+  searchMembersBySkills,
   autocomplete
 }
 

From f5bf453082ac1b0548ce50c0cd7d35ffef3a4ee1 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 25 May 2023 11:21:12 +1000
Subject: [PATCH 10/21] Deploy to dev

---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b5b509d..6d62c3e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -70,6 +70,7 @@ workflows:
               only:
                 - develop
                 - feature/email
+                - feature/emsi_skills_search
       # Production builds are exectuted only on tagged commits to the
       # master branch.
       - "build-prod":

From 3f82b5277e1500443f935c68158e393db272a702 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Fri, 26 May 2023 15:58:58 +1000
Subject: [PATCH 11/21] Allow for easier access to number of challenges won /
 placed and sorting by those fields

---
 src/services/SearchService.js | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 1def005..414107f 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -10,20 +10,22 @@ const eshelper = require('../common/eshelper')
 const logger = require('../common/logger')
 const errors = require('../common/errors')
 const { BOOLEAN_OPERATOR } = require('../../app-constants')
+const moment = require('moment')
 
 const MEMBER_FIELDS = ['userId', 'handle', 'handleLower', 'firstName', 'lastName',
   'status', 'addresses', 'photoURL', 'homeCountryCode', 'competitionCountryCode',
   'description', 'email', 'tracks', 'maxRating', 'wins', 'createdAt', 'createdBy',
-  'updatedAt', 'updatedBy', 'skills', 'stats', 'emsiSkills']
+  'updatedAt', 'updatedBy', 'skills', 'stats', 'emsiSkills',
+'numberOfChallengesWon', 'numberOfChallengesPlaced']
 
 const MEMBER_SORT_BY_FIELDS = ['userId', 'country', 'handle', 'firstName', 'lastName', 
-  'accountAge', 'numberOfChallengesWon', 
-  'numberOfChallengesPlaced']
+  'numberOfChallengesWon', 'numberOfChallengesPlaced']
 
 const MEMBER_AUTOCOMPLETE_FIELDS = ['userId', 'handle', 'handleLower',
   'status', 'email', 'createdAt', 'updatedAt']
 
-var MEMBER_STATS_FIELDS = ['userId', 'handle', 'handleLower', 'maxRating',
+var MEMBER_STATS_FIELDS = ['userId', 'handle', 'handleLower', 'maxRating', 
+  'numberOfChallengesWon', 'numberOfChallengesPlaced',
   'challenges', 'wins', 'DEVELOP', 'DESIGN', 'DATA_SCIENCE', 'COPILOT']
 
 const esClient = helper.getESClient()
@@ -111,6 +113,8 @@ async function fillMembers(docsMembers, query, fields) {
     // merge overall members and stats
     const mbrsSkillsStatsKeys = _.keyBy(mbrsSkillsStats, 'userId')
     const resultMbrsSkillsStats = _.map(resultMbrSkills, function (item) {
+      item.numberOfChallengesWon=0;
+      item.numberOfChallengesPlaced=0;
       if (mbrsSkillsStatsKeys[item.userId]) {
         item.stats = []
         if (mbrsSkillsStatsKeys[item.userId].maxRating) {
@@ -121,6 +125,9 @@ async function fillMembers(docsMembers, query, fields) {
             item.maxRating.ratingColor = helper.getRatingColor(item.maxRating.rating)
           }
         }
+        item.numberOfChallengesWon = mbrsSkillsStatsKeys[item.userId].wins
+        item.numberOfChallengesPlaced = mbrsSkillsStatsKeys[item.userId].challenges
+        
         // clean up stats fileds and filter on stats fields
         item.stats.push(_.pick(mbrsSkillsStatsKeys[item.userId], MEMBER_STATS_FIELDS))
       } else {
@@ -128,6 +135,7 @@ async function fillMembers(docsMembers, query, fields) {
       }
       return item
     })
+
     // sort the data
     results = _.orderBy(resultMbrsSkillsStats, ['handleLower'], [query.sort])
     // filter member based on fields
@@ -196,7 +204,9 @@ const searchMembersBySkillsWithOptions = async (currentUser, query, skillsFilter
 
   const membersSkillsDocs = await eshelper.searchMembersSkills(skillsFilter, skillsBooleanOperator, page, perPage, esClient)
   
-  return fillMembers(membersSkillsDocs, query, fields)
+  let response = await fillMembers(membersSkillsDocs, query, fields)
+  response.result = _.orderBy(response.result, sortBy, sortOrder)
+  return response
 }
 /**
  * members autocomplete.

From be926f5403c701239fcc7287f08327b33c2dbc20 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Mon, 29 May 2023 16:53:06 +1000
Subject: [PATCH 12/21] Fix pagination when searching

---
 src/services/SearchService.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 414107f..367d4cb 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -141,6 +141,7 @@ async function fillMembers(docsMembers, query, fields) {
     // filter member based on fields
     results = _.map(results, (item) => _.pick(item, fields))
   }
+  results = helper.paginate(results, query.perPage, query.page - 1)
   return { total: total, page: query.page, perPage: query.perPage, result: results }
 }
 

From b171bb582b4f8db734a185ce256c213ecd2384d2 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Mon, 29 May 2023 21:30:29 +1000
Subject: [PATCH 13/21] Fix sorting

---
 src/services/SearchService.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 367d4cb..8608cc6 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -137,7 +137,7 @@ async function fillMembers(docsMembers, query, fields) {
     })
 
     // sort the data
-    results = _.orderBy(resultMbrsSkillsStats, ['handleLower'], [query.sort])
+    results = _.orderBy(resultMbrsSkillsStats, [query.sortBy, 'handleLower'], [query.sortOrder] )
     // filter member based on fields
     results = _.map(results, (item) => _.pick(item, fields))
   }
@@ -252,7 +252,7 @@ autocomplete.schema = {
     page: Joi.page(),
     perPage: Joi.perPage(),
     size: Joi.size(),
-    sort: Joi.sort()
+    sortOrder: Joi.string().valid('asc', 'desc').default('desc')
   })
 }
 

From 406350fa8857d1659007f6a924a31282df424fd3 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 1 Jun 2023 17:43:15 +1000
Subject: [PATCH 14/21] Update to better scroll / scan all relevant records
 when searching by skill

---
 src/common/eshelper.js | 38 +++++++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/common/eshelper.js b/src/common/eshelper.js
index bc8bff9..56dfae9 100644
--- a/src/common/eshelper.js
+++ b/src/common/eshelper.js
@@ -186,12 +186,15 @@ async function getSuggestion (query, esClient, currentUser) {
  * @returns {Promise<*>}
  */
 async function searchMembersSkills (skillIds, skillsBooleanOperator, page, perPage, esClient) {
+  const searchResults = {hits:{hits:[]}}
+  const responseQueue = []
+
   // construct ES query for members skills
   const esQuerySkills = {
     index: config.get('ES.MEMBER_PROFILE_ES_INDEX'),
     type: config.get('ES.MEMBER_PROFILE_ES_TYPE'),
-    from: 0,
-    size: 100,
+    size: 10000,
+    scroll: '90s',
     body: {
       sort: [{ createdAt: { order: 'desc' } }],
       query: {
@@ -218,7 +221,7 @@ async function searchMembersSkills (skillIds, skillsBooleanOperator, page, perPa
       const matchPhrase = {}
       matchPhrase[`emsiSkills.emsiId`] = `${skillId}`
       shouldFilter.push({
-        match_phrase: matchPhrase// eslint-disable-line
+        match_phrase: matchPhrase // eslint-disable-line
       })
     }
   }
@@ -230,11 +233,36 @@ async function searchMembersSkills (skillIds, skillsBooleanOperator, page, perPa
   if (shouldFilter.length > 0) {
     esQuerySkills.body.query.bool.filter.bool.should = shouldFilter
   }
+  
   // search with constructed query
-  return esClient.search(esQuerySkills)
-}
+  const response = await esClient.search(esQuerySkills)
 
+  responseQueue.push(response)
+  while (responseQueue.length) {
+    const body = responseQueue.shift()
+    // collect the titles from this response
+    body.hits.hits.forEach(function (hit) {
+      searchResults.hits.hits.push(hit)
+      //searchResults.push(hit._source.quote)
+    })
+
+    // check to see if we have collected all of the quotes
+    if (body.hits.total === searchResults.hits.hits.length) {
+      console.log('Number of matches', searchResults.hits.hits.length)
+      searchResults.hits.total=body.hits.total
+      break
+    }
 
+    // get the next response if there are more quotes to fetch
+    responseQueue.push(
+      await esClient.scroll({
+        scroll_id: body._scroll_id,
+        scroll: '90s'
+      })
+    )
+  }
+  return searchResults
+}
 
 /**
  * Get total items

From 0ede71d7c5af2d57b07132f04bf34c48a970f7b6 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Fri, 2 Jun 2023 10:25:34 +1000
Subject: [PATCH 15/21] Fix bug with how stats are searched - we need to scroll
 those so that we can cover *all* search results for larger result sets

---
 src/common/eshelper.js        | 38 ++++++++++++++++++++++++++++++++---
 src/services/SearchService.js | 12 +++++++----
 2 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/src/common/eshelper.js b/src/common/eshelper.js
index 56dfae9..70f04e4 100644
--- a/src/common/eshelper.js
+++ b/src/common/eshelper.js
@@ -123,10 +123,15 @@ async function getMembersSkills (query, esClient) {
  * @returns {Object} members stats
  */
 async function getMembersStats (query, esClient) {
+  const searchResults = {hits:{hits:[]}}
+  const responseQueue = []
+
   // construct ES query for stats
   const esQueryStats = {
     index: config.get('ES.MEMBER_STATS_ES_INDEX'),
     type: config.get('ES.MEMBER_STATS_ES_TYPE'),
+    size: 10000,
+    scroll: '90s',
     body: {
       sort: [{ handleLower: { order: query.sort } }]
     }
@@ -141,9 +146,36 @@ async function getMembersStats (query, esClient) {
       filter: boolQueryStats
     }
   }
+
+
   // search with constructed query
-  const docsStats = await esClient.search(esQueryStats)
-  return docsStats
+  const response = await esClient.search(esQueryStats)
+
+  responseQueue.push(response)
+  while (responseQueue.length) {
+    const body = responseQueue.shift()
+    // collect the titles from this response
+    body.hits.hits.forEach(function (hit) {
+      searchResults.hits.hits.push(hit)
+      //searchResults.push(hit._source.quote)
+    })
+
+    // check to see if we have collected all of the quotes
+    if (body.hits.total === searchResults.hits.hits.length) {
+      console.log('Number of stat matches', searchResults.hits.hits.length)
+      searchResults.hits.total=body.hits.total
+      break
+    }
+
+    // get the next response if there are more quotes to fetch
+    responseQueue.push(
+      await esClient.scroll({
+        scroll_id: body._scroll_id,
+        scroll: '90s'
+      })
+    )
+  }
+  return searchResults
 }
 
 /**
@@ -248,7 +280,7 @@ async function searchMembersSkills (skillIds, skillsBooleanOperator, page, perPa
 
     // check to see if we have collected all of the quotes
     if (body.hits.total === searchResults.hits.hits.length) {
-      console.log('Number of matches', searchResults.hits.hits.length)
+      console.log('Number of members matching skills:', searchResults.hits.hits.length)
       searchResults.hits.total=body.hits.total
       break
     }
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 8608cc6..753660e 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -125,7 +125,10 @@ async function fillMembers(docsMembers, query, fields) {
             item.maxRating.ratingColor = helper.getRatingColor(item.maxRating.rating)
           }
         }
-        item.numberOfChallengesWon = mbrsSkillsStatsKeys[item.userId].wins
+        if(mbrsSkillsStatsKeys[item.userId].wins > item.numberOfChallengesWon){
+          item.numberOfChallengesWon = mbrsSkillsStatsKeys[item.userId].wins
+        }
+
         item.numberOfChallengesPlaced = mbrsSkillsStatsKeys[item.userId].challenges
         
         // clean up stats fileds and filter on stats fields
@@ -135,12 +138,13 @@ async function fillMembers(docsMembers, query, fields) {
       }
       return item
     })
-
-    // sort the data
-    results = _.orderBy(resultMbrsSkillsStats, [query.sortBy, 'handleLower'], [query.sortOrder] )
     // filter member based on fields
     results = _.map(results, (item) => _.pick(item, fields))
+
+    // sort the data
+    results = _.orderBy(resultMbrsSkillsStats, [query.sortBy, "handleLower"], [query.sortOrder] )
   }
+
   results = helper.paginate(results, query.perPage, query.page - 1)
   return { total: total, page: query.page, perPage: query.perPage, result: results }
 }

From 0b762ca2ccdc1a45e35085133d144c43eb447989 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Tue, 6 Jun 2023 13:02:07 +1000
Subject: [PATCH 16/21] Allow for copilots and managers to get member email
 address

---
 README.md                     |  2 +-
 config/default.js             | 14 +++++++-----
 src/services/MemberService.js |  5 ++++
 src/services/SearchService.js | 43 ++++++++++++++++-------------------
 4 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/README.md b/README.md
index d66015c..a2fa78d 100644
--- a/README.md
+++ b/README.md
@@ -51,9 +51,9 @@ The following parameters can be set in config files or in env variables:
 - EMAIL_VERIFY_DISAGREE_URL: email verify disagree URL
 - SCOPES: the configurable M2M token scopes, refer `config/default.js` for more details
 - MEMBER_SECURE_FIELDS: Member profile identifiable info fields, only admin, M2M, or member himself can fetch these fields
+- COMMUNICATION_SECURE_FIELDS: Member contact information, accessible by admins, managers, and copilots - anyone with a role in the AUTOCOMPLETE_ROLES array
 - MEMBER_TRAIT_SECURE_FIELDS: Member traits identifiable info fields, only admin, M2M, or member himself can fetch these fields
 - MISC_SECURE_FIELDS: Misc identifiable info fields, only admin, M2M, or member himself can fetch these fields
-- SEARCH_SECURE_FIELDS: Member Search identifiable info fields, only admin, M2M, or member himself can fetch these fields
 - STATISTICS_SECURE_FIELDS: Member Statistics identifiable info fields, only admin, M2M, or member himself can fetch these fields
 - HEALTH_CHECK_TIMEOUT: health check timeout in milliseconds
 
diff --git a/config/default.js b/config/default.js
index d12849d..4a4b77d 100644
--- a/config/default.js
+++ b/config/default.js
@@ -88,10 +88,17 @@ module.exports = {
     }
   },
 
+
+  // Member identifiable info fields, copilots, admins, or M2M can get these fields
+  // Anyone in the constants.AUTOCOMPLETE_ROLES will have access to these fields
+  COMMUNICATION_SECURE_FIELDS: process.env.COMMUNICATION_SECURE_FIELDS
+    ? process.env.COMMUNICATION_SECURE_FIELDS.split(',')
+    : ['firstName', 'lastName', 'email'],
+
   // Member identifiable info fields, only admin, M2M, or member himself can get these fields
   MEMBER_SECURE_FIELDS: process.env.MEMBER_SECURE_FIELDS
     ? process.env.MEMBER_SECURE_FIELDS.split(',')
-    : ['firstName', 'lastName', 'email', 'addresses', 'createdBy', 'updatedBy'],
+    : ['addresses', 'createdBy', 'updatedBy'],
 
   // Member traits identifiable info fields, only admin, M2M, or member himself can fetch these fields
   MEMBER_TRAIT_SECURE_FIELDS: process.env.MEMBER_TRAIT_SECURE_FIELDS
@@ -103,11 +110,6 @@ module.exports = {
     ? process.env.MISC_SECURE_FIELDS.split(',')
     : ['createdBy', 'updatedBy'],
 
-  // Member Search identifiable info fields, only admin, M2M, or member himself can fetch these fields
-  SEARCH_SECURE_FIELDS: process.env.SEARCH_SECURE_FIELDS
-    ? process.env.SEARCH_SECURE_FIELDS.split(',')
-    : ['firstName', 'lastName', 'email', 'addresses', 'createdBy', 'updatedBy'],
-
   // Member Statistics identifiable info fields, only admin, M2M, or member himself can fetch these fields
   STATISTICS_SECURE_FIELDS: process.env.STATISTICS_SECURE_FIELDS
     ? process.env.STATISTICS_SECURE_FIELDS.split(',')
diff --git a/src/services/MemberService.js b/src/services/MemberService.js
index 9ab48bf..330ae18 100644
--- a/src/services/MemberService.js
+++ b/src/services/MemberService.js
@@ -51,6 +51,11 @@ function omitMemberAttributes (currentUser, mb) {
   if (!helper.canManageMember(currentUser, mb)) {
     res = _.omit(res, config.MEMBER_SECURE_FIELDS)
   }
+  // If a user has one of the autocomplete role, allow them to see fname, lname, email
+  if(!helper.hasAutocompleteRole(currentUser)){
+    res = _.omit(res, config.COMMUNICATION_SECURE_FIELDS)
+  }
+
   return res
 }
 
diff --git a/src/services/SearchService.js b/src/services/SearchService.js
index 753660e..38bdd3e 100644
--- a/src/services/SearchService.js
+++ b/src/services/SearchService.js
@@ -30,6 +30,19 @@ var MEMBER_STATS_FIELDS = ['userId', 'handle', 'handleLower', 'maxRating',
 
 const esClient = helper.getESClient()
 
+function omitMemberAttributes (currentUser, query, allowedValues) {
+  // validate and parse fields param
+  let fields = helper.parseCommaSeparatedString(query.fields, allowedValues) || allowedValues
+  // if current user is not admin and not M2M, then exclude the admin/M2M only fields
+  if (!currentUser || (!currentUser.isMachine && !helper.hasAdminRole(currentUser))) {
+    fields = _.without(fields, ...config.MEMBER_SECURE_FIELDS)
+  }
+  // If the current user does not have an autocompleterole, remove the communication fields
+  if(!currentUser || (!currentUser.isMachine && !helper.hasAutocompleteRole(currentUser))){
+    fields = _.without(fields, ...config.COMMUNICATION_SECURE_FIELDS)
+  }
+  return fields
+}
 /**
  * Search members.
  * @param {Object} currentUser the user who performs operation
@@ -37,13 +50,7 @@ const esClient = helper.getESClient()
  * @returns {Object} the search result
  */
 async function searchMembers (currentUser, query) {
-  // validate and parse fields param
-  let fields = helper.parseCommaSeparatedString(query.fields, MEMBER_FIELDS) || MEMBER_FIELDS
-  // if current user is not admin and not M2M, then exclude the admin/M2M only fields
-  if (!currentUser || (!currentUser.isMachine && !helper.hasAdminRole(currentUser))) {
-    fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
-    MEMBER_STATS_FIELDS = _.without(MEMBER_STATS_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
-  }
+  fields = omitMemberAttributes(currentUser, query, MEMBER_FIELDS)
 
   if (query.email != null && query.email.length > 0) {
     if (currentUser == null) {
@@ -138,14 +145,15 @@ async function fillMembers(docsMembers, query, fields) {
       }
       return item
     })
-    // filter member based on fields
-    results = _.map(results, (item) => _.pick(item, fields))
 
     // sort the data
     results = _.orderBy(resultMbrsSkillsStats, [query.sortBy, "handleLower"], [query.sortOrder] )
   }
 
   results = helper.paginate(results, query.perPage, query.page - 1)
+  // filter member based on fields
+  results = _.map(results, (item) => _.pick(item, fields))
+
   return { total: total, page: query.page, perPage: query.perPage, result: results }
 }
 
@@ -189,13 +197,7 @@ searchMembersBySkills.schema = {
  * @returns {Promise<*[]|{total, perPage, numberOfPages: number, data: *[], page}>}
  */
 const searchMembersBySkillsWithOptions = async (currentUser, query, skillsFilter, skillsBooleanOperator, page, perPage, sortBy, sortOrder, esClient) => {
-  let fields = helper.parseCommaSeparatedString(query.fields, MEMBER_FIELDS) || MEMBER_FIELDS
-  // if current user is not admin and not M2M, then exclude the admin/M2M only fields
-  if (!currentUser || (!currentUser.isMachine && !helper.hasAdminRole(currentUser))) {
-    fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
-    MEMBER_STATS_FIELDS = _.without(MEMBER_STATS_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
-  }
-
+  fields = omitMemberAttributes(currentUser, query, MEMBER_FIELDS)
   const emptyResult = {
     total: 0,
     page,
@@ -220,13 +222,8 @@ const searchMembersBySkillsWithOptions = async (currentUser, query, skillsFilter
  * @returns {Object} the autocomplete result
  */
 async function autocomplete (currentUser, query) {
-  // validate and parse fields param
-  let fields = helper.parseCommaSeparatedString(query.fields, MEMBER_AUTOCOMPLETE_FIELDS) || MEMBER_AUTOCOMPLETE_FIELDS
-  // if current user is not autocomplete role and not M2M, then exclude the autocomplete/M2M only fields
-  if (!currentUser || (!currentUser.isMachine && !helper.hasAutocompleteRole(currentUser))) {
-    fields = _.without(fields, ...config.SEARCH_SECURE_FIELDS)
-    // MEMBER_AUTOCOMPLETE_FIELDS = _.without(MEMBER_AUTOCOMPLETE_FIELDS, ...config.STATISTICS_SECURE_FIELDS)
-  }
+  fields = omitMemberAttributes(currentUser, query, MEMBER_AUTOCOMPLETE_FIELDS)
+
   // get suggestion based on querys term
   const docsSuggestions = await eshelper.getSuggestion(query, esClient, currentUser)
   if (docsSuggestions.hasOwnProperty('suggest')) {

From f6e7d081f45f4cf2c5006668e921c0573d46c00f Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 8 Jun 2023 10:13:02 +1000
Subject: [PATCH 17/21] Postman updates

---
 docs/member-api.postman_collection.json  | 57 ++++++++++++++++++++++--
 docs/member-api.postman_environment.json | 18 ++++++--
 2 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/docs/member-api.postman_collection.json b/docs/member-api.postman_collection.json
index 627f4f0..f26f881 100644
--- a/docs/member-api.postman_collection.json
+++ b/docs/member-api.postman_collection.json
@@ -1,9 +1,8 @@
 {
 	"info": {
-		"_postman_id": "9f8d8103-0043-4959-b0cf-8e833845cc57",
+		"_postman_id": "515f8efb-72f4-4ccd-8a20-0543397f4e5c",
 		"name": "member-api",
-		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-		"_exporter_id": "10740"
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
 	},
 	"item": [
 		{
@@ -7169,6 +7168,58 @@
 								}
 							},
 							"response": []
+						},
+						{
+							"name": "search all members - by skill",
+							"request": {
+								"method": "GET",
+								"header": [],
+								"url": {
+									"raw": "{{URL}}/members/searchBySkills?skillId={{skill_id_1}}",
+									"host": [
+										"{{URL}}"
+									],
+									"path": [
+										"members",
+										"searchBySkills"
+									],
+									"query": [
+										{
+											"key": "skillId",
+											"value": "{{skill_id_1}}"
+										}
+									]
+								}
+							},
+							"response": []
+						},
+						{
+							"name": "search all members - by skills",
+							"request": {
+								"method": "GET",
+								"header": [],
+								"url": {
+									"raw": "{{URL}}/members/searchBySkills?skillId={{skill_id_1}}&skillId={{skil_id_2}}",
+									"host": [
+										"{{URL}}"
+									],
+									"path": [
+										"members",
+										"searchBySkills"
+									],
+									"query": [
+										{
+											"key": "skillId",
+											"value": "{{skill_id_1}}"
+										},
+										{
+											"key": "skillId",
+											"value": "{{skil_id_2}}"
+										}
+									]
+								}
+							},
+							"response": []
 						}
 					],
 					"event": [
diff --git a/docs/member-api.postman_environment.json b/docs/member-api.postman_environment.json
index 078b03f..5e3eb60 100644
--- a/docs/member-api.postman_environment.json
+++ b/docs/member-api.postman_environment.json
@@ -1,5 +1,5 @@
 {
-	"id": "e4ccf752-810c-4a8d-9a72-9d8535ac20d8",
+	"id": "0ed61d4c-ecad-41b4-b29f-ca3086939151",
 	"name": "member-api",
 	"values": [
 		{
@@ -41,9 +41,19 @@
 			"key": "m2m_update",
 			"value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczovL3RvcGNvZGVyLWRldi5hdXRoMC5jb20vIiwic3ViIjoiZW5qdzE4MTBlRHozWFR3U08yUm4yWTljUVRyc3BuM0JAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8vbTJtLnRvcGNvZGVyLWRldi5jb20vIiwiaWF0IjoxNTUwOTA2Mzg4LCJleHAiOjE2ODA5OTI3ODgsImF6cCI6ImVuancxODEwZUR6M1hUd1NPMlJuMlk5Y1FUcnNwbjNCIiwic2NvcGUiOiJ1cGRhdGU6bWVtYmVycyIsImd0eSI6ImNsaWVudC1jcmVkZW50aWFscyJ9.wImcvhkF9QPOCSEfZ01U-YxYM8NZi1yqgRmw3eiNn1Q",
 			"enabled": true
+		},
+		{
+			"key": "skill_id_1",
+			"value": "KS1200771D9CR9LB4MWW",
+			"enabled": true
+		},
+		{
+			"key": "skil_id_2",
+			"value": "KS121F45VPV8C9W3QFYH",
+			"enabled": true
 		}
 	],
 	"_postman_variable_scope": "environment",
-	"_postman_exported_at": "2020-02-14T13:59:07.169Z",
-	"_postman_exported_using": "Postman/7.13.0"
-}
+	"_postman_exported_at": "2023-06-08T00:12:44.668Z",
+	"_postman_exported_using": "Postman/8.5.1"
+}
\ No newline at end of file

From ea09ff57620d348ed5cf1b1b81b7f312cd8d00e7 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 8 Jun 2023 12:09:34 +1000
Subject: [PATCH 18/21] Updated swagger for new endpoint

---
 docs/swagger.yaml | 90 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 89 insertions(+), 1 deletion(-)

diff --git a/docs/swagger.yaml b/docs/swagger.yaml
index 64c15ca..b8cd630 100644
--- a/docs/swagger.yaml
+++ b/docs/swagger.yaml
@@ -55,7 +55,11 @@ info:
     
     ## Member Secure Fields
     
-    Member identifiable info fields, only admin, M2M, or member himself can get these fields - `firstName`, `lastName`, `email`, `addresses`, `createdBy`, `updatedBy`
+    Member identifiable info fields, only admin, M2M, or member himself can get these fields - `addresses`, `createdBy`, `updatedBy`
+    
+    ## Member Communication Fields 
+    
+    Member fields used for communication are accessible by managers, copilots, and admins.  These include: `firstName`, `lastName`, `email`
     
     ## Member Traits Secure Fields
     
@@ -646,6 +650,90 @@ paths:
           description: Internal server error
           schema:
             $ref: '#/definitions/ErrorModel'
+  '/members/searchBySkills':
+    get:
+      tags:
+        - Search
+      description: 
+        Search members by EMSI skill id(s) provided.  This API is used for the talent search.  By default, the results are sorted by the number of challenges won.
+      parameters:
+        - name: skillID
+          in: query
+          required: true
+          type: string
+          description: >
+            skillId=skillID1&skillId=skillID2
+            EMSI skill id(s) to use when filtering members.  Members who match all of the skill IDs provided will be returned.
+        - name: fields
+          in: query
+          required: false
+          type: string
+          description: >
+            fields=fieldName1,fieldName2,...,fieldN
+            
+            
+            parameter for choosing which fields of members profile that will be included in response.
+            
+            + userId - Select the field userId
+            
+            + handle - Select the field handle
+            
+            + firstName - Select the field firstName
+            
+            + lastName - Select the field lastName
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/perPage'
+        - name: sortOrder
+          in: query
+          required: false
+          type: string
+          description: sort by asc or desc
+        - name: sortBy
+          in: query
+          required: false
+          type: string
+          description: Field to sort by.  Options include 'userId', 'country', 'handle', 'firstName', 'lastName',  'numberOfChallengesWon', 'numberOfChallengesPlaced'
+      responses:
+        '200':
+          description: OK
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/MemberSearchDataItem'
+          headers:
+            X-Next-Page:
+              type: integer
+              description: The index of the next page
+            X-Page:
+              type: integer
+              description: The index of the current page (starting at 1)
+            X-Per-Page:
+              type: integer
+              description: The number of items to list per page
+            X-Prev-Page:
+              type: integer
+              description: The index of the previous page
+            X-Total:
+              type: integer
+              description: The total number of items
+            X-Total-Pages:
+              type: integer
+              description: The total number of pages
+            Link:
+              type: string
+              description: Pagination link header.
+        '400':
+          description: Bad request data
+          schema:
+            $ref: '#/definitions/ErrorModel'
+        '403':
+          description: No permission to access the API
+          schema:
+            $ref: '#/definitions/ErrorModel'
+        '500':
+          description: Internal server error
+          schema:
+            $ref: '#/definitions/ErrorModel'
   '/members/{handle}/traits':
     get:
       tags:

From 7498cac1c1ed86c7b59999e186ebd27d922e29c3 Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 8 Jun 2023 14:55:39 +1000
Subject: [PATCH 19/21] Lock down search by skills to just admins, for the time
 being

---
 src/routes.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/routes.js b/src/routes.js
index 7895e00..e0dfc23 100644
--- a/src/routes.js
+++ b/src/routes.js
@@ -28,6 +28,7 @@ module.exports = {
       controller: 'SearchController',
       method: 'searchMembersBySkills',
       auth: 'jwt',
+      access: constants.ADMIN_ROLES,
       scopes: [MEMBERS.READ, MEMBERS.ALL]
     }
   },

From 5cd47761ba00a7deaae42bb57f841fcc6ce9816a Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 8 Jun 2023 15:02:55 +1000
Subject: [PATCH 20/21] Fix import

---
 src/routes.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/routes.js b/src/routes.js
index e0dfc23..a7b0037 100644
--- a/src/routes.js
+++ b/src/routes.js
@@ -2,7 +2,7 @@
  * Contains all routes
  */
 
-// const constants = require('../app-constants')
+const constants = require('../app-constants')
 const { SCOPES: {
   MEMBERS
 } } = require('config')

From 9b08eea4b2500c9d8c3edef6afc6f0a9c44c177a Mon Sep 17 00:00:00 2001
From: Justin Gasper <jmgasper@gmail.com>
Date: Thu, 8 Jun 2023 15:09:22 +1000
Subject: [PATCH 21/21] Fix problem seen in dev

---
 src/common/helper.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/helper.js b/src/common/helper.js
index 8925c51..cbbe79e 100644
--- a/src/common/helper.js
+++ b/src/common/helper.js
@@ -131,7 +131,7 @@ function hasSearchByEmailRole (authUser) {
  * @returns {Boolean} whether the user has autocomplete role
  */
 function hasAutocompleteRole (authUser) {
-  if (!authUser.roles) {
+  if (!authUser || !authUser.roles) {
     return false
   }
   for (let i = 0; i < authUser.roles.length; i += 1) {