From 430b1d97794709d1c35d9dec0b77cb248eb8dff9 Mon Sep 17 00:00:00 2001 From: Trevor Richards Date: Thu, 5 Oct 2023 14:24:29 -0700 Subject: [PATCH 1/3] feat: add scheduler to remove expired EdxUser priv --- .../repository/EdxUserDistrictRepository.java | 3 + .../repository/EdxUserSchoolRepository.java | 3 + .../api/edx/schedulers/EdxUserScheduler.java | 41 ++++++++++ .../bc/gov/educ/api/edx/BaseEdxAPITest.java | 13 ++- .../api/edx/BaseSecureExchangeAPITest.java | 0 .../edx/schedulers/EdxUserSchedulerTest.java | 82 +++++++++++++++++++ api/src/test/resources/application.properties | 2 + 7 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 api/src/main/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserScheduler.java create mode 100644 api/src/test/java/ca/bc/gov/educ/api/edx/BaseSecureExchangeAPITest.java create mode 100644 api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java diff --git a/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserDistrictRepository.java b/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserDistrictRepository.java index 320b3c19..095be1bf 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserDistrictRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserDistrictRepository.java @@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -24,4 +25,6 @@ public interface EdxUserDistrictRepository extends JpaRepository findDistrictsByPermission(String permissionCode); + + List findAllByExpiryDateBefore(LocalDateTime dateTime); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserSchoolRepository.java b/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserSchoolRepository.java index 7657af66..28f17630 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserSchoolRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/edx/repository/EdxUserSchoolRepository.java @@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -26,4 +27,6 @@ public interface EdxUserSchoolRepository extends JpaRepository findAllBySchoolIDIn(List schoolIDs); + List findAllByExpiryDateBefore(LocalDateTime dateTime); + } diff --git a/api/src/main/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserScheduler.java b/api/src/main/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserScheduler.java new file mode 100644 index 00000000..195320ef --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserScheduler.java @@ -0,0 +1,41 @@ +package ca.bc.gov.educ.api.edx.schedulers; + +import ca.bc.gov.educ.api.edx.model.v1.EdxUserDistrictEntity; +import ca.bc.gov.educ.api.edx.model.v1.EdxUserSchoolEntity; +import ca.bc.gov.educ.api.edx.repository.*; +import net.javacrumbs.shedlock.core.LockAssert; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; + +@Component +public class EdxUserScheduler { + private final EdxUserSchoolRepository edxUserSchoolRepository; + private final EdxUserDistrictRepository edxUserDistrictRepository; + + public EdxUserScheduler( + EdxUserSchoolRepository edxUserSchoolRepository, + EdxUserDistrictRepository edxUserDistrictRepository + ) { + this.edxUserSchoolRepository = edxUserSchoolRepository; + this.edxUserDistrictRepository = edxUserDistrictRepository; + } + + @Scheduled(cron = "${scheduled.jobs.purge.edx.users.cron}") + @SchedulerLock(name = "PurgeEdxUsers", lockAtLeastFor = "PT4H", lockAtMostFor = "PT4H") + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void purgeExpiredEdxUsers() { + LockAssert.assertLocked(); + final LocalDateTime now = LocalDateTime.now(); + List expiredSchoolUsers = this.edxUserSchoolRepository.findAllByExpiryDateBefore(now); + List expiredDistrictUsers = this.edxUserDistrictRepository.findAllByExpiryDateBefore(now); + + this.edxUserSchoolRepository.deleteAll(expiredSchoolUsers); + this.edxUserDistrictRepository.deleteAll(expiredDistrictUsers); + } +} diff --git a/api/src/test/java/ca/bc/gov/educ/api/edx/BaseEdxAPITest.java b/api/src/test/java/ca/bc/gov/educ/api/edx/BaseEdxAPITest.java index fe935ac9..66ed944a 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/edx/BaseEdxAPITest.java +++ b/api/src/test/java/ca/bc/gov/educ/api/edx/BaseEdxAPITest.java @@ -48,7 +48,6 @@ public void before() { this.secureExchangeAPITestUtils.cleanDB(); } - protected EdxUserEntity createUserEntity(EdxUserRepository edxUserRepository, EdxPermissionRepository edxPermissionRepository, EdxRoleRepository edxRoleRepository, EdxUserSchoolRepository edxUserSchoolRepository, EdxUserDistrictRepository edxUserDistrictRepository) { var entity = edxUserRepository.save(getEdxUserEntity()); @@ -217,6 +216,12 @@ protected EdxUserSchool createEdxUserSchool(EdxUser edxUsr) { return edxUserSchool; } + protected EdxUserSchool createEdxUserSchool(EdxUser edxUsr, LocalDateTime expiryDate) { + EdxUserSchool edxUserSchool = this.createEdxUserSchool(edxUsr); + edxUserSchool.setExpiryDate(expiryDate.toString()); + return edxUserSchool; + } + protected EdxUserDistrict createEdxUserDistrict(EdxUser edxUsr) { EdxUserDistrict edxUserDistrict = new EdxUserDistrict(); edxUserDistrict.setEdxUserID(edxUsr.getEdxUserID()); @@ -224,6 +229,12 @@ protected EdxUserDistrict createEdxUserDistrict(EdxUser edxUsr) { return edxUserDistrict; } + protected EdxUserDistrict createEdxUserDistrict(EdxUser edxUsr, LocalDateTime expiryDate) { + EdxUserDistrict edxUserDistrict = this.createEdxUserDistrict(edxUsr); + edxUserDistrict.setExpiryDate(expiryDate.toString()); + return edxUserDistrict; + } + protected List createActivationCodeTableDataForSchoolUser(EdxActivationCodeRepository edxActivationCodeRepository, EdxPermissionRepository edxPermissionRepository, EdxRoleRepository edxRoleRepository, EdxActivationRoleRepository edxActivationRoleRepository, boolean isActive, UUID validationCode, Integer numberOfClicks, UUID schoolID) { List edxActivationCodeEntityList = new ArrayList<>(); EdxRoleEntity savedRoleEntity = createRoleAndPermissionData(edxPermissionRepository, edxRoleRepository); diff --git a/api/src/test/java/ca/bc/gov/educ/api/edx/BaseSecureExchangeAPITest.java b/api/src/test/java/ca/bc/gov/educ/api/edx/BaseSecureExchangeAPITest.java new file mode 100644 index 00000000..e69de29b diff --git a/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java b/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java new file mode 100644 index 00000000..b8940447 --- /dev/null +++ b/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java @@ -0,0 +1,82 @@ +package ca.bc.gov.educ.api.edx.schedulers; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; + +import ca.bc.gov.educ.api.edx.BaseEdxAPITest; +import ca.bc.gov.educ.api.edx.mappers.v1.EdxUserDistrictMapper; +import ca.bc.gov.educ.api.edx.mappers.v1.EdxUserMapper; +import ca.bc.gov.educ.api.edx.mappers.v1.EdxUserSchoolMapper; +import ca.bc.gov.educ.api.edx.model.v1.EdxUserDistrictEntity; +import ca.bc.gov.educ.api.edx.model.v1.EdxUserSchoolEntity; +import ca.bc.gov.educ.api.edx.repository.EdxUserDistrictRepository; +import ca.bc.gov.educ.api.edx.repository.EdxUserRepository; +import ca.bc.gov.educ.api.edx.repository.EdxUserSchoolRepository; +import ca.bc.gov.educ.api.edx.struct.v1.EdxUser; + +class EdxUserSchedulerTest extends BaseEdxAPITest { + @Autowired + private EdxUserSchoolRepository userSchoolRepository; + + @Autowired + private EdxUserDistrictRepository userDistrictRepository; + + @Autowired + private EdxUserRepository userRepository; + + @Autowired + private EdxUserScheduler scheduler; + + private EdxUserSchoolMapper userSchoolMapper = EdxUserSchoolMapper.mapper; + private EdxUserDistrictMapper userDistrictMapper = EdxUserDistrictMapper.mapper; + private EdxUserMapper userMapper = EdxUserMapper.mapper; + + @BeforeEach + public void setUp() { + // LockAssert.TestHelper.makeAllAssertsPass(true); + } + + @AfterEach + public void tearDown() { + this.userSchoolRepository.deleteAll(); + this.userDistrictRepository.deleteAll(); + this.userRepository.deleteAll(); + } + + @Test + void testPurgeExpiredUsers() { + LocalDateTime now = LocalDateTime.now().withNano(0); + EdxUser user = userMapper.toStructure(this.userRepository.save(userMapper.toModel(this.createEdxUser()))); + + List userSchools = List.of( + this.createEdxUserSchool(user, now.minusDays(1)), + this.createEdxUserSchool(user, now.plusDays(1)) + ).stream().map(userSchoolMapper::toModel).toList(); + + List userDistricts = List.of( + this.createEdxUserDistrict(user, now.minusDays(1)), + this.createEdxUserDistrict(user, now.plusDays(1)) + ).stream().map(userDistrictMapper::toModel).toList(); + + this.userSchoolRepository.saveAll(userSchools); + this.userDistrictRepository.saveAll(userDistricts); + scheduler.purgeExpiredEdxUsers(); + + List foundUserSchoolEntities = this.userSchoolRepository.findAll(); + assertThat(foundUserSchoolEntities).hasSize(1); + assertThat(foundUserSchoolEntities.get(0).getExpiryDate()).isAfter(now); + + List foundUserDistrictEntities = this.userDistrictRepository.findAll(); + assertThat(foundUserDistrictEntities).hasSize(1); + assertThat(foundUserDistrictEntities.get(0).getExpiryDate()).isAfter(now); + } + +} diff --git a/api/src/test/resources/application.properties b/api/src/test/resources/application.properties index 453a7eb3..378b0022 100644 --- a/api/src/test/resources/application.properties +++ b/api/src/test/resources/application.properties @@ -55,6 +55,8 @@ scheduled.jobs.extract.uncompleted.sagas.cron=- scheduled.jobs.extract.uncompleted.sagas.cron.lockAtLeastFor=55s scheduled.jobs.extract.uncompleted.sagas.cron.lockAtMostFor=58s +scheduled.jobs.purge.edx.users.cron=- + apis.endpoints.student.api=blah institute.api.url=blah From 2c43fea96bf821eaaab4d98b6bdb85f2ea986137 Mon Sep 17 00:00:00 2001 From: Trevor Richards Date: Thu, 5 Oct 2023 15:37:05 -0700 Subject: [PATCH 2/3] fix: put LockAssert.TestHelper back --- .../bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java b/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java index b8940447..51e13260 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java +++ b/api/src/test/java/ca/bc/gov/educ/api/edx/schedulers/EdxUserSchedulerTest.java @@ -4,7 +4,6 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.UUID; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.AfterEach; @@ -21,6 +20,7 @@ import ca.bc.gov.educ.api.edx.repository.EdxUserRepository; import ca.bc.gov.educ.api.edx.repository.EdxUserSchoolRepository; import ca.bc.gov.educ.api.edx.struct.v1.EdxUser; +import net.javacrumbs.shedlock.core.LockAssert; class EdxUserSchedulerTest extends BaseEdxAPITest { @Autowired @@ -41,7 +41,7 @@ class EdxUserSchedulerTest extends BaseEdxAPITest { @BeforeEach public void setUp() { - // LockAssert.TestHelper.makeAllAssertsPass(true); + LockAssert.TestHelper.makeAllAssertsPass(true); } @AfterEach From 7e814e12b7b4d5b76f67258b63fb3624601348e6 Mon Sep 17 00:00:00 2001 From: Trevor Richards Date: Thu, 5 Oct 2023 15:46:54 -0700 Subject: [PATCH 3/3] feat: add configmap for edx user purge cron --- tools/config/update-configmap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/config/update-configmap.sh b/tools/config/update-configmap.sh index 276bd487..26e652fe 100644 --- a/tools/config/update-configmap.sh +++ b/tools/config/update-configmap.sh @@ -413,6 +413,7 @@ fi SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON="0 0/1 * * * *" SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_LEAST_FOR="55s" SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_MOST_FOR="57s" +SCHEDULED_JOBS_PURGE_EDX_USERS_CRON="0 30 0 * * *" EDX_ACTIVATION_CODE_LENGTH="8" EDX_ACTIVATION_CODE_VALID_CHARACTERS="ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" echo