From d5520aee5d8b21fcffc0baf8cc86acc8887a9fbb Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sun, 10 Oct 2021 08:21:56 +0530 Subject: [PATCH] Display Completed Jobs (#109) * class and variables renamed to schedule* from delay* * display complete jobs in dashboard * fix message move message count #105 * Fixed #112 * * doc updated * README updates * Potential cross slot error in rename * remove doble tagging * fixes for url web prefix * added filter to select number of days/months/weeks to select data --- .travis.yml | 63 ------ CHANGELOG.md | 30 ++- README.md | 12 +- build.gradle | 2 +- rqueue-core/build.gradle | 2 +- .../rqueue/annotation/RqueueListener.java | 18 +- .../rqueue/common/RqueueRedisTemplate.java | 4 + .../sonus21/rqueue/config/RqueueConfig.java | 80 +++++-- .../config/RqueueListenerBaseConfig.java | 38 ++-- .../rqueue/config/RqueueSchedulerConfig.java | 14 +- .../rqueue/config/RqueueWebConfig.java | 11 + .../SimpleRqueueListenerContainerFactory.java | 24 +-- .../com/github/sonus21/rqueue/core/Job.java | 13 +- .../sonus21/rqueue/core/MessageScheduler.java | 204 +++++++++--------- ...a => ProcessingQueueMessageScheduler.java} | 4 +- .../core/ReactiveRqueueMessageEnqueuer.java | 28 +-- .../rqueue/core/RqueueMessageEnqueuer.java | 50 ++--- .../rqueue/core/RqueueMessageSender.java | 16 +- .../rqueue/core/RqueueMessageTemplate.java | 6 +- ...va => ScheduledQueueMessageScheduler.java} | 16 +- .../rqueue/core/context/DefaultContext.java | 6 +- .../rqueue/core/impl/BaseMessageSender.java | 17 +- .../rqueue/core/impl/MessageSweeper.java | 22 +- .../ReactiveRqueueMessageEnqueuerImpl.java | 3 +- .../core/impl/RqueueMessageManagerImpl.java | 2 +- .../core/impl/RqueueMessageSenderImpl.java | 2 +- .../core/impl/RqueueMessageTemplateImpl.java | 6 +- .../core/middleware/LoggingMiddleware.java | 4 +- .../rqueue/core/middleware/Middleware.java | 5 +- .../core/middleware/PermissionMiddleware.java | 3 +- .../core/middleware/ProfilerMiddleware.java | 2 +- .../rqueue/core/support/MessageProcessor.java | 2 +- .../rqueue/dao/RqueueMessageMetadataDao.java | 2 +- .../sonus21/rqueue/dao/RqueueStringDao.java | 7 + .../impl/RqueueMessageMetadataDaoImpl.java | 4 +- .../rqueue/dao/impl/RqueueStringDaoImpl.java | 21 ++ .../sonus21/rqueue/listener/JobImpl.java | 32 ++- .../listener/MessageProcessorHandler.java | 8 +- .../listener/PostProcessingHandler.java | 8 +- .../sonus21/rqueue/listener/QueueDetail.java | 27 +-- .../rqueue/listener/RqueueExecutor.java | 4 +- .../rqueue/listener/RqueueMessageHandler.java | 78 +++---- .../RqueueMessageListenerContainer.java | 131 +++++------ .../rqueue/listener/RqueueMessagePoller.java | 2 +- .../rqueue/listener/StrictPriorityPoller.java | 5 +- .../listener/WeightedPriorityPoller.java | 5 +- .../sonus21/rqueue/metrics/RqueueMetrics.java | 15 +- .../rqueue/metrics/RqueueQueueMetrics.java | 18 +- .../SystemUtil.java => models/Pair.java} | 19 +- .../rqueue/models/aggregator/QueueEvents.java | 2 +- .../rqueue/models/db/MessageMetadata.java | 1 + .../sonus21/rqueue/models/db/QueueConfig.java | 3 +- .../sonus21/rqueue/models/db/RqueueJob.java | 2 +- .../rqueue/models/enums/MessageStatus.java | 11 +- .../sonus21/rqueue/models/enums/NavTab.java | 1 + .../models/request/ChartDataRequest.java | 16 ++ .../models/request/DataDeleteRequest.java | 7 +- .../models/request/DateViewRequest.java | 2 +- .../models/response/DataSelectorResponse.java | 36 ++++ .../sonus21/rqueue/utils/Constants.java | 4 +- .../sonus21/rqueue/utils/DateTimeUtils.java | 21 +- .../sonus21/rqueue/utils/HttpUtils.java | 24 ++- .../sonus21/rqueue/utils/RedisUtils.java | 3 +- .../sonus21/rqueue/utils/StringUtils.java | 3 +- .../sonus21/rqueue/utils/ValueResolver.java | 3 +- .../ExponentialTaskExecutionBackOff.java | 10 +- .../backoff/FixedTaskExecutionBackOff.java | 2 +- .../utils/backoff/TaskExecutionBackOff.java | 7 +- .../rqueue/web/controller/BaseController.java | 35 +++ .../controller/BaseReactiveController.java | 36 ++++ .../ReactiveRqueueRestController.java | 111 +++++----- .../ReactiveRqueueViewController.java | 88 +++----- .../web/controller/RqueueRestController.java | 107 ++++----- .../web/controller/RqueueViewController.java | 84 +++----- ...=> RqueueJobMetricsAggregatorService.java} | 14 +- .../service/RqueueMessageMetadataService.java | 15 +- .../web/service/RqueueQDetailService.java | 8 +- .../web/service/RqueueUtilityService.java | 7 +- .../impl/RqueueDashboardChartServiceImpl.java | 42 ++-- .../RqueueMessageMetadataServiceImpl.java | 50 ++++- .../impl/RqueueQDetailServiceImpl.java | 123 ++++++++--- .../impl/RqueueSystemManagerServiceImpl.java | 83 ++++++- .../impl/RqueueUtilityServiceImpl.java | 94 +++++++- .../impl/RqueueViewControllerServiceImpl.java | 25 +-- .../spring-configuration-metadata.json | 27 ++- .../resources/public/rqueue/css/rqueue.css | 5 - .../main/resources/public/rqueue/js/rqueue.js | 31 ++- .../main/resources/templates/rqueue/base.html | 2 +- .../resources/templates/rqueue/index.html | 1 + .../templates/rqueue/latency_chart.html | 52 +++-- .../templates/rqueue/queue_detail.html | 2 +- .../templates/rqueue/stats_chart.html | 8 + .../common/RqueueLockManagerImplTest.java | 4 +- .../rqueue/config/RqueueConfigTest.java | 21 +- .../config/RqueueListenerBaseConfigTest.java | 12 +- .../converter/JsonMessageConverterTest.java | 60 +++--- .../core/MessageSchedulerDisabledTest.java | 15 +- .../rqueue/core/MessageSchedulerTest.java | 17 +- ... ProcessingQueueMessageSchedulerTest.java} | 15 +- .../rqueue/core/RqueueMessageSenderTest.java | 16 +- .../core/RqueueMessageTemplateTest.java | 10 +- ...> ScheduledQueueMessageSchedulerTest.java} | 67 +++--- .../impl/RqueueMessageEnqueuerImplTest.java | 12 +- .../impl/RqueueMessageManagerImplTest.java | 6 +- .../middleware/RedisLockMiddlewareTest.java | 4 +- .../rqueue/dao/RqueueSystemConfigDaoTest.java | 6 +- .../listener/ConcurrentListenerTest.java | 3 +- .../sonus21/rqueue/listener/JobImplTest.java | 69 +++++- .../listener/PriorityGroupListenerTest.java | 2 +- .../rqueue/listener/QueueDetailTest.java | 13 +- .../rqueue/listener/RqueueExecutorTest.java | 20 +- .../RqueueMessageListenerContainerTest.java | 68 +++--- .../rqueue/listener/RqueueMiddlewareTest.java | 11 +- .../rqueue/metrics/QueueCounterTest.java | 2 +- .../rqueue/metrics/RqueueCounterTest.java | 2 +- .../rqueue/metrics/RqueueMetricsTest.java | 30 +-- .../metrics/RqueueQueueMetricsTest.java | 96 +++++++++ .../models/request/ChartDataRequestTest.java | 37 ++++ .../rqueue/utils/DateTimeUtilsTest.java | 23 ++ .../sonus21/rqueue/utils/HttpUtilsTest.java | 38 ++++ .../sonus21/rqueue/utils/TestUtils.java | 6 +- .../RqueueDashboardChartServiceTest.java | 2 +- .../RqueueMessageMetadataServiceTest.java | 9 +- .../web/service/RqueueQDetailServiceTest.java | 17 +- .../RqueueSystemManagerServiceTest.java | 14 +- ...queueTaskMetricsAggregatorServiceTest.java | 20 +- .../RqueueSystemManagerServiceImplTest.java | 17 +- rqueue-spring-boot-example/JMeter.jmx | 2 +- rqueue-spring-boot-example/README.md | 34 +++ rqueue-spring-boot-example/grafana.json | 4 +- .../sonus21/rqueue/example/Controller.java | 30 ++- .../rqueue/example/RQueueApplication.java | 5 +- .../src/main/resources/application.properties | 5 +- rqueue-spring-boot-reactive-example/README.md | 31 +++ .../sonus21/task/executor/Controller.java | 37 +++- .../src/main/resources/application.properties | 4 +- ...eReactiveTaskExecutorApplicationTests.java | 4 +- rqueue-spring-boot-starter/build.gradle | 2 +- .../spring/boot/application/Application.java | 2 - .../ApplicationListenerDisabled.java | 2 - .../ApplicationWithCustomConfiguration.java | 2 - ...pplicationWithCustomMessageConverter.java} | 8 +- .../ApplicationWithMessageProcessor.java | 34 +++ .../application/ApplicationWithRestart.java | 2 - .../ApplicationWithTaskExecutionBackoff.java | 12 +- .../application/BootMetricApplication.java | 2 - .../MultiRedisSetupApplication.java | 2 - .../application/ProducerOnlyApplication.java | 2 - .../application/RedisClusterApplication.java | 2 - .../boot/reactive/ReactiveWebApplication.java | 7 +- .../tests/integration/ApplicationTest.java | 20 +- .../tests/integration/BootMetricsTest.java | 4 +- ...est.java => BootScheduledChannelTest.java} | 4 +- .../CustomMessageConverterTest.java | 4 +- .../integration/MessageProcessorTest.java | 134 ++++++++++++ .../integration/ReactiveRestApiTest.java | 84 ++++++-- .../integration/ReactiveWebDisabledTest.java | 139 ++++-------- ...eWebTest.java => ReactiveWebViewTest.java} | 6 +- .../test/UserBannedMessageListener.java | 3 +- ...pplicationWithCustomMessageConverter.java} | 2 +- .../ApplicationWithMessageProcessor.java | 61 ++++++ ...ry.java => ApplicationWithMiddleware.java} | 2 +- .../rqueue/test/common/SpringTestBase.java | 15 +- .../rqueue/test/entity/ConsumedMessage.java | 19 +- .../rqueue/test/tests/AllQueueMode.java | 14 +- .../rqueue/test/tests/BasicListenerTest.java | 20 +- .../test/tests/MessageChannelTests.java | 14 +- .../sonus21/rqueue/test/tests/MetricTest.java | 10 +- .../sonus21/rqueue/test/tests/RetryTests.java | 15 +- .../util/TestMessageConverterProvider.java | 2 +- .../test/util/TestMessageProcessor.java | 54 +++++ rqueue-spring-example/README.md | 31 +++ .../rqueue/spring/example/Controller.java | 43 +++- .../main/java/rqueue/spring/example/Main.java | 1 + .../spring/example/MessageListener.java | 62 +++++- rqueue-spring/build.gradle | 2 +- .../integration/DefaultListenerGroup.java | 36 ++-- .../integration/RqueueRestControllerTest.java | 102 +++++++-- ...edTest.java => RqueueWebDisabledTest.java} | 109 ++-------- .../tests/integration/SpringMetricTest.java | 4 +- .../WeightedMultiLevelQueueListener.java | 2 +- .../tests/unit/RqueueMessageConfigTest.java | 2 +- .../java/com/github/sonus21/TestBase.java | 4 +- .../sonus21/junit/RedisBootstrapper.java | 8 +- .../sonus21/junit/RedisBootstrapperBase.java | 16 +- .../src/main/resources/logback.xml | 2 +- test.sh | 14 ++ 187 files changed, 2807 insertions(+), 1559 deletions(-) delete mode 100644 .travis.yml rename rqueue-core/src/main/java/com/github/sonus21/rqueue/core/{ProcessingMessageScheduler.java => ProcessingQueueMessageScheduler.java} (95%) rename rqueue-core/src/main/java/com/github/sonus21/rqueue/core/{DelayedMessageScheduler.java => ScheduledQueueMessageScheduler.java} (74%) rename rqueue-core/src/main/java/com/github/sonus21/rqueue/{utils/SystemUtil.java => models/Pair.java} (69%) create mode 100644 rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataSelectorResponse.java create mode 100644 rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java create mode 100644 rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java rename rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/{RqueueTaskMetricsAggregatorService.java => RqueueJobMetricsAggregatorService.java} (96%) rename rqueue-core/src/test/java/com/github/sonus21/rqueue/core/{ProcessingMessageSchedulerTest.java => ProcessingQueueMessageSchedulerTest.java} (90%) rename rqueue-core/src/test/java/com/github/sonus21/rqueue/core/{DelayedMessageSchedulerTest.java => ScheduledQueueMessageSchedulerTest.java} (83%) create mode 100644 rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsTest.java create mode 100644 rqueue-core/src/test/java/com/github/sonus21/rqueue/utils/HttpUtilsTest.java rename rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/{MessageConverterApplication.java => ApplicationWithCustomMessageConverter.java} (78%) create mode 100644 rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationWithMessageProcessor.java rename rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/{BootDelayedChannelTest.java => BootScheduledChannelTest.java} (94%) create mode 100644 rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageProcessorTest.java rename rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/{ReactiveWebTest.java => ReactiveWebViewTest.java} (97%) rename rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/{CustomMessageConverterApplication.java => ApplicationWithCustomMessageConverter.java} (95%) create mode 100644 rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationWithMessageProcessor.java rename rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/{ApplicationWithContainerFactory.java => ApplicationWithMiddleware.java} (98%) create mode 100644 rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/util/TestMessageProcessor.java rename rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/{RqueueViewsDisabledTest.java => RqueueWebDisabledTest.java} (75%) create mode 100755 test.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a6d04692..00000000 --- a/.travis.yml +++ /dev/null @@ -1,63 +0,0 @@ -language: java -jdk: - - openjdk8 - -services: - - redis - -env: - global: - - ORG_GRADLE_PROJECT_sonatypeUsername=xxx - - ORG_GRADLE_PROJECT_sonatypePassword=xxx - - USER_NAME=rqueue - - REDIS_RUNNING=true - - CI_ENV=true - -cache: - directories: - - $HOME/.gradle - -before_script: - - mkdir 9000 9001 9002 9003 9004 9005 - - printf "port 9000 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9000/redis.conf - - printf "port 9001 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9001/redis.conf - - printf "port 9002 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9002/redis.conf - - printf "port 9003 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9003/redis.conf - - printf "port 9004 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9004/redis.conf - - printf "port 9005 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9005/redis.conf - - cd 9000 && redis-server ./redis.conf & - - cd 9001 && redis-server ./redis.conf & - - cd 9002 && redis-server ./redis.conf & - - cd 9003 && redis-server ./redis.conf & - - cd 9004 && redis-server ./redis.conf & - - cd 9005 && redis-server ./redis.conf & - - redis-cli --cluster create 127.0.0.1:9000 127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.0.0.1:9005 --cluster-replicas 1 --cluster-yes - - df -h - - lscpu - -jobs: - include: - - stage: spring-boot-2.1 - env: - - SPRING_BOOT_VERSION=2.1.0.RELEASE - - SPRING_VERSION=5.1.2.RELEASE - - SPRING_DATA_VERSION=2.1.2.RELEASE - - MICROMETER_VERSION=1.1.0 - script: - - ./gradlew codeCoverageReport - - stage: spring-boot-2.2 - env: - - SPRING_BOOT_VERSION=2.2.0.RELEASE - - SPRING_VERSION=5.2.0.RELEASE - - SPRING_DATA_VERSION=2.2.0.RELEASE - - MICROMETER_VERSION=1.3.2 - script: - - ./gradlew codeCoverageReport - - -notifications: - email: - - sonunitw12@gmail.com - -after_success: - - ./gradlew coveralls diff --git a/CHANGELOG.md b/CHANGELOG.md index 5722e39f..437c0613 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,28 @@ # [Rqueue] New and Notable Changes +### [2.10.0] - 10-Oct-2021 + +### Fixes + +* Fixes for post processor calls (post processor calls were not made) +* Fixes message move message count (by default 1000 messages are moved) +* Potential issue in rename collection +* More than one (-) sign in the dashboard +* Fixes for server context path. Rqueue end points would be served relative to x-forwarded-prefix/server.servlet.context-path + +### Features + +* Display completed jobs in the dashboard +* Option to choose number of days in the chart + ReactiveWebViewTest ### [2.9.0] - 30-Jul-2021 + ### Fixes -* Option to add rqueue web url prefix, the prefix is configured from application.properties file using -`rqueue.web.url.prefix=/my-application/`, now rqueue dashboard would be served -at `/my-application/rquque` instead of `/rqueue`, the configuration has higher priority than the -HTTP request header `x-forwarded-prefix`. - + +* Option to add rqueue web url prefix, the prefix is configured from application.properties file + using `rqueue.web.url.prefix=/my-application/`, now rqueue dashboard would be served + at `/my-application/rquque` instead of `/rqueue`, the configuration has higher priority than the + HTTP request header `x-forwarded-prefix`. * Custom message converter is not working * RedisCommandExecutionException : command arguments must be strings or integers @@ -196,7 +212,7 @@ Fixes: ### Fixed * An issue in the scheduler that's always scheduling job at the delay of 5 seconds. (this leads to - messages are not copied from delayed queue to main queue on high load) + messages are not copied from scheduled queue to main queue on high load) ## [1.2] - 03-Nov-2019 @@ -255,3 +271,5 @@ Fixes: [2.8.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core/2.8.0-RELEASE [2.9.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core/2.9.0-RELEASE + +[2.10.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core/2.10.0-RELEASE diff --git a/README.md b/README.md index 54bb6cb5..26c42558 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
Rqueue Logo -

Rqueue: Redis Queue, Task Queue, Delayed Queue for Spring and Spring Boot

+

Rqueue: Redis Queue, Task Queue, Scheduled Queue for Spring and Spring Boot

[![Build Status](https://circleci.com/gh/sonus21/rqueue/tree/master.svg?style=shield)](https://circleci.com/gh/sonus21/rqueue/tree/master) @@ -32,7 +32,7 @@ well, where all services code is in Spring. * **Automatic message serialization and deserialization** * **Message Multicasting** : Call multiple message listeners on every message * **Batch Message Polling** : Fetch multiple messages from Redis at once -* **Metrics** : In flight messages, waiting for consumption and delayed messages +* **Metrics** : In flight messages, waiting for consumption and scheduled messages * **Competing Consumers** : multiple messages can be consumed in parallel by different workers/listeners. * **Concurrency** : Concurrency of any listener can be configured @@ -71,14 +71,14 @@ Release Version: [Maven central](https://search.maven.org/search?q=g:com.github. * Add dependency * Gradle ```groovy - implementation 'com.github.sonus21:rqueue-spring-boot-starter:2.9.0-RELEASE' + implementation 'com.github.sonus21:rqueue-spring-boot-starter:2.10.0-RELEASE' ``` * Maven ```xml com.github.sonus21 rqueue-spring-boot-starter - 2.9.0-RELEASE + 2.10.0-RELEASE ``` @@ -91,14 +91,14 @@ Release Version: [Maven central](https://search.maven.org/search?q=g:com.github. * Add Dependency * Gradle ```groovy - implementation 'com.github.sonus21:rqueue-spring:2.9.0-RELEASE' + implementation 'com.github.sonus21:rqueue-spring:2.10.0-RELEASE' ``` * Maven ```xml com.github.sonus21 rqueue-spring - 2.9.0-RELEASE + 2.10.0-RELEASE ``` diff --git a/build.gradle b/build.gradle index 46380241..29727f72 100644 --- a/build.gradle +++ b/build.gradle @@ -70,7 +70,7 @@ ext { subprojects { group = 'com.github.sonus21' - version = '2.9.0-RELEASE' + version = '2.10.0-RELEASE' dependencies { // https://mvnrepository.com/artifact/org.springframework/spring-messaging diff --git a/rqueue-core/build.gradle b/rqueue-core/build.gradle index ef821974..60db6976 100644 --- a/rqueue-core/build.gradle +++ b/rqueue-core/build.gradle @@ -1,4 +1,4 @@ -ext.projectDescription = 'Asynchronous and delayed task executor' +ext.projectDescription = 'Asynchronous and scheduled task executor' ext.name = 'Rqueue Core Java' buildscript { apply from: "${rootDir}/gradle/code-signing.gradle" diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java index 8203c6e8..52369742 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java @@ -40,7 +40,6 @@ * public class MessageListener { * @RqueueListener( * value="${job.queue}", - * delayedQueue="true", * numRetries="3", * deadLetterQueue="#{job.dead.letter.queue}", * visibilityTimeout="30*60*1000") @@ -62,16 +61,6 @@ */ String[] value() default {}; - /** - * All queues are considered delayed for fast recovery, it can not be disabled even by setting - * this to false. - * - * @deprecated since 2.0 - * @return whether it's delayed queue or not. - */ - @Deprecated - String delayedQueue() default "true"; - /** * Number of times a message should be retried before it can be discarded or send it to dead * letter queue in case of consecutive failures. This is a global value for a consumer, each @@ -99,8 +88,9 @@ String deadLetterQueue() default ""; /** - * By default messages sent to dead letter queue are not consumable by listener. This flag is used - * to turn on the consumable feature of dead letter queue. + * By default, messages sent to dead letter queue are not consumable by listener. This flag is + * used to turn on the consumable feature of dead letter queue. If this is set to true then + * application should add message listener for the dead letter queue. * * @return true/false */ @@ -128,7 +118,7 @@ * listeners, that will violate exactly once processing. On the other hand if provided time is too * high then the message would be hidden from other consumers for a longer period. * - *

NOTE: This time is in milli seconds + *

NOTE: This time is in milliseconds * * @return visibilityTimeout visibility timeout */ diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java index 3ffeeb62..9883d4b1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java @@ -129,6 +129,10 @@ public Set> zrangeWithScore(String key, long start, long end) { return redisTemplate.opsForZSet().rangeWithScores(key, start, end); } + public void zremRangeByScore(String key, long min, long max) { + redisTemplate.opsForZSet().removeRangeByScore(key, min, max); + } + public Set getMembers(String key) { return redisTemplate.opsForSet().members(key); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java index 8252eb07..18ab6cdf 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java @@ -43,12 +43,12 @@ @Configuration public class RqueueConfig { + private static final String brokerId = UUID.randomUUID().toString(); + private static final AtomicLong counter = new AtomicLong(1); private final RedisConnectionFactory connectionFactory; private final ReactiveRedisConnectionFactory reactiveRedisConnectionFactory; private final boolean sharedConnection; private final int dbVersion; - private static final String brokerId = UUID.randomUUID().toString(); - private static final AtomicLong counter = new AtomicLong(1); @Value("${rqueue.reactive.enabled:false}") private boolean reactiveEnabled; @@ -83,11 +83,14 @@ public class RqueueConfig { @Value("${rqueue.simple.queue.prefix:}") private String simpleQueuePrefix; - @Value("${rqueue.delayed.queue.prefix:}") - private String delayedQueuePrefix; + @Value("${rqueue.scheduled.queue.prefix:}") + private String scheduledQueuePrefix; - @Value("${rqueue.delayed.queue.channel.prefix:}") - private String delayedQueueChannelPrefix; + @Value("${rqueue.completed.queue.prefix:}") + private String completedQueuePrefix; + + @Value("${rqueue.scheduled.queue.channel.prefix:}") + private String scheduledQueueChannelPrefix; @Value("${rqueue.processing.queue.name.prefix:}") private String processingQueuePrefix; @@ -139,6 +142,21 @@ public class RqueueConfig { @Value("${rqueue.internal.communication.channel.name.prefix:i-channel}") private String internalChannelNamePrefix; + @Value("${rqueue.completed.job.cleanup.interval:30000}") + private long completedJobCleanupIntervalInMs; + + public static String getBrokerId() { + return brokerId; + } + + public boolean messageInTerminalStateShouldBeStored() { + return getMessageDurabilityInTerminalStateInSecond() > 0; + } + + public long messageDurabilityInTerminalStateInMillisecond() { + return getMessageDurabilityInTerminalStateInSecond() * Constants.ONE_MILLI; + } + public String getInternalCommChannelName() { return prefix + internalChannelNamePrefix; } @@ -157,9 +175,9 @@ private String getSimpleQueueSuffix() { return "queue-v2::"; } - private String getDelayedQueueSuffix() { - if (!StringUtils.isEmpty(delayedQueuePrefix)) { - return delayedQueuePrefix; + private String getScheduledQueueSuffix() { + if (!StringUtils.isEmpty(scheduledQueuePrefix)) { + return scheduledQueuePrefix; } if (dbVersion == 2) { return "d-queue::"; @@ -167,9 +185,16 @@ private String getDelayedQueueSuffix() { return "d-queue-v2::"; } - private String getDelayedQueueChannelSuffix() { - if (!StringUtils.isEmpty(delayedQueueChannelPrefix)) { - return delayedQueueChannelPrefix; + private String getCompletedQueueSuffix() { + if (!StringUtils.isEmpty(completedQueuePrefix)) { + return completedQueuePrefix; + } + return "c-queue::"; + } + + private String getScheduledQueueChannelSuffix() { + if (!StringUtils.isEmpty(scheduledQueueChannelPrefix)) { + return scheduledQueueChannelPrefix; } if (dbVersion == 2) { return "d-channel::"; @@ -204,18 +229,22 @@ public String getQueueName(String queueName) { return prefix + getSimpleQueueSuffix() + getTaggedName(queueName); } - public String getDelayedQueueName(String queueName) { + public String getCompletedQueueName(String queueName) { + return prefix + getCompletedQueueSuffix() + getTaggedName(queueName); + } + + public String getScheduledQueueName(String queueName) { if (dbVersion == 1) { return "rqueue-delay::" + queueName; } - return prefix + getDelayedQueueSuffix() + getTaggedName(queueName); + return prefix + getScheduledQueueSuffix() + getTaggedName(queueName); } - public String getDelayedQueueChannelName(String queueName) { + public String getScheduledQueueChannelName(String queueName) { if (dbVersion == 1) { return "rqueue-channel::" + queueName; } - return prefix + getDelayedQueueChannelSuffix() + getTaggedName(queueName); + return prefix + getScheduledQueueChannelSuffix() + getTaggedName(queueName); } public String getProcessingQueueName(String queueName) { @@ -248,6 +277,18 @@ private String getTaggedName(String queueName) { if (!clusterMode) { return queueName; } + boolean left = false; + boolean right = false; + for (Character c : queueName.toCharArray()) { + if (c == '{') { + left = true; + } else if (c == '}') { + right = true; + } + } + if (left && right) { + return queueName; + } return "{" + queueName + "}"; } @@ -259,11 +300,12 @@ public String getJobsKey(String messageId) { return prefix + jobsCollectionNamePrefix + messageId; } - public String getDelDataName() { + public String getDelDataName(String queueName) { return prefix + delPrefix + brokerId + Constants.REDIS_KEY_SEPARATOR + + getTaggedName(queueName) + counter.incrementAndGet(); } @@ -271,10 +313,6 @@ public Duration getJobDurabilityInTerminalState() { return Duration.ofSeconds(jobDurabilityInTerminalStateInSecond); } - public static String getBrokerId() { - return brokerId; - } - public String getLibVersion() { if (StringUtils.isEmpty(version)) { ClassPathResource resource = diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java index 4ef77042..6cac1307 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java @@ -22,12 +22,12 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.common.impl.RqueueLockManagerImpl; import com.github.sonus21.rqueue.converter.MessageConverterProvider; -import com.github.sonus21.rqueue.core.DelayedMessageScheduler; -import com.github.sonus21.rqueue.core.ProcessingMessageScheduler; +import com.github.sonus21.rqueue.core.ProcessingQueueMessageScheduler; import com.github.sonus21.rqueue.core.RqueueBeanProvider; import com.github.sonus21.rqueue.core.RqueueInternalPubSubChannel; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; +import com.github.sonus21.rqueue.core.ScheduledQueueMessageScheduler; import com.github.sonus21.rqueue.core.impl.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.dao.impl.RqueueStringDaoImpl; @@ -55,12 +55,12 @@ * This is a base configuration class for Rqueue, that is used in Spring and Spring boot Rqueue libs * for configurations. This class creates required beans to work Rqueue library. * - *

It internally maintains two types of scheduled tasks for different functionality, for delayed - * queue messages have to be moved from ZSET to LIST, in other case to at least once message - * delivery guarantee, messages have to be moved from ZSET to LIST again, we expect very small - * number of messages in processing queue. Reason being we delete messages once it's consumed, but - * due to failure in listeners message might not be removed, whereas message in a delayed queue can - * be very high based on the use case. + *

It internally maintains two types of scheduled tasks for different functionality, for + * scheduled queue messages have to be moved from ZSET to LIST, in other case to at least once + * message delivery guarantee, messages have to be moved from ZSET to LIST again, we expect very + * small number of messages in processing queue. Reason being we delete messages once it's consumed, + * but due to failure in listeners message might not be removed, whereas message in a scheduled + * queue can be very high based on the use case. */ public abstract class RqueueListenerBaseConfig { @@ -68,6 +68,10 @@ public abstract class RqueueListenerBaseConfig { private static final String TEMPLATE_DIR = "templates/rqueue/"; private static final String TEMPLATE_SUFFIX = ".html"; + @Autowired(required = false) + protected final SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = + new SimpleRqueueListenerContainerFactory(); + @Value("${rqueue.reactive.enabled:false}") protected boolean reactiveEnabled; @@ -95,10 +99,6 @@ protected MessageConverterProvider getMessageConverterProvider() { } } - @Autowired(required = false) - protected final SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = - new SimpleRqueueListenerContainerFactory(); - /** * Create Rqueue configuration bean either from listener container factory or from bean factory. * 1st priority is given to container factory. This redis connection factory is used to connect to @@ -184,25 +184,25 @@ public RqueueRedisListenerContainerFactory rqueueRedisListenerContainerFactory() } /** - * This scheduler is used to pull messages from a delayed queue to their respective queue. + * This scheduler is used to pull messages from a scheduled queue to their respective queue. * Internally it moves messages from ZSET to LIST based on the priority and current time. * - * @return {@link DelayedMessageScheduler} object + * @return {@link ScheduledQueueMessageScheduler} object */ @Bean - public DelayedMessageScheduler delayedMessageScheduler() { - return new DelayedMessageScheduler(); + public ScheduledQueueMessageScheduler scheduledMessageScheduler() { + return new ScheduledQueueMessageScheduler(); } /** * This scheduler is used to pull messages from processing queue to their respective queue. * Internally it moves messages from ZSET to LIST based on the priority and current time. * - * @return {@link ProcessingMessageScheduler} object + * @return {@link ProcessingQueueMessageScheduler} object */ @Bean - public ProcessingMessageScheduler processingMessageScheduler() { - return new ProcessingMessageScheduler(); + public ProcessingQueueMessageScheduler processingMessageScheduler() { + return new ProcessingQueueMessageScheduler(); } @Bean diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueSchedulerConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueSchedulerConfig.java index 82e7083a..4189af2e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueSchedulerConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueSchedulerConfig.java @@ -56,15 +56,15 @@ public class RqueueSchedulerConfig { @Value("${rqueue.scheduler.redis.enabled:true}") private boolean redisEnabled; - // Number of threads used to process delayed queue messages - @Value("${rqueue.scheduler.delayed.message.thread.pool.size:3}") - private int delayedMessageThreadPoolSize; + // Number of threads used to process scheduled-queue messages + @Value("${rqueue.scheduler.scheduled.message.thread.pool.size:3}") + private int scheduledMessageThreadPoolSize; - // Number of threads used to process processing queue messages + // Number of threads used to process processing-queue messages @Value("${rqueue.scheduler.processing.message.thread.pool.size:1}") private int processingMessageThreadPoolSize; - // How frequently messages should be moved from delayed queues to source queue - @Value("${rqueue.scheduler.delayed.message.time.interval:5000}") - private long delayedMessageTimeIntervalInMilli; + // How frequently messages should be moved from scheduled queues to source queue + @Value("${rqueue.scheduler.scheduled.message.time.interval:5000}") + private long scheduledMessageTimeIntervalInMilli; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueWebConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueWebConfig.java index 3a1ac9ac..a854a41a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueWebConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueWebConfig.java @@ -16,6 +16,8 @@ package com.github.sonus21.rqueue.config; +import com.github.sonus21.rqueue.utils.HttpUtils; +import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; import org.springframework.beans.factory.annotation.Value; @@ -32,9 +34,14 @@ public class RqueueWebConfig { @Value("${rqueue.web.enable:true}") private boolean enable; + /** Base path for Rqueue web endpoints. Relative to server.servlet.context-path */ @Value("${rqueue.web.url.prefix:}") + @Getter(AccessLevel.NONE) private String urlPrefix; + @Value("${server.servlet.context-path:}") + private String servletContextPath; + @Value("${rqueue.web.max.message.move.count:1000}") private int maxMessageMoveCount; @@ -67,4 +74,8 @@ public class RqueueWebConfig { // lock duration for aggregate job, acquired per queue @Value("${rqueue.web.collect.statistic.aggregate.event.lock.duration:500}") private int aggregateEventLockDurationInMs; + + public String getUrlPrefix(String xForwardedPrefix) { + return HttpUtils.joinPath(xForwardedPrefix, servletContextPath, urlPrefix); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java index 5a1a6e07..0ec48857 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java @@ -51,9 +51,9 @@ @SuppressWarnings("WeakerAccess") public class SimpleRqueueListenerContainerFactory { + private final List middlewares = new LinkedList<>(); // The message converter provider that will return a message converter to convert messages to/from private MessageConverterProvider messageConverterProvider; - // Provide task executor, this can be used to provide some additional details like some threads // name, etc otherwise a default task executor would be created private AsyncTaskExecutor taskExecutor; @@ -65,14 +65,12 @@ public class SimpleRqueueListenerContainerFactory { private ReactiveRedisConnectionFactory reactiveRedisConnectionFactory; // Custom requeue message handler private RqueueMessageHandler rqueueMessageHandler; - - // Send message poll time when no messages are available + // polling interval (millisecond) when no messages are available private long pollingInterval = 200L; - // In case of failure how much time, we should wait for next job + // In case of job execution failure how long(in millisecond) this job should be delayed private long backOffTime = 5 * Constants.ONE_MILLI; - // Number of workers requires for execution + // Number of workers requires for listeners private Integer maxNumWorkers; - // This message processor would be called before a task can start execution. // It needs to be noted that this message processor would be called multiple time // In case of retry, so application should be able to handle that. @@ -88,9 +86,6 @@ public class SimpleRqueueListenerContainerFactory { private MessageProcessor postExecutionMessageProcessor; // Any custom message requeue message template. private RqueueMessageTemplate rqueueMessageTemplate; - - private final List middlewares = new LinkedList<>(); - // Any message headers that should be set, headers are NOT STORED in db so it should not be // changed, same header is used in serialized and deserialization process. private MessageHeaders messageHeaders; @@ -106,9 +101,10 @@ public class SimpleRqueueListenerContainerFactory { private boolean inspectAllBean = true; /** - * Whenever a consumer fails then the consumed message can be delayed for further consumption. The - * delay of that can be configured, by default same message would be retried in 5 seconds and this - * will continue due to default task interval. {@link + * Whenever a consumer fails then the consumed message can be scheduled for further retry. The + * delay of such retry can be configured based on the different configuration, by default same + * message would be retried in 5 seconds and this will continue till all retries are not exhausted + * due to default task interval. {@link * com.github.sonus21.rqueue.utils.backoff.FixedTaskExecutionBackOff#DEFAULT_INTERVAL} * * @see com.github.sonus21.rqueue.utils.backoff.ExponentialTaskExecutionBackOff @@ -429,7 +425,7 @@ public void setPreExecutionMessageProcessor(MessageProcessor preExecutionMessage /** * Get configured polling interval * - * @return the time in milli seconds + * @return the time in milliseconds */ public long getPollingInterval() { return pollingInterval; @@ -438,7 +434,7 @@ public long getPollingInterval() { /** * Set polling time interval, this controls the listener polling interval * - * @param pollingInterval time in milli seconds + * @param pollingInterval time in milliseconds */ public void setPollingInterval(long pollingInterval) { this.pollingInterval = pollingInterval; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/Job.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/Job.java index 516544c0..46b499e1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/Job.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/Job.java @@ -96,6 +96,13 @@ public interface Job { */ boolean updateVisibilityTimeout(Duration deltaDuration); + /** + * A message that was enqueued + * + * @return an object could be null if deserialization fail. + */ + Object getMessage(); + /** * There are times when message can not be deserialized from the string to Object, this can happen * when class information is missing. In such cases only raw message is available but application @@ -112,12 +119,6 @@ public interface Job { * @param message message object */ void setMessage(Object message); - /** - * A message that was enqueued - * - * @return an object could be null if deserialization fail. - */ - Object getMessage(); /** * MessageMetadata corresponding the enqueued message diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java index 150d4437..21354ede 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java @@ -54,8 +54,8 @@ public abstract class MessageScheduler implements DisposableBean, ApplicationListener { - @Autowired protected RqueueSchedulerConfig rqueueSchedulerConfig; private final Object monitor = new Object(); + @Autowired protected RqueueSchedulerConfig rqueueSchedulerConfig; @Autowired protected RqueueConfig rqueueConfig; private RedisScript redisScript; private MessageSchedulerListener messageSchedulerListener; @@ -190,111 +190,14 @@ private boolean isQueueActive(String queueName) { return val; } - private void addTask(MessageMoverTask timerTask, ScheduledTaskDetail scheduledTaskDetail) { - getLogger().debug("Timer: {}, Task: {}", timerTask, scheduledTaskDetail); - queueNameToScheduledTask.put(timerTask.getName(), scheduledTaskDetail); - } - - private void checkExistingTask( - ScheduledTaskDetail scheduledTaskDetail, - long currentTime, - QueueDetail queueDetail, - String zsetName) { - // run existing tasks continue - long existingDelay = scheduledTaskDetail.getStartTime() - currentTime; - Future submittedTask = scheduledTaskDetail.getFuture(); - boolean completedOrCancelled = submittedTask.isDone() || submittedTask.isCancelled(); - // tasks older than TASK_ALIVE_TIME are considered dead - if (!completedOrCancelled - && existingDelay < MIN_DELAY - && existingDelay > Constants.TASK_ALIVE_TIME) { - ThreadUtils.waitForTermination( - getLogger(), - submittedTask, - Constants.DEFAULT_SCRIPT_EXECUTION_TIME, - "LIST: {} ZSET: {}, Task: {} failed", - queueDetail.getQueueName(), - zsetName, - scheduledTaskDetail); - } - } - - private void scheduleTask( - long startTime, long currentTime, QueueDetail queueDetail, String zsetName) { - long requiredDelay = Math.max(1, startTime - currentTime); - long taskStartTime = startTime; - MessageMoverTask timerTask = - new MessageMoverTask( - queueDetail.getName(), - queueDetail.getQueueName(), - zsetName, - isProcessingQueue(queueDetail.getName())); - Future future; - if (requiredDelay < MIN_DELAY) { - future = scheduler.submit(timerTask); - taskStartTime = currentTime; - } else { - future = scheduler.schedule(timerTask, Instant.ofEpochMilli(currentTime + requiredDelay)); - } - addTask(timerTask, new ScheduledTaskDetail(taskStartTime, future)); - } - - private void scheduleNewTask( - QueueDetail queueDetail, String queueName, String zsetName, long startTime) { - MessageMoverTask timerTask = - new MessageMoverTask( - queueDetail.getName(), - queueDetail.getQueueName(), - zsetName, - isProcessingQueue(zsetName)); - Future future = - scheduler.schedule( - timerTask, Instant.ofEpochMilli(getNextScheduleTime(queueName, startTime))); - addTask(timerTask, new ScheduledTaskDetail(startTime, future)); - } - - private void updateLastScheduleTime(String queueName, long time) { - queueNameToLastMessageScheduleTime.put(queueName, time); - } - private long getLastScheduleTime(String queueName) { return queueNameToLastMessageScheduleTime.getOrDefault(queueName, 0L); } - private boolean shouldNotSchedule(String queueName, boolean forceSchedule) { - boolean isQueueActive = isQueueActive(queueName); - if (!isQueueActive || scheduler == null) { - return true; - } - long lastSeenTime = getLastScheduleTime(queueName); - long currentTime = System.currentTimeMillis(); - // ignore too frequents events - return !forceSchedule && currentTime - lastSeenTime < getMinDelay(); - } - protected ScheduledTaskDetail getScheduledTask(String queueName) { return queueNameToScheduledTask.get(queueName); } - private class QueueScheduler { - protected synchronized void schedule(String queueName, Long startTime, boolean forceSchedule) { - if (shouldNotSchedule(queueName, forceSchedule)) { - return; - } - long currentTime = System.currentTimeMillis(); - updateLastScheduleTime(queueName, currentTime); - ScheduledTaskDetail scheduledTaskDetail = getScheduledTask(queueName); - QueueDetail queueDetail = EndpointRegistry.get(queueName); - String zsetName = getZsetName(queueName); - if (scheduledTaskDetail == null || forceSchedule) { - scheduleTask(startTime, currentTime, queueDetail, zsetName); - return; - } - checkExistingTask(scheduledTaskDetail, currentTime, queueDetail, zsetName); - scheduleNewTask(queueDetail, queueName, zsetName, startTime); - } - } - protected void schedule(String queueName, Long startTime, boolean forceSchedule) { this.queueSchedulers.get(queueName).schedule(queueName, startTime, forceSchedule); } @@ -347,6 +250,107 @@ public void onApplicationEvent(RqueueBootstrapEvent event) { } } + protected long getMinDelay() { + return MIN_DELAY; + } + + private class QueueScheduler { + private void updateLastScheduleTime(String queueName, long time) { + queueNameToLastMessageScheduleTime.put(queueName, time); + } + + private void scheduleNewTask( + QueueDetail queueDetail, String queueName, String zsetName, long startTime) { + MessageMoverTask timerTask = + new MessageMoverTask( + queueDetail.getName(), + queueDetail.getQueueName(), + zsetName, + isProcessingQueue(zsetName)); + Future future = + scheduler.schedule( + timerTask, Instant.ofEpochMilli(getNextScheduleTime(queueName, startTime))); + addTask(timerTask, new ScheduledTaskDetail(startTime, future)); + } + + private void scheduleTask( + long startTime, long currentTime, QueueDetail queueDetail, String zsetName) { + long requiredDelay = Math.max(1, startTime - currentTime); + long taskStartTime = startTime; + MessageMoverTask timerTask = + new MessageMoverTask( + queueDetail.getName(), + queueDetail.getQueueName(), + zsetName, + isProcessingQueue(queueDetail.getName())); + Future future; + if (requiredDelay < MIN_DELAY) { + future = scheduler.submit(timerTask); + taskStartTime = currentTime; + } else { + future = scheduler.schedule(timerTask, Instant.ofEpochMilli(currentTime + requiredDelay)); + } + addTask(timerTask, new ScheduledTaskDetail(taskStartTime, future)); + } + + private boolean shouldNotSchedule(String queueName, boolean forceSchedule) { + boolean isQueueActive = isQueueActive(queueName); + if (!isQueueActive || scheduler == null) { + return true; + } + long lastSeenTime = getLastScheduleTime(queueName); + long currentTime = System.currentTimeMillis(); + // ignore too frequents events + return !forceSchedule && currentTime - lastSeenTime < getMinDelay(); + } + + protected synchronized void schedule(String queueName, Long startTime, boolean forceSchedule) { + if (shouldNotSchedule(queueName, forceSchedule)) { + return; + } + long currentTime = System.currentTimeMillis(); + updateLastScheduleTime(queueName, currentTime); + ScheduledTaskDetail scheduledTaskDetail = getScheduledTask(queueName); + QueueDetail queueDetail = EndpointRegistry.get(queueName); + String zsetName = getZsetName(queueName); + if (scheduledTaskDetail == null || forceSchedule) { + scheduleTask(startTime, currentTime, queueDetail, zsetName); + return; + } + checkExistingTask(scheduledTaskDetail, currentTime, queueDetail, zsetName); + scheduleNewTask(queueDetail, queueName, zsetName, startTime); + } + + private void addTask(MessageMoverTask timerTask, ScheduledTaskDetail scheduledTaskDetail) { + getLogger().debug("Timer: {}, Task: {}", timerTask, scheduledTaskDetail); + queueNameToScheduledTask.put(timerTask.getName(), scheduledTaskDetail); + } + + private void checkExistingTask( + ScheduledTaskDetail scheduledTaskDetail, + long currentTime, + QueueDetail queueDetail, + String zsetName) { + // run existing tasks continue + long existingDelay = scheduledTaskDetail.getStartTime() - currentTime; + Future submittedTask = scheduledTaskDetail.getFuture(); + boolean completedOrCancelled = submittedTask.isDone() || submittedTask.isCancelled(); + // tasks older than TASK_ALIVE_TIME are considered dead + if (!completedOrCancelled + && existingDelay < MIN_DELAY + && existingDelay > Constants.TASK_ALIVE_TIME) { + ThreadUtils.waitForTermination( + getLogger(), + submittedTask, + Constants.DEFAULT_SCRIPT_EXECUTION_TIME, + "LIST: {} ZSET: {}, Task: {} failed", + queueDetail.getQueueName(), + zsetName, + scheduledTaskDetail); + } + } + } + @ToString @AllArgsConstructor private class MessageMoverTask implements Runnable { @@ -383,10 +387,6 @@ public String getName() { } } - protected long getMinDelay() { - return MIN_DELAY; - } - private class MessageSchedulerListener implements MessageListener { private void handleMessage(String queueName, Long startTime) { long lastSeenTime = getLastScheduleTime(queueName); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageScheduler.java similarity index 95% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageScheduler.java index d8b54e3e..9a250647 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingMessageScheduler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageScheduler.java @@ -26,7 +26,7 @@ import org.slf4j.Logger; @Slf4j -public class ProcessingMessageScheduler extends MessageScheduler { +public class ProcessingQueueMessageScheduler extends MessageScheduler { private Map queueNameToDelay; @Override @@ -66,7 +66,7 @@ protected boolean isProcessingQueue(String queueName) { @Override protected String getThreadNamePrefix() { - return "processingMessageScheduler-"; + return "processingQueueMsgScheduler-"; } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ReactiveRqueueMessageEnqueuer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ReactiveRqueueMessageEnqueuer.java index 31d6fe61..734e01bc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ReactiveRqueueMessageEnqueuer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ReactiveRqueueMessageEnqueuer.java @@ -26,7 +26,7 @@ public interface ReactiveRqueueMessageEnqueuer { /** - * Enqueue a message on given queue without any delay, consume as soon as possible. + * Enqueue a message on given queue, consume as soon as possible. * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. @@ -35,7 +35,7 @@ public interface ReactiveRqueueMessageEnqueuer { Mono enqueue(String queueName, Object message); /** - * Enqueue a message on given queue without any delay, consume as soon as possible. + * Enqueue a message on given queue, consume as soon as possible. * * @param queueName on which queue message has to be send * @param messageId message id @@ -45,7 +45,7 @@ public interface ReactiveRqueueMessageEnqueuer { Mono enqueue(String queueName, String messageId, Object message); /** - * Enqueue unique message on a given queue without any delay, consume as soon as possible. + * Enqueue unique message on a given queue without, consume as soon as possible. * * @param queueName on which queue message has to be send * @param messageId the message id for uniqueness @@ -123,7 +123,7 @@ default Mono enqueueUniqueWithPriority( * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message id on successful enqueue otherwise null. */ Mono enqueueIn(String queueName, Object message, long delayInMilliSecs); @@ -135,7 +135,7 @@ default Mono enqueueUniqueWithPriority( * @param queueName on which queue message has to be send * @param messageId the message id, using which this message will be identified * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message was enqueue successfully or failed. */ Mono enqueueIn( @@ -200,9 +200,9 @@ default Mono enqueueIn( } /** - * Enqueue a message on given queue with delay, consume as soon as the delayed is expired. + * Enqueue a message on given queue with delay, consume as soon as the delay is expired. * - * @param queueName on which queue message has to be send + * @param queueName on which queue message has to be sent * @param messageId the message id for uniqueness * @param message message object it could be any arbitrary object. * @param delayInMillisecond total execution delay @@ -212,13 +212,13 @@ Mono enqueueUniqueIn( String queueName, String messageId, Object message, long delayInMillisecond); /** - * Enqueue a task that would be scheduled to run in the specified milli seconds. + * Enqueue a task that would be scheduled to run in the specified milliseconds. * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * @param delayInMilliSecs delay in milliseconds, this message would be only visible to the * listener when number of millisecond has elapsed. * @return message id on successful enqueue otherwise {@literal null} */ @@ -226,14 +226,14 @@ Mono enqueueInWithRetry( String queueName, Object message, int retryCount, long delayInMilliSecs); /** - * Enqueue a task that would be scheduled to run in the specified milli seconds. + * Enqueue a task that would be scheduled to run in the specified milliseconds. * - * @param queueName on which queue message has to be send + * @param queueName on which queue message has to be sent * @param messageId the message identifier * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * @param delayInMilliSecs delay in milliseconds, this message would be only visible to the * listener when number of millisecond has elapsed. * @return message was enqueue successfully or failed. */ @@ -247,7 +247,7 @@ Mono enqueueInWithRetry( * @param queueName on which queue message has to be send * @param priority the name of the priority level * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message id on successful enqueue otherwise {@literal null}. */ default Mono enqueueInWithPriority( @@ -264,7 +264,7 @@ default Mono enqueueInWithPriority( * @param priority the name of the priority level * @param messageId the message id * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message was enqueue successfully or failed. */ default Mono enqueueInWithPriority( diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java index 847ea4c5..ffb933d7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java @@ -71,7 +71,7 @@ public interface RqueueMessageEnqueuer { * * @param queueName on which queue message has to be send * @param messageId the message id for uniqueness - * @param message message object it could be any arbitrary object. + * @param message message object it could be any arbitrary object. * @return message id on successful enqueue otherwise null. */ boolean enqueueUnique(String queueName, String messageId, Object message); @@ -143,7 +143,7 @@ default boolean enqueueUniqueWithPriority( * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message id on successful enqueue otherwise null. */ String enqueueIn(String queueName, Object message, long delayInMilliSecs); @@ -155,7 +155,7 @@ default boolean enqueueUniqueWithPriority( * @param queueName on which queue message has to be send * @param messageId the message id, using which this message will be identified * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message was enqueue successfully or failed. */ boolean enqueueIn(String queueName, String messageId, Object message, long delayInMilliSecs); @@ -218,11 +218,11 @@ default boolean enqueueIn( } /** - * Enqueue a message on given queue with delay, consume as soon as the delayed is expired. + * Enqueue a message on given queue with delay, consume as soon as the scheduled is expired. * - * @param queueName on which queue message has to be send - * @param messageId the message id for uniqueness - * @param message message object it could be any arbitrary object. + * @param queueName on which queue message has to be send + * @param messageId the message id for uniqueness + * @param message message object it could be any arbitrary object. * @param delayInMillisecond total execution delay * @return message id on successful enqueue otherwise {@literal null}. */ @@ -230,13 +230,13 @@ boolean enqueueUniqueIn( String queueName, String messageId, Object message, long delayInMillisecond); /** - * Enqueue a task that would be scheduled to run in the specified milli seconds. + * Enqueue a task that would be scheduled to run in the specified milliseconds. * - * @param queueName on which queue message has to be send + * @param queueName on which queue message has to be sent * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * @param delayInMilliSecs delay in milliseconds, this message would be only visible to the * listener when number of millisecond has elapsed. * @return message id on successful enqueue otherwise {@literal null} */ @@ -244,14 +244,14 @@ String enqueueInWithRetry( String queueName, Object message, int retryCount, long delayInMilliSecs); /** - * Enqueue a task that would be scheduled to run in the specified milli seconds. + * Enqueue a task that would be scheduled to run in the specified milliseconds. * - * @param queueName on which queue message has to be send + * @param queueName on which queue message has to be sent * @param messageId the message identifier * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * @param delayInMilliSecs delay in milliseconds, this message would be only visible to the * listener when number of millisecond has elapsed. * @return message was enqueue successfully or failed. */ @@ -262,10 +262,10 @@ boolean enqueueInWithRetry( * Schedule a message on the given queue at the provided time. It will be executed as soon as the * given delay is elapse. * - * @param queueName on which queue message has to be send - * @param priority the name of the priority level - * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milliseconds * @return message id on successful enqueue otherwise {@literal null}. */ default String enqueueInWithPriority( @@ -278,11 +278,11 @@ default String enqueueInWithPriority( * Schedule a message on the given queue at the provided time. It will be executed as soon as the * given delay is elapse. * - * @param queueName on which queue message has to be send - * @param priority the name of the priority level - * @param messageId the message id - * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param messageId the message id + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milliseconds * @return message was enqueue successfully or failed. */ default boolean enqueueInWithPriority( @@ -476,9 +476,9 @@ default boolean enqueueAt(String queueName, String messageId, Object message, Da * Schedule unique messages on the given queue at the provided time. It will be available to * consume as soon as the given time is reached. * - * @param queueName on which queue message has to be send - * @param message message object it could be any arbitrary object. - * @param messageId a unique identifier for this message + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param messageId a unique identifier for this message * @param timeInMilliSeconds time at which this message has to be consumed. * @return message was enqueue successfully or failed. */ diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java index 33cb14e7..5bc94ca4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java @@ -75,7 +75,7 @@ default boolean put(String queueName, Object message) { * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message was submitted successfully or failed. */ default boolean put(String queueName, Object message, long delayInMilliSecs) { @@ -88,7 +88,7 @@ default boolean put(String queueName, Object message, long delayInMilliSecs) { * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message was submitted successfully or failed. */ boolean enqueueIn(String queueName, Object message, long delayInMilliSecs); @@ -193,7 +193,7 @@ default boolean put(String queueName, Object message, int retryCount) { * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * @param delayInMilliSecs delay in milliseconds, this message would be only visible to the * listener when number of millisecond has elapsed. * @return message was submitted successfully or failed. */ @@ -202,13 +202,13 @@ default boolean put(String queueName, Object message, int retryCount, long delay } /** - * Enqueue a task that would be scheduled to run in the specified milli seconds. + * Enqueue a task that would be scheduled to run in the specified milliseconds. * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * @param delayInMilliSecs delay in milliseconds, this message would be only visible to the * listener when number of millisecond has elapsed. * @return message was submitted successfully or failed. */ @@ -232,7 +232,7 @@ boolean enqueueInWithRetry( * @param queueName on which queue message has to be send * @param priority the name of the priority level * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds + * @param delayInMilliSecs delay in milliseconds * @return message was submitted successfully or failed. */ boolean enqueueInWithPriority( @@ -353,8 +353,8 @@ boolean moveMessageFromDeadLetterToQueue( Iterable getMessageConverters(); /** - * Find all messages stored on a given queue, it considers all the messages including delayed and - * non-delayed. + * Find all messages stored on a given queue, it considers all the messages including scheduled + * and non-scheduled. * * @param queueName queue name to be query for * @return list of messages. diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java index 3fb0873c..14651601 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java @@ -39,7 +39,7 @@ List pop( int count); Long addMessageWithDelay( - String delayQueueName, String delayQueueChannelName, RqueueMessage rqueueMessage); + String scheduleQueueName, String scheduleQueueChannelName, RqueueMessage rqueueMessage); void moveMessageWithDelay( String srcZsetName, String tgtZsetName, RqueueMessage src, RqueueMessage tgt, long delay); @@ -51,7 +51,7 @@ void moveMessageWithDelay( Boolean addToZset(String zsetName, RqueueMessage rqueueMessage, long score); List getAllMessages( - String queueName, String processingQueueName, String delayQueueName); + String queueName, String processingQueueName, String scheduleQueueName); Long getScore(String zsetName, RqueueMessage message); @@ -91,5 +91,5 @@ Long scheduleMessage( Mono addReactiveMessage(String queueName, RqueueMessage rqueueMessage); Flux addReactiveMessageWithDelay( - String delayedQueueName, String delayedQueueChannelName, RqueueMessage rqueueMessage); + String scheduledQueueName, String scheduledQueueChannelName, RqueueMessage rqueueMessage); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageScheduler.java similarity index 74% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageScheduler.java index ef9c0219..1a1484ef 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DelayedMessageScheduler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageScheduler.java @@ -20,7 +20,7 @@ import org.slf4j.Logger; @Slf4j -public class DelayedMessageScheduler extends MessageScheduler { +public class ScheduledQueueMessageScheduler extends MessageScheduler { @Override protected Logger getLogger() { @@ -31,38 +31,36 @@ protected Logger getLogger() { protected long getNextScheduleTime(String queueName, Long value) { long currentTime = System.currentTimeMillis(); if (value == null) { - return currentTime + rqueueSchedulerConfig.getDelayedMessageTimeIntervalInMilli(); + return currentTime + rqueueSchedulerConfig.getScheduledMessageTimeIntervalInMilli(); } if (value < currentTime) { return currentTime; } - return currentTime + rqueueSchedulerConfig.getDelayedMessageTimeIntervalInMilli(); + return currentTime + rqueueSchedulerConfig.getScheduledMessageTimeIntervalInMilli(); } @Override protected String getChannelName(String queueName) { - return EndpointRegistry.get(queueName).getDelayedQueueChannelName(); + return EndpointRegistry.get(queueName).getScheduledQueueChannelName(); } @Override protected String getZsetName(String queueName) { - return EndpointRegistry.get(queueName).getDelayedQueueName(); + return EndpointRegistry.get(queueName).getScheduledQueueName(); } @Override protected String getThreadNamePrefix() { - return "delayedMessageScheduler-"; + return "scheduledQueueMsgScheduler-"; } @Override protected int getThreadPoolSize() { - return rqueueSchedulerConfig.getDelayedMessageThreadPoolSize(); + return rqueueSchedulerConfig.getScheduledMessageThreadPoolSize(); } @Override protected boolean isProcessingQueue(String queueName) { return false; } - - } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/context/DefaultContext.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/context/DefaultContext.java index 9ba603d9..22caf1e6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/context/DefaultContext.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/context/DefaultContext.java @@ -24,8 +24,8 @@ */ public class DefaultContext implements Context { - private final Context parentContext; public static final Context EMPTY = new DefaultContext(null, null, null); + private final Context parentContext; private final Object key; private final Object value; @@ -39,8 +39,8 @@ private DefaultContext(Context parentContext, Object key, Object value) { * Create a new context from the parent context, null key is not allowed. * * @param parentContext parent context - * @param key key - * @param value value corresponding to this context + * @param key key + * @param value value corresponding to this context * @return a new context */ public static Context withValue(Context parentContext, Object key, Object value) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java index 7b7ff920..d0d021b6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java @@ -99,13 +99,13 @@ protected Object enqueue( } else { if (reactive) { return messageTemplate.addReactiveMessageWithDelay( - queueDetail.getDelayedQueueName(), - queueDetail.getDelayedQueueChannelName(), + queueDetail.getScheduledQueueName(), + queueDetail.getScheduledQueueChannelName(), rqueueMessage); } else { messageTemplate.addMessageWithDelay( - queueDetail.getDelayedQueueName(), - queueDetail.getDelayedQueueChannelName(), + queueDetail.getScheduledQueueName(), + queueDetail.getScheduledQueueChannelName(), rqueueMessage); } } @@ -179,8 +179,8 @@ protected void registerQueueInternal(String queueName, String... priorities) { .name(queueName) .active(false) .queueName(rqueueConfig.getQueueName(queueName)) - .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName)) - .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName)) + .scheduledQueueName(rqueueConfig.getScheduledQueueName(queueName)) + .scheduledQueueChannelName(rqueueConfig.getScheduledQueueChannelName(queueName)) .processingQueueName(rqueueConfig.getProcessingQueueName(queueName)) .processingQueueChannelName(rqueueConfig.getProcessingQueueChannelName(queueName)) .priority(priorityMap) @@ -193,8 +193,9 @@ protected void registerQueueInternal(String queueName, String... priorities) { .name(queueName + suffix) .active(false) .queueName(rqueueConfig.getQueueName(queueName) + suffix) - .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName) + suffix) - .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName) + suffix) + .scheduledQueueName(rqueueConfig.getScheduledQueueName(queueName) + suffix) + .scheduledQueueChannelName( + rqueueConfig.getScheduledQueueChannelName(queueName) + suffix) .processingQueueName(rqueueConfig.getProcessingQueueName(queueName) + suffix) .processingQueueChannelName( rqueueConfig.getProcessingQueueChannelName(queueName) + suffix) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java index 8e611234..035f37b7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java @@ -66,9 +66,8 @@ public static MessageSweeper getInstance( if (MessageSweeper.messageSweeper == null) { synchronized (MessageSweeper.class) { if (MessageSweeper.messageSweeper == null) { - MessageSweeper tmp = + MessageSweeper.messageSweeper = new MessageSweeper(rqueueConfig, messageTemplate, rqueueMessageMetadataDao); - MessageSweeper.messageSweeper = tmp; return MessageSweeper.messageSweeper; } return MessageSweeper.messageSweeper; @@ -85,25 +84,28 @@ public boolean deleteMessage(MessageDeleteRequest request) { List deleteJobData = new ArrayList<>(); QueueDetail detail = request.queueDetail; if (detail != null) { - String newQueueName = rqueueConfig.getDelDataName(); - String newDelayedZsetName = rqueueConfig.getDelDataName(); - String newProcessingZsetName = rqueueConfig.getDelDataName(); + String newQueueName = rqueueConfig.getDelDataName(detail.getQueueName()); + String newScheduledZsetName = rqueueConfig.getDelDataName(detail.getQueueName()); + String newProcessingZsetName = rqueueConfig.getDelDataName(detail.getQueueName()); messageTemplate.renameCollections( Arrays.asList( - detail.getQueueName(), detail.getDelayedQueueName(), detail.getProcessingQueueName()), - Arrays.asList(newQueueName, newDelayedZsetName, newProcessingZsetName)); + detail.getQueueName(), + detail.getScheduledQueueName(), + detail.getProcessingQueueName()), + Arrays.asList(newQueueName, newScheduledZsetName, newProcessingZsetName)); deleteJobData.add(new DeleteJobData(newQueueName, DataType.LIST)); - deleteJobData.add(new DeleteJobData(newDelayedZsetName, DataType.ZSET)); + deleteJobData.add(new DeleteJobData(newScheduledZsetName, DataType.ZSET)); deleteJobData.add(new DeleteJobData(newProcessingZsetName, DataType.ZSET)); } else { switch (request.dataType) { case LIST: - DeleteJobData data = new DeleteJobData(rqueueConfig.getDelDataName(), request.dataType); + DeleteJobData data = + new DeleteJobData(rqueueConfig.getDelDataName(request.dataName), request.dataType); messageTemplate.renameCollection(request.dataName, data.name); deleteJobData.add(data); break; case ZSET: - data = new DeleteJobData(rqueueConfig.getDelDataName(), request.dataType); + data = new DeleteJobData(rqueueConfig.getDelDataName(request.dataName), request.dataType); messageTemplate.renameCollection(request.dataName, data.name); deleteJobData.add(data); break; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java index b18c4065..b1f2bb20 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java @@ -306,8 +306,7 @@ public String call(Long a, Boolean b) { private static class BoolMonoConverter implements MonoConverter { - private BoolMonoConverter(RqueueMessage message) { - } + private BoolMonoConverter(RqueueMessage message) {} @Override public Boolean call(Long a, Boolean b) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java index 03134d76..065aa762 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java @@ -96,7 +96,7 @@ public List getAllRqueueMessage(String queueName) { return messageTemplate.getAllMessages( queueDetail.getQueueName(), queueDetail.getProcessingQueueName(), - queueDetail.getDelayedQueueName()); + queueDetail.getScheduledQueueName()); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java index a0462738..9a30d5f4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageSenderImpl.java @@ -117,7 +117,7 @@ public List getAllMessages(String queueName) { messageTemplate.getAllMessages( queueDetail.getQueueName(), queueDetail.getProcessingQueueName(), - queueDetail.getDelayedQueueName())) { + queueDetail.getScheduledQueueName())) { messages.add(RqueueMessageUtils.convertMessageToObject(message, messageConverter)); } return messages; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 1df1f2a4..aaeafb62 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -114,13 +114,13 @@ public Long addMessageWithDelay( @Override public Flux addReactiveMessageWithDelay( - String delayedQueueName, String delayedQueueChannelName, RqueueMessage rqueueMessage) { + String scheduledQueueName, String scheduledQueueChannelName, RqueueMessage rqueueMessage) { log.debug( - "AddReactiveMessageWithDelay Queue: {}, Message: {}", delayedQueueName, rqueueMessage); + "AddReactiveMessageWithDelay Queue: {}, Message: {}", scheduledQueueName, rqueueMessage); RedisScript script = getScript(ScriptType.ENQUEUE_MESSAGE); return reactiveScriptExecutor.execute( script, - Arrays.asList(delayedQueueName, delayedQueueChannelName), + Arrays.asList(scheduledQueueName, scheduledQueueChannelName), Arrays.asList(rqueueMessage, rqueueMessage.getProcessAt(), System.currentTimeMillis())); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/LoggingMiddleware.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/LoggingMiddleware.java index 55810fb6..614c4f7c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/LoggingMiddleware.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/LoggingMiddleware.java @@ -20,9 +20,7 @@ import java.util.concurrent.Callable; import lombok.extern.slf4j.Slf4j; -/** - * A simple logging middleware that logs queue and job id for visibility - */ +/** A simple logging middleware that logs queue and job id for visibility */ @Slf4j public class LoggingMiddleware implements Middleware { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/Middleware.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/Middleware.java index 2651d910..b67ece6e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/Middleware.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/Middleware.java @@ -35,8 +35,7 @@ * delete this job using one of the methods {@link Job#release(JobStatus, Serializable)}, {@link * Job#release(JobStatus, Serializable, Duration)} {@link Job#delete(JobStatus, Serializable)}. * - *

For example three middlewares [m1,m2,m3] are registered than m1 would be called first - * followed + *

For example three middlewares [m1,m2,m3] are registered than m1 would be called first followed * by m2, m3 and message handler {@link HandlerMiddleware}. Middleware m1 can either call m2 or skip * it, skipping call of m2 means this job will not be processed by either m2 or m3 and this job must * be either released or deleted. If m2 is called from m1 than m2 can either call m3 or skip it, if @@ -63,7 +62,7 @@ public interface Middleware { /** * Middleware handles that would be called * - * @param job job object + * @param job job object * @param next next middleware in chain * @throws Exception any exception */ diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/PermissionMiddleware.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/PermissionMiddleware.java index c82a25c0..92fa7cc7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/PermissionMiddleware.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/PermissionMiddleware.java @@ -26,8 +26,7 @@ *

If the given job does not have the right permission than this job would be released back to * pool for other consumers. * - *

NOTE: User's role was changed between and the said is no longer be able to perform - * some + *

NOTE: User's role was changed between and the said is no longer be able to perform some * action */ public interface PermissionMiddleware extends TimeProviderMiddleware { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/ProfilerMiddleware.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/ProfilerMiddleware.java index 5b5f76f3..f7a162b5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/ProfilerMiddleware.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/ProfilerMiddleware.java @@ -33,7 +33,7 @@ public class ProfilerMiddleware implements Middleware { /** * Report execution of the said job * - * @param job the running job + * @param job the running job * @param executionTime execution time */ protected void report(Job job, Duration executionTime) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java index 73132c30..07e67f14 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java @@ -58,7 +58,7 @@ default boolean process(Object message) { * then only message would be consumed otherwise it will be discarded, no further processing of * the message would be done. * - * @param message message + * @param message message * @param rqueueMessage rqueue message object * @return true/false. * @deprecated use {@link #process(Job)} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueMessageMetadataDao.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueMessageMetadataDao.java index b88a85d7..c59bb8d7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueMessageMetadataDao.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueMessageMetadataDao.java @@ -28,7 +28,7 @@ public interface RqueueMessageMetadataDao { List findAll(Collection ids); - void save(MessageMetadata messageMetadata, Duration duration); + void save(MessageMetadata messageMetadata, Duration ttl); void delete(String id); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueStringDao.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueStringDao.java index 7a2d1aff..b8baeeeb 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueStringDao.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/RqueueStringDao.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import org.springframework.data.redis.connection.DataType; +import org.springframework.data.redis.core.ZSetOperations.TypedTuple; public interface RqueueStringDao { Map> readFromLists(List keys); @@ -52,4 +53,10 @@ public interface RqueueStringDao { DataType type(String key); Boolean deleteIfSame(String key, String value); + + void addToOrderedSetWithScore(String key, String value, long score); + + List> readFromOrderedSetWithScoreBetween(String key, long start, long end); + + void deleteAll(String key, long min, long max); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java index ee604756..7e99408b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java @@ -73,11 +73,11 @@ public void deleteAll(Collection ids) { } @Override - public Mono saveReactive(MessageMetadata messageMetadata, Duration duration) { + public Mono saveReactive(MessageMetadata messageMetadata, Duration ttl) { Assert.notNull(messageMetadata.getId(), "messageMetadata id cannot be null"); return reactiveRedisTemplate .template() .opsForValue() - .set(messageMetadata.getId(), messageMetadata, duration); + .set(messageMetadata.getId(), messageMetadata, ttl); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java index 7d734ba7..c95a22cb 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java @@ -33,6 +33,7 @@ import java.util.Objects; import java.util.Set; import org.springframework.data.redis.connection.DataType; +import org.springframework.data.redis.core.ZSetOperations.TypedTuple; import org.springframework.data.redis.core.script.DefaultScriptExecutor; import org.springframework.data.redis.core.script.RedisScript; import org.springframework.util.CollectionUtils; @@ -165,4 +166,24 @@ public DataType type(String key) { public Boolean deleteIfSame(String key, String value) { return scriptExecutor.execute(delIfSameScript, Collections.singletonList(key), value); } + + @Override + public void addToOrderedSetWithScore(String key, String value, long score) { + redisTemplate.zadd(key, value, score); + } + + @Override + public List> readFromOrderedSetWithScoreBetween( + String key, long start, long end) { + Set> messages = redisTemplate.zrangeWithScore(key, start, end); + if (CollectionUtils.isEmpty(messages)) { + return Collections.emptyList(); + } + return new ArrayList<>(messages); + } + + @Override + public void deleteAll(String key, long min, long max) { + redisTemplate.zremRangeByScore(key, min, max); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java index ee9e268d..10fcc9d7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java @@ -49,9 +49,9 @@ public class JobImpl implements Job { private final RqueueConfig rqueueConfig; private final QueueDetail queueDetail; private final RqueueJob rqueueJob; - private Object userMessage; private final PostProcessingHandler postProcessingHandler; private final boolean isPeriodicJob; + private Object userMessage; private Context context = DefaultContext.EMPTY; private Boolean released; private Boolean deleted; @@ -85,6 +85,7 @@ public JobImpl( private void save() { if (rqueueConfig.isJobEnabled() && !isPeriodicJob) { + // tracking intermediate job status Duration ttl = expiry; if (getMessageMetadata().getStatus().isTerminalState()) { ttl = rqueueConfig.getJobDurabilityInTerminalState(); @@ -153,13 +154,13 @@ public boolean updateVisibilityTimeout(Duration deltaDuration) { } @Override - public void setMessage(Object message) { - this.userMessage = message; + public Object getMessage() { + return userMessage; } @Override - public Object getMessage() { - return userMessage; + public void setMessage(Object message) { + this.userMessage = message; } @Override @@ -279,23 +280,20 @@ private int getFailureCountInternal() { } void updateMessageStatus(MessageStatus messageStatus) { - Duration messageMetaExpiry; - boolean deleteMessage = false; if (messageStatus.isTerminalState()) { - messageMetaExpiry = - Duration.ofSeconds(rqueueConfig.getMessageDurabilityInTerminalStateInSecond()); - if (messageMetaExpiry.isZero() || messageMetaExpiry.isNegative()) { - deleteMessage = true; + if (rqueueConfig.messageInTerminalStateShouldBeStored()) { + this.messageMetadataService.saveMessageMetadataForQueue( + queueDetail.getCompletedQueueName(), + getMessageMetadata(), + rqueueConfig.messageDurabilityInTerminalStateInMillisecond()); + } else { + this.messageMetadataService.delete(rqueueJob.getMessageMetadata().getId()); } } else { - messageMetaExpiry = Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute()); + this.messageMetadataService.save( + getMessageMetadata(), Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute())); } setMessageStatus(messageStatus); - if (deleteMessage) { - this.messageMetadataService.delete(rqueueJob.getMessageMetadata().getId()); - } else { - this.messageMetadataService.save(rqueueJob.getMessageMetadata(), messageMetaExpiry); - } save(); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java index c161c56d..af0f0806 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MessageProcessorHandler.java @@ -16,7 +16,7 @@ package com.github.sonus21.rqueue.listener; -import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.Job; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.models.enums.MessageStatus; import com.github.sonus21.rqueue.utils.PrefixLogger; @@ -42,7 +42,7 @@ class MessageProcessorHandler extends PrefixLogger { this.postExecutionMessageProcessor = postExecutionMessageProcessor; } - void handleMessage(RqueueMessage rqueueMessage, Object userMessage, MessageStatus status) { + void handleMessage(Job job, MessageStatus status) { MessageProcessor messageProcessor = null; switch (status) { case DELETED: @@ -62,8 +62,8 @@ void handleMessage(RqueueMessage rqueueMessage, Object userMessage, MessageStatu } if (messageProcessor != null) { try { - log(Level.DEBUG, "Calling {} processor for {}", null, status, rqueueMessage); - messageProcessor.process(userMessage, rqueueMessage); + log(Level.DEBUG, "Calling {} processor for {}", null, status, job.getRqueueMessage()); + messageProcessor.process(job); } catch (Exception e) { log(Level.ERROR, "Message processor {} call failed", e, status); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java index 38a9e89d..68cc3938 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java @@ -122,7 +122,7 @@ private void deleteMessage(JobImpl job, MessageStatus status, int failureCount) rqueueMessageTemplate.removeElementFromZset( job.getQueueDetail().getProcessingQueueName(), rqueueMessage); rqueueMessage.setFailureCount(failureCount); - messageProcessorHandler.handleMessage(rqueueMessage, job.getMessage(), status); + messageProcessorHandler.handleMessage(job, status); publishEvent(job, job.getRqueueMessage(), status); } @@ -164,7 +164,7 @@ private void moveMessageToDlq(JobImpl job, int failureCount) { newMessage.updateReEnqueuedAt(); QueueDetail queueDetail = job.getQueueDetail(); Object userMessage = job.getMessage(); - messageProcessorHandler.handleMessage(newMessage, userMessage, MessageStatus.MOVED_TO_DLQ); + messageProcessorHandler.handleMessage(job, MessageStatus.MOVED_TO_DLQ); if (queueDetail.isDeadLetterConsumerEnabled()) { QueueConfig queueConfig = rqueueSystemConfigDao.getConfigByName(queueDetail.getDeadLetterQueueName(), true); @@ -189,7 +189,7 @@ private void moveMessageToDlq(JobImpl job, int failureCount) { ? FixedTaskExecutionBackOff.DEFAULT_INTERVAL : backOff; moveMessageToQueue( - queueDetail, queueConfig.getDelayedQueueName(), rqueueMessage, newMessage, backOff); + queueDetail, queueConfig.getScheduledQueueName(), rqueueMessage, newMessage, backOff); } } else { moveMessageToQueue( @@ -211,7 +211,7 @@ RqueueMessage parkMessageForRetry( } else { rqueueMessageTemplate.moveMessageWithDelay( queueDetail.getProcessingQueueName(), - queueDetail.getDelayedQueueName(), + queueDetail.getScheduledQueueName(), rqueueMessage, newMessage, delay); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java index 30690409..4c2ad647 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java @@ -44,30 +44,24 @@ @ToString public class QueueDetail extends SerializableBase { + private static final long serialVersionUID = 9153752084449974622L; // visibility timeout in milliseconds private final long visibilityTimeout; - - private static final long serialVersionUID = 9153752084449974622L; private final String name; private final int numRetry; @Builder.Default private final QueueType type = QueueType.QUEUE; private final String queueName; private final String deadLetterQueueName; private final boolean deadLetterConsumerEnabled; + private final String completedQueueName; private final String processingQueueName; private final String processingQueueChannelName; - private final String delayedQueueName; - private final String delayedQueueChannelName; + private final String scheduledQueueName; + private final String scheduledQueueChannelName; private final boolean active; private final Concurrency concurrency; private final boolean systemGenerated; private final int batchSize; - - public enum QueueType { - QUEUE, - STREAM - } - private Map priority; private String priorityGroup; @@ -86,8 +80,9 @@ public QueueConfig toConfig() { .name(name) .numRetry(numRetry) .queueName(queueName) - .delayedQueueName(delayedQueueName) + .scheduledQueueName(scheduledQueueName) .processingQueueName(processingQueueName) + .completedQueueName(completedQueueName) .visibilityTimeout(visibilityTimeout) .createdOn(System.currentTimeMillis()) .updatedOn(System.currentTimeMillis()) @@ -140,8 +135,9 @@ private QueueDetail cloneQueueDetail( .queueName(queueName + suffix) .processingQueueName(processingQueueName + suffix) .processingQueueChannelName(processingQueueChannelName + suffix) - .delayedQueueName(delayedQueueName + suffix) - .delayedQueueChannelName(delayedQueueChannelName + suffix) + .scheduledQueueName(scheduledQueueName + suffix) + .scheduledQueueChannelName(scheduledQueueChannelName + suffix) + .completedQueueName(completedQueueName + suffix) .active(active) .batchSize(batchSize) .systemGenerated(systemGenerated) @@ -154,4 +150,9 @@ private QueueDetail cloneQueueDetail( public Duration visibilityDuration() { return Duration.ofMillis(visibilityTimeout); } + + public enum QueueType { + QUEUE, + STREAM + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index 5212333a..08f59bc5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -47,12 +47,12 @@ class RqueueExecutor extends MessageContainerBase { private final RqueueMessage rqueueMessage; private final RqueueBeanProvider beanProvider; private final QueueDetail queueDetail; + private final List middlewareList; private boolean updatedToProcessing; private JobImpl job; private ExecutionStatus status; private Throwable error; private int failureCount; - private final List middlewareList; RqueueExecutor( RqueueBeanProvider rqueueBeanProvider, @@ -330,7 +330,7 @@ private void processPeriodicMessage() { beanProvider .getRqueueMessageTemplate() .scheduleMessage( - job.getQueueDetail().getDelayedQueueName(), + job.getQueueDetail().getScheduledQueueName(), messageKey, newMessage, expiryInSeconds)); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java index b0b94e81..74e640cc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java @@ -258,18 +258,6 @@ private void executeMatches(Set matchesIn, Message message, String loo } } - @AllArgsConstructor - static class HandlerMethodWithPrimary { - - HandlerMethod method; - boolean primary; - - @Override - public String toString() { - return method.toString(); - } - } - private MappingInformation getMappingInformation( Method method, MappingInformation mappingInformation) { RqueueHandler rqueueHandler = AnnotationUtils.findAnnotation(method, RqueueHandler.class); @@ -425,33 +413,6 @@ private Map resolvePriority(RqueueListener rqueueListener) { return Collections.unmodifiableMap(priorityMap); } - @AllArgsConstructor - @EqualsAndHashCode - private static class Match { - - private final MappingInformation information; - private final HandlerMethodWithPrimary handlerMethod; - } - - private class MultiHandler extends RetryableRunnable { - - private final Match match; - private final Message message; - private final String lookupDestination; - - protected MultiHandler(Match match, Message message, String lookupDestination) { - super(log, ""); - this.match = match; - this.message = message; - this.lookupDestination = lookupDestination; - } - - @Override - public void start() { - handleMatch(match.information, match.handlerMethod.method, lookupDestination, message); - } - } - private boolean resolveConsumerEnabled(RqueueListener rqueueListener) { return ValueResolver.resolveToBoolean( getApplicationContext(), rqueueListener.deadLetterQueueListenerEnabled()); @@ -586,4 +547,43 @@ protected void processHandlerMethodException( public MessageConverter getMessageConverter() { return this.messageConverter; } + + @AllArgsConstructor + static class HandlerMethodWithPrimary { + + HandlerMethod method; + boolean primary; + + @Override + public String toString() { + return method.toString(); + } + } + + @AllArgsConstructor + @EqualsAndHashCode + private static class Match { + + private final MappingInformation information; + private final HandlerMethodWithPrimary handlerMethod; + } + + private class MultiHandler extends RetryableRunnable { + + private final Match match; + private final Message message; + private final String lookupDestination; + + protected MultiHandler(Match match, Message message, String lookupDestination) { + super(log, ""); + this.match = match; + this.message = message; + this.lookupDestination = lookupDestination; + } + + @Override + public void start() { + handleMatch(match.information, match.handlerMethod.method, lookupDestination, message); + } + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java index 1d140caf..5d4a4e0d 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java @@ -75,9 +75,16 @@ public class RqueueMessageListenerContainer public static final String EVENT_SOURCE = "RqueueMessageListenerContainer"; private static final String DEFAULT_THREAD_NAME_PREFIX = ClassUtils.getShortName(RqueueMessageListenerContainer.class); + final QueueStateMgr queueStateMgr = new QueueStateMgr(); private final Object lifecycleMgr = new Object(); private final RqueueMessageTemplate rqueueMessageTemplate; private final RqueueMessageHandler rqueueMessageHandler; + private final Map queueRunningState = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> scheduledFutureByQueue = + new ConcurrentHashMap<>(); + private final Map queueThreadMap = new ConcurrentHashMap<>(); + @Autowired protected RqueueBeanProvider rqueueBeanProvider; + List middlewares; private MessageProcessor discardMessageProcessor; private MessageProcessor deadLetterQueueMessageProcessor; private MessageProcessor manualDeletionMessageProcessor; @@ -85,13 +92,7 @@ public class RqueueMessageListenerContainer private MessageProcessor preExecutionMessageProcessor; private TaskExecutionBackOff taskExecutionBackOff = new FixedTaskExecutionBackOff(); private PostProcessingHandler postProcessingHandler; - final QueueStateMgr queueStateMgr = new QueueStateMgr(); - private final Map queueRunningState = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> scheduledFutureByQueue = - new ConcurrentHashMap<>(); - private final Map queueThreadMap = new ConcurrentHashMap<>(); private AsyncTaskExecutor taskExecutor; - @Autowired protected RqueueBeanProvider rqueueBeanProvider; private Integer maxNumWorkers; private String beanName; private boolean defaultTaskExecutor = false; @@ -102,7 +103,6 @@ public class RqueueMessageListenerContainer private long pollingInterval = 200L; private int phase = Integer.MAX_VALUE; private PriorityMode priorityMode; - List middlewares; private MessageHeaders messageHeaders; public RqueueMessageListenerContainer( @@ -112,10 +112,10 @@ public RqueueMessageListenerContainer( this.rqueueMessageHandler = rqueueMessageHandler; this.rqueueMessageTemplate = rqueueMessageTemplate; this.discardMessageProcessor = new MessageProcessor() {}; - this.deadLetterQueueMessageProcessor = this.discardMessageProcessor; - this.manualDeletionMessageProcessor = this.discardMessageProcessor; - this.postExecutionMessageProcessor = this.discardMessageProcessor; - this.preExecutionMessageProcessor = this.discardMessageProcessor; + this.deadLetterQueueMessageProcessor = new MessageProcessor() {}; + this.manualDeletionMessageProcessor = new MessageProcessor() {}; + this.postExecutionMessageProcessor = new MessageProcessor() {}; + this.preExecutionMessageProcessor = new MessageProcessor() {}; } public RqueueMessageTemplate getRqueueMessageTemplate() { @@ -389,9 +389,10 @@ private List getQueueDetail(String queue, MappingInformation mappin .name(queue) .queueName(rqueueConfig.getQueueName(queue)) .processingQueueName(rqueueConfig.getProcessingQueueName(queue)) - .delayedQueueName(rqueueConfig.getDelayedQueueName(queue)) + .completedQueueName(rqueueConfig.getCompletedQueueName(queue)) + .scheduledQueueName(rqueueConfig.getScheduledQueueName(queue)) .processingQueueChannelName(rqueueConfig.getProcessingQueueChannelName(queue)) - .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queue)) + .scheduledQueueChannelName(rqueueConfig.getScheduledQueueChannelName(queue)) .deadLetterQueueName(mappingInformation.getDeadLetterQueueName()) .visibilityTimeout(mappingInformation.getVisibilityTimeout()) .deadLetterConsumerEnabled(mappingInformation.isDeadLetterConsumerEnabled()) @@ -566,56 +567,6 @@ protected void doStop() { waitForRunningQueuesToStop(); } - class QueueStateMgr { - Set pausedQueues = ConcurrentHashMap.newKeySet(); - - boolean isQueueActive(String queueName) { - return queueRunningState.getOrDefault(queueName, false); - } - - boolean isQueuePaused(String queueName) { - return pausedQueues.contains(queueName); - } - - void pauseUnpauseQueue(String queue, boolean pause) { - if (pause && pausedQueues.contains(queue)) { - log.error("Duplicate pause called {}", queue); - return; - } - if (!pause && !pausedQueues.contains(queue)) { - log.error("Queue is not paused but unpause is requested {}", queue); - return; - } - if (pause) { - pause(queue); - } else { - unpause(queue); - } - RqueueQueuePauseEvent event = new RqueueQueuePauseEvent(EVENT_SOURCE, queue, pause); - rqueueBeanProvider.getApplicationEventPublisher().publishEvent(event); - } - - private void unpause(String queue) { - log.info("Queue '{}' action unpause", queue); - pausedQueues.remove(queue); - } - - private void pause(String queue) { - log.info("Queue '{}' action pause", queue); - pausedQueues.add(queue); - } - - void pauseQueueIfRequired(QueueConfig config) { - if (config == null) { - // new queue - return; - } - if (config.isPaused()) { - pause(config.getName()); - } - } - } - private void waitForRunningQueuesToStop() { for (Map.Entry entry : queueRunningState.entrySet()) { String queueName = entry.getKey(); @@ -725,11 +676,61 @@ public List getMiddleWares() { return middlewares; } + public MessageHeaders getMessageHeaders() { + return messageHeaders; + } + public void setMessageHeaders(MessageHeaders messageHeaders) { this.messageHeaders = messageHeaders; } - public MessageHeaders getMessageHeaders() { - return messageHeaders; + class QueueStateMgr { + Set pausedQueues = ConcurrentHashMap.newKeySet(); + + boolean isQueueActive(String queueName) { + return queueRunningState.getOrDefault(queueName, false); + } + + boolean isQueuePaused(String queueName) { + return pausedQueues.contains(queueName); + } + + void pauseUnpauseQueue(String queue, boolean pause) { + if (pause && pausedQueues.contains(queue)) { + log.error("Duplicate pause called {}", queue); + return; + } + if (!pause && !pausedQueues.contains(queue)) { + log.error("Queue is not paused but unpause is requested {}", queue); + return; + } + if (pause) { + pause(queue); + } else { + unpause(queue); + } + RqueueQueuePauseEvent event = new RqueueQueuePauseEvent(EVENT_SOURCE, queue, pause); + rqueueBeanProvider.getApplicationEventPublisher().publishEvent(event); + } + + private void unpause(String queue) { + log.info("Queue '{}' action unpause", queue); + pausedQueues.remove(queue); + } + + private void pause(String queue) { + log.info("Queue '{}' action pause", queue); + pausedQueues.add(queue); + } + + void pauseQueueIfRequired(QueueConfig config) { + if (config == null) { + // new queue + return; + } + if (config.isPaused()) { + pause(config.getName()); + } + } } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java index ab647254..a7892d3d 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java @@ -30,10 +30,10 @@ import org.springframework.util.CollectionUtils; abstract class RqueueMessagePoller extends MessageContainerBase { - private final PostProcessingHandler postProcessingHandler; final List middlewares; final long pollingInterval; final long backoffTime; + private final PostProcessingHandler postProcessingHandler; private final RqueueBeanProvider rqueueBeanProvider; private final MessageHeaders messageHeaders; List queues; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java index 5e874d20..4a781451 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java @@ -32,14 +32,13 @@ import org.springframework.messaging.MessageHeaders; class StrictPriorityPoller extends RqueueMessagePoller { + private static final String ALL_QUEUES_ARE_INELIGIBLE = "\uD83D\uDE1F"; + private static final String ALL_QUEUES_ARE_INACTIVE = "\uD83D\uDC4B"; private final Map queueNameToDetail; private final Map queueNameToThread; private final Map queueDeactivationTime = new HashMap<>(); private final Map lastFetchedTime = new HashMap<>(); - private static final String ALL_QUEUES_ARE_INELIGIBLE = "\uD83D\uDE1F"; - private static final String ALL_QUEUES_ARE_INACTIVE = "\uD83D\uDC4B"; - StrictPriorityPoller( String groupName, final List queueDetails, diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java index 19be5423..1d411fc6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java @@ -31,6 +31,8 @@ import org.springframework.messaging.MessageHeaders; class WeightedPriorityPoller extends RqueueMessagePoller { + private static final int ALL_QUEUES_ARE_INELIGIBLE = -1; + private static final int ALL_QUEUES_ARE_INACTIVE = -2; private final Map queueNameToThread; private final Map queueNameToDetail; private final List queueDetailList; @@ -39,9 +41,6 @@ class WeightedPriorityPoller extends RqueueMessagePoller { private float[] probability; private int currentIndex = 0; - private static final int ALL_QUEUES_ARE_INELIGIBLE = -1; - private static final int ALL_QUEUES_ARE_INACTIVE = -2; - WeightedPriorityPoller( String groupName, final List queueDetails, diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java index b6638b9c..6ce05805 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java @@ -30,14 +30,13 @@ /** * RqueueMetrics register metrics related to queue. A queue can have 4 types of metrics like - * queue.size, processing.queue.size and other two depends on the queue configurations. For delayed - * queue messages can be in delayed queue because time has not reached. Some messages can be in dead - * letter queue if dead letter queue is configured. + * queue.size, processing.queue.size and scheduled.queue.size. Some messages can be in dead letter + * queue if dead letter queue is configured. */ public class RqueueMetrics implements RqueueMetricsRegistry { static final String QUEUE_KEY = "key"; private static final String QUEUE_SIZE = "queue.size"; - private static final String DELAYED_QUEUE_SIZE = "delayed.queue.size"; + private static final String SCHEDULED_QUEUE_SIZE = "scheduled.queue.size"; private static final String PROCESSING_QUEUE_SIZE = "processing.queue.size"; private static final String DEAD_LETTER_QUEUE_SIZE = "dead.letter.queue.size"; private final QueueCounter queueCounter; @@ -78,9 +77,11 @@ private void monitor() { .description("The number of entries in the processing queue") .register(meterRegistry); Gauge.builder( - DELAYED_QUEUE_SIZE, queueDetail, c -> size(queueDetail.getDelayedQueueName(), true)) - .tags(queueTags.and(QUEUE_KEY, queueDetail.getDelayedQueueName())) - .description("The number of entries waiting in the delayed queue") + SCHEDULED_QUEUE_SIZE, + queueDetail, + c -> size(queueDetail.getScheduledQueueName(), true)) + .tags(queueTags.and(QUEUE_KEY, queueDetail.getScheduledQueueName())) + .description("The number of entries waiting in the scheduled queue") .register(meterRegistry); if (queueDetail.isDlqSet()) { Builder builder = diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java index 8194b2a9..b3e25689 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java @@ -50,8 +50,8 @@ public long getPendingMessageCount(String queue) { } /** - * Get number of messages waiting in delayed queue, these messages would move to pending queue as - * soon as the scheduled time is reach. + * Get number of messages waiting in scheduled queue, these messages would move to pending queue + * as soon as the scheduled time is reach. * * @param queue queue name * @return -1 if queue is not registered otherwise message count @@ -59,7 +59,7 @@ public long getPendingMessageCount(String queue) { public long getScheduledMessageCount(String queue) { try { QueueDetail queueDetail = EndpointRegistry.get(queue); - return redisTemplate.getZsetSize(queueDetail.getDelayedQueueName()); + return redisTemplate.getZsetSize(queueDetail.getScheduledQueueName()); } catch (QueueDoesNotExist e) { return -1; } @@ -83,7 +83,7 @@ public long getProcessingMessageCount(String queue) { /** * Get number of messages waiting for consumption * - * @param queue queue name + * @param queue queue name * @param priority priority of this queue * @return -1 if queue is not registered otherwise message count */ @@ -97,17 +97,17 @@ public long getPendingMessageCount(String queue, String priority) { } /** - * Get number of messages waiting in delayed queue, these messages would move to pending queue as - * soon as the scheduled time is reach. + * Get number of messages waiting in scheduled queue, these messages would move to pending queue + * as soon as the scheduled time is reach. * - * @param queue queue name + * @param queue queue name * @param priority priority of this queue * @return -1 if queue is not registered otherwise message count */ public long getScheduledMessageCount(String queue, String priority) { try { QueueDetail queueDetail = EndpointRegistry.get(queue, priority); - return redisTemplate.getZsetSize(queueDetail.getDelayedQueueName()); + return redisTemplate.getZsetSize(queueDetail.getScheduledQueueName()); } catch (QueueDoesNotExist e) { return -1; } @@ -116,7 +116,7 @@ public long getScheduledMessageCount(String queue, String priority) { /** * Get number of messages those are currently being processed * - * @param queue queue name + * @param queue queue name * @param priority priority of this queue * @return -1 if queue is not registered otherwise message count */ diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/SystemUtil.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/Pair.java similarity index 69% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/SystemUtil.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/models/Pair.java index ceb5773d..67deb345 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/SystemUtil.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/Pair.java @@ -14,12 +14,19 @@ * */ -package com.github.sonus21.rqueue.utils; +package com.github.sonus21.rqueue.models; -public final class SystemUtil { - private SystemUtil() {} +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; - public static int cpuCount() { - return Runtime.getRuntime().availableProcessors(); - } +@AllArgsConstructor +@NoArgsConstructor +@Setter +@Getter +public class Pair { + + private S first; + private T second; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/aggregator/QueueEvents.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/aggregator/QueueEvents.java index 32bacbc5..b156ebc3 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/aggregator/QueueEvents.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/aggregator/QueueEvents.java @@ -23,7 +23,7 @@ public class QueueEvents { - public List rqueueExecutionEvents; + public final List rqueueExecutionEvents; private final Long createdAt; public QueueEvents(RqueueExecutionEvent event) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java index 8eb2383b..dff9a3a2 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java @@ -43,6 +43,7 @@ public class MessageMetadata extends SerializableBase { private RqueueMessage rqueueMessage; // Rqueue message current status private MessageStatus status; + private long updatedOn; public MessageMetadata(String id, MessageStatus messageStatus) { this.id = id; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java index bd1ad4fe..507741f6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/QueueConfig.java @@ -48,8 +48,9 @@ public class QueueConfig extends SerializableBase { private String id; private String name; private String queueName; + private String completedQueueName; private String processingQueueName; - private String delayedQueueName; + private String scheduledQueueName; private int numRetry; private long visibilityTimeout; private boolean paused; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/RqueueJob.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/RqueueJob.java index b94256f7..4a85747f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/RqueueJob.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/RqueueJob.java @@ -46,7 +46,7 @@ public class RqueueJob extends SerializableBase { private String messageId; - // currently being consumed message + // being consumed message private RqueueMessage rqueueMessage; // Message metadata for this message, metadata can have different RqueueMessage than currently // being consumed diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/MessageStatus.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/MessageStatus.java index ebed7b73..f881666e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/MessageStatus.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/MessageStatus.java @@ -25,22 +25,23 @@ public enum MessageStatus { // Message is just enqueued ENQUEUED(false, JobStatus.UNKNOWN), - // Currently this message is being processed + // this message is being processed PROCESSING(false, JobStatus.PROCESSING), // Message was deleted DELETED(true, JobStatus.SUCCESS), - // Message was ignored by pre processor + // Message was ignored by pre-processor IGNORED(true, JobStatus.SUCCESS), // Message was successful consumed SUCCESSFUL(true, JobStatus.SUCCESS), // Message moved to dead letter queue MOVED_TO_DLQ(true, JobStatus.SUCCESS), /** - * Message was discarded due to retry limit or {@link com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff#STOP} - * was returned by task execution backoff method. + * Message was discarded due to retry limit exceeded or {@link + * com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff#STOP} was returned by task + * execution backoff method. */ DISCARDED(true, JobStatus.SUCCESS), - // Execution has failed, it will retry later + // Execution has failed, it will be retried later FAILED(false, JobStatus.FAILED); private final boolean terminalState; private final JobStatus jobStatus; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/NavTab.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/NavTab.java index 6e215817..7452fbbc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/NavTab.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/enums/NavTab.java @@ -28,6 +28,7 @@ public enum NavTab { RUNNING("Running"), PENDING("Pending"), SCHEDULED("Scheduled"), + COMPLETED("Completed"), UTILITY("Utility"); private final String description; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/ChartDataRequest.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/ChartDataRequest.java index d03735e9..b8b0d577 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/ChartDataRequest.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/ChartDataRequest.java @@ -16,11 +16,13 @@ package com.github.sonus21.rqueue.models.request; +import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.models.SerializableBase; import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.enums.ChartDataType; import com.github.sonus21.rqueue.models.enums.ChartType; import com.github.sonus21.rqueue.models.response.ChartDataResponse; +import com.github.sonus21.rqueue.utils.Constants; import java.util.List; import lombok.Getter; import lombok.NoArgsConstructor; @@ -35,6 +37,7 @@ public class ChartDataRequest extends SerializableBase { private static final long serialVersionUID = 7727090378318819986L; private ChartType type; private String queue; + private int number; private AggregationType aggregationType; private List dateTypes; @@ -43,6 +46,19 @@ public ChartDataRequest(ChartType chartType, AggregationType aggregationType) { this.aggregationType = aggregationType; } + public int numberOfDays(RqueueWebConfig rqueueWebConfig) { + int n = this.number; + if (aggregationType == AggregationType.MONTHLY) { + n = this.number * Constants.DAYS_IN_A_MONTH; + } else if (aggregationType == AggregationType.WEEKLY) { + n = this.number * Constants.DAYS_IN_A_WEEK; + } + if (n <= 0 || n > rqueueWebConfig.getHistoryDay()) { + return rqueueWebConfig.getHistoryDay(); + } + return n; + } + public ChartDataResponse validate() { ChartDataResponse chartDataResponse = new ChartDataResponse(); if (getType() == null) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DataDeleteRequest.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DataDeleteRequest.java index 24e044a1..1b8e8e88 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DataDeleteRequest.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DataDeleteRequest.java @@ -32,7 +32,10 @@ @EqualsAndHashCode(callSuper = true) public class DataDeleteRequest extends SerializableBase { @JsonProperty("queue") - @NotEmpty private String queueName; + @NotEmpty + private String queueName; + @JsonProperty("data_set") - @NotEmpty private String datasetName; + @NotEmpty + private String datasetName; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DateViewRequest.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DateViewRequest.java index 8e6f6041..e3d2a6f8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DateViewRequest.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/request/DateViewRequest.java @@ -35,7 +35,7 @@ public class DateViewRequest extends SerializableBase { private @NotNull DataType type; private @NotEmpty String name; - private String key; + private String key; @JsonProperty("page") private int pageNumber = 0; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataSelectorResponse.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataSelectorResponse.java new file mode 100644 index 00000000..6540aa73 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataSelectorResponse.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.models.response; + +import com.github.sonus21.rqueue.models.Pair; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@ToString(callSuper = true) +@SuppressWarnings("java:S2160") +@NoArgsConstructor +@AllArgsConstructor +public class DataSelectorResponse extends BaseResponse { + private String title; + private List> data; +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Constants.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Constants.java index 28da189d..02623531 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Constants.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Constants.java @@ -19,8 +19,6 @@ import com.github.sonus21.rqueue.config.RqueueConfig; public final class Constants { - private Constants() {} - public static final String BLANK = ""; public static final long ONE_MILLI = 1000; public static final int ONE_MILLI_INT = 1000; @@ -57,6 +55,8 @@ private Constants() {} public static final long MINIMUM_JOB_PERIOD = 1000L; public static final String QUEUE_CRUD_LOCK_KEY_PREFIX = "q-crud::"; + private Constants() {} + public static String getQueueCrudLockKey(RqueueConfig rqueueConfig, String queueName) { return rqueueConfig.getLockKey(QUEUE_CRUD_LOCK_KEY_PREFIX + queueName); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/DateTimeUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/DateTimeUtils.java index 6f346f6d..cb7f3b2c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/DateTimeUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/DateTimeUtils.java @@ -26,8 +26,7 @@ public final class DateTimeUtils { private static final DateTimeFormatter simple = DateTimeFormatter.ofPattern("dd MMM yyyy HH:mm"); - DateTimeUtils() { - } + DateTimeUtils() {} private static String hourString(long hour) { if (hour > 1) { @@ -83,7 +82,13 @@ private static String formatHour(long hours, long minutes, long seconds) { "%s, %s, %s", hourString(hours), minuteString(minutes), secondString(seconds)); } - public static String milliToHumanRepresentation(long millis) { + public static String milliToHumanRepresentation(long millisecond) { + long millis = millisecond; + String prefix = ""; + if (millis < 0) { + prefix = "- "; + millis = -1 * millis; + } long seconds = millis / Constants.ONE_MILLI; long minutes = seconds / Constants.SECONDS_IN_A_MINUTE; seconds = seconds % Constants.SECONDS_IN_A_MINUTE; // remaining seconds @@ -92,18 +97,18 @@ public static String milliToHumanRepresentation(long millis) { long days = hours / Constants.HOURS_IN_A_DAY; hours = hours % Constants.HOURS_IN_A_DAY; // remaining hours if (days != 0) { - return formatDay(days, hours, minutes, seconds); + return prefix + formatDay(days, hours, minutes, seconds); } if (hours != 0) { - return formatHour(hours, minutes, seconds); + return prefix + formatHour(hours, minutes, seconds); } if (minutes != 0) { if (seconds == 0) { - return minuteString(minutes); + return prefix + minuteString(minutes); } - return String.format("%s, %s", minuteString(minutes), secondString(seconds)); + return prefix + String.format("%s, %s", minuteString(minutes), secondString(seconds)); } - return secondString(seconds); + return prefix + secondString(seconds); } public static String formatMilliToString(Long milli) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java index 7eb9965e..7805424a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java @@ -26,8 +26,7 @@ @Slf4j public final class HttpUtils { - private HttpUtils() { - } + private HttpUtils() {} private static SimpleClientHttpRequestFactory getRequestFactory(RqueueConfig rqueueConfig) { SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); @@ -53,4 +52,25 @@ public static T readUrl(RqueueConfig rqueueConfig, String url, Class claz return null; } } + + public static String joinPath(String... components) { + StringBuilder sb = new StringBuilder(); + for (String comp : components) { + if (StringUtils.isEmpty(comp) || comp.equals(Constants.FORWARD_SLASH)) { + continue; + } + sb.append(Constants.FORWARD_SLASH); + if (comp.endsWith(Constants.FORWARD_SLASH) && comp.startsWith(Constants.FORWARD_SLASH)) { + sb.append(comp, 1, comp.length() - 1); + } else if (comp.endsWith(Constants.FORWARD_SLASH)) { + sb.append(comp, 0, comp.length() - 1); + } else if (comp.startsWith(Constants.FORWARD_SLASH)) { + sb.append(comp.substring(1)); + } else { + sb.append(comp); + } + } + sb.append(Constants.FORWARD_SLASH); + return sb.toString(); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java index b9cf4a74..cf0249e5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java @@ -79,8 +79,7 @@ public ReactiveRedisTemplate getRedisTemplate( } }; - private RedisUtils() { - } + private RedisUtils() {} public static RedisTemplate getRedisTemplate( RedisConnectionFactory redisConnectionFactory) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java index 07e0705a..2e948964 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java @@ -20,8 +20,7 @@ public final class StringUtils { - StringUtils() { - } + StringUtils() {} public static boolean isEmpty(String string) { if (string == null) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java index 9aea56c2..c93f38c6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java @@ -30,8 +30,7 @@ public final class ValueResolver { - private ValueResolver() { - } + private ValueResolver() {} @SuppressWarnings("unchecked") public static T parseStringUsingSpel(String val, Class t) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java index 8e58ecff..267eb08a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java @@ -116,7 +116,7 @@ public long getInitialInterval() { /** * Set the initial interval in milliseconds. * - * @param initialInterval initial interval in milli seconds. + * @param initialInterval initial interval in milliseconds. */ public void setInitialInterval(long initialInterval) { checkInitialInterval(initialInterval); @@ -171,18 +171,16 @@ public long getMaxInterval() { } /** - * Set the max interval in milli seconds + * Set the max interval in milliseconds * - * @param maxInterval value in milli seconds + * @param maxInterval value in milliseconds */ public void setMaxInterval(long maxInterval) { checkMaxInterval(initialInterval, maxInterval); this.maxInterval = maxInterval; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public long nextBackOff(Object message, RqueueMessage rqueueMessage, int failureCount) { if (failureCount >= getMaxRetries(message, rqueueMessage, failureCount)) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java index 547474dd..9e7979b1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java @@ -58,7 +58,7 @@ public FixedTaskExecutionBackOff(long interval, int maxRetries) { /** * Return the delay between two attempts in milliseconds. * - * @return the configured interval in milli seconds. + * @return the configured interval in milliseconds. */ public long getInterval() { return this.interval; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/TaskExecutionBackOff.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/TaskExecutionBackOff.java index b4b32a58..460c6c17 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/TaskExecutionBackOff.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/backoff/TaskExecutionBackOff.java @@ -28,12 +28,11 @@ public interface TaskExecutionBackOff { /** * Return the number of milliseconds to wait for the same message to be consumed by other - * consumers. ({@value #STOP}) to indicate that no further enqueue should be made for the - * message. + * consumers. ({@value #STOP}) to indicate that no further enqueue should be made for the message. * - * @param message message that's fetched + * @param message message that's fetched * @param rqueueMessage raw message - * @param failureCount number of times this message has failed. + * @param failureCount number of times this message has failed. * @return backoff in the millisecond. */ long nextBackOff(Object message, RqueueMessage rqueueMessage, int failureCount); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java new file mode 100644 index 00000000..bf5c8148 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.web.controller; + +import com.github.sonus21.rqueue.config.RqueueWebConfig; +import javax.servlet.http.HttpServletResponse; + +public class BaseController { + private final RqueueWebConfig rqueueWebConfig; + + public BaseController(RqueueWebConfig rqueueWebConfig) { + this.rqueueWebConfig = rqueueWebConfig; + } + + protected boolean isEnable(HttpServletResponse response) { + if (!rqueueWebConfig.isEnable()) { + response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + } + return rqueueWebConfig.isEnable(); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java new file mode 100644 index 00000000..8381860c --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.web.controller; + +import com.github.sonus21.rqueue.config.RqueueWebConfig; +import org.springframework.http.HttpStatus; +import org.springframework.http.server.reactive.ServerHttpResponse; + +public class BaseReactiveController { + protected final RqueueWebConfig rqueueWebConfig; + + public BaseReactiveController(RqueueWebConfig rqueueWebConfig) { + this.rqueueWebConfig = rqueueWebConfig; + } + + protected boolean isEnabled(ServerHttpResponse response) { + if (!rqueueWebConfig.isEnable()) { + response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); + } + return rqueueWebConfig.isEnable(); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java index b1950b44..420efd6c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java @@ -18,6 +18,7 @@ import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.exception.ProcessingException; +import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.request.ChartDataRequest; import com.github.sonus21.rqueue.models.request.DataDeleteRequest; import com.github.sonus21.rqueue.models.request.DataTypeRequest; @@ -31,6 +32,7 @@ import com.github.sonus21.rqueue.models.response.ChartDataResponse; import com.github.sonus21.rqueue.models.response.DataViewResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; +import com.github.sonus21.rqueue.models.response.DataSelectorResponse; import com.github.sonus21.rqueue.models.response.StringResponse; import com.github.sonus21.rqueue.utils.ReactiveEnabled; import com.github.sonus21.rqueue.web.service.RqueueDashboardChartService; @@ -42,7 +44,6 @@ import javax.validation.constraints.NotEmpty; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Conditional; -import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; @@ -56,13 +57,12 @@ @RestController @Conditional(ReactiveEnabled.class) @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue/api/v1") -public class ReactiveRqueueRestController { +public class ReactiveRqueueRestController extends BaseReactiveController { private final RqueueDashboardChartService rqueueDashboardChartService; private final RqueueQDetailService rqueueQDetailService; private final RqueueUtilityService rqueueUtilityService; private final RqueueSystemManagerService rqueueQManagerService; - private final RqueueWebConfig rqueueWebConfig; private final RqueueJobService rqueueJobService; @Autowired @@ -73,11 +73,11 @@ public ReactiveRqueueRestController( RqueueSystemManagerService rqueueQManagerService, RqueueWebConfig rqueueWebConfig, RqueueJobService rqueueJobService) { + super(rqueueWebConfig); this.rqueueDashboardChartService = rqueueDashboardChartService; this.rqueueQDetailService = rqueueQDetailService; this.rqueueUtilityService = rqueueUtilityService; this.rqueueQManagerService = rqueueQManagerService; - this.rqueueWebConfig = rqueueWebConfig; this.rqueueJobService = rqueueJobService; } @@ -85,11 +85,10 @@ public ReactiveRqueueRestController( @ResponseBody public Mono getDashboardData( @RequestBody @Valid ChartDataRequest chartDataRequest, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueDashboardChartService.getReactiveDashBoardData(chartDataRequest); } - return rqueueDashboardChartService.getReactiveDashBoardData(chartDataRequest); + return null; } @GetMapping("jobs") @@ -97,109 +96,111 @@ public Mono getDashboardData( public Mono getJobs( @RequestParam(name = "message-id") @NotEmpty String messageId, ServerHttpResponse response) throws ProcessingException { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueJobService.getReactiveJobs(messageId); } - return rqueueJobService.getReactiveJobs(messageId); + return null; } @PostMapping("queue-data") @ResponseBody public Mono exploreQueue( @RequestBody @Valid QueueExploreRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueQDetailService.getReactiveExplorePageData( + request.getSrc(), + request.getName(), + request.getType(), + request.getPageNumber(), + request.getItemPerPage()); } - return rqueueQDetailService.getReactiveExplorePageData( - request.getSrc(), - request.getName(), - request.getType(), - request.getPageNumber(), - request.getItemPerPage()); + return null; } @PostMapping("view-data") @ResponseBody public Mono viewData( @RequestBody @Valid DateViewRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueQDetailService.viewReactiveData( + request.getName(), + request.getType(), + request.getKey(), + request.getPageNumber(), + request.getItemPerPage()); } - return rqueueQDetailService.viewReactiveData( - request.getName(), - request.getType(), - request.getKey(), - request.getPageNumber(), - request.getItemPerPage()); + return null; } @PostMapping("delete-message") @ResponseBody public Mono deleteMessage( @RequestBody @Valid MessageDeleteRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueUtilityService.deleteReactiveMessage( + request.getQueueName(), request.getMessageId()); } - return rqueueUtilityService.deleteReactiveMessage( - request.getQueueName(), request.getMessageId()); + return null; } @PostMapping("delete-queue") @ResponseBody public Mono deleteQueue( @RequestBody @Valid DataTypeRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueQManagerService.deleteReactiveQueue(request.getName()); } - return rqueueQManagerService.deleteReactiveQueue(request.getName()); + return null; } @PostMapping("delete-queue-part") @ResponseBody public Mono deleteAll( @RequestBody @Valid DataDeleteRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueUtilityService.makeEmptyReactive( + request.getQueueName(), request.getDatasetName()); } - return rqueueUtilityService.makeEmptyReactive(request.getQueueName(), request.getDatasetName()); + return null; } @PostMapping("data-type") @ResponseBody public Mono dataType( @Valid @RequestBody DataTypeRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueUtilityService.getReactiveDataType(request.getName()); } - return rqueueUtilityService.getReactiveDataType(request.getName()); + return null; } @PostMapping("move-data") @ResponseBody public Mono dataType( @RequestBody @Valid MessageMoveRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueUtilityService.moveReactiveMessage(request); } - return rqueueUtilityService.moveReactiveMessage(request); + return null; } @PostMapping("pause-unpause-queue") @ResponseBody public Mono pauseUnpauseQueue( @RequestBody @Valid PauseUnpauseQueueRequest request, ServerHttpResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + return rqueueUtilityService.reactivePauseUnpauseQueue(request); } - return rqueueUtilityService.reactivePauseUnpauseQueue(request); + return null; + } + + @GetMapping("aggregate-data-selector") + @ResponseBody + public Mono aggregateDataCounter( + @RequestParam AggregationType type, ServerHttpResponse response) { + if (isEnabled(response)) { + return rqueueUtilityService.reactiveAggregateDataCounter(type); + } + return null; } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java index 97620f51..603c853a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java @@ -16,16 +16,13 @@ package com.github.sonus21.rqueue.web.controller; -import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.utils.ReactiveEnabled; -import com.github.sonus21.rqueue.utils.StringUtils; import com.github.sonus21.rqueue.web.service.RqueueViewControllerService; import java.util.Locale; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Conditional; -import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.stereotype.Controller; @@ -40,51 +37,42 @@ @Controller @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue") @Conditional(ReactiveEnabled.class) -public class ReactiveRqueueViewController { - +public class ReactiveRqueueViewController extends BaseReactiveController { private final ViewResolver rqueueViewResolver; private final RqueueViewControllerService rqueueViewControllerService; - private final RqueueWebConfig rqueueWebConfig; @Autowired public ReactiveRqueueViewController( - RqueueConfig rqueueConfig, RqueueWebConfig rqueueWebConfig, RqueueViewControllerService rqueueViewControllerService, @Qualifier("reactiveRqueueViewResolver") ViewResolver rqueueViewResolver) { + super(rqueueWebConfig); this.rqueueViewResolver = rqueueViewResolver; this.rqueueViewControllerService = rqueueViewControllerService; - this.rqueueWebConfig = rqueueWebConfig; } private String xForwardedPrefix(ServerHttpRequest request) { - String prefix = rqueueWebConfig.getUrlPrefix(); - if (StringUtils.isEmpty(prefix)) { - return request.getHeaders().getFirst("x-forwarded-prefix"); - } - return prefix; + return request.getHeaders().getFirst("x-forwarded-prefix"); } @GetMapping public Mono index(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.index(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("index", Locale.ENGLISH); } - rqueueViewControllerService.index(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("index", Locale.ENGLISH); + return null; } @GetMapping("queues") public Mono queues(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.queues(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("queues", Locale.ENGLISH); } - rqueueViewControllerService.queues(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("queues", Locale.ENGLISH); + return null; } @GetMapping("queues/{queueName}") @@ -94,66 +82,60 @@ public Mono queueDetail( ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.queueDetail(model, xForwardedPrefix(request), queueName); + return rqueueViewResolver.resolveViewName("queue_detail", Locale.ENGLISH); } - rqueueViewControllerService.queueDetail(model, xForwardedPrefix(request), queueName); - return rqueueViewResolver.resolveViewName("queue_detail", Locale.ENGLISH); + return null; } @GetMapping("running") public Mono running(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.running(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.running(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("scheduled") public Mono scheduled(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.scheduled(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.scheduled(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("dead") public Mono dead(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.dead(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.dead(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("pending") public Mono pending(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.pending(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.pending(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("utility") public Mono utility(Model model, ServerHttpRequest request, ServerHttpResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatusCode(HttpStatus.SERVICE_UNAVAILABLE); - return null; + if (isEnabled(response)) { + rqueueViewControllerService.utility(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("utility", Locale.ENGLISH); } - rqueueViewControllerService.utility(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("utility", Locale.ENGLISH); + return null; } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java index e4323c17..0775ee5a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java @@ -18,6 +18,7 @@ import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.exception.ProcessingException; +import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.request.ChartDataRequest; import com.github.sonus21.rqueue.models.request.DataDeleteRequest; import com.github.sonus21.rqueue.models.request.DataTypeRequest; @@ -31,6 +32,7 @@ import com.github.sonus21.rqueue.models.response.ChartDataResponse; import com.github.sonus21.rqueue.models.response.DataViewResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; +import com.github.sonus21.rqueue.models.response.DataSelectorResponse; import com.github.sonus21.rqueue.models.response.StringResponse; import com.github.sonus21.rqueue.utils.ReactiveDisabled; import com.github.sonus21.rqueue.web.service.RqueueDashboardChartService; @@ -54,12 +56,11 @@ @RestController @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue/api/v1") @Conditional(ReactiveDisabled.class) -public class RqueueRestController { +public class RqueueRestController extends BaseController { private final RqueueDashboardChartService rqueueDashboardChartService; private final RqueueQDetailService rqueueQDetailService; private final RqueueUtilityService rqueueUtilityService; private final RqueueSystemManagerService rqueueQManagerService; - private final RqueueWebConfig rqueueWebConfig; private final RqueueJobService rqueueJobService; @Autowired @@ -70,11 +71,11 @@ public RqueueRestController( RqueueSystemManagerService rqueueQManagerService, RqueueWebConfig rqueueWebConfig, RqueueJobService rqueueJobService) { + super(rqueueWebConfig); this.rqueueDashboardChartService = rqueueDashboardChartService; this.rqueueQDetailService = rqueueQDetailService; this.rqueueUtilityService = rqueueUtilityService; this.rqueueQManagerService = rqueueQManagerService; - this.rqueueWebConfig = rqueueWebConfig; this.rqueueJobService = rqueueJobService; } @@ -82,11 +83,10 @@ public RqueueRestController( @ResponseBody public ChartDataResponse getDashboardData( @RequestBody @Valid ChartDataRequest chartDataRequest, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueDashboardChartService.getDashboardChartData(chartDataRequest); } - return rqueueDashboardChartService.getDashboardChartData(chartDataRequest); + return null; } @GetMapping("jobs") @@ -94,108 +94,109 @@ public ChartDataResponse getDashboardData( public DataViewResponse getJobs( @RequestParam(name = "message-id") @NotEmpty String messageId, HttpServletResponse response) throws ProcessingException { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueJobService.getJobs(messageId); } - return rqueueJobService.getJobs(messageId); + return null; } @PostMapping("queue-data") @ResponseBody public DataViewResponse exploreQueue( @Valid @RequestBody QueueExploreRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueQDetailService.getExplorePageData( + request.getSrc(), + request.getName(), + request.getType(), + request.getPageNumber(), + request.getItemPerPage()); } - return rqueueQDetailService.getExplorePageData( - request.getSrc(), - request.getName(), - request.getType(), - request.getPageNumber(), - request.getItemPerPage()); + return null; } @PostMapping("view-data") @ResponseBody public DataViewResponse viewData( @RequestBody @Valid DateViewRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueQDetailService.viewData( + request.getName(), + request.getType(), + request.getKey(), + request.getPageNumber(), + request.getItemPerPage()); } - return rqueueQDetailService.viewData( - request.getName(), - request.getType(), - request.getKey(), - request.getPageNumber(), - request.getItemPerPage()); + return null; } @PostMapping("delete-message") @ResponseBody public BooleanResponse deleteMessage( @Valid @RequestBody MessageDeleteRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueUtilityService.deleteMessage(request.getQueueName(), request.getMessageId()); } - return rqueueUtilityService.deleteMessage(request.getQueueName(), request.getMessageId()); + return null; } @PostMapping("delete-queue") @ResponseBody public BaseResponse deleteQueue( @Valid @RequestBody DataTypeRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueQManagerService.deleteQueue(request.getName()); } - return rqueueQManagerService.deleteQueue(request.getName()); + return null; } @PostMapping("delete-queue-part") @ResponseBody public BooleanResponse deleteAll( @RequestBody @Valid DataDeleteRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueUtilityService.makeEmpty(request.getQueueName(), request.getDatasetName()); } - return rqueueUtilityService.makeEmpty(request.getQueueName(), request.getDatasetName()); + return null; } @PostMapping("data-type") @ResponseBody public StringResponse dataType( @RequestBody @Valid DataTypeRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueUtilityService.getDataType(request.getName()); } - return rqueueUtilityService.getDataType(request.getName()); + return null; } @PostMapping("move-data") @ResponseBody public MessageMoveResponse dataType( @RequestBody @Valid MessageMoveRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueUtilityService.moveMessage(request); } - return rqueueUtilityService.moveMessage(request); + return null; } @PostMapping("pause-unpause-queue") @ResponseBody public BaseResponse pauseUnpauseQueue( @RequestBody @Valid PauseUnpauseQueueRequest request, HttpServletResponse response) { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + return rqueueUtilityService.pauseUnpauseQueue(request); } - return rqueueUtilityService.pauseUnpauseQueue(request); + return null; + } + + @GetMapping("aggregate-data-selector") + @ResponseBody + public DataSelectorResponse aggregateDataCounter( + @RequestParam AggregationType type, HttpServletResponse response) { + if (isEnable(response)) { + return rqueueUtilityService.aggregateDataCounter(type); + } + return null; } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java index 98f0798b..58623cff 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java @@ -18,7 +18,6 @@ import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.utils.ReactiveDisabled; -import com.github.sonus21.rqueue.utils.StringUtils; import com.github.sonus21.rqueue.web.service.RqueueViewControllerService; import com.mitchellbosecke.pebble.spring.servlet.PebbleViewResolver; import java.util.Locale; @@ -37,9 +36,8 @@ @Controller @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue") @Conditional(ReactiveDisabled.class) -public class RqueueViewController { +public class RqueueViewController extends BaseController { - private final RqueueWebConfig rqueueWebConfig; private final PebbleViewResolver rqueueViewResolver; private final RqueueViewControllerService rqueueViewControllerService; @@ -48,39 +46,33 @@ public RqueueViewController( @Qualifier("rqueueViewResolver") PebbleViewResolver rqueueViewResolver, RqueueWebConfig rqueueWebConfig, RqueueViewControllerService rqueueViewControllerService) { + super(rqueueWebConfig); this.rqueueViewControllerService = rqueueViewControllerService; - this.rqueueWebConfig = rqueueWebConfig; this.rqueueViewResolver = rqueueViewResolver; } private String xForwardedPrefix(HttpServletRequest request) { - String prefix = rqueueWebConfig.getUrlPrefix(); - if (StringUtils.isEmpty(prefix)) { - return request.getHeader("x-forwarded-prefix"); - } - return prefix; + return request.getHeader("x-forwarded-prefix"); } @GetMapping public View index(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.index(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("index", Locale.ENGLISH); } - rqueueViewControllerService.index(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("index", Locale.ENGLISH); + return null; } @GetMapping("queues") public View queues(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.queues(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("queues", Locale.ENGLISH); } - rqueueViewControllerService.queues(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("queues", Locale.ENGLISH); + return null; } @GetMapping("queues/{queueName}") @@ -90,66 +82,60 @@ public View queueDetail( HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.queueDetail(model, xForwardedPrefix(request), queueName); + return rqueueViewResolver.resolveViewName("queue_detail", Locale.ENGLISH); } - rqueueViewControllerService.queueDetail(model, xForwardedPrefix(request), queueName); - return rqueueViewResolver.resolveViewName("queue_detail", Locale.ENGLISH); + return null; } @GetMapping("running") public View running(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.running(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.running(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("scheduled") public View scheduled(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.scheduled(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.scheduled(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("dead") public View dead(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.dead(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.dead(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("pending") public View pending(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.pending(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); } - rqueueViewControllerService.pending(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("running", Locale.ENGLISH); + return null; } @GetMapping("utility") public View utility(Model model, HttpServletRequest request, HttpServletResponse response) throws Exception { - if (!rqueueWebConfig.isEnable()) { - response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return null; + if (isEnable(response)) { + rqueueViewControllerService.utility(model, xForwardedPrefix(request)); + return rqueueViewResolver.resolveViewName("utility", Locale.ENGLISH); } - rqueueViewControllerService.utility(model, xForwardedPrefix(request)); - return rqueueViewResolver.resolveViewName("utility", Locale.ENGLISH); + return null; } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java similarity index 96% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorService.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java index fa391589..2e7fee03 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java @@ -51,12 +51,12 @@ import org.springframework.context.ApplicationListener; import org.springframework.context.SmartLifecycle; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; -import org.springframework.stereotype.Service; +import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -@Service +@Component @Slf4j -public class RqueueTaskMetricsAggregatorService +public class RqueueJobMetricsAggregatorService implements ApplicationListener, DisposableBean, SmartLifecycle { private final RqueueConfig rqueueConfig; private final RqueueWebConfig rqueueWebConfig; @@ -71,7 +71,7 @@ public class RqueueTaskMetricsAggregatorService private List> eventAggregatorTasks; @Autowired - public RqueueTaskMetricsAggregatorService( + public RqueueJobMetricsAggregatorService( RqueueConfig rqueueConfig, RqueueWebConfig rqueueWebConfig, RqueueLockManager rqueueLockManager, @@ -265,10 +265,10 @@ private void saveAggregateData( } private Map aggregate(QueueEvents events) { - List queueRqueueExecutionEvents = events.rqueueExecutionEvents; - RqueueExecutionEvent queueRqueueExecutionEvent = queueRqueueExecutionEvents.get(0); + List executionEvents = events.rqueueExecutionEvents; + RqueueExecutionEvent queueRqueueExecutionEvent = executionEvents.get(0); Map localDateTasksStatMap = new HashMap<>(); - for (RqueueExecutionEvent event : queueRqueueExecutionEvents) { + for (RqueueExecutionEvent event : executionEvents) { LocalDate date = DateTimeUtils.localDateFromMilli(queueRqueueExecutionEvent.getTimestamp()); TasksStat stat = localDateTasksStatMap.getOrDefault(date, new TasksStat()); aggregate(event, stat); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java index 4b0c8c33..b2f04462 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java @@ -21,6 +21,7 @@ import java.time.Duration; import java.util.Collection; import java.util.List; +import org.springframework.data.redis.core.ZSetOperations.TypedTuple; import reactor.core.publisher.Mono; public interface RqueueMessageMetadataService { @@ -32,13 +33,21 @@ public interface RqueueMessageMetadataService { List findAll(Collection ids); - void save(MessageMetadata messageMetadata, Duration duration); + void save(MessageMetadata messageMetadata, Duration ttl); MessageMetadata getByMessageId(String queueName, String messageId); - void deleteMessage(String queueName, String messageId, Duration duration); + void deleteMessage(String queueName, String messageId, Duration ttl); MessageMetadata getOrCreateMessageMetadata(RqueueMessage rqueueMessage); - Mono saveReactive(MessageMetadata messageMetadata, Duration duration); + Mono saveReactive(MessageMetadata messageMetadata, Duration ttl); + + List> readMessageMetadataForQueue( + String queueName, long start, long end); + + void saveMessageMetadataForQueue( + String queueName, MessageMetadata messageMetadata, Long ttlInMillisecond); + + void deleteQueueMessages(String queueName, long before); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java index d12cfcb9..9fd079ac 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java @@ -48,9 +48,9 @@ DataViewResponse viewData( List> getDeadLetterTasks(); - Mono getReactiveExplorePageData(String src, String name, DataType type, - int pageNumber, int itemPerPage); + Mono getReactiveExplorePageData( + String src, String name, DataType type, int pageNumber, int itemPerPage); - Mono viewReactiveData(String name, DataType type, String key, int pageNumber, - int itemPerPage); + Mono viewReactiveData( + String name, DataType type, String key, int pageNumber, int itemPerPage); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java index d6191e3d..642da40c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java @@ -16,13 +16,15 @@ package com.github.sonus21.rqueue.web.service; +import com.github.sonus21.rqueue.models.Pair; +import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.request.MessageMoveRequest; import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; import com.github.sonus21.rqueue.models.response.BaseResponse; import com.github.sonus21.rqueue.models.response.BooleanResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; +import com.github.sonus21.rqueue.models.response.DataSelectorResponse; import com.github.sonus21.rqueue.models.response.StringResponse; -import org.springframework.data.util.Pair; import reactor.core.publisher.Mono; public interface RqueueUtilityService { @@ -49,4 +51,7 @@ public interface RqueueUtilityService { BaseResponse pauseUnpauseQueue(PauseUnpauseQueueRequest request); + Mono reactiveAggregateDataCounter(AggregationType type); + + DataSelectorResponse aggregateDataCounter(AggregationType type); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java index 4f1666de..1dc1206f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java @@ -274,60 +274,69 @@ private List> createChartData( } private List> aggregateMonthly( - List queueStatisticsList, List chartDataTypeList) { + int numberOfDays, + List queueStatisticsList, + List chartDataTypeList) { Map monthToChartDataType = aggregateData( (date, today) -> (int) Math.floor((today.toEpochDay() - LocalDate.parse(date).toEpochDay()) / 30.0f), queueStatisticsList, chartDataTypeList, - getCount(Constants.DAYS_IN_A_MONTH)); + getCount(numberOfDays, Constants.DAYS_IN_A_MONTH)); return createChartData("Monthly", chartDataTypeList, monthToChartDataType); } private List> aggregateDaily( - List queueStatisticsList, List chartDataTypes) { + int numberOfDays, + List queueStatisticsList, + List chartDataTypes) { Map dayToChartDataType = aggregateData( (date, today) -> (int) (today.toEpochDay() - LocalDate.parse(date).toEpochDay()), queueStatisticsList, chartDataTypes, - getCount(1)); + getCount(numberOfDays, 1)); return createChartData("Daily", chartDataTypes, dayToChartDataType); } - private int getCount(int factor) { + private int getCount(int numberOfDays, int factor) { if (factor == 1) { - return rqueueWebConfig.getHistoryDay(); + return numberOfDays; } - return (int) Math.ceil(rqueueWebConfig.getHistoryDay() * 1.0f / factor); + return (int) Math.ceil(numberOfDays * 1.0f / factor); } private List> aggregateWeekly( - List queueStatisticsList, List chartDataTypeList) { + int numberOfDays, + List queueStatisticsList, + List chartDataTypeList) { Map weekToChartDataType = aggregateData( (date, today) -> (int) Math.floor((today.toEpochDay() - LocalDate.parse(date).toEpochDay()) / 7.0f), queueStatisticsList, chartDataTypeList, - getCount(Constants.DAYS_IN_A_WEEK)); + getCount(numberOfDays, Constants.DAYS_IN_A_WEEK)); return createChartData("Weekly", chartDataTypeList, weekToChartDataType); } private List> getChartData( - Collection ids, AggregationType type, List chartDataTypes) { + int numberOfDays, + Collection ids, + AggregationType type, + List chartDataTypes) { List queueStatistics = rqueueQStatsDao.findAll(ids); if (queueStatistics == null) { queueStatistics = new ArrayList<>(); } switch (type) { case DAILY: - return aggregateDaily(queueStatistics, getChartDataType(chartDataTypes)); + return aggregateDaily(numberOfDays, queueStatistics, getChartDataType(chartDataTypes)); case WEEKLY: - return aggregateWeekly(queueStatistics, getChartDataType(chartDataTypes)); + return aggregateWeekly(numberOfDays, queueStatistics, getChartDataType(chartDataTypes)); case MONTHLY: - return aggregateMonthly(queueStatistics, getChartDataType(chartDataTypes)); + return aggregateMonthly(numberOfDays, queueStatistics, getChartDataType(chartDataTypes)); default: throw new UnknownSwitchCase(type.name()); } @@ -336,7 +345,11 @@ private List> getChartData( private ChartDataResponse getQueueStats(ChartDataRequest chartDataRequest) { Collection ids = getQueueStatsId(chartDataRequest); List> rows = - getChartData(ids, chartDataRequest.getAggregationType(), chartDataRequest.getDateTypes()); + getChartData( + chartDataRequest.numberOfDays(rqueueWebConfig), + ids, + chartDataRequest.getAggregationType(), + chartDataRequest.getDateTypes()); ChartDataResponse response = new ChartDataResponse(); response.setData(rows); response.setHTitle(chartDataRequest.getAggregationType().getDescription()); @@ -366,6 +379,7 @@ private ChartDataResponse getQueueLatency(ChartDataRequest chartDataRequest) { Collection ids = getQueueStatsId(chartDataRequest); List> rows = getChartData( + chartDataRequest.numberOfDays(rqueueWebConfig), ids, chartDataRequest.getAggregationType(), Collections.singletonList(ChartDataType.EXECUTION)); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java index 066928c2..5cf19982 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java @@ -19,16 +19,21 @@ import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.dao.RqueueMessageMetadataDao; +import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; import java.util.Collection; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.DefaultTypedTuple; +import org.springframework.data.redis.core.ZSetOperations.TypedTuple; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @@ -36,10 +41,13 @@ @Slf4j public class RqueueMessageMetadataServiceImpl implements RqueueMessageMetadataService { private final RqueueMessageMetadataDao rqueueMessageMetadataDao; + private final RqueueStringDao rqueueStringDao; @Autowired - public RqueueMessageMetadataServiceImpl(RqueueMessageMetadataDao rqueueMessageMetadataDao) { + public RqueueMessageMetadataServiceImpl( + RqueueMessageMetadataDao rqueueMessageMetadataDao, RqueueStringDao rqueueStringDao) { this.rqueueMessageMetadataDao = rqueueMessageMetadataDao; + this.rqueueStringDao = rqueueStringDao; } @Override @@ -101,4 +109,44 @@ public MessageMetadata getOrCreateMessageMetadata(RqueueMessage rqueueMessage) { public Mono saveReactive(MessageMetadata messageMetadata, Duration duration) { return rqueueMessageMetadataDao.saveReactive(messageMetadata, duration); } + + @Override + public List> readMessageMetadataForQueue( + String queueName, long start, long end) { + List> metaIds = + rqueueStringDao.readFromOrderedSetWithScoreBetween(queueName, start, end); + Map metaIdToScoreMap = + metaIds.stream().collect(Collectors.toMap(TypedTuple::getValue, TypedTuple::getScore)); + List messageMetadata = findAll(metaIdToScoreMap.keySet()); + return messageMetadata.stream() + .map( + metadata -> { + Double score = metaIdToScoreMap.get(metadata.getId()); + if (score == null) { + return null; + } else { + return new DefaultTypedTuple<>(metadata, score); + } + }) + .filter(Objects::nonNull) + .sorted( + Comparator.comparingLong( + (DefaultTypedTuple e1) -> + -(Objects.requireNonNull(e1.getValue()).getUpdatedOn()))) + .collect(Collectors.toList()); + } + + @Override + public void saveMessageMetadataForQueue( + String queueName, MessageMetadata messageMetadata, Long ttlInMillisecond) { + messageMetadata.setUpdatedOn(System.currentTimeMillis()); + save(messageMetadata, Duration.ofMillis(ttlInMillisecond)); + rqueueStringDao.addToOrderedSetWithScore( + queueName, messageMetadata.getId(), -(System.currentTimeMillis() + ttlInMillisecond)); + } + + @Override + public void deleteQueueMessages(String queueName, long before) { + rqueueStringDao.deleteAll(queueName, -before, 0); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java index 2eafb76f..f8ca3cab 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java @@ -20,6 +20,7 @@ import static com.google.common.collect.Lists.newArrayList; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; @@ -52,6 +53,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -67,17 +69,20 @@ public class RqueueQDetailServiceImpl implements RqueueQDetailService { private final RqueueMessageTemplate rqueueMessageTemplate; private final RqueueSystemManagerService rqueueSystemManagerService; private final RqueueMessageMetadataService rqueueMessageMetadataService; + private final RqueueConfig rqueueConfig; @Autowired public RqueueQDetailServiceImpl( @Qualifier("stringRqueueRedisTemplate") RqueueRedisTemplate stringRqueueRedisTemplate, RqueueMessageTemplate rqueueMessageTemplate, RqueueSystemManagerService rqueueSystemManagerService, - RqueueMessageMetadataService rqueueMessageMetadataService) { + RqueueMessageMetadataService rqueueMessageMetadataService, + RqueueConfig rqueueConfig) { this.stringRqueueRedisTemplate = stringRqueueRedisTemplate; this.rqueueMessageTemplate = rqueueMessageTemplate; this.rqueueSystemManagerService = rqueueSystemManagerService; this.rqueueMessageMetadataService = rqueueMessageMetadataService; + this.rqueueConfig = rqueueConfig; } @Override @@ -105,12 +110,13 @@ public List> getQueueDataStructureDetail(QueueCon NavTab.RUNNING, new RedisDataDetail( processingQueueName, DataType.ZSET, running == null ? 0 : running))); - String timeQueueName = queueConfig.getDelayedQueueName(); - Long scheduled = stringRqueueRedisTemplate.getZsetSize(timeQueueName); + String scheduledQueueName = queueConfig.getScheduledQueueName(); + Long scheduled = stringRqueueRedisTemplate.getZsetSize(scheduledQueueName); queueRedisDataDetails.add( new HashMap.SimpleEntry<>( NavTab.SCHEDULED, - new RedisDataDetail(timeQueueName, DataType.ZSET, scheduled == null ? 0 : scheduled))); + new RedisDataDetail( + scheduledQueueName, DataType.ZSET, scheduled == null ? 0 : scheduled))); if (!CollectionUtils.isEmpty(queueConfig.getDeadLetterQueues())) { for (DeadLetterQueue dlq : queueConfig.getDeadLetterQueues()) { if (!dlq.isConsumerEnabled()) { @@ -128,6 +134,17 @@ public List> getQueueDataStructureDetail(QueueCon } } } + if (rqueueConfig.messageInTerminalStateShouldBeStored() + && !StringUtils.isEmpty(queueConfig.getCompletedQueueName())) { + Long completed = stringRqueueRedisTemplate.getZsetSize(queueConfig.getCompletedQueueName()); + queueRedisDataDetails.add( + new HashMap.SimpleEntry<>( + NavTab.COMPLETED, + new RedisDataDetail( + queueConfig.getCompletedQueueName(), + DataType.ZSET, + completed == null ? 0 : completed))); + } return queueRedisDataDetails; } @@ -205,18 +222,22 @@ private void addActionsIfRequired( String src, String name, DataType type, - boolean delayedQueue, + boolean scheduledQueue, boolean deadLetterQueue, + boolean completionQueue, DataViewResponse response) { if (deadLetterQueue) { response.addAction( new Action(ActionType.DELETE, String.format("dead letter queue '%s'", name))); + } else if (completionQueue) { + response.addAction( + new Action(ActionType.DELETE, String.format("Completed messages for queue '%s'", name))); } else if (type == DataType.LIST) { response.addAction( new Action(ActionType.DELETE, String.format("pending messages for queue '%s'", src))); - } else if (delayedQueue) { + } else if (scheduledQueue) { response.addAction( - new Action(ActionType.DELETE, String.format("delayed messages for queue '%s'", src))); + new Action(ActionType.DELETE, String.format("scheduled messages for queue '%s'", src))); } } @@ -226,18 +247,27 @@ public DataViewResponse getExplorePageData( QueueConfig queueConfig = rqueueSystemManagerService.getQueueConfig(src); DataViewResponse response = new DataViewResponse(); boolean deadLetterQueue = queueConfig.isDeadLetterQueue(name); - boolean timeQueue = queueConfig.getDelayedQueueName().equals(name); - setHeadersIfRequired(deadLetterQueue, type, response, pageNumber); - addActionsIfRequired(src, name, type, timeQueue, deadLetterQueue, response); + boolean scheduledQueue = queueConfig.getScheduledQueueName().equals(name); + boolean completionQueue = name.equals(queueConfig.getCompletedQueueName()); + setHeadersIfRequired(deadLetterQueue, completionQueue, type, response, pageNumber); + addActionsIfRequired( + src, name, type, scheduledQueue, deadLetterQueue, completionQueue, response); switch (type) { case ZSET: - if (timeQueue) { + if (scheduledQueue) { + response.setRows( + buildRows( + readFromZset(name, pageNumber, itemPerPage), new ZsetRowBuilder(true, false))); + } else if (completionQueue) { response.setRows( - buildRows(readFromZset(name, pageNumber, itemPerPage), new ZsetRowBuilder(true))); + buildRows( + readFromMessageMetadataStore(name, pageNumber, itemPerPage), + new ZsetRowBuilder(false, true))); } else { response.setRows( buildRows( - readFromZetWithScore(name, pageNumber, itemPerPage), new ZsetRowBuilder(false))); + readFromZetWithScore(name, pageNumber, itemPerPage), + new ZsetRowBuilder(false, false))); } break; case LIST: @@ -251,6 +281,24 @@ public DataViewResponse getExplorePageData( return response; } + private List> readFromMessageMetadataStore( + String name, int pageNumber, int itemPerPage) { + long start = pageNumber * (long) itemPerPage; + long end = start + itemPerPage - 1; + List> mes = + rqueueMessageMetadataService.readMessageMetadataForQueue(name, start, end); + if (CollectionUtils.isEmpty(mes)) { + return Collections.emptyList(); + } + return mes.stream() + .map( + e -> + new DefaultTypedTuple<>( + Objects.requireNonNull(e.getValue()).getRqueueMessage(), + (double) e.getValue().getUpdatedOn())) + .collect(Collectors.toList()); + } + private DataViewResponse responseForSet(String name) { List items = new ArrayList<>(stringRqueueRedisTemplate.getMembers(name)); DataViewResponse response = new DataViewResponse(); @@ -332,18 +380,24 @@ public DataViewResponse viewData( } private void setHeadersIfRequired( - boolean deadLetterQueue, DataType type, DataViewResponse response, int pageNumber) { + boolean deadLetterQueue, + boolean completedQueue, + DataType type, + DataViewResponse response, + int pageNumber) { if (pageNumber != 0) { return; } List headers = newArrayList("Id", "Message", "Type"); - if (DataType.ZSET == type) { + if (DataType.ZSET == type && !completedQueue) { headers.add("Time Left"); } - if (!deadLetterQueue) { - headers.add("Action"); - } else { + if (deadLetterQueue) { headers.add("AddedOn"); + } else if (completedQueue) { + headers.add("CompletedOn"); + } else { + headers.add("Action"); } response.setHeaders(headers); } @@ -407,7 +461,7 @@ public List> getScheduledTasks() { stringRqueueRedisTemplate.getRedisTemplate(), ((connection, keySerializer, valueSerializer) -> { for (QueueConfig queueConfig : queueConfigs) { - connection.zCard(keySerializer.serialize(queueConfig.getDelayedQueueName())); + connection.zCard(keySerializer.serialize(queueConfig.getScheduledQueueName())); } })); } @@ -415,7 +469,7 @@ public List> getScheduledTasks() { for (int i = 0; i < queueConfigs.size(); i++) { QueueConfig queueConfig = queueConfigs.get(i); rows.add( - Arrays.asList(queueConfig.getName(), queueConfig.getDelayedQueueName(), result.get(i))); + Arrays.asList(queueConfig.getName(), queueConfig.getScheduledQueueName(), result.get(i))); } return rows; } @@ -506,7 +560,7 @@ default TableRow getRow(RqueueMessage rqueueMessage) { TableRow row(RqueueMessage rqueueMessage, boolean deleted, Double score); } - class ListRowBuilder implements RowBuilder { + private static class ListRowBuilder implements RowBuilder { private final boolean deadLetterQueue; ListRowBuilder(boolean deadLetterQueue) { @@ -530,32 +584,41 @@ public TableRow row(RqueueMessage rqueueMessage, boolean deleted, Double score) } } - class ZsetRowBuilder implements RowBuilder { + private static class ZsetRowBuilder implements RowBuilder { private final long currentTime; - private final boolean timeQueue; + private final boolean scheduledQueue; + private final boolean completionQueue; - ZsetRowBuilder(boolean timeQueue) { - this.timeQueue = timeQueue; + ZsetRowBuilder(boolean scheduledQueue, boolean completionQueue) { + this.scheduledQueue = scheduledQueue; + this.completionQueue = completionQueue; this.currentTime = System.currentTimeMillis(); } @Override public TableRow row(RqueueMessage rqueueMessage, boolean deleted, Double score) { TableRow row = getRow(rqueueMessage); - if (timeQueue) { + if (scheduledQueue) { row.addColumn( new TableColumn( DateTimeUtils.milliToHumanRepresentation( rqueueMessage.getProcessAt() - currentTime))); + } else if (completionQueue) { + row.addColumn( + new TableColumn( + DateTimeUtils.milliToHumanRepresentation( + System.currentTimeMillis() - score.longValue()))); } else { row.addColumn( new TableColumn( DateTimeUtils.milliToHumanRepresentation(score.longValue() - currentTime))); } - if (!deleted) { - row.addColumn(new TableColumn(TableColumnType.ACTION, ActionType.DELETE)); - } else { - row.addColumn(new TableColumn(Constants.BLANK)); + if (!completionQueue) { + if (!deleted) { + row.addColumn(new TableColumn(TableColumnType.ACTION, ActionType.DELETE)); + } else { + row.addColumn(new TableColumn(Constants.BLANK)); + } } return row; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java index eaa3fe4b..489ad27c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java @@ -27,6 +27,8 @@ import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.models.response.BaseResponse; +import com.github.sonus21.rqueue.utils.RetryableRunnable; +import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; import java.util.ArrayList; import java.util.Arrays; @@ -34,7 +36,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -42,19 +48,24 @@ import reactor.core.publisher.Mono; @Service +@Slf4j public class RqueueSystemManagerServiceImpl implements RqueueSystemManagerService { private final RqueueConfig rqueueConfig; private final RqueueStringDao rqueueStringDao; private final RqueueSystemConfigDao rqueueSystemConfigDao; + private final RqueueMessageMetadataService rqueueMessageMetadataService; + private ScheduledExecutorService executorService; @Autowired public RqueueSystemManagerServiceImpl( RqueueConfig rqueueConfig, RqueueStringDao rqueueStringDao, - RqueueSystemConfigDao rqueueSystemConfigDao) { + RqueueSystemConfigDao rqueueSystemConfigDao, + RqueueMessageMetadataService rqueueMessageMetadataService) { this.rqueueConfig = rqueueConfig; this.rqueueStringDao = rqueueStringDao; this.rqueueSystemConfigDao = rqueueSystemConfigDao; + this.rqueueMessageMetadataService = rqueueMessageMetadataService; } private List queueKeys(QueueConfig queueConfig) { @@ -63,7 +74,7 @@ private List queueKeys(QueueConfig queueConfig) { queueConfig.getQueueName(), queueConfig.getProcessingQueueName(), rqueueConfig.getQueueStatisticsKey(queueConfig.getName())); - keys.add(queueConfig.getDelayedQueueName()); + keys.add(queueConfig.getScheduledQueueName()); if (queueConfig.hasDeadLetterQueue()) { for (DeadLetterQueue queue : queueConfig.getDeadLetterQueues()) { keys.add(queue.getName()); @@ -103,6 +114,18 @@ private QueueConfig createOrUpdateConfig(QueueConfig queueConfig, QueueDetail qu if (queueDetail.isDlqSet()) { updated = systemQueueConfig.addDeadLetterQueue(queueDetail.getDeadLetterQueue()); } + + // for older queue this needs to be set + if (systemQueueConfig.getScheduledQueueName() == null) { + systemQueueConfig.setScheduledQueueName(queueDetail.getScheduledQueueName()); + updated = true; + } + + if (systemQueueConfig.getCompletedQueueName() == null) { + systemQueueConfig.setCompletedQueueName(queueDetail.getCompletedQueueName()); + updated = true; + } + updated = systemQueueConfig.updateVisibilityTimeout(queueDetail.getVisibilityTimeout()) || updated; updated = @@ -158,6 +181,30 @@ public void onApplicationEvent(RqueueBootstrapEvent event) { } createOrUpdateConfigs(queueDetails); } + handleEventForCleanup(event); + } + + private void handleEventForCleanup(RqueueBootstrapEvent event) { + if (executorService != null) { + executorService.shutdown(); + } + if (rqueueConfig.isProducer()) { + return; + } + if (event.isStartup() && rqueueConfig.messageInTerminalStateShouldBeStored()) { + executorService = Executors.newSingleThreadScheduledExecutor(); + List queueConfigList = + rqueueSystemConfigDao.getConfigByNames(EndpointRegistry.getActiveQueues()); + int i = 0; + for (QueueConfig queueConfig : queueConfigList) { + CleanCompletedJobs completedJobs = new CleanCompletedJobs(queueConfig.getName()); + executorService.scheduleAtFixedRate( + completedJobs, + i, + rqueueConfig.getCompletedJobCleanupIntervalInMs(), + TimeUnit.MILLISECONDS); + } + } } @Override @@ -204,4 +251,36 @@ public QueueConfig getQueueConfig(String queueName) { public Mono deleteReactiveQueue(String queueName) { return Mono.just(deleteQueue(queueName)); } + + private class CleanCompletedJobs extends RetryableRunnable { + private final String queueName; + + protected CleanCompletedJobs(String queueName) { + super(log, ""); + this.queueName = queueName; + } + + @Override + public void start() { + QueueConfig queueConfig = rqueueSystemConfigDao.getConfigByName(queueName, true); + if (queueConfig == null) { + log.error("Queue config does not exist {}", queueName); + return; + } + if (queueConfig.isPaused()) { + log.debug("Queue {} is paused", queueName); + return; + } + if (queueConfig.getCompletedQueueName() == null) { + log.error("Queue completed queue name is not set {}", queueName); + return; + } + + long endTime = + System.currentTimeMillis() - rqueueConfig.messageDurabilityInTerminalStateInMillisecond(); + log.debug("Deleting completed messages for queue: {}, before: {}", endTime, queueName); + rqueueMessageMetadataService.deleteQueueMessages( + queueConfig.getCompletedQueueName(), endTime); + } + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java index 2e1a46fc..cbd70de6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java @@ -28,13 +28,16 @@ import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.exception.UnknownSwitchCase; import com.github.sonus21.rqueue.models.MessageMoveResult; +import com.github.sonus21.rqueue.models.Pair; import com.github.sonus21.rqueue.models.db.QueueConfig; +import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.enums.DataType; import com.github.sonus21.rqueue.models.request.MessageMoveRequest; import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; import com.github.sonus21.rqueue.models.response.BaseResponse; import com.github.sonus21.rqueue.models.response.BooleanResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; +import com.github.sonus21.rqueue.models.response.DataSelectorResponse; import com.github.sonus21.rqueue.models.response.StringResponse; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.StringUtils; @@ -42,10 +45,11 @@ import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import java.time.Duration; import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @@ -199,7 +203,7 @@ public Pair getLatestVersion() { } } } - return Pair.of(releaseLink, latestVersion); + return new Pair<>(releaseLink, latestVersion); } @Override @@ -249,4 +253,90 @@ public BaseResponse pauseUnpauseQueue(PauseUnpauseQueueRequest request) { } return response; } + + private List> getDailyDateCounter() { + List> dateSelector = new LinkedList<>(); + int[] dates = new int[] {1, 2, 3, 4, 6, 7}; + int step = 15; + int stepAfter = 15; + int i = 1; + dateSelector.add(new Pair<>("0", "Select")); + // 84 days + while (i <= rqueueWebConfig.getHistoryDay()) { + if (i >= stepAfter) { + if (i <= rqueueWebConfig.getHistoryDay()) { + dateSelector.add(new Pair<>(String.valueOf(i), String.format("Last %d days", i))); + } + i += step; + } else { + for (int date : dates) { + if (date == i) { + String suffix = "day"; + if (i != 1) { + suffix = "days"; + } + String msg = String.format("Last %d %s", date, suffix); + dateSelector.add(new Pair<>(String.valueOf(date), msg)); + break; + } + } + i += 1; + } + } + return dateSelector; + } + + @Override + public Mono reactiveAggregateDataCounter(AggregationType type) { + return Mono.just(aggregateDataCounter(type)); + } + + @Override + public DataSelectorResponse aggregateDataCounter(AggregationType type) { + String title = ""; + List> data; + if (type == AggregationType.DAILY) { + data = getDailyDateCounter(); + title = "Select Number of Days"; + } else if (type == AggregationType.WEEKLY) { + data = getWeeklyDateCounter(); + title = "Select Number of Weeks"; + } else { + data = getMonthlyDateCounter(); + title = "Select Number of Months"; + } + return new DataSelectorResponse(title, data); + } + + private List> getMonthlyDateCounter() { + List> dateSelector = new LinkedList<>(); + dateSelector.add(new Pair<>("0", "Select")); + int nMonths = + (int) Math.ceil(rqueueWebConfig.getHistoryDay() / (double) Constants.DAYS_IN_A_MONTH); + for (int month = 1; month <= nMonths; month++) { + String suffix = "month"; + if (month != 1) { + suffix = "months"; + } + String msg = String.format("Last %d %s", month, suffix); + dateSelector.add(new Pair<>(String.valueOf(month), msg)); + } + return dateSelector; + } + + private List> getWeeklyDateCounter() { + List> dateSelector = new LinkedList<>(); + dateSelector.add(new Pair<>("0", "Select")); + int nWeek = + (int) Math.ceil(rqueueWebConfig.getHistoryDay() / (double) Constants.DAYS_IN_A_WEEK); + for (int week = 1; week <= nWeek; week++) { + String suffix = "week"; + if (week != 1) { + suffix = "weeks"; + } + String msg = String.format("Last %d %s", week, suffix); + dateSelector.add(new Pair<>(String.valueOf(week), msg)); + } + return dateSelector; + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java index a26ef5cb..1fbc3b11 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java @@ -17,14 +17,14 @@ package com.github.sonus21.rqueue.web.service.impl; import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.config.RqueueWebConfig; +import com.github.sonus21.rqueue.models.Pair; import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.enums.ChartDataType; import com.github.sonus21.rqueue.models.enums.DataType; import com.github.sonus21.rqueue.models.enums.NavTab; import com.github.sonus21.rqueue.models.response.RedisDataDetail; -import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.utils.StringUtils; import com.github.sonus21.rqueue.web.service.RqueueQDetailService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; import com.github.sonus21.rqueue.web.service.RqueueUtilityService; @@ -36,8 +36,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map.Entry; +import javax.servlet.http.HttpServletResponse; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.springframework.ui.Model; @@ -45,6 +45,7 @@ public class RqueueViewControllerServiceImpl implements RqueueViewControllerService { private final RqueueConfig rqueueConfig; + private final RqueueWebConfig rqueueWebConfig; private final RqueueQDetailService rqueueQDetailService; private final RqueueUtilityService rqueueUtilityService; private final RqueueSystemManagerService rqueueSystemManagerService; @@ -52,10 +53,12 @@ public class RqueueViewControllerServiceImpl implements RqueueViewControllerServ @Autowired public RqueueViewControllerServiceImpl( RqueueConfig rqueueConfig, + RqueueWebConfig rqueueWebConfig, RqueueQDetailService rqueueQDetailService, RqueueUtilityService rqueueUtilityService, RqueueSystemManagerService rqueueSystemManagerService) { this.rqueueConfig = rqueueConfig; + this.rqueueWebConfig = rqueueWebConfig; this.rqueueQDetailService = rqueueQDetailService; this.rqueueUtilityService = rqueueUtilityService; this.rqueueSystemManagerService = rqueueSystemManagerService; @@ -76,17 +79,7 @@ private void addBasicDetails(Model model, String xForwardedPrefix) { "time", OffsetDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); model.addAttribute("timeInMilli", System.currentTimeMillis()); model.addAttribute("version", rqueueConfig.getLibVersion()); - String prefix = Constants.FORWARD_SLASH; - if (!StringUtils.isEmpty(xForwardedPrefix)) { - if (xForwardedPrefix.endsWith(Constants.FORWARD_SLASH)) { - xForwardedPrefix = xForwardedPrefix.substring(0, xForwardedPrefix.length() - 1); - } - prefix = xForwardedPrefix + prefix; - } - if (!prefix.startsWith(Constants.FORWARD_SLASH)) { - prefix = Constants.FORWARD_SLASH + prefix; - } - model.addAttribute("urlPrefix", prefix); + model.addAttribute("urlPrefix", rqueueWebConfig.getUrlPrefix(xForwardedPrefix)); } @Override @@ -95,6 +88,8 @@ public void index(Model model, String xForwardedPrefix) { addNavData(model, null); model.addAttribute("title", "Rqueue Dashboard"); model.addAttribute("aggregatorTypes", Arrays.asList(AggregationType.values())); + model.addAttribute( + "aggregatorDateCounter", rqueueUtilityService.aggregateDataCounter(AggregationType.DAILY)); model.addAttribute("typeSelectors", ChartDataType.getActiveCharts()); } @@ -122,6 +117,8 @@ public void queueDetail(Model model, String xForwardedPrefix, String queueName) model.addAttribute("title", "Queue: " + queueName); model.addAttribute("queueName", queueName); model.addAttribute("aggregatorTypes", Arrays.asList(AggregationType.values())); + model.addAttribute( + "aggregatorDateCounter", rqueueUtilityService.aggregateDataCounter(AggregationType.DAILY)); model.addAttribute("typeSelectors", ChartDataType.getActiveCharts()); model.addAttribute("queueActions", queueActions); model.addAttribute("queueRedisDataDetails", queueRedisDataDetail); diff --git a/rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json b/rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json index 0fdde871..2194d82f 100644 --- a/rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json +++ b/rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json @@ -45,6 +45,13 @@ "type": "java.lang.Boolean", "defaultValue": false }, + { + "sourceType": "com.github.sonus21.rqueue.config.RqueueConfig", + "name": "rqueue.completed.job.cleanup.interval", + "description": "How frequently completed jobs should be disabled in millisecond", + "type": "java.lang.Long", + "defaultValue": 30000 + }, { "sourceType": "com.github.sonus21.rqueue.config.RqueueConfig", "name": "rqueue.latest.version.check.enabled", @@ -110,15 +117,15 @@ }, { "sourceType": "com.github.sonus21.rqueue.config.RqueueConfig", - "name": "rqueue.delayed.queue.prefix", - "description": "Rqueue Delayed Queue Name Prefix", + "name": "rqueue.scheduled.queue.prefix", + "description": "Rqueue Scheduled Queue Name Prefix", "type": "java.lang.String", "defaultValue": "d-queue::" }, { "sourceType": "com.github.sonus21.rqueue.config.RqueueConfig", - "name": "rqueue.delayed.queue.channel.prefix", - "description": "Rqueue Delayed Queue Channel Name Prefix", + "name": "rqueue.scheduled.queue.channel.prefix", + "description": "Rqueue Scheduled Queue Channel Name Prefix", "type": "java.lang.String", "defaultValue": "d-channel::" }, @@ -270,8 +277,8 @@ }, { "sourceType": "com.github.sonus21.rqueue.config.RqueueSchedulerConfig", - "name": "rqueue.scheduler.delayed.message.thread.pool.size", - "description": "Number of threads should be used to pull delayed message to original queue", + "name": "rqueue.scheduler.scheduled.message.thread.pool.size", + "description": "Number of threads should be used to pull scheduled message to original queue", "type": "java.lang.Integer", "defaultValue": 3 }, @@ -284,8 +291,8 @@ }, { "sourceType": "com.github.sonus21.rqueue.config.RqueueSchedulerConfig", - "name": "rqueue.scheduler.delayed.message.time.interval", - "description": "How frequently messages should be moved from delayed queues to source queue (millisecond) ", + "name": "rqueue.scheduler.scheduled.message.time.interval", + "description": "How frequently messages should be moved from scheduled queues to source queue (millisecond) ", "type": "java.lang.Long", "defaultValue": 5000 }, @@ -348,14 +355,14 @@ { "sourceType": "com.github.sonus21.rqueue.config.RqueueWebConfig", "name": "rqueue.web.collect.statistic.aggregate.shutdown.wait.time", - "description": "How long Rqueue should wait for aggregate job to complete o shutdown (mili seconds)", + "description": "How long Rqueue should wait for aggregate job to complete before shutdown (milliseconds)", "type": "java.lang.Integer", "defaultValue": 500 }, { "sourceType": "com.github.sonus21.rqueue.config.RqueueWebConfig", "name": "rqueue.web.collect.statistic.aggregate.event.lock.duration", - "description": "How long Rqueue should wait for Redis lock when aggregation data is being saved(mili seconds)", + "description": "How long Rqueue should wait for Redis lock when aggregation data is being saved(milliseconds)", "type": "java.lang.Integer", "defaultValue": 500 } diff --git a/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css b/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css index 664321c3..07efcbc6 100644 --- a/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css +++ b/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css @@ -421,11 +421,6 @@ section { display: none; } -.latency-chart-form { - margin-top: 250px; - margin-left: -50px; -} - .message-move-form, .explore-data-form { border: 1px solid; diff --git a/rqueue-core/src/main/resources/public/rqueue/js/rqueue.js b/rqueue-core/src/main/resources/public/rqueue/js/rqueue.js index dc554536..63d6b34e 100644 --- a/rqueue-core/src/main/resources/public/rqueue/js/rqueue.js +++ b/rqueue-core/src/main/resources/public/rqueue/js/rqueue.js @@ -111,15 +111,36 @@ function refreshStatsChart(chartParams, div_id) { }); chartParams["aggregationType"] = aggregatorType; chartParams["dateTypes"] = types; + chartParams['number'] = $('#stats-nday :selected').val(); drawChart(chartParams, div_id); } function refreshLatencyChart(chartParams, div_id) { chartParams['aggregationType'] = $( '#latency-aggregator-type :selected').val(); + chartParams['number'] = $('#latency-nday :selected').val(); drawChart(chartParams, div_id); } +function refreshAggregatorSelector(element_selector, type) { + ajaxRequest( + getAbsoluteUrl('rqueue/api/v1/aggregate-data-selector?type=' + type), + 'GET', "", + function (response) { + let title = response.title; + let data = response.data; + let element = $(element_selector); + element.empty(); + for (let i = 0; i < data.length; i++) { + element.append($('