diff --git a/.env.example b/.env.example index 545f0fa14..4ed48f603 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,2 @@ COMPOSE_PROJECT_NAME=yii2-queue COMPOSE_FILE=tests/docker-compose.yml - -AWS_SQS_ENABLED= -AWS_SQS_URL= -AWS_KEY= -AWS_SECRET= -AWS_REGION= -AWS_SQS_FIFO_ENABLED= -AWS_SQS_FIFO_URL= -AWS_SQS_FIFO_MESSAGE_GROUP_ID= diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36ec37335..8dadc45d9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,14 +23,6 @@ on: env: COMPOSE_PROJECT_NAME: yii2-queue COMPOSE_FILE: tests/docker-compose.yml - AWS_SQS_ENABLED: ${{ secrets.AWS_SQS_ENABLED }} - AWS_SQS_URL: ${{ secrets.AWS_SQS_URL }} - AWS_KEY: ${{ secrets.AWS_KEY }} - AWS_SECRET: ${{ secrets.AWS_SECRET }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_SQS_FIFO_ENABLED: ${{ secrets.AWS_SQS_FIFO_ENABLED }} - AWS_SQS_FIFO_URL: ${{ secrets.AWS_SQS_FIFO_URL }} - AWS_SQS_FIFO_MESSAGE_GROUP_ID: ${{ secrets.AWS_SQS_FIFO_MESSAGE_GROUP_ID }} jobs: phpunit: name: PHP ${{ matrix.php }} diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index d9d868552..be42e8f9a 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -25,14 +25,6 @@ name: static analysis env: COMPOSE_PROJECT_NAME: yii2-queue COMPOSE_FILE: tests/docker-compose.yml - AWS_SQS_ENABLED: ${{ secrets.AWS_SQS_ENABLED }} - AWS_SQS_URL: ${{ secrets.AWS_SQS_URL }} - AWS_KEY: ${{ secrets.AWS_KEY }} - AWS_SECRET: ${{ secrets.AWS_SECRET }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_SQS_FIFO_ENABLED: ${{ secrets.AWS_SQS_FIFO_ENABLED }} - AWS_SQS_FIFO_URL: ${{ secrets.AWS_SQS_FIFO_URL }} - AWS_SQS_FIFO_MESSAGE_GROUP_ID: ${{ secrets.AWS_SQS_FIFO_MESSAGE_GROUP_ID }} jobs: psalm: name: PHP ${{ matrix.php }} diff --git a/Makefile b/Makefile index 138991334..7f260cc75 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,11 @@ help: ## Display help information build: ## Build an image from a docker-compose file. Params: {{ v=8.1 }}. Default latest PHP 8.1 @cp -n .env.example .env PHP_VERSION=$(filter-out $@,$(v)) docker-compose up -d --build + make create-sqs-queue + make create-sqs-fifo-queue test: ## Run tests. Params: {{ v=8.1 }}. Default latest PHP 8.1 - PHP_VERSION=$(filter-out $@,$(v)) docker-compose build --pull yii2-queue-php + make build PHP_VERSION=$(filter-out $@,$(v)) docker-compose run yii2-queue-php vendor/bin/phpunit --coverage-clover coverage.xml make down @@ -35,3 +37,9 @@ clean: clean-all: clean sudo rm -rf tests/runtime/.composer* + +create-sqs-queue: ## Create SQS queue + docker exec yii2-queue-localstack sh -c "awslocal sqs create-queue --queue-name yii2-queue" + +create-sqs-fifo-queue: ## Create SQS FIFO queue + docker exec yii2-queue-localstack sh -c 'awslocal sqs create-queue --queue-name yii2-queue.fifo --attributes "FifoQueue=true"' diff --git a/docs/guide-ru/driver-sqs.md b/docs/guide-ru/driver-sqs.md index c7332a9dc..f99aa818d 100644 --- a/docs/guide-ru/driver-sqs.md +++ b/docs/guide-ru/driver-sqs.md @@ -10,7 +10,7 @@ ```php return [ 'bootstrap' => [ - 'queue', // The component registers own console commands + 'queue', // Компонент регистрирует собственные консольные команды ], 'components' => [ 'queue' => [ @@ -24,6 +24,30 @@ return [ ]; ``` +Пример настройки для FIFO очередей: + +```php +return [ + 'bootstrap' => [ + 'queue', // Компонент регистрирует собственные консольные команды + ], + 'components' => [ + 'queue' => [ + 'class' => \yii\queue\sqs\Queue::class, + 'url' => '', + 'key' => '', + 'secret' => '', + 'region' => '', + 'messageGroupId' => '', + ], + ], +]; +``` + +Идентификатор группы сообщений требуется SQS для очередей FIFO. Вы можете настроить свои собственные или использовать значение "default". + +Идентификатор дедупликации генерируется автоматически, поэтому независимо от того, активировали ли вы дедупликацию на основе содержимого в очереди SQS или нет, этот идентификатор будет использоваться. + Консоль ------- diff --git a/docs/guide-zh-CN/driver-sqs.md b/docs/guide-zh-CN/driver-sqs.md new file mode 100644 index 000000000..504050415 --- /dev/null +++ b/docs/guide-zh-CN/driver-sqs.md @@ -0,0 +1,86 @@ +AWS SQS Driver +============ + +The driver uses AWS SQS to store queue data. + +You have to add `aws/aws-sdk-php` extension to your application in order to use it. + +Configuration example for standard queues: + +```php +return [ + 'bootstrap' => [ + 'queue', // The component registers own console commands + ], + 'components' => [ + 'queue' => [ + 'class' => \yii\queue\sqs\Queue::class, + 'url' => '', + 'key' => '', + 'secret' => '', + 'region' => '', + ], + ], +]; +``` + +Configuration example for FIFO queues: + +```php +return [ + 'bootstrap' => [ + 'queue', // The component registers own console commands + ], + 'components' => [ + 'queue' => [ + 'class' => \yii\queue\sqs\Queue::class, + 'url' => '', + 'key' => '', + 'secret' => '', + 'region' => '', + 'messageGroupId' => '', + ], + ], +]; +``` + +The message group ID is required by SQS for FIFO queues. You can configure your own or use the "default" value. + +The deduplication ID is generated automatically, so no matter if you have activated content-based deduplication in the SQS queue or not, this ID will be used. + +Console +------- + +Console command is used to execute tasks. + +```sh +yii queue/listen [timeout] +``` + +`listen` command launches a daemon which infinitely queries the queue. If there are new tasks +they're immediately obtained and executed. `timeout` parameter is number of seconds to wait a job. +It uses SQS "Long Polling" feature, that holds a connection between client and a queue. + +**Important:** `timeout` parameter for SQS driver must be in range between 0 and 20 seconds. + +This method is most efficient when command is properly daemonized via +[supervisor](worker.md#supervisor) or [systemd](worker.md#systemd). + +```sh +yii queue/run +``` + +`run` command obtains and executes tasks in a loop until queue is empty. Works well with +[cron](worker.md#cron). + +`run` and `listen` commands have options: + +- `--verbose`, `-v`: print executing statuses into console. +- `--isolate`: each task is executed in a separate child process. +- `--color`: highlighting for verbose mode. + +```sh +yii queue/clear +``` + +`clear` command clears a queue. diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 5babaa3c2..821636bb0 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -8,6 +8,7 @@ cacheDirectory=".phpunit.cache" displayDetailsOnIncompleteTests="true" displayDetailsOnSkippedTests="true" + executionOrder="random" > diff --git a/psalm.xml b/psalm.xml index 8a030fcb1..a4f56374a 100644 --- a/psalm.xml +++ b/psalm.xml @@ -16,7 +16,6 @@ - diff --git a/src/drivers/sqs/Command.php b/src/drivers/sqs/Command.php index 5bd14c529..3597e0af5 100644 --- a/src/drivers/sqs/Command.php +++ b/src/drivers/sqs/Command.php @@ -48,9 +48,6 @@ public function actionRun(): ?int */ public function actionListen(int $timeout = 3): ?int { - if (!is_numeric($timeout)) { - throw new Exception('Timeout must be numeric.'); - } if ($timeout < 1 || $timeout > 20) { throw new Exception('Timeout must be between 1 and 20'); } diff --git a/src/drivers/sqs/Queue.php b/src/drivers/sqs/Queue.php index 1618d0e8e..21aad3237 100644 --- a/src/drivers/sqs/Queue.php +++ b/src/drivers/sqs/Queue.php @@ -15,6 +15,7 @@ use yii\base\NotSupportedException; use yii\queue\cli\Queue as CliQueue; use yii\queue\serializers\JsonSerializer; +use yii\queue\serializers\SerializerInterface; /** * SQS Queue. @@ -64,12 +65,12 @@ class Queue extends CliQueue * Json serializer by default. * @inheritdoc */ - public string|array|\yii\queue\serializers\SerializerInterface $serializer = JsonSerializer::class; + public string|array|SerializerInterface $serializer = JsonSerializer::class; /** - * @var SqsClient + * @var SqsClient|null */ - private SqsClient $_client; + private ?SqsClient $client = null; /** * Listens queue and runs each job. @@ -173,7 +174,7 @@ public function status($id): int * @return bool * @since 2.2.1 */ - public function handle(string $id, string $message, int $ttr, int $attempt) + public function handle(string $id, string $message, int $ttr, int $attempt): bool { return $this->handleMessage($id, $message, $ttr, $attempt); } @@ -213,8 +214,8 @@ protected function pushMessage(string $payload, int $ttr, int $delay, mixed $pri */ protected function getClient(): SqsClient { - if ($this->_client) { - return $this->_client; + if (null !== $this->client) { + return $this->client; } if ($this->key !== null && $this->secret !== null) { @@ -228,11 +229,11 @@ protected function getClient(): SqsClient $credentials = CredentialProvider::defaultProvider(); } - $this->_client = new SqsClient([ + $this->client = new SqsClient([ 'credentials' => $credentials, 'region' => $this->region, 'version' => $this->version, ]); - return $this->_client; + return $this->client; } } diff --git a/tests/app/config/main.php b/tests/app/config/main.php index 047508603..7b719c946 100644 --- a/tests/app/config/main.php +++ b/tests/app/config/main.php @@ -31,6 +31,8 @@ 'amqpInteropQueue', 'beanstalkQueue', 'stompQueue', + 'sqsQueue', + 'sqsFifoQueue', ], 'components' => [ 'syncQueue' => [ @@ -116,6 +118,21 @@ 'class' => StompQueue::class, 'host' => getenv('ACTIVEMQ_HOST') ?: 'localhost', ], + 'sqsQueue' => [ + 'class' => SqsQueue::class, + 'url' => getenv('AWS_SQS_URL'), + 'key' => getenv('AWS_KEY'), + 'secret' => getenv('AWS_SECRET'), + 'region' => getenv('AWS_REGION'), + ], + 'sqsFifoQueue' => [ + 'class' => SqsQueue::class, + 'url' => getenv('AWS_SQS_FIFO_URL'), + 'key' => getenv('AWS_KEY'), + 'secret' => getenv('AWS_SECRET'), + 'region' => getenv('AWS_REGION'), + 'messageGroupId' => getenv('AWS_SQS_FIFO_MESSAGE_GROUP_ID'), + ], ], ]; @@ -127,27 +144,4 @@ ]; } -if (getenv('AWS_SQS_ENABLED')) { - $config['bootstrap'][] = 'sqsQueue'; - $config['components']['sqsQueue'] = [ - 'class' => SqsQueue::class, - 'url' => getenv('AWS_SQS_URL'), - 'key' => getenv('AWS_KEY'), - 'secret' => getenv('AWS_SECRET'), - 'region' => getenv('AWS_REGION'), - ]; -} - -if (getenv('AWS_SQS_FIFO_ENABLED')) { - $config['bootstrap'][] = 'sqsFifoQueue'; - $config['components']['sqsFifoQueue'] = [ - 'class' => SqsQueue::class, - 'url' => getenv('AWS_SQS_FIFO_URL'), - 'key' => getenv('AWS_KEY'), - 'secret' => getenv('AWS_SECRET'), - 'region' => getenv('AWS_REGION'), - 'messageGroupId' => getenv('AWS_SQS_FIFO_MESSAGE_GROUP_ID'), - ]; -} - return $config; diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 1f22950d3..84fd09812 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -32,15 +32,13 @@ services: BEANSTALK_HOST: beanstalk GEARMAN_HOST: gearmand COMPOSER_ALLOW_SUPERUSER: 1 - AWS_SQS_ENABLED: ${AWS_SQS_ENABLED} - AWS_KEY: ${AWS_KEY} - AWS_SECRET: ${AWS_SECRET} - AWS_REGION: ${AWS_REGION} - AWS_SQS_URL: ${AWS_SQS_URL} ACTIVEMQ_HOST: activemq - AWS_SQS_FIFO_ENABLED: ${AWS_SQS_FIFO_ENABLED} - AWS_SQS_FIFO_URL: ${AWS_SQS_FIFO_URL} - AWS_SQS_FIFO_MESSAGE_GROUP_ID: ${AWS_SQS_FIFO_MESSAGE_GROUP_ID} + AWS_KEY: ${AWS_KEY:-admin} + AWS_SECRET: ${AWS_SECRET:-admin} + AWS_REGION: ${AWS_REGION:-us-east-1} + AWS_SQS_URL: ${AWS_SQS_URL:-http://localstack:4566/000000000000/yii2-queue} + AWS_SQS_FIFO_URL: ${AWS_SQS_FIFO_URL:-http://localstack:4566/000000000000/yii2-queue.fifo} + AWS_SQS_FIFO_MESSAGE_GROUP_ID: ${AWS_SQS_FIFO_MESSAGE_GROUP_ID:-default} depends_on: - mysql - postgres @@ -49,6 +47,7 @@ services: - beanstalk - gearmand - activemq + - localstack networks: net: {} @@ -118,7 +117,22 @@ services: networks: net: {} + # https://hub.docker.com/r/localstack/localstack + localstack: + container_name: yii2-queue-localstack + image: localstack/localstack + restart: always + ports: + - "4566:4566" + environment: + AWS_DEFAULT_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: admin + SERVICES: sqs + DISABLE_CORS_CHECKS: 1 + networks: + net: {} + networks: net: name: yii2_queue_net - diff --git a/tests/drivers/sqs/FifoQueueTest.php b/tests/drivers/sqs/FifoQueueTest.php index bdf87582b..5e59219b3 100644 --- a/tests/drivers/sqs/FifoQueueTest.php +++ b/tests/drivers/sqs/FifoQueueTest.php @@ -10,6 +10,7 @@ namespace tests\drivers\sqs; +use Aws\Sqs\Exception\SqsException; use tests\app\RetryJob; use tests\drivers\CliTestCase; use Yii; @@ -40,17 +41,17 @@ public function testListen(): void public function testFifoQueueDoesNotSupportPerMessageDelays(): void { + $this->expectException(SqsException::class); $this->startProcess(['php', 'yii', 'queue/listen', '1']); $job = $this->createSimpleJob(); - $this->setExpectedException('\Aws\Sqs\Exception\SqsException'); $this->getQueue()->delay(2)->push($job); } public function testRetry(): void { $this->startProcess(['php', 'yii', 'queue/listen', '1']); - $job = new RetryJob(['uid' => uniqid()]); + $job = new RetryJob(['uid' => uniqid('', true)]); $this->getQueue()->push($job); sleep(6); @@ -60,10 +61,6 @@ public function testRetry(): void public function testClear(): void { - if (!getenv('AWS_SQS_FIFO_CLEAR_TEST_ENABLED')) { - $this->markTestSkipped(__METHOD__ . ' is disabled'); - } - $this->getQueue()->push($this->createSimpleJob()); $this->runProcess(['php', 'yii', 'queue/clear', '--interactive=0']); } @@ -75,13 +72,4 @@ protected function getQueue(): Queue { return Yii::$app->sqsFifoQueue; } - - protected function setUp(): void - { - if (!getenv('AWS_SQS_FIFO_ENABLED')) { - $this->markTestSkipped('AWS SQS FIFO tests are disabled'); - } - - parent::setUp(); - } } diff --git a/tests/drivers/sqs/QueueTest.php b/tests/drivers/sqs/QueueTest.php index 253674326..acfb14b36 100644 --- a/tests/drivers/sqs/QueueTest.php +++ b/tests/drivers/sqs/QueueTest.php @@ -52,7 +52,7 @@ public function testLater(): void public function testRetry(): void { $this->startProcess(['php', 'yii', 'queue/listen', '1']); - $job = new RetryJob(['uid' => uniqid()]); + $job = new RetryJob(['uid' => uniqid('', true)]); $this->getQueue()->push($job); sleep(6); @@ -62,10 +62,6 @@ public function testRetry(): void public function testClear(): void { - if (!getenv('AWS_SQS_CLEAR_TEST_ENABLED')) { - $this->markTestSkipped(__METHOD__ . ' is disabled'); - } - $this->getQueue()->push($this->createSimpleJob()); $this->runProcess(['php', 'yii', 'queue/clear', '--interactive=0']); } @@ -77,13 +73,4 @@ protected function getQueue(): Queue { return Yii::$app->sqsQueue; } - - protected function setUp(): void - { - if (!getenv('AWS_SQS_ENABLED')) { - $this->markTestSkipped('AWS SQS tests are disabled'); - } - - parent::setUp(); - } }