From 34229af21e02496b3bcf93ede9e3279b7dd12916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Pineau?= Date: Mon, 14 Dec 2020 17:09:04 +0100 Subject: [PATCH] [Messenger] Be able to get raw data when a message in not decodable by the PHP Serializer --- .../Resources/config/console.php | 3 + .../Command/AbstractFailedMessagesCommand.php | 14 +++- .../Command/FailedMessagesRemoveCommand.php | 8 ++- .../Command/FailedMessagesRetryCommand.php | 48 +++++++++---- .../Command/FailedMessagesShowCommand.php | 68 ++++++++++-------- .../Stamp/MessageDecodingFailedStamp.php | 19 +++++ .../Serialization/PhpSerializerTest.php | 19 +++-- ...SerializerWithClassNotFoundSupportTest.php | 72 +++++++++++++++++++ .../Transport/Serialization/PhpSerializer.php | 30 ++++++-- 9 files changed, 225 insertions(+), 56 deletions(-) create mode 100644 src/Symfony/Component/Messenger/Stamp/MessageDecodingFailedStamp.php create mode 100644 src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerWithClassNotFoundSupportTest.php diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/console.php b/src/Symfony/Bundle/FrameworkBundle/Resources/config/console.php index 6407f3e244400..d64cd058e61f8 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Resources/config/console.php +++ b/src/Symfony/Bundle/FrameworkBundle/Resources/config/console.php @@ -191,6 +191,7 @@ service('messenger.routable_message_bus'), service('event_dispatcher'), service('logger'), + service('messenger.transport.native_php_serializer')->nullOnInvalid(), ]) ->tag('console.command') @@ -198,6 +199,7 @@ ->args([ abstract_arg('Default failure receiver name'), abstract_arg('Receivers'), + service('messenger.transport.native_php_serializer')->nullOnInvalid(), ]) ->tag('console.command') @@ -205,6 +207,7 @@ ->args([ abstract_arg('Default failure receiver name'), abstract_arg('Receivers'), + service('messenger.transport.native_php_serializer')->nullOnInvalid(), ]) ->tag('console.command') diff --git a/src/Symfony/Component/Messenger/Command/AbstractFailedMessagesCommand.php b/src/Symfony/Component/Messenger/Command/AbstractFailedMessagesCommand.php index 62f01fc03f051..3f502e78b215e 100644 --- a/src/Symfony/Component/Messenger/Command/AbstractFailedMessagesCommand.php +++ b/src/Symfony/Component/Messenger/Command/AbstractFailedMessagesCommand.php @@ -21,12 +21,14 @@ use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\Exception\InvalidArgumentException; use Symfony\Component\Messenger\Stamp\ErrorDetailsStamp; +use Symfony\Component\Messenger\Stamp\MessageDecodingFailedStamp; use Symfony\Component\Messenger\Stamp\RedeliveryStamp; use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp; use Symfony\Component\Messenger\Stamp\TransportMessageIdStamp; use Symfony\Component\Messenger\Transport\Receiver\ListableReceiverInterface; use Symfony\Component\Messenger\Transport\Receiver\MessageCountAwareInterface; use Symfony\Component\Messenger\Transport\Receiver\ReceiverInterface; +use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer; use Symfony\Component\VarDumper\Caster\Caster; use Symfony\Component\VarDumper\Caster\TraceStub; use Symfony\Component\VarDumper\Cloner\ClonerInterface; @@ -44,13 +46,15 @@ abstract class AbstractFailedMessagesCommand extends Command protected const DEFAULT_TRANSPORT_OPTION = 'choose'; protected $failureTransports; + protected ?PhpSerializer $phpSerializer; private ?string $globalFailureReceiverName; - public function __construct(?string $globalFailureReceiverName, ServiceProviderInterface $failureTransports) + public function __construct(?string $globalFailureReceiverName, ServiceProviderInterface $failureTransports, PhpSerializer $phpSerializer = null) { $this->failureTransports = $failureTransports; $this->globalFailureReceiverName = $globalFailureReceiverName; + $this->phpSerializer = $phpSerializer; parent::__construct(); } @@ -78,6 +82,8 @@ protected function displaySingleMessage(Envelope $envelope, SymfonyStyle $io) $lastRedeliveryStamp = $envelope->last(RedeliveryStamp::class); /** @var ErrorDetailsStamp|null $lastErrorDetailsStamp */ $lastErrorDetailsStamp = $envelope->last(ErrorDetailsStamp::class); + /** @var MessageDecodingFailedStamp|null $lastMessageDecodingFailedStamp */ + $lastMessageDecodingFailedStamp = $envelope->last(MessageDecodingFailedStamp::class); $rows = [ ['Class', \get_class($envelope->getMessage())], @@ -126,12 +132,18 @@ protected function displaySingleMessage(Envelope $envelope, SymfonyStyle $io) if ($io->isVeryVerbose()) { $io->title('Message:'); + if (null !== $lastMessageDecodingFailedStamp) { + $io->error('The message could not be decoded. See below an APPROXIMATIVE representation of the class.'); + } $dump = new Dumper($io, null, $this->createCloner()); $io->writeln($dump($envelope->getMessage())); $io->title('Exception:'); $flattenException = $lastErrorDetailsStamp?->getFlattenException(); $io->writeln(null === $flattenException ? '(no data)' : $dump($flattenException)); } else { + if (null !== $lastMessageDecodingFailedStamp) { + $io->error('The message could not be decoded.'); + } $io->writeln(' Re-run command with -vv to see more message & error details.'); } } diff --git a/src/Symfony/Component/Messenger/Command/FailedMessagesRemoveCommand.php b/src/Symfony/Component/Messenger/Command/FailedMessagesRemoveCommand.php index 2b62af3ce4bb2..34243b3d35c0b 100644 --- a/src/Symfony/Component/Messenger/Command/FailedMessagesRemoveCommand.php +++ b/src/Symfony/Component/Messenger/Command/FailedMessagesRemoveCommand.php @@ -74,7 +74,13 @@ private function removeMessages(string $failureTransportName, array $ids, Receiv } foreach ($ids as $id) { - $envelope = $receiver->find($id); + $this->phpSerializer?->enableClassNotFoundCreation(); + try { + $envelope = $receiver->find($id); + } finally { + $this->phpSerializer?->enableClassNotFoundCreation(false); + } + if (null === $envelope) { $io->error(sprintf('The message with id "%s" was not found.', $id)); continue; diff --git a/src/Symfony/Component/Messenger/Command/FailedMessagesRetryCommand.php b/src/Symfony/Component/Messenger/Command/FailedMessagesRetryCommand.php index abd9ab14e7300..af70141733cb9 100644 --- a/src/Symfony/Component/Messenger/Command/FailedMessagesRetryCommand.php +++ b/src/Symfony/Component/Messenger/Command/FailedMessagesRetryCommand.php @@ -23,11 +23,12 @@ use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\Messenger\Event\WorkerMessageReceivedEvent; use Symfony\Component\Messenger\EventListener\StopWorkerOnMessageLimitListener; -use Symfony\Component\Messenger\Exception\LogicException; use Symfony\Component\Messenger\MessageBusInterface; +use Symfony\Component\Messenger\Stamp\MessageDecodingFailedStamp; use Symfony\Component\Messenger\Transport\Receiver\ListableReceiverInterface; use Symfony\Component\Messenger\Transport\Receiver\ReceiverInterface; use Symfony\Component\Messenger\Transport\Receiver\SingleMessageReceiver; +use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer; use Symfony\Component\Messenger\Worker; use Symfony\Contracts\Service\ServiceProviderInterface; @@ -41,13 +42,13 @@ class FailedMessagesRetryCommand extends AbstractFailedMessagesCommand private MessageBusInterface $messageBus; private ?LoggerInterface $logger; - public function __construct(?string $globalReceiverName, ServiceProviderInterface $failureTransports, MessageBusInterface $messageBus, EventDispatcherInterface $eventDispatcher, LoggerInterface $logger = null) + public function __construct(?string $globalReceiverName, ServiceProviderInterface $failureTransports, MessageBusInterface $messageBus, EventDispatcherInterface $eventDispatcher, LoggerInterface $logger = null, PhpSerializer $phpSerializer = null) { $this->eventDispatcher = $eventDispatcher; $this->messageBus = $messageBus; $this->logger = $logger; - parent::__construct($globalReceiverName, $failureTransports); + parent::__construct($globalReceiverName, $failureTransports, $phpSerializer); } protected function configure(): void @@ -133,23 +134,23 @@ private function runInteractive(string $failureTransportName, SymfonyStyle $io, // to be temporarily "acked", even if the user aborts // handling the message while (true) { - $ids = []; - foreach ($receiver->all(1) as $envelope) { - ++$count; - - $id = $this->getMessageId($envelope); - if (null === $id) { - throw new LogicException(sprintf('The "%s" receiver is able to list messages by id but the envelope is missing the TransportMessageIdStamp stamp.', $failureTransportName)); + $envelopes = []; + $this->phpSerializer?->enableClassNotFoundCreation(); + try { + foreach ($receiver->all(1) as $envelope) { + ++$count; + $envelopes[] = $envelope; } - $ids[] = $id; + } finally { + $this->phpSerializer?->enableClassNotFoundCreation(false); } // break the loop if all messages are consumed - if (0 === \count($ids)) { + if (0 === \count($envelopes)) { break; } - $this->retrySpecificIds($failureTransportName, $ids, $io, $shouldForce); + $this->retrySpecificEnvelops($envelopes, $failureTransportName, $io, $shouldForce); } } else { // get() and ask messages one-by-one @@ -171,6 +172,10 @@ private function runWorker(string $failureTransportName, ReceiverInterface $rece $this->displaySingleMessage($envelope, $io); + if ($envelope->last(MessageDecodingFailedStamp::class)) { + throw new \RuntimeException(sprintf('The message with id "%s" could not decoded, it can only be shown or removed.', $this->getMessageId($envelope) ?? '?')); + } + $shouldHandle = $shouldForce || $io->confirm('Do you want to retry (yes) or delete this message (no)?'); if ($shouldHandle) { @@ -207,7 +212,12 @@ private function retrySpecificIds(string $failureTransportName, array $ids, Symf } foreach ($ids as $id) { - $envelope = $receiver->find($id); + $this->phpSerializer?->enableClassNotFoundCreation(); + try { + $envelope = $receiver->find($id); + } finally { + $this->phpSerializer?->enableClassNotFoundCreation(false); + } if (null === $envelope) { throw new RuntimeException(sprintf('The message "%s" was not found.', $id)); } @@ -216,4 +226,14 @@ private function retrySpecificIds(string $failureTransportName, array $ids, Symf $this->runWorker($failureTransportName, $singleReceiver, $io, $shouldForce); } } + + private function retrySpecificEnvelops(array $envelopes, string $failureTransportName, SymfonyStyle $io, bool $shouldForce) + { + $receiver = $this->getReceiver($failureTransportName); + + foreach ($envelopes as $envelope) { + $singleReceiver = new SingleMessageReceiver($receiver, $envelope); + $this->runWorker($failureTransportName, $singleReceiver, $io, $shouldForce); + } + } } diff --git a/src/Symfony/Component/Messenger/Command/FailedMessagesShowCommand.php b/src/Symfony/Component/Messenger/Command/FailedMessagesShowCommand.php index b8752d902e2c9..df9c308223e2c 100644 --- a/src/Symfony/Component/Messenger/Command/FailedMessagesShowCommand.php +++ b/src/Symfony/Component/Messenger/Command/FailedMessagesShowCommand.php @@ -96,29 +96,29 @@ private function listMessages(?string $failedTransportName, SymfonyStyle $io, in $io->comment(sprintf('Displaying only \'%s\' messages', $classFilter)); } - foreach ($envelopes as $envelope) { - $currentClassName = \get_class($envelope->getMessage()); - - if ($classFilter && $classFilter !== $currentClassName) { - continue; - } - - /** @var RedeliveryStamp|null $lastRedeliveryStamp */ - $lastRedeliveryStamp = $envelope->last(RedeliveryStamp::class); - /** @var ErrorDetailsStamp|null $lastErrorDetailsStamp */ - $lastErrorDetailsStamp = $envelope->last(ErrorDetailsStamp::class); - - $errorMessage = ''; - if (null !== $lastErrorDetailsStamp) { - $errorMessage = $lastErrorDetailsStamp->getExceptionMessage(); + $this->phpSerializer?->enableClassNotFoundCreation(); + try { + foreach ($envelopes as $envelope) { + $currentClassName = \get_class($envelope->getMessage()); + + if ($classFilter && $classFilter !== $currentClassName) { + continue; + } + + /** @var RedeliveryStamp|null $lastRedeliveryStamp */ + $lastRedeliveryStamp = $envelope->last(RedeliveryStamp::class); + /** @var ErrorDetailsStamp|null $lastErrorDetailsStamp */ + $lastErrorDetailsStamp = $envelope->last(ErrorDetailsStamp::class); + + $rows[] = [ + $this->getMessageId($envelope), + $currentClassName, + null === $lastRedeliveryStamp ? '' : $lastRedeliveryStamp->getRedeliveredAt()->format('Y-m-d H:i:s'), + $lastErrorDetailsStamp?->getExceptionMessage() ?? '', + ]; } - - $rows[] = [ - $this->getMessageId($envelope), - $currentClassName, - null === $lastRedeliveryStamp ? '' : $lastRedeliveryStamp->getRedeliveredAt()->format('Y-m-d H:i:s'), - $errorMessage, - ]; + } finally { + $this->phpSerializer?->enableClassNotFoundCreation(false); } $rowsCount = \count($rows); @@ -148,14 +148,19 @@ private function listMessagesPerClass(?string $failedTransportName, SymfonyStyle $countPerClass = []; - foreach ($envelopes as $envelope) { - $c = \get_class($envelope->getMessage()); + $this->phpSerializer?->enableClassNotFoundCreation(); + try { + foreach ($envelopes as $envelope) { + $c = \get_class($envelope->getMessage()); - if (!isset($countPerClass[$c])) { - $countPerClass[$c] = [$c, 0]; - } + if (!isset($countPerClass[$c])) { + $countPerClass[$c] = [$c, 0]; + } - ++$countPerClass[$c][1]; + ++$countPerClass[$c][1]; + } + } finally { + $this->phpSerializer?->enableClassNotFoundCreation(false); } if (0 === \count($countPerClass)) { @@ -171,7 +176,12 @@ private function showMessage(?string $failedTransportName, string $id, SymfonySt { /** @var ListableReceiverInterface $receiver */ $receiver = $this->getReceiver($failedTransportName); - $envelope = $receiver->find($id); + $this->phpSerializer?->enableClassNotFoundCreation(); + try { + $envelope = $receiver->find($id); + } finally { + $this->phpSerializer?->enableClassNotFoundCreation(false); + } if (null === $envelope) { throw new RuntimeException(sprintf('The message "%s" was not found.', $id)); } diff --git a/src/Symfony/Component/Messenger/Stamp/MessageDecodingFailedStamp.php b/src/Symfony/Component/Messenger/Stamp/MessageDecodingFailedStamp.php new file mode 100644 index 0000000000000..66e4778fbd47a --- /dev/null +++ b/src/Symfony/Component/Messenger/Stamp/MessageDecodingFailedStamp.php @@ -0,0 +1,19 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Messenger\Stamp; + +/** + * @author Grégoire Pineau + */ +class MessageDecodingFailedStamp implements StampInterface +{ +} diff --git a/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerTest.php b/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerTest.php index b31914a9b07f7..c83606a59fdb3 100644 --- a/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerTest.php +++ b/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerTest.php @@ -22,7 +22,7 @@ class PhpSerializerTest extends TestCase { public function testEncodedIsDecodable() { - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $envelope = new Envelope(new DummyMessage('Hello')); @@ -36,7 +36,7 @@ public function testDecodingFailsWithMissingBodyKey() $this->expectException(MessageDecodingFailedException::class); $this->expectExceptionMessage('Encoded envelope should have at least a "body", or maybe you should implement your own serializer'); - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $serializer->decode([]); } @@ -46,7 +46,7 @@ public function testDecodingFailsWithBadFormat() $this->expectException(MessageDecodingFailedException::class); $this->expectExceptionMessageMatches('/Could not decode/'); - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $serializer->decode([ 'body' => '{"message": "bar"}', @@ -58,7 +58,7 @@ public function testDecodingFailsWithBadBase64Body() $this->expectException(MessageDecodingFailedException::class); $this->expectExceptionMessageMatches('/Could not decode/'); - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $serializer->decode([ 'body' => 'x', @@ -70,7 +70,7 @@ public function testDecodingFailsWithBadClass() $this->expectException(MessageDecodingFailedException::class); $this->expectExceptionMessageMatches('/class "ReceivedSt0mp" not found/'); - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $serializer->decode([ 'body' => 'O:13:"ReceivedSt0mp":0:{}', @@ -79,7 +79,7 @@ public function testDecodingFailsWithBadClass() public function testEncodedSkipsNonEncodeableStamps() { - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $envelope = new Envelope(new DummyMessage('Hello'), [ new DummyPhpSerializerNonSendableStamp(), @@ -91,7 +91,7 @@ public function testEncodedSkipsNonEncodeableStamps() public function testNonUtf8IsBase64Encoded() { - $serializer = new PhpSerializer(); + $serializer = $this->createPhpSerializer(); $envelope = new Envelope(new DummyMessage("\xE9")); @@ -99,6 +99,11 @@ public function testNonUtf8IsBase64Encoded() $this->assertTrue((bool) preg_match('//u', $encoded['body']), 'Encodes non-UTF8 payloads'); $this->assertEquals($envelope, $serializer->decode($encoded)); } + + protected function createPhpSerializer(): PhpSerializer + { + return new PhpSerializer(); + } } class DummyPhpSerializerNonSendableStamp implements NonSendableStampInterface diff --git a/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerWithClassNotFoundSupportTest.php b/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerWithClassNotFoundSupportTest.php new file mode 100644 index 0000000000000..5c3f1fe48e24b --- /dev/null +++ b/src/Symfony/Component/Messenger/Tests/Transport/Serialization/PhpSerializerWithClassNotFoundSupportTest.php @@ -0,0 +1,72 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Messenger\Tests\Transport\Serialization; + +use Symfony\Component\Messenger\Envelope; +use Symfony\Component\Messenger\Exception\MessageDecodingFailedException; +use Symfony\Component\Messenger\Stamp\MessageDecodingFailedStamp; +use Symfony\Component\Messenger\Tests\Fixtures\DummyMessage; +use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer; + +class PhpSerializerWithClassNotFoundSupportTest extends PhpSerializerTest +{ + public function testDecodingFailsWithBadClass() + { + $this->expectException(MessageDecodingFailedException::class); + + $serializer = $this->createPhpSerializer(); + + $serializer->decode([ + 'body' => 'O:13:"ReceivedSt0mp":0:{}', + ]); + } + + public function testDecodingFailsButCreateClassNotFound() + { + $serializer = $this->createPhpSerializer(); + + $encodedEnvelope = $serializer->encode(new Envelope(new DummyMessage('Hello'))); + // Simulate a change in the code base + $encodedEnvelope['body'] = str_replace('DummyMessage', 'OupsyMessage', $encodedEnvelope['body']); + + $envelope = $serializer->decode($encodedEnvelope); + + $lastMessageDecodingFailedStamp = $envelope->last(MessageDecodingFailedStamp::class); + $this->assertInstanceOf(MessageDecodingFailedStamp::class, $lastMessageDecodingFailedStamp); + $message = $envelope->getMessage(); + // The class does not exist, so we cannot use anything else. The only + // purpose of this feature is to aim debugging (so dumping value) + ob_start(); + var_dump($message); + $content = ob_get_clean(); + // remove object ID + $content = preg_replace('/#\d+/', '', $content); + $expected = << + string(55) "Symfony\Component\Messenger\Tests\Fixtures\OupsyMessage" + ["message":"Symfony\Component\Messenger\Tests\Fixtures\OupsyMessage":private]=> + string(5) "Hello" + } + + EOT; + $this->assertEquals($expected, $content); + } + + protected function createPhpSerializer(): PhpSerializer + { + $serializer = new PhpSerializer(); + $serializer->enableClassNotFoundCreation(); + + return $serializer; + } +} diff --git a/src/Symfony/Component/Messenger/Transport/Serialization/PhpSerializer.php b/src/Symfony/Component/Messenger/Transport/Serialization/PhpSerializer.php index aecb98391429f..b9a4edb44f851 100644 --- a/src/Symfony/Component/Messenger/Transport/Serialization/PhpSerializer.php +++ b/src/Symfony/Component/Messenger/Transport/Serialization/PhpSerializer.php @@ -13,6 +13,7 @@ use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\Exception\MessageDecodingFailedException; +use Symfony\Component\Messenger\Stamp\MessageDecodingFailedStamp; use Symfony\Component\Messenger\Stamp\NonSendableStampInterface; /** @@ -20,6 +21,13 @@ */ class PhpSerializer implements SerializerInterface { + private bool $createClassNotFound = false; + + public function enableClassNotFoundCreation(bool $enable = true): void + { + $this->createClassNotFound = $enable; + } + public function decode(array $encodedEnvelope): Envelope { if (empty($encodedEnvelope['body'])) { @@ -50,14 +58,19 @@ public function encode(Envelope $envelope): array ]; } - private function safelyUnserialize(string $contents) + private function safelyUnserialize(string $contents): Envelope { if ('' === $contents) { throw new MessageDecodingFailedException('Could not decode an empty message using PHP serialization.'); } $signalingException = new MessageDecodingFailedException(sprintf('Could not decode message using PHP serialization: %s.', $contents)); - $prevUnserializeHandler = ini_set('unserialize_callback_func', self::class.'::handleUnserializeCallback'); + + if ($this->createClassNotFound) { + $prevUnserializeHandler = ini_set('unserialize_callback_func', null); + } else { + $prevUnserializeHandler = ini_set('unserialize_callback_func', self::class.'::handleUnserializeCallback'); + } $prevErrorHandler = set_error_handler(function ($type, $msg, $file, $line, $context = []) use (&$prevErrorHandler, $signalingException) { if (__FILE__ === $file) { throw $signalingException; @@ -67,13 +80,22 @@ private function safelyUnserialize(string $contents) }); try { - $meta = unserialize($contents); + /** @var Envelope */ + $envelope = unserialize($contents); } finally { restore_error_handler(); ini_set('unserialize_callback_func', $prevUnserializeHandler); } - return $meta; + if (!$envelope instanceof Envelope) { + throw $signalingException; + } + + if ($envelope->getMessage() instanceof \__PHP_Incomplete_Class) { + $envelope = $envelope->with(new MessageDecodingFailedStamp()); + } + + return $envelope; } /**