8000 Merge branch '4.4' into 5.2 · symfony/cache@d690163 · GitHub
  • [go: up one dir, main page]
    Skip to content

    Commit d690163

    Browse files
    nicolas-grekasnicolas-grekas
    committed
    Merge branch '4.4' into 5.2
    * 4.4: [ErrorHandler] fix handling buffered SilencedErrorContext [HttpClient] fix Psr18Client when allow_url_fopen=0 [DependencyInjection] Add support of PHP enumerations [Cache] handle prefixed redis connections when clearing pools [Cache] fix eventual consistency when using RedisTagAwareAdapter with a cluster [Validator][Translation] Add ExpressionLanguageSyntax en and fr [HttpKernel] [HttpCache] Keep s-maxage=0 from ESI sub-responses [Cache] Disable locking on Windows by default [DependencyInjection] Fix binding "iterable $foo" when using the PHP-DSL [Config] fix tracking default values that reference the parent class [DependencyInjection] fix accepted types on FactoryTrait::factory() [VarDumper] Fix tests for PHP 8.1 [Mailer] fix encoding of addresses using SmtpTransport [MonologBridge] Fix the server:log help --filter sample
    2 parents aaab9c4 + fcdbaf8 commit d690163

    19 files changed

    +160
    -98
    lines changed

    ‎Adapter/RedisTagAwareAdapter.php

    Lines changed: 70 additions & 78 deletions
    Original file line numberDiff line numberDiff line change
    @@ -23,17 +23,13 @@
    2323
    use Symfony\Component\Cache\Traits\RedisTrait;
    2424

    2525
    /**
    26-
    * Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using RENAME+SMEMBERS.
    26+
    * Stores tag id <> cache id relationship as a Redis Set.
    2727
    *
    2828
    * Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
    2929
    * if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
    3030
    * relationship survives eviction (cache cleanup when Redis runs out of memory).
    3131
    *
    32-
    * Requirements:
    33-
    * - Client: PHP Redis or Predis
    34-
    * Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
    35-
    * - Server: Redis 2.8+
    36-
    * Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
    32+
    * Redis server 2.8+ with any `volatile-*` eviction policy, OR `noeviction` if you're sure memory will NEVER fill up
    3733
    *
    3834
    * Design limitations:
    3935
    * - Max 4 billion cache keys per cache tag as limited by Redis Set datatype.
    @@ -49,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
    4945
    {
    5046
    use RedisTrait;
    5147

    52-
    /**
    53-
    * Limits for how many keys are deleted in batch.
    54-
    */
    55-
    private const BULK_DELETE_LIMIT = 10000;
    56-
    5748
    /**
    5849
    * On cache items without a lifetime set, we set it to 100 days. This is to make sure cache items are
    5950
    * preferred to be evicted over tag Sets, if eviction policy is configured according to requirements.
    @@ -96,7 +87,7 @@ protected function doSave(array $values, int $lifetime, array $addTagData = [],
    9687
    {
    9788
    $eviction = $this->getRedisEvictionPolicy();
    9889
    if ('noeviction' !== $eviction && 0 !== strpos($eviction, 'volatile-')) {
    99-
    throw new LogicException(sprintf('Redis maxmemory-policy setting "%s" is *not* supported by RedisTagAwareAdapter, use "noeviction" or "volatile-*" eviction policies.', $eviction));
    90+
    throw new LogicException(sprintf('Redis maxmemory-policy setting "%s" is *not* supported by RedisTagAwareAdapter, use "noeviction" or "volatile-*" eviction policies.', $eviction));
    10091
    }
    10192

    10293
    // serialize values
    @@ -163,15 +154,9 @@ protected function doDeleteYieldTags(array $ids): iterable
    163154
    return v:sub(14, 13 + v:byte(13) + v:byte(12) * 256 + v:byte(11) * 65536)
    164155
    EOLUA;
    165156

    166-
    if ($this->redis instanceof \Predis\ClientInterface) {
    167-
    $evalArgs = [$lua, 1, &$id];
    168-
    } else {
    169-
    $evalArgs = [$lua, [&$id], 1];
    170-
    }
    171-
    172-
    $results = $this->pipeline(function () use ($ids, &$id, $evalArgs) {
    157+
    $results = $this->pipeline(function () use ($ids, $lua) {
    173158
    foreach ($ids as $id) {
    174-
    yield 'eval' => $evalArgs;
    159+
    yield 'eval' => $this->redis instanceof \Predis\ClientInterface ? [$lua, 1, $id] : [$lua, [$id], 1];
    175160
    }
    176161
    });
    177162

    @@ -189,12 +174,15 @@ protected function doDeleteYieldTags(array $ids): iterable
    189174
    */
    190175
    protected function doDeleteTagRelations(array $tagData): bool
    191176
    {
    192-
    $this->pipeline(static function () use ($tagData) {
    177+
    $results = $this->pipeline(static function () use ($tagData) {
    193178
    foreach ($tagData as $tagId => $idList) {
    194179
    array_unshift($idList, $tagId);
    195180
    yield 'sRem' => $idList;
    196181
    }
    197-
    })->rewind();
    182+
    });
    183+
    foreach ($results as $result) {
    184+
    // no-op
    185+
    }
    198186

    199187
    return true;
    200188
    }
    @@ -204,77 +192,81 @@ protected function doDeleteTagRelations(array $tagData): bool
    204192
    */
    205193
    protected function doInvalidate(array $tagIds): bool
    206194
    {
    207-
    if (!$this->redis instanceof \Predis\ClientInterface || !$this->redis->getConnection() instanceof PredisCluster) {
    208-
    $movedTagSetIds = $this->renameKeys($this->redis, $tagIds);
    209-
    } else {
    210-
    $clusterConnection = $this->redis->getConnection();
    211-
    $tagIdsByConnection = new \SplObjectStorage();
    212-
    $movedTagSetIds = [];
    195+
    // This script scans the set of items linked to tag: it empties the set
    196+
    // and removes the linked items. When the set is still not empty after
    197+
    // the scan, it means we're in cluster mode and that the linked items
    198+
    // are on other nodes: we move the links to a temporary set and we
    199+
    // gargage collect that set from the client side.
    213200

    214-
    foreach ($tagIds as $id) {
    215-
    $connection = $clusterConnection->getConnectionByKey($id);
    216-
    $slot = $tagIdsByConnection[$connection] ?? $tagIdsByConnection[$connection] = new \ArrayObject();
    217-
    $slot[] = $id;
    218-
    }
    201+
    $lua = <<<'EOLUA'
    202+
    local cursor = '0'
    203+
    local id = KEYS[1]
    204+
    repeat
    205+
    local result = redis.call('SSCAN', id, cursor, 'COUNT', 5000);
    206+
    cursor = result[1];
    207+
    local rems = {}
    208+
    209+
    for _, v in ipairs(result[2]) do
    210+
    local ok, _ = pcall(redis.call, 'DEL', ARGV[1]..v)
    211+
    if ok then
    212+
    table.insert(rems, v)
    213+
    end
    214+
    end
    215+
    if 0 < #rems then
    216+
    redis.call('SREM', id, unpack(rems))
    217+
    end
    218+
    until '0' == cursor;
    219+
    220+
    redis.call('SUNIONSTORE', '{'..id..'}'..id, id)
    221+
    redis.call('DEL', id)
    222+
    223+
    return redis.call('SSCAN', '{'..id..'}'..id, '0', 'COUNT', 5000)
    224+
    EOLUA;
    219225

    220-
    foreach ($tagIdsByConnection as $connection) {
    221-
    $slot = $tagIdsByConnection[$connection];
    222-
    $movedTagSetIds = array_merge($movedTagSetIds, $this->renameKeys(new $this->redis($connection, $this->redis->getOptions()), $slot->getArrayCopy()));
    226+
    $results = $this->pipeline(function () use ($tagIds, $lua) {
    227+
    if ($this->redis instanceof \Predis\ClientInterface) {
    228+
    $prefix = $this->redis->getOptions()->prefix ? $this->redis->getOptions()->prefix->getPrefix() : '';
    229+
    } elseif (\is_array($prefix = $this->redis->getOption(\Redis::OPT_PREFIX) ?? '')) {
    230+
    $prefix = current($prefix);
    223231
    }
    224-
    }
    225232

    226-
    // No Sets found
    227-
    if (!$movedTagSetIds) {
    228-
    return false;
    229-
    }
    230-
    231-
    // Now safely take the time to read the keys in each set and collect ids we need to delete
    232-
    $tagIdSets = $this->pipeline(static function () use ($movedTagSetIds) {
    233-
    foreach ($movedTagSetIds as $movedTagId) {
    234-
    yield 'sMembers' => [$movedTagId];
    233+
    foreach ($tagIds as $id) {
    234+
    yield 'eval' => $this->redis instanceof \Predis\ClientInterface ? [$lua, 1, $id, $prefix] : [$lua, [$id, $prefix], 1];
    235235
    }
    236236
    });
    237237

    238-
    // Return combination of the temporary Tag Set ids and their values (cache ids)
    239-
    $ids = array_merge($movedTagSetIds, ...iterator_to_array($tagIdSets, false));
    238+
    $lua = <<<'EOLUA'
    239+
    local id = KEYS[1]
    240+
    local cursor = table.remove(ARGV)
    241+
    redis.call('SREM', '{'..id..'}'..id, unpack(ARGV))
    240242
    241-
    // Delete cache in chunks to avoid overloading the connection
    242-
    foreach (array_chunk(array_unique($ids), self::BULK_DELETE_LIMIT) as $chunkIds) {
    243-
    $this->doDelete($chunkIds);
    244-
    }
    243+
    return redis.call('SSCAN', '{'..id..'}'..id, cursor, 'COUNT', 5000)
    244+
    EOLUA;
    245245

    246-
    return true;
    247-
    }
    246+
    foreach ($results as $id => [$cursor, $ids]) {
    247+
    while ($ids || '0' !== $cursor) {
    248+
    $this->doDelete($ids);
    248249

    249-
    /**
    250-
    * Renames several keys in order to be able to operate on them without risk of race conditions.
    251-
    *
    252-
    * Filters out keys that do not exist before returning new keys.
    253-
    *
    254-
    * @see https://redis.io/commands/rename
    255-
    * @see https://redis.io/topics/cluster-spec#keys-hash-tags
    256-
    *
    257-
    * @return array Filtered list of the valid moved keys (only those that existed)
    258-
    */
    259-
    private function renameKeys($redis, array $ids): array
    260-
    {
    261-
    $newIds = [];
    262-
    $uniqueToken = bin2hex(random_bytes(10));
    250+
    $evalArgs = [$id, $cursor];
    251+
    array_splice($evalArgs, 1, 0, $ids);
    263252

    264-
    $results = $this->pipeline(static function () use ($ids, $uniqueToken) {
    265-
    foreach ($ids as $id) {
    266-
    yield 'rename' => [$id, '{'.$id.'}'.$uniqueToken];
    267-
    }
    268-
    }, $redis);
    253+
    if ($this->redis instanceof \Predis\ClientInterface) {
    254+
    array_unshift($evalArgs, $lua, 1);
    255+
    } else {
    256+
    $evalArgs = [$lua, $evalArgs, 1];
    257+
    }
    269258

    270-
    foreach ($results as $id => $result) {
    271-
    if (true === $result || ($result instanceof Status && Status::get('OK') === $result)) {
    272-
    // Only take into account if ok (key existed), will be false on phpredis if it did not exist
    273-
    $newIds[] = '{'.$id.'}'.$uniqueToken;
    259+
    $results = $this->pipeline(function () use ($evalArgs) {
    260+
    yield 'eval' => $evalArgs;
    261+
    });
    262+
    263+
    foreach ($results as [$cursor, $ids]) {
    264+
    // no-op
    265+
    }
    274266
    }
    275267
    }
    276268

    277-
    return $newIds;
    269+
    return true;
    278270
    }
    279271

    280272
    private function getRedisEvictionPolicy(): string

    ‎LockRegistry.php

    Lines changed: 6 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -27,7 +27,7 @@
    2727
    final class LockRegistry
    2828
    {
    2929
    private static $openedFiles = [];
    30-
    private static $lockedFiles = [];
    30+
    private static $lockedFiles;
    3131

    3232
    /**
    3333
    * The number of items in this list controls the max number of concurrent processes.
    @@ -82,6 +82,11 @@ public static function setFiles(array $files): array
    8282

    8383
    public static function compute(callable $callback, ItemInterface $item, bool &$save, CacheInterface $pool, \Closure $setMetadata = null, LoggerInterface $logger = null)
    8484
    {
    85+
    if ('\\' === \DIRECTORY_SEPARATOR && null === self::$lockedFiles) {
    86+
    // disable locking on Windows by default
    87+
    self::$files = self::$lockedFiles = [];
    88+
    }
    89+
    8590
    $key = self::$files ? abs(crc32($item->getKey())) % \count(self::$files) : -1;
    8691

    8792
    if ($key < 0 || (self::$lockedFiles[$key] ?? false) || !$lock = self::open($key)) {

    ‎Tests/Adapter/AbstractRedisAdapterTest.php

    Lines changed: 15 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -24,7 +24,7 @@ abstract class AbstractRedisAdapterTest extends AdapterTestCase
    2424

    2525
    protected static $redis;
    2626

    27-
    public function createCachePool(int $defaultLifetime = 0): CacheItemPoolInterface
    27+
    public function createCachePool(int $defaultLifetime = 0, string $testMethod = null): CacheItemPoolInterface
    2828
    {
    2929
    return new RedisAdapter(self::$redis, str_replace('\\', '.', __CLASS__), $defaultLifetime);
    3030
    }
    @@ -45,4 +45,18 @@ public static function tearDownAfterClass(): void
    4545
    {
    4646
    self::$redis = null;
    4747
    }
    48+
    49+
    /**
    50+
    * @runInSeparateProcess
    51+
    */
    52+
    public function testClearWithPrefix()
    53+
    {
    54+
    $cache = $this->createCachePool(0, __FUNCTION__);
    55+
    56+
    $cache->save($cache->getItem('foo')->set('bar'));
    57+
    $this->assertTrue($cache->hasItem('foo'));
    58+
    59+
    $cache->clear();
    60+
    $this->assertFalse($cache->hasItem('foo'));
    61+
    }
    4862
    }

    ‎Tests/Adapter/PredisAdapterTest.php

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -22,7 +22,7 @@ class PredisAdapterTest extends AbstractRedisAdapterTest
    2222
    public static function setUpBeforeClass(): void
    2323
    {
    2424
    parent::setUpBeforeClass();
    25-
    self::$redis = new \Predis\Client(['host' => getenv('REDIS_HOST')]);
    25+
    self::$redis = new \Predis\Client(['host' => getenv('REDIS_HOST')], ['prefix' => 'prefix_']);
    2626
    }
    2727

    2828
    public function testCreateConnection()

    ‎Tests/Adapter/PredisClusterAdapterTest.php

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -19,7 +19,7 @@ class PredisClusterAdapterTest extends AbstractRedisAdapterTest
    1919
    public static function setUpBeforeClass(): void
    2020
    {
    2121
    parent::setUpBeforeClass();
    22-
    self::$redis = new \Predis\Client([['host' => getenv('REDIS_HOST')]]);
    22+
    self::$redis = new \Predis\Client([['host' => getenv('REDIS_HOST')]], ['prefix' => 'prefix_']);
    2323
    }
    2424

    2525
    public static function tearDownAfterClass(): void

    ‎Tests/Adapter/PredisRedisClusterAdapterTest.php

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -24,7 +24,7 @@ public static function setUpBeforeClass(): void
    2424
    self::markTestSkipped('REDIS_CLUSTER_HOSTS env var is not defined.');
    2525
    }
    2626

    27-
    self::$redis = RedisAdapter::createConnection('redis:?host['.str_replace(' ', ']&host[', $hosts).']', ['class' => \Predis\Client::class, 'redis_cluster' => true]);
    27+
    self::$redis = RedisAdapter::createConnection('redis:?host['.str_replace(' ', ']&host[', $hosts).']', ['class' => \Predis\Client::class, 'redis_cluster' => true, 'prefix' => 'prefix_']);
    2828
    }
    2929

    3030
    public static function tearDownAfterClass(): void

    ‎Tests/Adapter/PredisTagAwareAdapterTest.php

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -27,7 +27,7 @@ protected function setUp(): void
    2727
    $this->skippedTests['testTagItemExpiry'] = 'Testing expiration slows down the test suite';
    2828
    }
    2929

    30-
    public function createCachePool(int $defaultLifetime = 0): CacheItemPoolInterface
    30+
    public function createCachePool(int $defaultLifetime = 0, string $testMethod = null): CacheItemPoolInterface
    3131
    {
    3232
    $this->assertInstanceOf(\Predis\Client::class, self::$redis);
    3333
    $adapter = new RedisTagAwareAdapter(self::$redis, str_replace('\\', '.', __CLASS__), $defaultLifetime);

    ‎Tests/Adapter/PredisTagAwareClusterAdapterTest.php

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -27,7 +27,7 @@ protected function setUp(): void
    2727
    $this->skippedTests['testTagItemExpiry'] = 'Testing expiration slows down the test suite';
    2828
    }
    2929

    30-
    public function createCachePool(int $defaultLifetime = 0): CacheItemPoolInterface
    30+
    public function createCachePool(int $defaultLifetime = 0, string $testMethod = null): CacheItemPoolInterface
    3131
    {
    3232
    $this->assertInstanceOf(\Predis\Client::class, self::$redis);
    3333
    $adapter = new RedisTagAwareAdapter(self::$redis, str_replace('\\', '.', __CLASS__), $defaultLifetime);

    ‎Tests/Adapter/RedisAdapterSentinelTest.php

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -32,7 +32,7 @@ public static function setUpBeforeClass(): void
    3232
    self::markTestSkipped('REDIS_SENTINEL_SERVICE env var is not defined.');
    3333
    }
    3434

    35-
    self::$redis = AbstractAdapter::createConnection('redis:?host['.str_replace(' ', ']&host[', $hosts).']', ['redis_sentinel' => $service]);
    35+
    self::$redis = AbstractAdapter::createConnection('redis:?host['.str_replace(' ', ']&host[', $hosts).']', ['redis_sentinel' => $service, 'prefix' => 'prefix_']);
    3636
    }
    3737

    3838
    public function testInvalidDSNHasBothClusterAndSentinel()

    ‎Tests/Adapter/RedisAdapterTest.php

    Lines changed: 6 additions & 2 deletions
    Original file line numberDiff line numberDiff line change
    @@ -28,9 +28,13 @@ public static function setUpBeforeClass(): void
    2828
    self::$redis = AbstractAdapter::createConnection('redis://'.getenv('REDIS_HOST'), ['lazy' => true]);
    2929
    }
    3030

    31-
    public function createCachePool(int $defaultLifetime = 0): CacheItemPoolInterface
    31+
    public function createCachePool(int $defaultLifetime = 0, string $testMethod = null): CacheItemPoolInterface
    3232
    {
    33-
    $adapter = parent::createCachePool($defaultLifetime);
    33+
    if ('testClearWithPrefix' === $testMethod && \defined('Redis::SCAN_PREFIX')) {
    34+
    self::$redis->setOption(\Redis::OPT_SCAN, \Redis::SCAN_PREFIX);
    35+
    }
    36+
    37+
    $adapter = parent::createCachePool($defaultLifetime, $testMethod);
    3438
    $this->assertInstanceOf(RedisProxy::class, self::$redis);
    3539

    3640
    return $adapter;

    0 commit comments

    Comments
     (0)
    0