From b6c53114921b08f6f2bf396b37caee8ee3c7c3df Mon Sep 17 00:00:00 2001 From: Thomas Calvet Date: Fri, 11 Jun 2021 18:26:40 +0200 Subject: [PATCH] [Filesystem] Add RetryableFilesystem --- .../Component/Filesystem/Filesystem.php | 140 ++---------- .../Filesystem/FilesystemInterface.php | 204 ++++++++++++++++++ .../Filesystem/Retry/GenericRetryStrategy.php | 71 ++++++ .../Retry/RetryStrategyInterface.php | 20 ++ .../Filesystem/RetryableFilesystem.php | 191 ++++++++++++++++ 5 files changed, 507 insertions(+), 119 deletions(-) create mode 100644 src/Symfony/Component/Filesystem/FilesystemInterface.php create mode 100644 src/Symfony/Component/Filesystem/Retry/GenericRetryStrategy.php create mode 100644 src/Symfony/Component/Filesystem/Retry/RetryStrategyInterface.php create mode 100644 src/Symfony/Component/Filesystem/RetryableFilesystem.php diff --git a/src/Symfony/Component/Filesystem/Filesystem.php b/src/Symfony/Component/Filesystem/Filesystem.php index d4083feef6e24..fe5a172675e60 100644 --- a/src/Symfony/Component/Filesystem/Filesystem.php +++ b/src/Symfony/Component/Filesystem/Filesystem.php @@ -20,19 +20,12 @@ * * @author Fabien Potencier */ -class Filesystem +class Filesystem implements FilesystemInterface { private static $lastError; /** - * Copies a file. - * - * If the target file is older than the origin file, it's always overwritten. - * If the target file is newer, it is overwritten only when the - * $overwriteNewerFiles option is set to true. - * - * @throws FileNotFoundException When originFile doesn't exist - * @throws IOException When copy fails + * {@inheritdoc} */ public function copy(string $originFile, string $targetFile, bool $overwriteNewerFiles = false) { @@ -80,11 +73,7 @@ public function copy(string $originFile, string $targetFile, bool $overwriteNewe } /** - * Creates a directory recursively. - * - * @param string|iterable $dirs The directory path - * - * @throws IOException On any directory creation failure + * {@inheritdoc} */ public function mkdir($dirs, int $mode = 0777) { @@ -100,11 +89,7 @@ public function mkdir($dirs, int $mode = 0777) } /** - * Checks the existence of files or directories. - * - * @param string|iterable $files A filename, an array of files, or a \Traversable instance to check - * - * @return bool true if the file exists, false otherwise + * {@inheritdoc} */ public function exists($files) { @@ -124,13 +109,7 @@ public function exists($files) } /** - * Sets access and modification time of file. - * - * @param string|iterable $files A filename, an array of files, or a \Traversable instance to create - * @param int|null $time The touch time as a Unix timestamp, if not supplied the current system time is used - * @param int|null $atime The access time as a Unix timestamp, if not supplied the current system time is used - * - * @throws IOException When touch fails + * {@inheritdoc} */ public function touch($files, int $time = null, int $atime = null) { @@ -142,11 +121,7 @@ public function touch($files, int $time = null, int $atime = null) } /** - * Removes files or directories. - * - * @param string|iterable $files A filename, an array of files, or a \Traversable instance to remove - * - * @throws IOException When removal fails + * {@inheritdoc} */ public function remove($files) { @@ -206,14 +181,7 @@ private static function doRemove(array $files, bool $isRecursive): void } /** - * Change mode for an array of files or directories. - * - * @param string|iterable $files A filename, an array of files, or a \Traversable instance to change mode - * @param int $mode The new mode (octal) - * @param int $umask The mode mask (octal) - * @param bool $recursive Whether change the mod recursively or not - * - * @throws IOException When the change fails + * {@inheritdoc} */ public function chmod($files, int $mode, int $umask = 0000, bool $recursive = false) { @@ -228,13 +196,7 @@ public function chmod($files, int $mode, int $umask = 0000, bool $recursive = fa } /** - * Change the owner of an array of files or directories. - * - * @param string|iterable $files A filename, an array of files, or a \Traversable instance to change owner - * @param string|int $user A user name or number - * @param bool $recursive Whether change the owner recursively or not - * - * @throws IOException When the change fails + * {@inheritdoc} */ public function chown($files, $user, bool $recursive = false) { @@ -255,13 +217,7 @@ public function chown($files, $user, bool $recursive = false) } /** - * Change the group of an array of files or directories. - * - * @param string|iterable $files A filename, an array of files, or a \Traversable instance to change group - * @param string|int $group A group name or number - * @param bool $recursive Whether change the group recursively or not - * - * @throws IOException When the change fails + * {@inheritdoc} */ public function chgrp($files, $group, bool $recursive = false) { @@ -282,10 +238,7 @@ public function chgrp($files, $group, bool $recursive = false) } /** - * Renames a file or a directory. - * - * @throws IOException When target file or directory already exists - * @throws IOException When origin cannot be renamed + * {@inheritdoc} */ public function rename(string $origin, string $target, bool $overwrite = false) { @@ -307,9 +260,7 @@ public function rename(string $origin, string $target, bool $overwrite = false) } /** - * Tells whether a file exists and is readable. - * - * @throws IOException When windows path is longer than 258 characters + * {@inheritdoc} */ private function isReadable(string $filename): bool { @@ -323,9 +274,7 @@ private function isReadable(string $filename): bool } /** - * Creates a symbolic link or copy a directory. - * - * @throws IOException When symlink fails + * {@inheritdoc} */ public function symlink(string $originDir, string $targetDir, bool $copyOnWindows = false) { @@ -355,12 +304,7 @@ public function symlink(string $originDir, string $targetDir, bool $copyOnWindow } /** - * Creates a hard link, or several hard links to a file. - * - * @param string|string[] $targetFiles The target file(s) - * - * @throws FileNotFoundException When original file is missing or not a file - * @throws IOException When link fails, including if link already exists + * {@inheritdoc} */ public function hardlink(string $originFile, $targetFiles) { @@ -387,7 +331,7 @@ public function hardlink(string $originFile, $targetFiles) } /** - * @param string $linkType Name of the link type, typically 'symbolic' or 'hard' + * {@inheritdoc} */ private function linkException(string $origin, string $target, string $linkType) { @@ -400,17 +344,7 @@ private function linkException(string $origin, string $target, string $linkType) } /** - * Resolves links in paths. - * - * With $canonicalize = false (default) - * - if $path does not exist or is not a link, returns null - * - if $path is a link, returns the next direct target of the link without considering the existence of the target - * - * With $canonicalize = true - * - if $path does not exist, returns null - * - if $path exists, returns its absolute fully resolved final version - * - * @return string|null + * {@inheritdoc} */ public function readlink(string $path, bool $canonicalize = false) { @@ -438,9 +372,7 @@ public function readlink(string $path, bool $canonicalize = false) } /** - * Given an existing path, convert it to a path relative to a given starting path. - * - * @return string Path of target relative to starting path + * {@inheritdoc} */ public function makePathRelative(string $endPath, string $startPath) { @@ -514,21 +446,7 @@ public function makePathRelative(string $endPath, string $startPath) } /** - * Mirrors a directory to another. - * - * Copies files and directories from the origin directory into the target directory. By default: - * - * - existing files in the target directory will be overwritten, except if they are newer (see the `override` option) - * - files in the target directory that do not exist in the source directory will not be deleted (see the `delete` option) - * - * @param \Traversable|null $iterator Iterator that filters which files and directories to copy, if null a recursive iterator is created - * @param array $options An array of boolean options - * Valid options are: - * - $options['override'] If true, target files newer than origin files are overwritten (see copy(), defaults to false) - * - $options['copy_on_windows'] Whether to copy files instead of links on Windows (see symlink(), defaults to false) - * - $options['delete'] Whether to delete files that are not in the source directory (defaults to false) - * - * @throws IOException When file type is unknown + * {@inheritdoc} */ public function mirror(string $originDir, string $targetDir, \Traversable $iterator = null, array $options = []) { @@ -587,9 +505,7 @@ public function mirror(string $originDir, string $targetDir, \Traversable $itera } /** - * Returns whether the file path is an absolute path. - * - * @return bool + * {@inheritdoc} */ public function isAbsolutePath(string $file) { @@ -603,13 +519,7 @@ public function isAbsolutePath(string $file) } /** - * Creates a temporary file with support for custom stream wrappers. - * - * @param string $prefix The prefix of the generated temporary filename - * Note: Windows uses only the first three characters of prefix - * @param string $suffix The suffix of the generated temporary filename - * - * @return string The new temporary filename (with path), or throw an exception on failure + * {@inheritdoc} */ public function tempnam(string $dir, string $prefix/*, string $suffix = ''*/) { @@ -651,11 +561,7 @@ public function tempnam(string $dir, string $prefix/*, string $suffix = ''*/) } /** - * Atomically dumps content into a file. - * - * @param string|resource $content The data to write into the file - * - * @throws IOException if the file cannot be written to + * {@inheritdoc} */ public function dumpFile(string $filename, $content) { @@ -693,11 +599,7 @@ public function dumpFile(string $filename, $content) } /** - * Appends content to an existing file. - * - * @param string|resource $content The content to append - * - * @throws IOException If the file is not writable + * {@inheritdoc} */ public function appendToFile(string $filename, $content) { diff --git a/src/Symfony/Component/Filesystem/FilesystemInterface.php b/src/Symfony/Component/Filesystem/FilesystemInterface.php new file mode 100644 index 0000000000000..0c82e9a81c6e0 --- /dev/null +++ b/src/Symfony/Component/Filesystem/FilesystemInterface.php @@ -0,0 +1,204 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Filesystem; + +use Symfony\Component\Filesystem\Exception\FileNotFoundException; +use Symfony\Component\Filesystem\Exception\IOException; + +interface FilesystemInterface +{ + /** + * Copies a file. + * + * If the target file is older than the origin file, it's always overwritten. + * If the target file is newer, it is overwritten only when the + * $overwriteNewerFiles option is set to true. + * + * @throws FileNotFoundException When originFile doesn't exist + * @throws IOException When copy fails + */ + public function copy(string $originFile, string $targetFile, bool $overwriteNewerFiles = false); + + /** + * Creates a directory recursively. + * + * @param string|iterable $dirs The directory path + * + * @throws IOException On any directory creation failure + */ + public function mkdir($dirs, int $mode = 0777); + + /** + * Checks the existence of files or directories. + * + * @param string|iterable $files A filename, an array of files, or a \Traversable instance to check + * + * @return bool true if the file exists, false otherwise + */ + public function exists($files); + + /** + * Sets access and modification time of file. + * + * @param string|iterable $files A filename, an array of files, or a \Traversable instance to create + * @param int|null $time The touch time as a Unix timestamp, if not supplied the current system time is used + * @param int|null $atime The access time as a Unix timestamp, if not supplied the current system time is used + * + * @throws IOException When touch fails + */ + public function touch($files, int $time = null, int $atime = null); + + /** + * Removes files or directories. + * + * @param string|iterable $files A filename, an array of files, or a \Traversable instance to remove + * + * @throws IOException When removal fails + */ + public function remove($files); + + /** + * Change mode for an array of files or directories. + * + * @param string|iterable $files A filename, an array of files, or a \Traversable instance to change mode + * @param int $mode The new mode (octal) + * @param int $umask The mode mask (octal) + * @param bool $recursive Whether change the mod recursively or not + * + * @throws IOException When the change fails + */ + public function chmod($files, int $mode, int $umask = 0000, bool $recursive = false); + + /** + * Change the owner of an array of files or directories. + * + * @param string|iterable $files A filename, an array of files, or a \Traversable instance to change owner + * @param string|int $user A user name or number + * @param bool $recursive Whether change the owner recursively or not + * + * @throws IOException When the change fails + */ + public function chown($files, $user, bool $recursive = false); + + /** + * Change the group of an array of files or directories. + * + * @param string|iterable $files A filename, an array of files, or a \Traversable instance to change group + * @param string|int $group A group name or number + * @param bool $recursive Whether change the group recursively or not + * + * @throws IOException When the change fails + */ + public function chgrp($files, $group, bool $recursive = false); + + /** + * Renames a file or a directory. + * + * @throws IOException When target file or directory already exists + * @throws IOException When origin cannot be renamed + */ + public function rename(string $origin, string $target, bool $overwrite = false); + + /** + * Creates a symbolic link or copy a directory. + * + * @throws IOException When symlink fails + */ + public function symlink(string $originDir, string $targetDir, bool $copyOnWindows = false); + + /** + * Creates a hard link, or several hard links to a file. + * + * @param string|string[] $targetFiles The target file(s) + * + * @throws FileNotFoundException When original file is missing or not a file + * @throws IOException When link fails, including if link already exists + */ + public function hardlink(string $originFile, $targetFiles); + + /** + * Resolves links in paths. + * + * With $canonicalize = false (default) + * - if $path does not exist or is not a link, returns null + * - if $path is a link, returns the next direct target of the link without considering the existence of the target + * + * With $canonicalize = true + * - if $path does not exist, returns null + * - if $path exists, returns its absolute fully resolved final version + * + * @return string|null + */ + public function readlink(string $path, bool $canonicalize = false); + + /** + * Given an existing path, convert it to a path relative to a given starting path. + * + * @return string Path of target relative to starting path + */ + public function makePathRelative(string $endPath, string $startPath); + + /** + * Mirrors a directory to another. + * + * Copies files and directories from the origin directory into the target directory. By default: + * + * - existing files in the target directory will be overwritten, except if they are newer (see the `override` option) + * - files in the target directory that do not exist in the source directory will not be deleted (see the `delete` option) + * + * @param \Traversable|null $iterator Iterator that filters which files and directories to copy, if null a recursive iterator is created + * @param array $options An array of boolean options + * Valid options are: + * - $options['override'] If true, target files newer than origin files are overwritten (see copy(), defaults to false) + * - $options['copy_on_windows'] Whether to copy files instead of links on Windows (see symlink(), defaults to false) + * - $options['delete'] Whether to delete files that are not in the source directory (defaults to false) + * + * @throws IOException When file type is unknown + */ + public function mirror(string $originDir, string $targetDir, \Traversable $iterator = null, array $options = []); + + /** + * Returns whether the file path is an absolute path. + * + * @return bool + */ + public function isAbsolutePath(string $file); + + /** + * Creates a temporary file with support for custom stream wrappers. + * + * @param string $prefix The prefix of the generated temporary filename + * Note: Windows uses only the first three characters of prefix + * @param string $suffix The suffix of the generated temporary filename + * + * @return string The new temporary filename (with path), or throw an exception on failure + */ + public function tempnam(string $dir, string $prefix/*, string $suffix = ''*/); + + /** + * Atomically dumps content into a file. + * + * @param string|resource $content The data to write into the file + * + * @throws IOException if the file cannot be written to + */ + public function dumpFile(string $filename, $content); + + /** + * Appends content to an existing file. + * + * @param string|resource $content The content to append + * + * @throws IOException If the file is not writable + */ + public function appendToFile(string $filename, $content); +} diff --git a/src/Symfony/Component/Filesystem/Retry/GenericRetryStrategy.php b/src/Symfony/Component/Filesystem/Retry/GenericRetryStrategy.php new file mode 100644 index 0000000000000..aa0323337de6a --- /dev/null +++ b/src/Symfony/Component/Filesystem/Retry/GenericRetryStrategy.php @@ -0,0 +1,71 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Filesystem\Retry; + +use Symfony\Component\HttpClient\Exception\InvalidArgumentException; + +class GenericRetryStrategy implements RetryStrategyInterface +{ + private $delayMs; + private $multiplier; + private $maxDelayMs; + private $jitter; + + /** + * @param int $delayMs Amount of time to delay (or the initial value when multiplier is used) + * @param float $multiplier Multiplier to apply to the delay each time a retry occurs + * @param int $maxDelayMs Maximum delay to allow (0 means no maximum) + * @param float $jitter Probability of randomness int delay (0 = none, 1 = 100% random) + */ + public function __construct(int $delayMs = 1000, float $multiplier = 2.0, int $maxDelayMs = 0, float $jitter = 0.1) + { + if ($delayMs < 0) { + throw new InvalidArgumentException(sprintf('Delay must be greater than or equal to zero: "%s" given.', $delayMs)); + } + $this->delayMs = $delayMs; + + if ($multiplier < 1) { + throw new InvalidArgumentException(sprintf('Multiplier must be greater than or equal to one: "%s" given.', $multiplier)); + } + $this->multiplier = $multiplier; + + if ($maxDelayMs < 0) { + throw new InvalidArgumentException(sprintf('Max delay must be greater than or equal to zero: "%s" given.', $maxDelayMs)); + } + $this->maxDelayMs = $maxDelayMs; + + if ($jitter < 0 || $jitter > 1) { + throw new InvalidArgumentException(sprintf('Jitter must be between 0 and 1: "%s" given.', $jitter)); + } + $this->jitter = $jitter; + } + + /** + * {@inheritdoc} + */ + public function getDelay(int $retryCount): int + { + $delay = $this->delayMs * $this->multiplier ** $retryCount; + dump($delay); + + if ($this->jitter > 0) { + $randomness = $delay * $this->jitter; + $delay = $delay + random_int(-$randomness, +$randomness); + } + + if ($delay > $this->maxDelayMs && 0 !== $this->maxDelayMs) { + return $this->maxDelayMs; + } + + return (int) $delay; + } +} diff --git a/src/Symfony/Component/Filesystem/Retry/RetryStrategyInterface.php b/src/Symfony/Component/Filesystem/Retry/RetryStrategyInterface.php new file mode 100644 index 0000000000000..0f486c2d0fcdd --- /dev/null +++ b/src/Symfony/Component/Filesystem/Retry/RetryStrategyInterface.php @@ -0,0 +1,20 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Filesystem\Retry; + +interface RetryStrategyInterface +{ + /** + * Returns the time to wait in milliseconds. + */ + public function getDelay(int $retryCount): int; +} diff --git a/src/Symfony/Component/Filesystem/RetryableFilesystem.php b/src/Symfony/Component/Filesystem/RetryableFilesystem.php new file mode 100644 index 0000000000000..c2d5dce8f0ff2 --- /dev/null +++ b/src/Symfony/Component/Filesystem/RetryableFilesystem.php @@ -0,0 +1,191 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Filesystem; + +use Symfony\Component\Filesystem\Exception\IOExceptionInterface; +use Symfony\Component\Filesystem\Retry\GenericRetryStrategy; +use Symfony\Component\Filesystem\Retry\RetryStrategyInterface; + +class RetryableFilesystem implements FilesystemInterface +{ + private $filesystem; + private $strategy; + private $maxRetries; + + public function __construct(Filesystem $filesystem = null, RetryStrategyInterface $strategy = null, int $maxRetries = 3) + { + $this->filesystem = $filesystem ?? new Filesystem(); + $this->strategy = $strategy ?? new GenericRetryStrategy(); + $this->maxRetries = $maxRetries; + } + + /** + * {@inheritdoc} + */ + public function copy(string $originFile, string $targetFile, bool $overwriteNewerFiles = false) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function mkdir($dirs, int $mode = 0777) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function exists($files) + { + return $this->filesystem->exists(...\func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function touch($files, int $time = null, int $atime = null) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function remove($files) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function chmod($files, int $mode, int $umask = 0000, bool $recursive = false) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function chown($files, $user, bool $recursive = false) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function chgrp($files, $group, bool $recursive = false) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function rename(string $origin, string $target, bool $overwrite = false) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function symlink(string $originDir, string $targetDir, bool $copyOnWindows = false) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function hardlink(string $originFile, $targetFiles) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function readlink(string $path, bool $canonicalize = false) + { + return $this->filesystem->readlink(...\func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function makePathRelative(string $endPath, string $startPath) + { + return $this->filesystem->makePathRelative(...\func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function mirror(string $originDir, string $targetDir, \Traversable $iterator = null, array $options = []) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function isAbsolutePath(string $file) + { + return $this->filesystem->isAbsolutePath(...\func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function tempnam(string $dir, string $prefix/*, string $suffix = ''*/) + { + return $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function dumpFile(string $filename, $content) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + /** + * {@inheritdoc} + */ + public function appendToFile(string $filename, $content) + { + $this->retry(__FUNCTION__, \func_get_args()); + } + + private function retry(string $method, array $args) + { + $retryCount = 0; + $i = 0; + while (true) { + try { + return $this->filesystem->$method(...$args); + } catch (IOExceptionInterface $e) { + if ($retryCount >= $this->maxRetries) { + throw $e; + } + + usleep($this->strategy->getDelay($retryCount++) * 1000); + } + } + } +}