From 974a381b5a72684301df83f0c1c7fe2ee7ac8054 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Wed, 12 Feb 2020 15:28:03 +0100 Subject: [PATCH 01/33] Implemented changes to binaries. Binaries now accept a 'bootstrap' argument, allowing the developer to load a custom bootstrap from the project they're working on. This allows Async to run in the same environment as the project it's part of. --- bin/supervisor | 61 ++++++--- bin/worker | 125 ++++++++++++++++++ composer.json | 2 +- config.tasks.php | 2 +- .../Async/Executors/ShellExecutor.php | 4 +- .../Async/{Worker.php => ShellWorker.php} | 12 +- src/FuzeWorks/Async/Tasks.php | 6 +- supervisor.php | 70 ---------- worker.php | 68 ---------- 9 files changed, 184 insertions(+), 166 deletions(-) create mode 100644 bin/worker rename src/FuzeWorks/Async/{Worker.php => ShellWorker.php} (95%) delete mode 100644 supervisor.php delete mode 100644 worker.php diff --git a/bin/supervisor b/bin/supervisor index 28fda4b..62f174e 100644 --- a/bin/supervisor +++ b/bin/supervisor @@ -40,7 +40,9 @@ use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TasksException; use FuzeWorks\Exception\InvalidArgumentException; use FuzeWorks\Exception\LibraryException; +use FuzeWorks\Factory; +// First perform a PHP version check if (version_compare('7.1.0', PHP_VERSION, '>')) { fwrite( STDERR, @@ -63,35 +65,56 @@ $autoloaders = [ ]; foreach ($autoloaders as $file) if (file_exists($file)) - require($file); + require_once($file); + +// If a bootstrap is provided, use that one +$arguments = getopt('', ['bootstrap:']); +if (!isset($arguments['bootstrap']) || empty($arguments['bootstrap'])) +{ + fwrite(STDERR, "Could not load supervisor. No bootstrap provided."); + die(1); +} +// Load the file. If it doesn't exist, fail. +$file = $arguments['bootstrap']; +if (!file_exists($file)) +{ + fwrite(STDERR, "Could not load supervisor. Provided bootstrap doesn't exist."); + die(1); +} + +// Load the bootstrap +/** @var Factory $container */ +$container = require($file); + +// Check if container is a Factory +if (!$container instanceof Factory) +{ + fwrite(STDERR, "Could not load supervisor. Provided bootstrap is not a valid bootstrap."); + die(1); +} + +// Check if the Async library is already loaded. If not, load it. +// @todo: Better check in libraries for existing library try { - // Open configurator - $configurator = new FuzeWorks\Configurator(); - - // Set up basic settings - $configurator->setTimeZone('Europe/Amsterdam'); - $configurator->setTempDirectory(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'temp'); - $configurator->setLogDirectory(dirname(__FILE__). DIRECTORY_SEPARATOR . 'log'); - - // Add Async library - $configurator->deferComponentClassMethod('libraries', 'addLibraryClass', null, 'async', '\FuzeWorks\Async\Tasks'); - - // Debug - $configurator->enableDebugMode()->setDebugAddress('ALL'); - - // Create container - $container = $configurator->createContainer(); - - // RUN THE APP /** @var Tasks $lib */ $lib = $container->libraries->get('async'); +} catch (LibraryException $e) { + $container->libraries->addLibraryClass('async', '\FuzeWorks\Async\Tasks'); + /** @var Tasks $lib */ + $lib = $container->libraries->get('async'); +} +// And finally, run the supervisor +try { $supervisor = $lib->getSuperVisor(); while ($supervisor->cycle() === SuperVisor::RUNNING) { usleep(250000); } + + // Write results + fwrite(STDOUT, "SuperVisor finished scheduled tasks."); } catch (InvalidArgumentException | TasksException | LibraryException $e) { fwrite(STDERR, sprintf('FuzeWorks Async could not load.' . PHP_EOL . 'Exception: ' . $e->getMessage() . PHP_EOL) diff --git a/bin/worker b/bin/worker new file mode 100644 index 0000000..ecac264 --- /dev/null +++ b/bin/worker @@ -0,0 +1,125 @@ +#!/usr/bin/env php +')) { + fwrite( + STDERR, + sprintf( + 'FuzeWorks Async requires PHP 7.1 or higher.' . PHP_EOL . + 'You are using PHP %s (%s).' . PHP_EOL, + PHP_VERSION, + PHP_BINARY + ) + ); + + die(1); +} + +// First load composer +$autoloaders = [ + __DIR__ . '/../../autoload.php', + __DIR__ . '/../vendor/autoload.php', + __DIR__ . '/vendor/autoload.php' +]; +foreach ($autoloaders as $file) + if (file_exists($file)) + require_once($file); + +// If a bootstrap is provided, use that one +$arguments = getopt('', ['bootstrap:']); +if (!isset($arguments['bootstrap']) || empty($arguments['bootstrap'])) +{ + fwrite(STDERR, "Could not load worker. No bootstrap provided."); + die(1); +} + + +// Load the file. If it doesn't exist, fail. +$file = $arguments['bootstrap']; +if (!file_exists($file)) +{ + fwrite(STDERR, "Could not load worker. Provided bootstrap doesn't exist."); + die(1); +} + +// Load the bootstrap +/** @var Factory $container */ +$container = require($file); + +// Check if container is a Factory +if (!$container instanceof Factory) +{ + fwrite(STDERR, "Could not load worker. Provided bootstrap is not a valid bootstrap."); + die(1); +} + +// Check if the Async library is already loaded. If not, load it. +// @todo: Better check in libraries for existing library +try { + /** @var Tasks $lib */ + $lib = $container->libraries->get('async'); +} catch (LibraryException $e) { + $container->libraries->addLibraryClass('async', '\FuzeWorks\Async\Tasks'); + /** @var Tasks $lib */ + $lib = $container->libraries->get('async'); +} + +// Fetch arguments for the worker +$arguments = getopt("t:p::"); +if (!isset($arguments['t'])) +{ + fwrite(STDERR, "Could not load worker. No taskID provided."); + die(1); +} + +// Prepare arguments +$taskID = base64_decode($arguments['t']); +$post = isset($arguments['p']); + +// RUN THE APP +$worker = $lib->getWorker(); +$worker->run($taskID, $post); + +fwrite(STDOUT,'Finished task \'' . $taskID . "'"); +?> \ No newline at end of file diff --git a/composer.json b/composer.json index 2a3620e..9698f4f 100644 --- a/composer.json +++ b/composer.json @@ -22,5 +22,5 @@ "FuzeWorks\\Async\\": "src/FuzeWorks/Async" } }, - "bin": ["bin/supervisor"] + "bin": ["bin/supervisor", "bin/worker"] } \ No newline at end of file diff --git a/config.tasks.php b/config.tasks.php index 21f14d5..7a00481 100644 --- a/config.tasks.php +++ b/config.tasks.php @@ -48,6 +48,6 @@ return array( 'type' => 'ShellExecutor', // For ShellExecutor, first parameter is the file location of the worker script - 'parameters' => [dirname(__FILE__) . DS . 'worker.php'] + 'parameters' => [dirname(__FILE__) . DS . 'bin' . DS . 'worker'] ] ); \ No newline at end of file diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index c8ad1e4..69e5c18 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -60,7 +60,7 @@ class ShellExecutor implements Executor $this->binary = PHP_BINDIR . DS . 'php'; if (!file_exists($workerFile)) - throw new TasksException("Could not construct ShellExecutor. Worker script does not exist."); + throw new TasksException("Could not construct ShellExecutor. ShellWorker script does not exist."); $this->worker = $workerFile; } @@ -77,7 +77,7 @@ class ShellExecutor implements Executor public function startTask(Task $task, bool $post = false): Task { // First prepare the command used to spawn workers - $commandString = "$this->binary $this->worker %s ".($post ? 'post' : 'run')." $this->stdout $this->stderr & echo $!"; + $commandString = "$this->binary $this->worker -t %s ".($post ? 'p' : '')." $this->stdout $this->stderr & echo $!"; // Then execute the command using the base64_encoded string of the taskID $output = $this->shellExec($commandString, [base64_encode($task->getId())]); diff --git a/src/FuzeWorks/Async/Worker.php b/src/FuzeWorks/Async/ShellWorker.php similarity index 95% rename from src/FuzeWorks/Async/Worker.php rename to src/FuzeWorks/Async/ShellWorker.php index c110e59..a614541 100644 --- a/src/FuzeWorks/Async/Worker.php +++ b/src/FuzeWorks/Async/ShellWorker.php @@ -42,7 +42,7 @@ use FuzeWorks\Exception\EventException; use FuzeWorks\Logger; use FuzeWorks\Priority; -class Worker +class ShellWorker { /** @@ -75,7 +75,11 @@ class Worker public function run(string $taskID, bool $post = false) { // First fetch the task - $task = $this->taskStorage->getTaskById($taskID); + try { + $task = $this->taskStorage->getTaskById($taskID); + } catch (TasksException $e) { + throw new TasksException("Could not run worker. Task not found."); + } // Fire a taskHandleEvent /** @var TaskHandleEvent $event */ @@ -159,6 +163,10 @@ class Worker $errors = $this->getErrors(); $this->output('', $errors); + // If no task is set yet, abort error logging to task + if (is_null($this->task)) + return; + try { // Write to TaskStorage if (!$this->post) diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index 1a8bbab..9a8cbe2 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -94,12 +94,12 @@ class Tasks implements iLibrary } /** - * @return Worker + * @return ShellWorker * @throws TasksException */ - public function getWorker(): Worker + public function getWorker(): ShellWorker { - return new Worker($this->getTaskStorage()); + return new ShellWorker($this->getTaskStorage()); } /** diff --git a/supervisor.php b/supervisor.php deleted file mode 100644 index 8fc8427..0000000 --- a/supervisor.php +++ /dev/null @@ -1,70 +0,0 @@ -setTimeZone('Europe/Amsterdam'); -$configurator->setTempDirectory(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'temp'); -$configurator->setLogDirectory(dirname(__FILE__). DIRECTORY_SEPARATOR . 'log'); - -// Add Async library -$configurator->deferComponentClassMethod('libraries', 'addLibraryClass', null, 'async', '\FuzeWorks\Async\Tasks'); - -// Debug -$configurator->enableDebugMode()->setDebugAddress('ALL'); - -// Create container -$container = $configurator->createContainer(); - -// Add lib -Logger::enableScreenLog(); - -// RUN THE APP -/** @var Tasks $lib */ -$lib = $container->libraries->get('async'); - -$supervisor = $lib->getSuperVisor(); -while ($supervisor->cycle() === SuperVisor::RUNNING) { - usleep(250000); -} \ No newline at end of file diff --git a/worker.php b/worker.php deleted file mode 100644 index 3439954..0000000 --- a/worker.php +++ /dev/null @@ -1,68 +0,0 @@ -setTimeZone('Europe/Amsterdam'); -$configurator->setTempDirectory(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'temp'); -$configurator->setLogDirectory(dirname(__FILE__). DIRECTORY_SEPARATOR . 'log'); - -// Add Async library -$configurator->deferComponentClassMethod('libraries', 'addLibraryClass', null, 'async', '\FuzeWorks\Async\Tasks'); - -// Debug -$configurator->enableDebugMode()->setDebugAddress('ALL'); - -// Create container -$container = $configurator->createContainer(); - -// Prepare arguments -$script = array_shift($argv); -$taskID = array_shift($argv); -$taskID = base64_decode($taskID); -$mode = trim(array_shift($argv)); -$post = ($mode === 'post' ? true : false); - -// RUN THE APP -/** @var Tasks $lib */ -$lib = $container->libraries->get('async'); -$worker = $lib->getWorker(); -$worker->run($taskID, $post); \ No newline at end of file -- 2.40.1 From ab198e9ef1abd16eea31434950047340ffdd634a Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Thu, 13 Feb 2020 14:06:23 +0100 Subject: [PATCH 02/33] Updated config format. --- config.tasks.php | 22 +++++++++++++++++-- .../Async/Executors/ShellExecutor.php | 7 ++++-- src/FuzeWorks/Async/TaskStorage.php | 8 +++++++ .../Async/TaskStorage/ArrayTaskStorage.php | 10 ++++----- src/FuzeWorks/Async/Tasks.php | 4 ++-- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/config.tasks.php b/config.tasks.php index 7a00481..922e272 100644 --- a/config.tasks.php +++ b/config.tasks.php @@ -42,12 +42,30 @@ return array( 'type' => 'ArrayTaskStorage', // For ArrayTaskStorage, first parameter is the file location of the array storage - 'parameters' => [dirname(__FILE__) . DS . 'storage.php'] + 'parameters' => [ + 'filename' => dirname(__FILE__) . DS . 'storage.php' + ], + + // For RedisTaskStorage, parameters are connection properties + #'parameters' => [ + # // Type can be 'tcp' or 'unix' + # 'socket_type' => 'tcp', + # // If socket_type == 'unix', set the socket here + # 'socket' => null, + # // If socket_type == 'tcp', set the host here + # 'host' => 'localhost', + # + # 'password' => null, + # 'port' => 6379, + # 'timeout' => 0 + #] ], 'Executor' => [ 'type' => 'ShellExecutor', // For ShellExecutor, first parameter is the file location of the worker script - 'parameters' => [dirname(__FILE__) . DS . 'bin' . DS . 'worker'] + 'parameters' => [ + 'workerFile' => dirname(__FILE__) . DS . 'bin' . DS . 'worker' + ] ] ); \ No newline at end of file diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index 69e5c18..30c5556 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -51,11 +51,14 @@ class ShellExecutor implements Executor /** * ShellExecutor constructor. * - * @param string $workerFile The worker script that shall run individual tasks + * @param array $parameters * @throws TasksException */ - public function __construct(string $workerFile) + public function __construct(array $parameters) { + // Fetch workerFile + $workerFile = $parameters['workerFile']; + // First determine the PHP binary $this->binary = PHP_BINDIR . DS . 'php'; diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index 08f1f4d..6548240 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -39,6 +39,14 @@ namespace FuzeWorks\Async; interface TaskStorage { + /** + * TaskStorage constructor. + * + * @throws TasksException + * @param array $parameters from config file + */ + public function __construct(array $parameters); + /** * Add a task to the TaskStorage. * diff --git a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php index cd3197b..bbc0269 100644 --- a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php @@ -60,13 +60,13 @@ class ArrayTaskStorage implements TaskStorage protected $tasks = []; /** - * ArrayTaskStorage constructor. - * - * @param string $fileName Name of the Storage file - * @throws TasksException + * @inheritDoc */ - public function __construct(string $fileName) + public function __construct(array $parameters) { + // Load the filename for this taskStorage + $fileName = $parameters['filename']; + if (!file_exists($fileName)) throw new TasksException("Could not construct ArrayTaskStorage. Storage file '$fileName' doesn't exist."); diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index 9a8cbe2..18df927 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -116,7 +116,7 @@ class Tasks implements iLibrary if (!class_exists($class, true)) throw new TasksException("Could not get TaskStorage. Type of '$class' not found."); - $object = new $class(...$parameters); + $object = new $class($parameters); if (!$object instanceof TaskStorage) throw new TasksException("Could not get TaskStorage. Type '$class' is not instanceof TaskStorage."); @@ -137,7 +137,7 @@ class Tasks implements iLibrary if (!class_exists($class, true)) throw new TasksException("Could not get Executor. Type of '$class' not found."); - $object = new $class(...$parameters); + $object = new $class($parameters); if (!$object instanceof Executor) throw new TasksException("Could not get Executor. Type '$class' is not instanceof Executor."); -- 2.40.1 From f229be030591db51ca393a52a632b841147fd765 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 14 Feb 2020 13:49:27 +0100 Subject: [PATCH 03/33] Implemented basic RedisTaskStorage. - Fixed bug where worker is not provided with bootstrap by ShellExecutor. - Added composer and Redis to Dockerfile - Added more output to ParallelSuperVisor --- Dockerfile | 8 +- bin/supervisor | 8 +- composer.json | 6 +- .../Async/Executors/ShellExecutor.php | 7 +- .../Async/Supervisors/ParallelSuperVisor.php | 10 +- .../Async/TaskStorage/RedisTaskStorage.php | 278 ++++++++++++++++++ src/FuzeWorks/Async/Tasks.php | 11 +- 7 files changed, 314 insertions(+), 14 deletions(-) create mode 100644 src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php diff --git a/Dockerfile b/Dockerfile index 5a4a7d0..f42a7bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,9 @@ FROM php:7.3-cli-buster RUN apt-get update &&\ apt-get install --no-install-recommends --assume-yes --quiet procps ca-certificates curl git &&\ rm -rf /var/lib/apt/lists/* -# PDO -RUN docker-php-ext-install pdo_mysql \ No newline at end of file + +# Install Redis +RUN pecl install redis-5.1.1 && docker-php-ext-enable redis + +# Install Composer +RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer \ No newline at end of file diff --git a/bin/supervisor b/bin/supervisor index 62f174e..4ffa970 100644 --- a/bin/supervisor +++ b/bin/supervisor @@ -77,8 +77,8 @@ if (!isset($arguments['bootstrap']) || empty($arguments['bootstrap'])) // Load the file. If it doesn't exist, fail. -$file = $arguments['bootstrap']; -if (!file_exists($file)) +$bootstrap = $arguments['bootstrap']; +if (!file_exists($bootstrap)) { fwrite(STDERR, "Could not load supervisor. Provided bootstrap doesn't exist."); die(1); @@ -86,7 +86,7 @@ if (!file_exists($file)) // Load the bootstrap /** @var Factory $container */ -$container = require($file); +$container = require($bootstrap); // Check if container is a Factory if (!$container instanceof Factory) @@ -108,7 +108,7 @@ try { // And finally, run the supervisor try { - $supervisor = $lib->getSuperVisor(); + $supervisor = $lib->getSuperVisor($bootstrap); while ($supervisor->cycle() === SuperVisor::RUNNING) { usleep(250000); } diff --git a/composer.json b/composer.json index 9698f4f..2e5c849 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,11 @@ "require": { "php": ">=7.2.0", "fuzeworks/core": "~1.2.0", - "ext-json": "*" + "ext-json": "*", + "ext-redis": "*" + }, + "require-dev": { + "fuzeworks/tracycomponent": "~1.2.0" }, "autoload": { "psr-4": { diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index 30c5556..87df219 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -45,22 +45,25 @@ class ShellExecutor implements Executor private $binary; private $worker; + private $bootstrapFile; private $stdout = "> /dev/null"; private $stderr = "2> /dev/null"; /** * ShellExecutor constructor. * + * @param string $bootstrapFile * @param array $parameters * @throws TasksException */ - public function __construct(array $parameters) + public function __construct(string $bootstrapFile, array $parameters) { // Fetch workerFile $workerFile = $parameters['workerFile']; // First determine the PHP binary $this->binary = PHP_BINDIR . DS . 'php'; + $this->bootstrapFile = $bootstrapFile; if (!file_exists($workerFile)) throw new TasksException("Could not construct ShellExecutor. ShellWorker script does not exist."); @@ -80,7 +83,7 @@ class ShellExecutor implements Executor public function startTask(Task $task, bool $post = false): Task { // First prepare the command used to spawn workers - $commandString = "$this->binary $this->worker -t %s ".($post ? 'p' : '')." $this->stdout $this->stderr & echo $!"; + $commandString = "$this->binary $this->worker --bootstrap=".$this->bootstrapFile." -t %s ".($post ? 'p' : '')." $this->stdout $this->stderr & echo $!"; // Then execute the command using the base64_encoded string of the taskID $output = $this->shellExec($commandString, [base64_encode($task->getId())]); diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index e70ac1f..0408ec8 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -80,7 +80,7 @@ class ParallelSuperVisor implements SuperVisor for ($i=0;$itasks);$i++) { $task = $this->tasks[$i]; - + // PENDING: should start if not constrained if ($task->getStatus() === Task::PENDING) { @@ -97,6 +97,7 @@ class ParallelSuperVisor implements SuperVisor // Modify the task in TaskStorage $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } // DELAYED: If task is delayed, and enough time has passed, change the status back to pending @@ -104,6 +105,7 @@ class ParallelSuperVisor implements SuperVisor { $task->setStatus(Task::PENDING); $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } // CANCELLED/COMPLETED: remove the task if requested to do so @@ -139,6 +141,7 @@ class ParallelSuperVisor implements SuperVisor // If any changes have been made, they should be written to TaskStorage $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } // FAILED: if a process has failed, attempt to rety if requested to do so @@ -161,6 +164,7 @@ class ParallelSuperVisor implements SuperVisor $task = $this->executor->startTask($task); $task->setStatus(Task::RUNNING); $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); continue; } } @@ -176,6 +180,7 @@ class ParallelSuperVisor implements SuperVisor $task->setStatus(Task::CANCELLED); $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } // SUCCESS: if a task has succeeded, see if it needs a postHandler @@ -191,6 +196,7 @@ class ParallelSuperVisor implements SuperVisor $task->setStatus(Task::COMPLETED); $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } // POST: when a task is currently running in it's postHandler @@ -226,6 +232,7 @@ class ParallelSuperVisor implements SuperVisor // If any changes have been made, they should be written to TaskStorage $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } } @@ -263,6 +270,7 @@ class ParallelSuperVisor implements SuperVisor // Save changes to TaskStorage $this->taskStorage->modifyTask($task); + fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } } diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php new file mode 100644 index 0000000..8f0d000 --- /dev/null +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -0,0 +1,278 @@ +conn = new Redis(); + + // Afterwards connect to server + $socketType = $parameters['socket_type']; + if ($socketType == 'unix') + $success = $this->conn->connect($parameters['socket']); + elseif ($socketType == 'tcp') + $success = $this->conn->connect($parameters['host'], $parameters['port'], $parameters['timeout']); + else + $success = false; + + // If unsuccessful, return false + if (!$success) + throw new TasksException("Could not construct RedisTaskStorage. Failed to connect."); + + // Otherwise attempt authentication, if needed + if (isset($parameters['password']) && !$this->conn->auth($parameters['password'])) + throw new TasksException("Could not construct RedisTaskStorage. Authentication failure."); + } catch (RedisException $e) { + throw new TasksException("Could not construct RedisTaskStorage. RedisException thrown: '" . $e->getMessage() . "'"); + } + } + + /** + * @inheritDoc + * @throws TasksException + */ + public function addTask(Task $task): bool + { + // Check if the task doesn't exist yet + $taskId = $task->getId(); + + // Query the index + $isMember = $this->conn->sIsMember($this->indexSet, $taskId); + if ($isMember) + throw new TasksException("Could not add Task to TaskStorage. Task '$taskId' already exists."); + + // Serialize the task and save it + $taskData = serialize($task); + $this->conn->set($this->key_prefix . $taskId, $taskData); + $this->conn->sAdd($this->indexSet, $taskId); + + return true; + } + + /** + * @inheritDoc + */ + public function readTasks(): array + { + return $this->refreshTasks(); + } + + /** + * @inheritDoc + */ + public function refreshTasks() + { + // First fetch an array of all tasks in the set + $taskList = $this->conn->sMembers($this->indexSet); + + // Go over each taskId and fetch the specific task + $tasks = []; + foreach ($taskList as $taskId) + $tasks[] = unserialize($this->conn->get($this->key_prefix . $taskId)); + + return $tasks; + } + + /** + * @inheritDoc + */ + public function getTaskById(string $identifier): Task + { + // Query the index + $isMember = $this->conn->sIsMember($this->indexSet, $identifier); + if (!$isMember) + throw new TasksException("Could not get task by ID. Task not found."); + + // Fetch the task + /** @var Task $task */ + $task = unserialize($this->conn->get($this->key_prefix . $identifier)); + + // Return the task + return $task; + } + + /** + * @inheritDoc + */ + public function modifyTask(Task $task): bool + { + // First get the task ID + $taskId = $task->getId(); + + // Check if it exists + $isMember = $this->conn->sIsMember($this->indexSet, $taskId); + if (!$isMember) + throw new TasksException("Could not modify task. Task '$taskId' already exists."); + + // And write the data + $taskData = serialize($task); + return $this->conn->set($this->key_prefix . $taskId, $taskData); + } + + /** + * @inheritDoc + * @throws TasksException + */ + public function deleteTask(Task $task): bool + { + // First get the task ID + $taskId = $task->getId(); + + // Check if it exists + $isMember = $this->conn->sIsMember($this->indexSet, $taskId); + if (!$isMember) + throw new TasksException("Could not modify task. Task '$taskId' already exists."); + + // Delete the key + $this->conn->del($this->key_prefix . $taskId); + $this->conn->sRem($this->indexSet, $taskId); + + // Remove all task output and post output + $settings = $task->getRetrySettings(); + $maxRetries = $settings['maxRetries']; + for ($i=0;$i<=$maxRetries;$i++) + { + // First remove all possible task output + if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $i)) + $this->conn->del($this->key_prefix . $taskId . '_output_' . $i); + + if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $i)) + $this->conn->del($this->key_prefix . $taskId . '_post_' . $i); + } + + return true; + } + + /** + * @inheritDoc + */ + public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + { + // First get the task ID + $taskId = $task->getId(); + + // Check if the key already exists + if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt)) + throw new TasksException("Could not write task output. Output already written."); + + // Prepare contents + $contents = ['taskId' => $task->getId(), 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode]; + $data = serialize($contents); + + // Write contents + return $this->conn->set($this->key_prefix . $taskId . '_output_' . $attempt, $data); + } + + /** + * @inheritDoc + */ + public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + { + // First get the task ID + $taskId = $task->getId(); + + // Check if the key already exists + if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt)) + throw new TasksException("Could not write post output. Output already written."); + + // Prepare contents + $contents = ['taskId' => $task->getId(), 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode]; + $data = serialize($contents); + + // Write contents + return $this->conn->set($this->key_prefix . $taskId . '_post_' . $attempt, $data); + } + + /** + * @inheritDoc + */ + public function readTaskOutput(Task $task, int $attempt = 0): ?array + { + // First get the task ID + $taskId = $task->getId(); + + // Check if the key already exists + if (!$this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt)) + return null; + + // Load and convert the data + $data = $this->conn->get($this->key_prefix . $taskId . '_output_' . $attempt); + $data = unserialize($data); + + return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + + /** + * @inheritDoc + */ + public function readPostOutput(Task $task, int $attempt = 0): ?array + { + // First get the task ID + $taskId = $task->getId(); + + // Check if the key already exists + if (!$this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt)) + return null; + + // Load and convert the data + $data = $this->conn->get($this->key_prefix . $taskId . '_post_' . $attempt); + $data = unserialize($data); + + return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } +} \ No newline at end of file diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index 18df927..fde3962 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -75,14 +75,16 @@ class Tasks implements iLibrary } /** + * @param string $bootstrapFile + * @return SuperVisor * @throws TasksException */ - public function getSuperVisor(): SuperVisor + public function getSuperVisor(string $bootstrapFile): SuperVisor { $cfg = $this->cfg->get('SuperVisor'); $class = 'FuzeWorks\Async\Supervisors\\' . $cfg['type']; $parameters = isset($cfg['parameters']) && is_array($cfg['parameters']) ? $cfg['parameters'] : []; - array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor()); + array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor($bootstrapFile)); if (!class_exists($class, true)) throw new TasksException("Could not get SuperVisor. Type of '$class' not found."); @@ -126,10 +128,11 @@ class Tasks implements iLibrary /** * Fetch the Executor based on the configured type * + * @param string $bootstrapFile * @return Executor * @throws TasksException */ - protected function getExecutor(): Executor + protected function getExecutor(string $bootstrapFile): Executor { $cfg = $this->cfg->get('Executor'); $class = 'FuzeWorks\Async\Executors\\' . $cfg['type']; @@ -137,7 +140,7 @@ class Tasks implements iLibrary if (!class_exists($class, true)) throw new TasksException("Could not get Executor. Type of '$class' not found."); - $object = new $class($parameters); + $object = new $class($bootstrapFile, $parameters); if (!$object instanceof Executor) throw new TasksException("Could not get Executor. Type '$class' is not instanceof Executor."); -- 2.40.1 From b60ac207860a2efbb14892bd1e7575a29aac53c9 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 14 Feb 2020 15:30:42 +0100 Subject: [PATCH 04/33] Add 'addTasks' method to `Tasks` class --- src/FuzeWorks/Async/Tasks.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index fde3962..e3ebae6 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -68,7 +68,7 @@ class Tasks implements iLibrary * @return bool * @throws TasksException */ - public function queueTask(Task $task): bool + public function addTask(Task $task): bool { $taskStorage = $this->getTaskStorage(); return $taskStorage->addTask($task); -- 2.40.1 From 820624e1803b8d6a4aec5c757107bddfd0b794c1 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Wed, 19 Feb 2020 00:42:48 +0100 Subject: [PATCH 05/33] Started implementing ControllerHandler. ControllerHandler is a standard utility handler for MVCR Controllers. This allows the user to create a task which is handled by a standardized controller. Not finished yet! Needs some love. --- Dockerfile | 2 +- composer.json | 3 +- .../Async/Handler/ControllerHandler.php | 175 ++++++++++++++++++ 3 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 src/FuzeWorks/Async/Handler/ControllerHandler.php diff --git a/Dockerfile b/Dockerfile index f42a7bf..3e4fd7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM php:7.3-cli-buster RUN apt-get update &&\ - apt-get install --no-install-recommends --assume-yes --quiet procps ca-certificates curl git &&\ + apt-get install --no-install-recommends --assume-yes --quiet procps ca-certificates curl git unzip &&\ rm -rf /var/lib/apt/lists/* # Install Redis diff --git a/composer.json b/composer.json index 2e5c849..6c8aa73 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,8 @@ "ext-redis": "*" }, "require-dev": { - "fuzeworks/tracycomponent": "~1.2.0" + "fuzeworks/tracycomponent": "~1.2.0", + "fuzeworks/mvcr": "~1.2.0" }, "autoload": { "psr-4": { diff --git a/src/FuzeWorks/Async/Handler/ControllerHandler.php b/src/FuzeWorks/Async/Handler/ControllerHandler.php new file mode 100644 index 0000000..e90f248 --- /dev/null +++ b/src/FuzeWorks/Async/Handler/ControllerHandler.php @@ -0,0 +1,175 @@ +setArguments($task); + + // First we fetch the controller + $controller = $this->getController($this->controllerName); + + // Check if method exists + if (!method_exists($controller, $this->controllerMethod)) + throw new TasksException("Could not handle task. Method '$this->controllerMethod' not found on controller."); + + // Call method and collect output + $this->output = $controller->{$this->controllerMethod}(...$args); + return true; + } + + /** + * @inheritDoc + */ + public function getOutput() + { + return $this->output; + } + + /** + * @inheritDoc + * @throws TasksException + */ + public function postHandler(Task $task) + { + // Set the arguments + $args = $this->setArguments($task); + + // Abort if no postMethod exists + if (is_null($this->postMethod)) + throw new TasksException("Could not handle task. No post method provided."); + + // First we fetch the controller + $controller = $this->getController($this->controllerName); + + // Check if method exists + if (!method_exists($controller, $this->postMethod)) + throw new TasksException("Could not handle task. Post method '$this->postMethod' not found on controller."); + + // Call method and collect output + $this->postOutput = $controller->{$this->postMethod}($task); + return true; + } + + /** + * @inheritDoc + */ + public function getPostOutput() + { + return $this->postOutput; + } + + /** + * Set the arguments of this handler using the provided task + * + * @param Task $task + * @return array + * @throws TasksException + */ + private function setArguments(Task $task): array + { + // Direct arguments + $args = $task->getArguments(); + if (count($args) < 3) + throw new TasksException("Could not handle task. Not enough arguments provided."); + + // First argument: controllerName + $this->controllerName = $args[0]; + $this->controllerMethod = $args[1]; + $this->postMethod = isset($args[2]) ? $args[2] : null; + + return !isset($args[2]) ? [] : array_slice(func_get_args(), 3); + } + + /** + * @param string $controllerName + * @return Controller + * @throws TasksException + */ + private function getController(string $controllerName): Controller + { + // First load the controllers component + try { + /** @var Controllers $controllers */ + $controllers = Factory::getInstance('controllers'); + + // Load the requested controller + return $controllers->get($controllerName); + } catch (FactoryException $e) { + throw new TasksException("Could not get controller. FuzeWorks\MVCR is not installed!"); + } catch (ControllerException $e) { + throw new TasksException("Could not get controller. Controller threw exception: '" . $e->getMessage() . "'"); + } catch (NotFoundException $e) { + throw new TasksException("Could not get controller. Controller was not found."); + } + } +} \ No newline at end of file -- 2.40.1 From b3982f2b2e90570bc2173edb5af78c3c80625c51 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Wed, 19 Feb 2020 16:42:35 +0100 Subject: [PATCH 06/33] ControllerHandler now works. Next up is a distinction between Task and Process status. --- src/FuzeWorks/Async/Executors/ShellExecutor.php | 2 +- src/FuzeWorks/Async/Handler/ControllerHandler.php | 12 ++++++------ src/FuzeWorks/Async/TaskStorage.php | 9 +++++++++ .../Async/TaskStorage/ArrayTaskStorage.php | 9 +++++++++ .../Async/TaskStorage/RedisTaskStorage.php | 13 +++++++++++++ 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index 87df219..c6706c9 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -83,7 +83,7 @@ class ShellExecutor implements Executor public function startTask(Task $task, bool $post = false): Task { // First prepare the command used to spawn workers - $commandString = "$this->binary $this->worker --bootstrap=".$this->bootstrapFile." -t %s ".($post ? 'p' : '')." $this->stdout $this->stderr & echo $!"; + $commandString = "$this->binary $this->worker --bootstrap=".$this->bootstrapFile." -t %s ".($post ? '-p' : '')." $this->stdout $this->stderr & echo $!"; // Then execute the command using the base64_encoded string of the taskID $output = $this->shellExec($commandString, [base64_encode($task->getId())]); diff --git a/src/FuzeWorks/Async/Handler/ControllerHandler.php b/src/FuzeWorks/Async/Handler/ControllerHandler.php index e90f248..ae8f1fd 100644 --- a/src/FuzeWorks/Async/Handler/ControllerHandler.php +++ b/src/FuzeWorks/Async/Handler/ControllerHandler.php @@ -83,7 +83,7 @@ class ControllerHandler implements Handler throw new TasksException("Could not handle task. Method '$this->controllerMethod' not found on controller."); // Call method and collect output - $this->output = $controller->{$this->controllerMethod}(...$args); + $this->output = call_user_func_array([$controller, $this->controllerMethod], $args); return true; } @@ -102,7 +102,7 @@ class ControllerHandler implements Handler public function postHandler(Task $task) { // Set the arguments - $args = $this->setArguments($task); + $this->setArguments($task); // Abort if no postMethod exists if (is_null($this->postMethod)) @@ -116,7 +116,7 @@ class ControllerHandler implements Handler throw new TasksException("Could not handle task. Post method '$this->postMethod' not found on controller."); // Call method and collect output - $this->postOutput = $controller->{$this->postMethod}($task); + $this->postOutput = call_user_func_array([$controller, $this->postMethod], [$task]); return true; } @@ -135,11 +135,11 @@ class ControllerHandler implements Handler * @return array * @throws TasksException */ - private function setArguments(Task $task): array + public function setArguments(Task $task): array { // Direct arguments $args = $task->getArguments(); - if (count($args) < 3) + if (count($args) < 2) throw new TasksException("Could not handle task. Not enough arguments provided."); // First argument: controllerName @@ -147,7 +147,7 @@ class ControllerHandler implements Handler $this->controllerMethod = $args[1]; $this->postMethod = isset($args[2]) ? $args[2] : null; - return !isset($args[2]) ? [] : array_slice(func_get_args(), 3); + return !array_key_exists(2, $args) ? [] : array_slice($args, 3); } /** diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index 6548240..a5f049f 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -152,4 +152,13 @@ interface TaskStorage * @return array|null */ public function readPostOutput(Task $task, int $attempt = 0): ?array; + + /** + * Reset the TaskStorage. + * + * Remove all tasks and their output from the storage so the TaskStorage begins anew. + * + * @return bool + */ + public function reset(): bool; } \ No newline at end of file diff --git a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php index bbc0269..eb6713f 100644 --- a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php @@ -262,6 +262,15 @@ class ArrayTaskStorage implements TaskStorage return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; } + /** + * @inheritDoc + */ + public function reset(): bool + { + // @todo Implement + return false; + } + private function commit() { $this->data = ['tasks' => []]; diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 8f0d000..8d7c41a 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -275,4 +275,17 @@ class RedisTaskStorage implements TaskStorage return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; } + + /** + * @inheritDoc + * @throws TasksException + */ + public function reset(): bool + { + // First get a list of all tasks + foreach ($this->readTasks() as $task) + $this->deleteTask($task); + + return true; + } } \ No newline at end of file -- 2.40.1 From 3499eed38816fa7212118ebb64b153bb3bdfd3e2 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 18:26:30 +0200 Subject: [PATCH 07/33] Started implementing Drone --- .drone.yml | 11 + .gitignore | 2 + Dockerfile | 1 + composer.json | 2 +- src/FuzeWorks/Async/Task.php | 45 +- src/FuzeWorks/Async/TaskStorage.php | 8 +- .../Async/TaskStorage/DummyTaskStorage.php | 227 ++++++++++ test/base/TaskStorageTest.php | 427 ++++++++++++++++++ test/base/TaskTest.php | 244 ++++++++++ test/bootstrap.php | 61 +++ test/phpunit.xml | 29 ++ test/temp/placeholder | 0 12 files changed, 1014 insertions(+), 43 deletions(-) create mode 100644 .drone.yml create mode 100644 src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php create mode 100644 test/base/TaskStorageTest.php create mode 100644 test/base/TaskTest.php create mode 100644 test/bootstrap.php create mode 100644 test/phpunit.xml create mode 100644 test/temp/placeholder diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..ace247f --- /dev/null +++ b/.drone.yml @@ -0,0 +1,11 @@ +kind: pipeline +type: docker +name: test + +steps: + - name: composer + image: composer:latest + commands: + - composer install + + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 952ad32..1d58abf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ composer.lock .idea/ log/ vendor/ +build/ +test/temp/ diff --git a/Dockerfile b/Dockerfile index 3e4fd7e..9c6702d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update &&\ # Install Redis RUN pecl install redis-5.1.1 && docker-php-ext-enable redis +RUN pecl install xdebug && docker-php-ext-enable xdebug # Install Composer RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer \ No newline at end of file diff --git a/composer.json b/composer.json index 6c8aa73..93e04a2 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,7 @@ "ext-redis": "*" }, "require-dev": { - "fuzeworks/tracycomponent": "~1.2.0", + "phpunit/phpunit": "^9", "fuzeworks/mvcr": "~1.2.0" }, "autoload": { diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index 3367929..49c3445 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -170,11 +170,6 @@ class Task */ protected $attributes = []; - /** - * @var Process - */ - protected $process; - /* -------- Some settings ------------ */ protected $retryOnFail = false; @@ -375,9 +370,9 @@ class Task /** - * @todo Handle output from multiple attempts * @param string $output * @param string $errors + * @todo Handle output from multiple attempts */ public function setOutput(string $output, string $errors) { @@ -386,9 +381,9 @@ class Task } /** - * @todo Handle output from multiple attempts * @param string $output * @param string $errors + * @todo Handle output from multiple attempts */ public function setPostOutput(string $output, string $errors) { @@ -396,39 +391,6 @@ class Task $this->postErrors = $errors; } - /** - * Sets the initial process for this task. - * - * To be set by Executor - * - * @param Process $process - */ - public function setProcess(Process $process) - { - $this->process = $process; - } - - /** - * Returns the initial process for this task - * - * @return Process|null - */ - public function getProcess(): ?Process - { - return $this->process; - } - - public function removeProcess(): bool - { - if ($this->process instanceof Process) - { - $this->process = null; - return true; - } - - return false; - } - /** * Set whether this task should retry after a failure, and how many times * @@ -495,7 +457,8 @@ class Task * @param $value * @return bool */ - private function isSerializable ($value) { + private function isSerializable($value) + { $return = true; $arr = array($value); diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index a5f049f..9dfdee0 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -132,9 +132,12 @@ interface TaskStorage * * $attempt refers to $task->getRetries(). If 0, it is the initial attempt. If > 0, it seeks a retry output. * + * Returns null because that is a very valid response. Oftentimes output will need to be checked and its undesirable + * to always throw an exception for expected behaviour. + * * @param Task $task * @param int $attempt - * @return array + * @return array|null */ public function readTaskOutput(Task $task, int $attempt = 0): ?array; @@ -147,6 +150,9 @@ interface TaskStorage * * $attempt refers to $task->getRetries(). If 0, it is the initial attempt. If > 0, it seeks a retry output. * + * Returns null because that is a very valid response. Oftentimes output will need to be checked and its undesirable + * to always throw an exception for expected behaviour. + * * @param Task $task * @param int $attempt * @return array|null diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php new file mode 100644 index 0000000..149e880 --- /dev/null +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -0,0 +1,227 @@ +tasks = array(); + $this->taskOutput = array(); + } + + /** + * @inheritDoc + * @throws TasksException + */ + public function addTask(Task $task): bool + { + // Check if the already exists + $taskId = $task->getId(); + foreach ($this->tasks as $t) + { + if ($t->getId() === $taskId) + throw new TasksException("Could not add Task to TaskStorage. Task '$taskId' already exists."); + } + + $this->tasks[] = $task; + return true; + } + + /** + * @inheritDoc + */ + public function readTasks(): array + { + return $this->tasks; + } + + /** + * @inheritDoc + */ + public function refreshTasks() + {// Ignore + } + + /** + * @inheritDoc + */ + public function getTaskById(string $identifier): Task + { + foreach ($this->tasks as $t) + if ($t->getId() === $identifier) + return $t; + + throw new TasksException("Could not get task by id. Task not found."); + } + + /** + * @inheritDoc + */ + public function modifyTask(Task $task): bool + { + $taskId = $task->getId(); + for ($i=0;$itasks);$i++) + { + if ($this->tasks[$i]->getId() === $taskId) + { + $this->tasks[$i] = $task; + return true; + } + } + + throw new TasksException("Could not modify task. Task '$taskId' doesn't exist."); + } + + /** + * @inheritDoc + * @throws TasksException + */ + public function deleteTask(Task $task): bool + { + $taskId = $task->getId(); + for ($i=0;$itasks);$i++) + { + if ($this->tasks[$i]->getId() === $taskId) + { + // Remove the task from the main storage + unset($this->tasks[$i]); + return true; + } + } + + throw new TasksException("Could not delete task. Task '$taskId' doesn't exist."); + } + + /** + * @inheritDoc + */ + public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + { + if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) + throw new TasksException("Could not write task output. Output already written."); + + $this->taskOutput[$task->getId()]['task'][$attempt] = [ + 'output' => $output, + 'errors' => $errors, + 'statusCode' => $statusCode + ]; + + return true; + } + + /** + * @inheritDoc + */ + public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + { + if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) + throw new TasksException("Could not write task post output. Output already written."); + + $this->taskOutput[$task->getId()]['post'][$attempt] = [ + 'output' => $output, + 'errors' => $errors, + 'statusCode' => $statusCode + ]; + + return true; + } + + /** + * @inheritDoc + */ + public function readTaskOutput(Task $task, int $attempt = 0): ?array + { + if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) + return $this->taskOutput[$task->getId()]['task'][$attempt]; + + return null; + } + + /** + * @inheritDoc + */ + public function readPostOutput(Task $task, int $attempt = 0): ?array + { + if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) + return $this->taskOutput[$task->getId()]['post'][$attempt]; + + return null; + } + + /** + * @inheritDoc + */ + public function reset(): bool + { + $this->tasks = []; + $this->taskOutput = []; + return true; + } +} \ No newline at end of file diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php new file mode 100644 index 0000000..4c8005e --- /dev/null +++ b/test/base/TaskStorageTest.php @@ -0,0 +1,427 @@ +taskStorage = new DummyTaskStorage([]); + } + + public function testDummyTaskStorageClass() + { + $this->assertInstanceOf('FuzeWorks\Async\TaskStorage\DummyTaskStorage', $this->taskStorage); + } + + /* ---------------------------------- Writing and reading tasks ----------------------- */ + + /** + * @depends testDummyTaskStorageClass + */ + public function testAddAndReadTasks() + { + // Prepare a dummy task + $dummyTask = new Task('testAddTask', 'none'); + + // Nothing is written yet so it should be empty + $this->assertEmpty($this->taskStorage->readTasks()); + + // Write task to storage and test properties of readTasks + $this->assertTrue($this->taskStorage->addTask($dummyTask)); + $output = $this->taskStorage->readTasks(); + $this->assertContains($dummyTask, $output); + + // Test if the properties match + $this->assertEquals('testAddTask', $output[0]->getId()); + $this->assertEquals('none', $output[0]->getHandlerClass()); + } + + /** + * @depends testAddAndReadTasks + */ + public function testAddExistingTask() + { + // Prepare a dummy task + $dummyTask = new Task('testAddExistingTask', 'none'); + + // First check that the task storage starts empty + $this->assertEmpty($this->taskStorage->readTasks()); + + // Then add the first task + $this->assertTrue($this->taskStorage->addTask($dummyTask)); + + // But then add another task, which should raise an exception + $this->expectException(TasksException::class); + $this->taskStorage->addTask($dummyTask); + } + + /** + * @depends testAddAndReadTasks + */ + public function testGetTaskById() + { + // Prepare a dummy task + $dummyTask1 = new Task('testGetTaskById1', 'none'); + $dummyTask2 = new Task('testGetTaskById2', 'none'); + + // First we add both tasks + $this->assertEmpty($this->taskStorage->readTasks()); + $this->assertTrue($this->taskStorage->addTask($dummyTask1)); + $this->assertTrue($this->taskStorage->addTask($dummyTask2)); + + // Afterwards, we attempt to get the separate tasks + $retrievedTask1 = $this->taskStorage->getTaskById('testGetTaskById1'); + $retrievedTask2 = $this->taskStorage->getTaskById('testGetTaskById2'); + $this->assertInstanceOf('FuzeWorks\Async\Task', $retrievedTask1); + $this->assertInstanceOf('FuzeWorks\Async\Task', $retrievedTask2); + + // Assert they have the values we seek + $this->assertEquals('testGetTaskById1', $retrievedTask1->getId()); + $this->assertEquals('testGetTaskById2', $retrievedTask2->getId()); + + // Test they are not the same + $this->assertNotSame($retrievedTask1, $retrievedTask2); + + // And test they are the initial dummy tasks + $this->assertSame($dummyTask1, $retrievedTask1); + $this->assertSame($dummyTask2, $retrievedTask2); + } + + /** + * @depends testGetTaskById + */ + public function testGetTaskByIdNotFound() + { + // Prepare a dummy task + $dummyTask = new Task('testGetTaskByIdNotFound', 'none'); + + // First we add the task + $this->assertEmpty($this->taskStorage->readTasks()); + $this->assertTrue($this->taskStorage->addTask($dummyTask)); + + // Afterwards we check if we can get this task + $this->assertInstanceOf('FuzeWorks\Async\Task', $this->taskStorage->getTaskById('testGetTaskByIdNotFound')); + + // And afterwards we check if an exception is raised if none exist + $this->expectException(TasksException::class); + $this->taskStorage->getTaskById('DoesNotExist'); + } + + /** + * @depends testGetTaskById + */ + public function testModifyTask() + { + // Prepare a dummy task + $dummyTask = new Task('testModifyTask', 'none'); + $dummyTask->setStatus(Task::RUNNING); + + // First we add the task + $this->assertEmpty($this->taskStorage->readTasks()); + $this->assertTrue($this->taskStorage->addTask($dummyTask)); + + // Afterwards we check if this task has the known details + $this->assertEquals(Task::RUNNING, $this->taskStorage->getTaskById('testModifyTask')->getStatus()); + + // Then we change the task + $dummyTask->setStatus(Task::FAILED); + $this->assertTrue($this->taskStorage->modifyTask($dummyTask)); + + // And check if the details have been changed + $this->assertEquals(Task::FAILED, $this->taskStorage->getTaskById('testModifyTask')->getStatus()); + } + + /** + * @depends testModifyTask + */ + public function testModifyTaskNotFound() + { + // Prepare a dummy task + $dummyTask = new Task('testModifyTaskNotFound', 'none'); + + // Attempt to change this task, which does not exist. + $this->expectException(TasksException::class); + $this->taskStorage->modifyTask($dummyTask); + } + + /** + * @depends testGetTaskById + */ + public function testDeleteTask() + { + // Prepare a dummy task + $dummyTask = new Task('testDeleteTask', 'none'); + + // Add the task to the storage + $this->assertEmpty($this->taskStorage->readTasks()); + $this->assertTrue($this->taskStorage->addTask($dummyTask)); + + // Test that it exists + $this->assertSame($dummyTask, $this->taskStorage->getTaskById('testDeleteTask')); + + // Then remove the task + $this->assertTrue($this->taskStorage->deleteTask($dummyTask)); + + // And test that it can't be found + $this->expectException(TasksException::class); + $this->taskStorage->getTaskById('testDeleteTask'); + } + + /** + * @depends testDeleteTask + */ + public function testDeleteTaskNotFound() + { + // Prepare a dummy task + $dummyTask = new Task('testDeleteTaskNotFound', 'none'); + + // Attempt to delete this task, which does not exist. + $this->expectException(TasksException::class); + $this->taskStorage->deleteTask($dummyTask); + } + + /* ---------------------------------- Writing and reading task output ----------------- */ + + /** + * @depends testDummyTaskStorageClass + */ + public function testWriteAndReadTaskOutput() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteTaskOutput', 'none'); + + // First write the task output + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0, 0)); + + // Then try to read the output + $output = $this->taskStorage->readTaskOutput($dummyTask, 0); + $this->assertEquals('output', $output['output']); + $this->assertEquals('errors', $output['errors']); + $this->assertEquals(0, $output['statusCode']); + } + + /** + * @depends testWriteAndReadTaskOutput + */ + public function testWriteAndReadTaskOutputAttempts() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskOutputAttempts', 'none'); + + // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last + // to make sure the default is not selected by accident by the TaskStorage + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output2', 'errors2', 102, 2)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output0', 'errors0', 100, 0)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output1', 'errors1', 101, 1)); + + // Attempt to load the first output + $output0 = $this->taskStorage->readTaskOutput($dummyTask, 0); + $this->assertEquals('output0', $output0['output']); + $this->assertEquals('errors0', $output0['errors']); + $this->assertEquals(100, $output0['statusCode']); + + // Attempt to load the second output + $output1 = $this->taskStorage->readTaskOutput($dummyTask, 1); + $this->assertEquals('output1', $output1['output']); + $this->assertEquals('errors1', $output1['errors']); + $this->assertEquals(101, $output1['statusCode']); + + // Attempt to load the third output + $output2 = $this->taskStorage->readTaskOutput($dummyTask, 2); + $this->assertEquals('output2', $output2['output']); + $this->assertEquals('errors2', $output2['errors']); + $this->assertEquals(102, $output2['statusCode']); + + // Attempt to load the default output + $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals('output0', $output['output']); + $this->assertEquals('errors0', $output['errors']); + $this->assertEquals(100, $output['statusCode']); + } + + /** + * @depends testWriteAndReadTaskOutput + */ + public function testWriteAndReadTaskOutputAlreadyExists() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskOutputAlreadyExists', 'none'); + + // Write a first time + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 100, 0)); + + // And write it a second time + $this->expectException(TasksException::class); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 100, 0)); + } + + /** + * @depends testWriteAndReadTaskOutput + */ + public function testWriteAndReadTaskOutputNotExist() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskOutputNotExist', 'none'); + + $this->assertNull($this->taskStorage->readTaskOutput($dummyTask)); + } + + /* ---------------------------------- Writing and reading task post output ------------ */ + + /** + * @depends testDummyTaskStorageClass + */ + public function testWriteAndReadTaskPostOutput() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskPostOutput', 'none'); + + // First write the task output + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'postOutput', 'errors', 0, 0)); + + // Then try to read the output + $output = $this->taskStorage->readPostOutput($dummyTask, 0); + $this->assertEquals('postOutput', $output['output']); + $this->assertEquals('errors', $output['errors']); + $this->assertEquals(0, $output['statusCode']); + } + + /** + * @depends testWriteAndReadTaskPostOutput + */ + public function testWriteAndReadTaskPostOutputAttempts() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskPostOutputAttempts', 'none'); + + // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last + // to make sure the default is not selected by accident by the TaskStorage + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output2', 'errors2', 102, 2)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output0', 'errors0', 100, 0)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output1', 'errors1', 101, 1)); + + // Attempt to load the first output + $output0 = $this->taskStorage->readPostOutput($dummyTask, 0); + $this->assertEquals('output0', $output0['output']); + $this->assertEquals('errors0', $output0['errors']); + $this->assertEquals(100, $output0['statusCode']); + + // Attempt to load the second output + $output1 = $this->taskStorage->readPostOutput($dummyTask, 1); + $this->assertEquals('output1', $output1['output']); + $this->assertEquals('errors1', $output1['errors']); + $this->assertEquals(101, $output1['statusCode']); + + // Attempt to load the third output + $output2 = $this->taskStorage->readPostOutput($dummyTask, 2); + $this->assertEquals('output2', $output2['output']); + $this->assertEquals('errors2', $output2['errors']); + $this->assertEquals(102, $output2['statusCode']); + + // Attempt to load the default output + $output = $this->taskStorage->readPostOutput($dummyTask); + $this->assertEquals('output0', $output['output']); + $this->assertEquals('errors0', $output['errors']); + $this->assertEquals(100, $output['statusCode']); + } + + /** + * @depends testWriteAndReadTaskPostOutput + */ + public function testWriteAndReadTaskPostOutputAlreadyExists() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskPostOutputAlreadyExists', 'none'); + + // Write a first time + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output', 'errors', 100, 0)); + + // And write it a second time + $this->expectException(TasksException::class); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output', 'errors', 100, 0)); + } + + /** + * @depends testWriteAndReadTaskPostOutput + */ + public function testWriteAndReadTaskPostOutputNotExist() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskPostOutputNotExist', 'none'); + + $this->assertNull($this->taskStorage->readPostOutput($dummyTask)); + } + + /* ---------------------------------- Data persistence and resets --------- ------------ */ + + /** + * @depends testAddAndReadTasks + * @depends testWriteAndReadTaskOutput + * @depends testWriteAndReadTaskPostOutput + */ + public function testReset() + { + // Prepare a dummy task + $dummyTask = new Task('testReset', 'none'); + + // Add the task and some output + $this->assertTrue($this->taskStorage->addTask($dummyTask)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 100)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'postOutput', 'errors', 100)); + + // Then reset the data + $this->assertTrue($this->taskStorage->reset()); + + // And test if the data is actually gone + $this->assertNull($this->taskStorage->readTaskOutput($dummyTask)); + $this->assertNull($this->taskStorage->readPostOutput($dummyTask)); + $this->expectException(TasksException::class); + $this->taskStorage->getTaskById('testReset'); + } + +} diff --git a/test/base/TaskTest.php b/test/base/TaskTest.php new file mode 100644 index 0000000..d10d7c5 --- /dev/null +++ b/test/base/TaskTest.php @@ -0,0 +1,244 @@ +assertInstanceOf('FuzeWorks\Async\Task', $dummyTask); + } + + /* ---------------------------------- Basic variables tests --------------------------- */ + + /** + * @depends testClass + */ + public function testBaseVariables() + { + // Create dummy task + $dummyTask = new Task('testBaseVariables', 'someThing', true); + + // test the values + $this->assertEquals('testBaseVariables', $dummyTask->getId()); + $this->assertEquals('someThing', $dummyTask->getHandlerClass()); + $this->assertTrue($dummyTask->getUsePostHandler()); + } + + /** + * @depends testBaseVariables + */ + public function testArguments() + { + // Create task without arguments + $dummyTask1 = new Task('testArguments1', 'none', true); + $this->assertEmpty($dummyTask1->getArguments()); + + // Now create a task with some arguments + $dummyTask2 = new Task('testArguments2', 'none', true, 'some', 'arguments'); + $this->assertEquals(['some', 'arguments'], $dummyTask2->getArguments()); + } + + /** + * @depends testBaseVariables + */ + public function testPostHandler() + { + // Create dummy tasks + $dummyTask1 = new Task('testPostHandler1', 'someThing', true); + $dummyTask2 = new Task('testPostHandler2', 'someThing', false); + + $this->assertTrue($dummyTask1->getUsePostHandler()); + $this->assertFalse($dummyTask2->getUsePostHandler()); + } + + /** + * @depends testBaseVariables + */ + public function testConstraints() + { + // First create a mock constraint + $stub = $this->createMock(Constraint::class); + + // Then add it to the task + $dummyTask = new Task('testConstraints', 'someThing', false); + $dummyTask->addConstraint($stub); + + // Assert it exists + $this->assertEquals([$stub], $dummyTask->getConstraints()); + } + + /** + * @depends testBaseVariables + */ + public function testStatusCodes() + { + // Create dummy task + $dummyTask = new Task('testStatusCodes', 'someThing', true); + + for ($i = 1; $i <= 9; $i++) { + $dummyTask->setStatus($i); + $this->assertEquals($i, $dummyTask->getStatus()); + } + } + + /** + * @depends testBaseVariables + */ + public function testDelayTime() + { + // Create dummy task + $dummyTask = new Task('testDelayTime', 'someThing', true); + + $this->assertEquals(0, $dummyTask->getDelayTime()); + $dummyTask->setDelayTime(1000); + $this->assertEquals(1000, $dummyTask->getDelayTime()); + } + + /** + * @depends testBaseVariables + */ + public function testAttributes() + { + // Create dummy task + $dummyTask = new Task('testAttributes', 'someThing', true); + + // First test a non-existing attribute + $this->assertNull($dummyTask->attribute('testKey')); + + // Now add it and test if it is there + $dummyTask->addAttribute('testKey', 'SomeContent'); + $this->assertEquals('SomeContent', $dummyTask->attribute('testKey')); + } + + /** + * @depends testBaseVariables + */ + public function testOutputsAndErrors() + { + // Create dummy task + $dummyTask = new Task('testOutputsAndErrors', 'someThing', true); + + // Check if non are filled + $this->assertNull($dummyTask->getOutput()); + $this->assertNull($dummyTask->getPostOutput()); + $this->assertNull($dummyTask->getErrors()); + $this->assertNull($dummyTask->getPostErrors()); + + // Then write some data to the task + $dummyTask->setOutput('SomeOutput', 'SomeErrors'); + $dummyTask->setPostOutput('SomePostOutput', 'SomePostErrors'); + + // And check again + $this->assertEquals('SomeOutput', $dummyTask->getOutput()); + $this->assertEquals('SomePostOutput', $dummyTask->getPostOutput()); + $this->assertEquals('SomeErrors', $dummyTask->getErrors()); + $this->assertEquals('SomePostErrors', $dummyTask->getPostErrors()); + } + + /** + * @depends testBaseVariables + */ + public function testRetrySettings() + { + // Create dummy task + $dummyTask = new Task('testRetrySettings', 'someThing', true); + + // Test starting position + $this->assertEquals([ + 'retryOnFail' => false, + 'maxRetries' => 2, + 'retryPFailures' => true, + 'retryRFailures' => true, + 'retryPostFailures' => true + ], $dummyTask->getRetrySettings()); + + // Then change the settings + $dummyTask->setRetrySettings(true, 30, false, false, false); + + // And test the new positions + $this->assertEquals([ + 'retryOnFail' => true, + 'maxRetries' => 30, + 'retryPFailures' => false, + 'retryRFailures' => false, + 'retryPostFailures' => false + ], $dummyTask->getRetrySettings()); + } + + /** + * @depends testBaseVariables + */ + public function testRetries() + { + // Create dummy task + $dummyTask = new Task('testRetries', 'someThing', true); + + // First test the starting position + $this->assertEquals(0, $dummyTask->getRetries()); + + // Then add one and test + $dummyTask->addRetry(); + $this->assertEquals(1, $dummyTask->getRetries()); + + // Then reset it and test + $dummyTask->resetRetries(); + $this->assertEquals(0, $dummyTask->getRetries()); + } + + public function testGetStatusType() + { + $this->assertEquals('Task::PENDING', Task::getStatusType(Task::PENDING)); + $this->assertEquals('Task::RUNNING', Task::getStatusType(Task::RUNNING)); + $this->assertEquals('Task::FAILED', Task::getStatusType(Task::FAILED)); + $this->assertEquals('Task::PFAILED', Task::getStatusType(Task::PFAILED)); + $this->assertEquals('Task::SUCCESS', Task::getStatusType(Task::SUCCESS)); + $this->assertEquals('Task::POST', Task::getStatusType(Task::POST)); + $this->assertEquals('Task::COMPLETED', Task::getStatusType(Task::COMPLETED)); + $this->assertEquals('Task::DELAYED', Task::getStatusType(Task::DELAYED)); + $this->assertEquals('Task::CANCELLED', Task::getStatusType(Task::CANCELLED)); + $this->assertFalse(Task::getStatusType(10)); + } +} diff --git a/test/bootstrap.php b/test/bootstrap.php new file mode 100644 index 0000000..55992ee --- /dev/null +++ b/test/bootstrap.php @@ -0,0 +1,61 @@ +setTimeZone('Europe/Amsterdam'); +$configurator->setTempDirectory(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'temp'); +$configurator->setLogDirectory(dirname(__FILE__). DIRECTORY_SEPARATOR . 'temp'); + +$configurator->addComponent(new \FuzeWorks\MVCRComponent()); + +// Add Async library +$configurator->deferComponentClassMethod('libraries', 'addLibraryClass', null, 'async', '\FuzeWorks\Async\Tasks'); + +// Debug +$configurator->addDirectory(dirname(__FILE__), 'controllers', Priority::HIGH); + +// Create container +$container = $configurator->createContainer(); +Logger::enableScreenLog(); +return $container; \ No newline at end of file diff --git a/test/phpunit.xml b/test/phpunit.xml new file mode 100644 index 0000000..ac4f16d --- /dev/null +++ b/test/phpunit.xml @@ -0,0 +1,29 @@ + + + + + ./base/ + + + + + + ../ + + ../vendor/ + ../test/ + ../src/Layout/ + ../src/Config/ + + + + \ No newline at end of file diff --git a/test/temp/placeholder b/test/temp/placeholder new file mode 100644 index 0000000..e69de29 -- 2.40.1 From 29c3db2159269428d30d48e5be82f03469c2f572 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 18:28:17 +0200 Subject: [PATCH 08/33] Try again --- .drone.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index ace247f..76c5e07 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,6 +6,4 @@ steps: - name: composer image: composer:latest commands: - - composer install - - \ No newline at end of file + - composer install \ No newline at end of file -- 2.40.1 From 9fa714559335029d23028594b7794be872ac60a3 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 18:30:06 +0200 Subject: [PATCH 09/33] Try with a modified environment. --- composer.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/composer.json b/composer.json index 93e04a2..828f268 100644 --- a/composer.json +++ b/composer.json @@ -22,6 +22,11 @@ "phpunit/phpunit": "^9", "fuzeworks/mvcr": "~1.2.0" }, + "config": { + "platform": { + "ext-redis": "1" + } + }, "autoload": { "psr-4": { "FuzeWorks\\Async\\": "src/FuzeWorks/Async" -- 2.40.1 From f1aef62615d83d718f8e334f7025b863db4413bd Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 18:35:15 +0200 Subject: [PATCH 10/33] Attempt to run a PHPUnit batch --- .drone.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 76c5e07..133cc08 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,4 +6,9 @@ steps: - name: composer image: composer:latest commands: - - composer install \ No newline at end of file + - composer install + + - name: base + image: phpunit:7.3 + commands: + - vendor/bin/phpunit -c test/phpunit.xml \ No newline at end of file -- 2.40.1 From 5145d65d1daafbb05bbf8487a077f6c64b76bdf1 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 18:40:45 +0200 Subject: [PATCH 11/33] Now try with an added service --- .drone.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 133cc08..1a97357 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,13 +2,25 @@ kind: pipeline type: docker name: test +services: + - name: cache + image: redis + steps: - name: composer image: composer:latest commands: - composer install - - name: base + - name: basetest image: phpunit:7.3 commands: - - vendor/bin/phpunit -c test/phpunit.xml \ No newline at end of file + - vendor/bin/phpunit -c test/phpunit.xml + + - name: redistest + image: phpunit:7.3 + commands: + - vendor/bin/phpunit -c test/phpunit.xml + environment: + SUPERVISOR: ParallelSuperVisor + TASKSTORAGE: RedisTaskStorage \ No newline at end of file -- 2.40.1 From 463966064065723404e641a77839014a4e3e41eb Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 21:19:35 +0200 Subject: [PATCH 12/33] Added separate environments for DummyTaskStorage and RedisTaskStorage. System now uses environment variables imported through Docker. See test/config.tasks.php for all environment options available. --- .drone.yml | 3 +- .../Async/TaskStorage/DummyTaskStorage.php | 6 +++ .../Async/TaskStorage/RedisTaskStorage.php | 8 ++++ src/FuzeWorks/Async/Tasks.php | 1 - test/base/TaskStorageTest.php | 48 +++++++++++++++---- test/bootstrap.php | 4 +- config.tasks.php => test/config.tasks.php | 43 ++++++++++------- 7 files changed, 81 insertions(+), 32 deletions(-) rename config.tasks.php => test/config.tasks.php (65%) diff --git a/.drone.yml b/.drone.yml index 1a97357..d75d4da 100644 --- a/.drone.yml +++ b/.drone.yml @@ -23,4 +23,5 @@ steps: - vendor/bin/phpunit -c test/phpunit.xml environment: SUPERVISOR: ParallelSuperVisor - TASKSTORAGE: RedisTaskStorage \ No newline at end of file + TASKSTORAGE: RedisTaskStorage + TASKSTORAGE_REDIS_HOST: cache \ No newline at end of file diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index 149e880..9056328 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -164,6 +164,9 @@ class DummyTaskStorage implements TaskStorage */ public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool { + // First check if the task exists + $task = $this->getTaskById($task->getId()); + if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) throw new TasksException("Could not write task output. Output already written."); @@ -181,6 +184,9 @@ class DummyTaskStorage implements TaskStorage */ public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool { + // First check if the task exists + $task = $this->getTaskById($task->getId()); + if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) throw new TasksException("Could not write task post output. Output already written."); diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 8d7c41a..76aeb55 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -206,6 +206,9 @@ class RedisTaskStorage implements TaskStorage // First get the task ID $taskId = $task->getId(); + // Check if the task exists + $task = $this->getTaskById($taskId); + // Check if the key already exists if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt)) throw new TasksException("Could not write task output. Output already written."); @@ -226,6 +229,9 @@ class RedisTaskStorage implements TaskStorage // First get the task ID $taskId = $task->getId(); + // Check if the task exists + $task = $this->getTaskById($taskId); + // Check if the key already exists if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt)) throw new TasksException("Could not write post output. Output already written."); @@ -286,6 +292,8 @@ class RedisTaskStorage implements TaskStorage foreach ($this->readTasks() as $task) $this->deleteTask($task); + $this->refreshTasks(); + return true; } } \ No newline at end of file diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index e3ebae6..bd76183 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -59,7 +59,6 @@ class Tasks implements iLibrary { /** @var Config $config */ $config = Factory::getInstance('config'); - $config->addComponentPath(dirname(__FILE__, 4), Priority::LOW); $this->cfg = $config->getConfig('tasks'); } diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index 4c8005e..1ef5d92 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -35,6 +35,7 @@ */ use FuzeWorks\Async\Task; +use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TasksException; use FuzeWorks\Async\TaskStorage; use FuzeWorks\Async\TaskStorage\DummyTaskStorage; @@ -50,12 +51,14 @@ class TaskStorageTest extends TestCase public function setUp(): void { - $this->taskStorage = new DummyTaskStorage([]); + $tasks = new Tasks(); + $this->taskStorage = $tasks->getTaskStorage(); + $this->taskStorage->reset(); } public function testDummyTaskStorageClass() { - $this->assertInstanceOf('FuzeWorks\Async\TaskStorage\DummyTaskStorage', $this->taskStorage); + $this->assertInstanceOf('FuzeWorks\Async\TaskStorage', $this->taskStorage); } /* ---------------------------------- Writing and reading tasks ----------------------- */ @@ -74,11 +77,15 @@ class TaskStorageTest extends TestCase // Write task to storage and test properties of readTasks $this->assertTrue($this->taskStorage->addTask($dummyTask)); $output = $this->taskStorage->readTasks(); - $this->assertContains($dummyTask, $output); + $this->assertCount(1, $output); + + // Get first + $task = $output[0]; + $this->assertEquals($dummyTask, $task); // Test if the properties match - $this->assertEquals('testAddTask', $output[0]->getId()); - $this->assertEquals('none', $output[0]->getHandlerClass()); + $this->assertEquals('testAddTask', $task->getId()); + $this->assertEquals('none', $task->getHandlerClass()); } /** @@ -125,11 +132,11 @@ class TaskStorageTest extends TestCase $this->assertEquals('testGetTaskById2', $retrievedTask2->getId()); // Test they are not the same - $this->assertNotSame($retrievedTask1, $retrievedTask2); + $this->assertNotEquals($retrievedTask1, $retrievedTask2); // And test they are the initial dummy tasks - $this->assertSame($dummyTask1, $retrievedTask1); - $this->assertSame($dummyTask2, $retrievedTask2); + $this->assertEquals($dummyTask1, $retrievedTask1); + $this->assertEquals($dummyTask2, $retrievedTask2); } /** @@ -202,7 +209,7 @@ class TaskStorageTest extends TestCase $this->assertTrue($this->taskStorage->addTask($dummyTask)); // Test that it exists - $this->assertSame($dummyTask, $this->taskStorage->getTaskById('testDeleteTask')); + $this->assertEquals($dummyTask, $this->taskStorage->getTaskById('testDeleteTask')); // Then remove the task $this->assertTrue($this->taskStorage->deleteTask($dummyTask)); @@ -233,9 +240,10 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskOutput() { // Prepare a dummy task - $dummyTask = new Task('testWriteTaskOutput', 'none'); + $dummyTask = new Task('testWriteAndReadTaskOutput', 'none'); // First write the task output + $this->taskStorage->addTask($dummyTask); $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0, 0)); // Then try to read the output @@ -245,6 +253,19 @@ class TaskStorageTest extends TestCase $this->assertEquals(0, $output['statusCode']); } + /** + * @depends testWriteAndReadTaskOutput + */ + public function testWriteAndReadTaskOutputTaskNotExist() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadTaskOutputTaskNotExist', 'none'); + + // Write output while the task does not exist yet, expect exception + $this->expectException(TasksException::class); + $this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0, 0); + } + /** * @depends testWriteAndReadTaskOutput */ @@ -252,6 +273,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskOutputAttempts', 'none'); + $this->taskStorage->addTask($dummyTask); // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last // to make sure the default is not selected by accident by the TaskStorage @@ -291,6 +313,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskOutputAlreadyExists', 'none'); + $this->taskStorage->addTask($dummyTask); // Write a first time $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 100, 0)); @@ -307,6 +330,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskOutputNotExist', 'none'); + $this->taskStorage->addTask($dummyTask); $this->assertNull($this->taskStorage->readTaskOutput($dummyTask)); } @@ -320,6 +344,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskPostOutput', 'none'); + $this->taskStorage->addTask($dummyTask); // First write the task output $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'postOutput', 'errors', 0, 0)); @@ -338,6 +363,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskPostOutputAttempts', 'none'); + $this->taskStorage->addTask($dummyTask); // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last // to make sure the default is not selected by accident by the TaskStorage @@ -377,6 +403,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskPostOutputAlreadyExists', 'none'); + $this->taskStorage->addTask($dummyTask); // Write a first time $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output', 'errors', 100, 0)); @@ -393,6 +420,7 @@ class TaskStorageTest extends TestCase { // Prepare a dummy task $dummyTask = new Task('testWriteAndReadTaskPostOutputNotExist', 'none'); + $this->taskStorage->addTask($dummyTask); $this->assertNull($this->taskStorage->readPostOutput($dummyTask)); } diff --git a/test/bootstrap.php b/test/bootstrap.php index 55992ee..875d2c5 100644 --- a/test/bootstrap.php +++ b/test/bootstrap.php @@ -52,8 +52,8 @@ $configurator->addComponent(new \FuzeWorks\MVCRComponent()); // Add Async library $configurator->deferComponentClassMethod('libraries', 'addLibraryClass', null, 'async', '\FuzeWorks\Async\Tasks'); -// Debug -$configurator->addDirectory(dirname(__FILE__), 'controllers', Priority::HIGH); +// Add test directory so that config.tasks.php can be loaded +$configurator->addDirectory(dirname(__FILE__), 'config', Priority::HIGH); // Create container $container = $configurator->createContainer(); diff --git a/config.tasks.php b/test/config.tasks.php similarity index 65% rename from config.tasks.php rename to test/config.tasks.php index 922e272..19744d0 100644 --- a/config.tasks.php +++ b/test/config.tasks.php @@ -33,35 +33,42 @@ * * @version Version 1.0.0 */ + +use FuzeWorks\Core; + return array( + // Add a file lock + 'lock' => true, + + // Which SuperVisor should be used 'SuperVisor' => [ - 'type' => 'ParallelSuperVisor', + 'type' => Core::getEnv('SUPERVISOR_TYPE', 'ParallelSuperVisor'), 'parameters' => [] ], 'TaskStorage' => [ - 'type' => 'ArrayTaskStorage', + 'type' => Core::getEnv('TASKSTORAGE_TYPE', 'DummyTaskStorage'), // For ArrayTaskStorage, first parameter is the file location of the array storage - 'parameters' => [ - 'filename' => dirname(__FILE__) . DS . 'storage.php' - ], + #'parameters' => [ + # 'filename' => dirname(__FILE__) . DS . 'storage.php' + #], // For RedisTaskStorage, parameters are connection properties - #'parameters' => [ - # // Type can be 'tcp' or 'unix' - # 'socket_type' => 'tcp', - # // If socket_type == 'unix', set the socket here - # 'socket' => null, - # // If socket_type == 'tcp', set the host here - # 'host' => 'localhost', - # - # 'password' => null, - # 'port' => 6379, - # 'timeout' => 0 - #] + 'parameters' => [ + // Type can be 'tcp' or 'unix' + 'socket_type' => Core::getEnv('TASKSTORAGE_REDIS_SOCKET_TYPE', 'tcp'), + // If socket_type == 'unix', set the socket here + 'socket' => Core::getEnv('TASKSTORAGE_REDIS_SOCKET', null), + // If socket_type == 'tcp', set the host here + 'host' => Core::getEnv('TASKSTORAGE_REDIS_HOST', '127.0.0.1'), + // And some standard settings + 'password' => Core::getEnv('TASKSTORAGE_REDIS_PASSWORD', null), + 'port' => Core::getEnv('TASKSTORAGE_REDIS_PORT', 6379), + 'timeout' => Core::getEnv('TASKSTORAGE_REDIS_TIMEOUT', 0), + ] ], 'Executor' => [ - 'type' => 'ShellExecutor', + 'type' => Core::getEnv('EXECUTOR_TYPE', 'ShellExecutor'), // For ShellExecutor, first parameter is the file location of the worker script 'parameters' => [ -- 2.40.1 From 19c84bc0d64cce0b56f11a9dce7457d42b4e799c Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 21:33:26 +0200 Subject: [PATCH 13/33] Temporarily check if Redis works --- src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 76aeb55..0f5824e 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -80,6 +80,8 @@ class RedisTaskStorage implements TaskStorage } catch (RedisException $e) { throw new TasksException("Could not construct RedisTaskStorage. RedisException thrown: '" . $e->getMessage() . "'"); } + + var_dump("Redis activated!"); } /** -- 2.40.1 From 99bae3c100b3b6b4a66de7d07f76776882a20639 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 21:34:48 +0200 Subject: [PATCH 14/33] Now? --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index d75d4da..eb16825 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,6 +22,6 @@ steps: commands: - vendor/bin/phpunit -c test/phpunit.xml environment: - SUPERVISOR: ParallelSuperVisor - TASKSTORAGE: RedisTaskStorage + SUPERVISOR_TYPE: ParallelSuperVisor + TASKSTORAGE_TYPE: RedisTaskStorage TASKSTORAGE_REDIS_HOST: cache \ No newline at end of file -- 2.40.1 From e2101bf0598065ffa5ed3a18406e619592314459 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 21:36:20 +0200 Subject: [PATCH 15/33] And remove the Redis debug again --- src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 0f5824e..76aeb55 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -80,8 +80,6 @@ class RedisTaskStorage implements TaskStorage } catch (RedisException $e) { throw new TasksException("Could not construct RedisTaskStorage. RedisException thrown: '" . $e->getMessage() . "'"); } - - var_dump("Redis activated!"); } /** -- 2.40.1 From 33c81e6c182734f10bba3760752951250ed090b7 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 15 May 2020 21:41:39 +0200 Subject: [PATCH 16/33] Now with coverage --- .drone.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index eb16825..0061faa 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,12 +15,14 @@ steps: - name: basetest image: phpunit:7.3 commands: - - vendor/bin/phpunit -c test/phpunit.xml + - vendor/bin/phpunit -c test/phpunit.xml --coverage-text + environment: + TASKSTORAGE_TYPE: DummyTaskStorage - name: redistest image: phpunit:7.3 commands: - - vendor/bin/phpunit -c test/phpunit.xml + - vendor/bin/phpunit -c test/phpunit.xml --coverage-text environment: SUPERVISOR_TYPE: ParallelSuperVisor TASKSTORAGE_TYPE: RedisTaskStorage -- 2.40.1 From 72cd7637b516205ddf9dcc060837ac5611a9060b Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Sat, 16 May 2020 19:12:22 +0200 Subject: [PATCH 17/33] Implemented many unit tests. --- .drone.yml | 4 +- composer.json | 1 + src/FuzeWorks/Async/Executor.php | 26 +- .../Async/Executors/ShellExecutor.php | 55 +-- .../Async/Supervisors/ParallelSuperVisor.php | 5 +- src/FuzeWorks/Async/Task.php | 14 + test/base/ShellExecutorTest.php | 221 +++++++++ test/base/TaskStorageTest.php | 3 + test/base/TaskTest.php | 8 + test/bootstrap.php | 9 +- test/mock/Handlers/ArgumentedHandler.php | 85 ++++ .../Handlers/TestStartAndReadTasksHandler.php | 76 +++ .../mock/Handlers/TestStopTaskHandler.php | 51 +- test/phpunit.xml | 1 + test/system/ParallelSuperVisorTest.php | 464 ++++++++++++++++++ 15 files changed, 957 insertions(+), 66 deletions(-) create mode 100644 test/base/ShellExecutorTest.php create mode 100644 test/mock/Handlers/ArgumentedHandler.php create mode 100644 test/mock/Handlers/TestStartAndReadTasksHandler.php rename src/FuzeWorks/Async/Process.php => test/mock/Handlers/TestStopTaskHandler.php (76%) create mode 100644 test/system/ParallelSuperVisorTest.php diff --git a/.drone.yml b/.drone.yml index 0061faa..088a992 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,14 +15,14 @@ steps: - name: basetest image: phpunit:7.3 commands: - - vendor/bin/phpunit -c test/phpunit.xml --coverage-text + - vendor/bin/phpunit -c test/phpunit.xml --coverage-php test/temp/covbase.cov environment: TASKSTORAGE_TYPE: DummyTaskStorage - name: redistest image: phpunit:7.3 commands: - - vendor/bin/phpunit -c test/phpunit.xml --coverage-text + - vendor/bin/phpunit -c test/phpunit.xml --coverage-php test/temp/covredis.cov environment: SUPERVISOR_TYPE: ParallelSuperVisor TASKSTORAGE_TYPE: RedisTaskStorage diff --git a/composer.json b/composer.json index 828f268..e608d04 100644 --- a/composer.json +++ b/composer.json @@ -20,6 +20,7 @@ }, "require-dev": { "phpunit/phpunit": "^9", + "phpunit/phpcov": "^7", "fuzeworks/mvcr": "~1.2.0" }, "config": { diff --git a/src/FuzeWorks/Async/Executor.php b/src/FuzeWorks/Async/Executor.php index 19a3199..de2c336 100644 --- a/src/FuzeWorks/Async/Executor.php +++ b/src/FuzeWorks/Async/Executor.php @@ -36,9 +36,25 @@ namespace FuzeWorks\Async; +/** + * Interface Executor + * + * + * @todo Implement ListRunningTasks + * @package FuzeWorks\Async + */ interface Executor { + /** + * Executor constructor. + * + * Parameters are a unique array which can differ for each Executor. + * + * @param array $parameters + */ + public function __construct(array $parameters); + // Control methods /** * Start executing a task. @@ -57,16 +73,12 @@ interface Executor * Returns the task since it makes modifications. Has to be modified in TaskStorage by SuperVisor. * * @param Task $task - * @return Task + * @param bool $harshly True to KILL a process + * @return Task|null Returns modified Task on success, or null if no PID is found */ - public function stopTask(Task $task): Task; + public function stopTask(Task $task, bool $harshly = false): ?Task; // Task info public function getTaskRunning(Task $task): bool; public function getTaskStats(Task $task): ?array; - public function getTaskExitCode(Task $task): ?int; - - // All tasks info - public function getRunningTasks(): array; - } \ No newline at end of file diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index c6706c9..e1e24d6 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -56,22 +56,21 @@ class ShellExecutor implements Executor * @param array $parameters * @throws TasksException */ - public function __construct(string $bootstrapFile, array $parameters) + public function __construct(array $parameters) { // Fetch workerFile - $workerFile = $parameters['workerFile']; + $this->worker = $parameters['workerFile']; + if (!file_exists($this->worker)) + throw new TasksException("Could not construct ShellExecutor. ShellWorker script does not exist."); // First determine the PHP binary $this->binary = PHP_BINDIR . DS . 'php'; - $this->bootstrapFile = $bootstrapFile; - - if (!file_exists($workerFile)) - throw new TasksException("Could not construct ShellExecutor. ShellWorker script does not exist."); - - $this->worker = $workerFile; + $this->bootstrapFile = $parameters['bootstrapFile']; + if (!file_exists($this->bootstrapFile)) + throw new TasksException("Could not construct ShellExecutor. No bootstrap file found."); } - private function shellExec($format, array $parameters = []) + protected function shellExec($format, array $parameters = []) { $parameters = array_map("escapeshellarg", $parameters); array_unshift($parameters, $format); @@ -88,15 +87,28 @@ class ShellExecutor implements Executor // Then execute the command using the base64_encoded string of the taskID $output = $this->shellExec($commandString, [base64_encode($task->getId())]); $pid = intval($output[0]); - $task->setProcess(new Process($pid)); + $task->addAttribute('pid', $pid); // And finally return the task return $task; } - public function stopTask(Task $task): Task + public function stopTask(Task $task, bool $harshly = false): ?Task { - // TODO: Implement stopTask() method. + // First prepare the kill command + $commandString = "kill " . ($harshly ? '-9 ' : '') . "%s"; + + // Fetch the process ID from the task + $pid = $task->attribute('pid'); + if (is_null($pid)) + return null; + + // Then execute the command + $this->shellExec($commandString, [$pid]); + if (!$this->getTaskRunning($task)) + $task->removeAttribute('pid'); + + return $task; } public function getTaskRunning(Task $task): bool @@ -111,13 +123,10 @@ class ShellExecutor implements Executor $commandString = "ps -o pid,%%cpu,%%mem,state,start -p %s | sed 1d"; // First we must determine what process is used. - $process = $task->getProcess(); - if (is_null($process)) + $pid = $task->attribute('pid'); + if (is_null($pid)) return null; - // Then using that process we determine the ProcessID - $pid = $process->getPid(); - // And we execute the commandString to fetch info on the process $output = $this->shellExec($commandString, [$pid]); @@ -141,16 +150,6 @@ class ShellExecutor implements Executor return null; // Finally, return the Task information - return $parts; - } - - public function getTaskExitCode(Task $task): int - { - // TODO: Implement getTaskExitCode() method. - } - - public function getRunningTasks(): array - { - // TODO: Implement getRunningTasks() method. + return ['pid' => (int) $parts[0], 'cpu' => (float) $parts[1], 'mem' => (float) $parts[2], 'state' => $parts[3], 'start' => $parts[4]]; } } \ No newline at end of file diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index 0408ec8..f3a36d0 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -111,6 +111,7 @@ class ParallelSuperVisor implements SuperVisor // CANCELLED/COMPLETED: remove the task if requested to do so elseif ($task->getStatus() === Task::COMPLETED || $task->getStatus() === Task::CANCELLED) { + // @todo Remove old tasks automatically } // RUNNING: check if task is still running. If not, set result based on output @@ -216,7 +217,9 @@ class ParallelSuperVisor implements SuperVisor $task->addRetry(); $task = $this->executor->startTask($task, true); } - elseif ($settings['maxRetries'] <= $task->getRetries()) + elseif ($settings['retryOnFail'] === true && $settings['retryPostFailures'] === true && $settings['maxRetries'] <= $task->getRetries()) + $task->setStatus(Task::CANCELLED); + else $task->setStatus(Task::CANCELLED); } // @todo Retry after $max_Time diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index 49c3445..4672f46 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -338,6 +338,20 @@ class Task $this->attributes[$key] = $value; } + /** + * Remove an attribute from a Task + * + * @param string $key + * @throws TasksException + */ + public function removeAttribute(string $key) + { + if (!isset($this->attributes[$key])) + throw new TasksException("Could not remove Task '$this->taskId' attribute '$key'. Not found."); + + unset($this->attributes[$key]); + } + /** * Return the output of this task execution * diff --git a/test/base/ShellExecutorTest.php b/test/base/ShellExecutorTest.php new file mode 100644 index 0000000..831b905 --- /dev/null +++ b/test/base/ShellExecutorTest.php @@ -0,0 +1,221 @@ +taskStorage = $tasks->getTaskStorage(); + $this->taskStorage->reset(); + + // And load the ShellExecutor using the execution settings + $this->executor = new ShellExecutor([ + 'bootstrapFile' => dirname(__DIR__) . DIRECTORY_SEPARATOR . 'bootstrap.php', + 'workerFile' => dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'worker' + ]); + } + + public function testClass() + { + $this->assertInstanceOf('FuzeWorks\Async\Executors\ShellExecutor', $this->executor); + } + + /** + * @depends testClass + */ + public function testNoWorkerFile() + { + $this->expectException(TasksException::class); + new ShellExecutor(['bootstrapFile' => dirname(__DIR__) . DIRECTORY_SEPARATOR . 'bootstrap.php']); + } + + /** + * @depends testClass + */ + public function testNoBoostrapFile() + { + $this->expectException(TasksException::class); + new ShellExecutor(['workerFile' => dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'worker']); + } + + /** + * @depends testClass + */ + public function testInvalidWorkerFile() + { + $this->expectException(TasksException::class); + new ShellExecutor([ + 'bootstrapFile' => dirname(__DIR__) . DIRECTORY_SEPARATOR . 'bootstrap.php', + 'workerFile' => 'not_found' + ]); + } + + /** + * @depends testClass + */ + public function testInvalidBootstrapFile() + { + $this->expectException(TasksException::class); + new ShellExecutor([ + 'bootstrapFile' => 'not_found', + 'workerFile' => dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'worker' + ]); + } + + /* ---------------------------------- Writing and reading tasks ----------------------- */ + + /** + * @depends testClass + * @covers ::startTask + * @covers ::getTaskRunning + */ + public function testStartAndReadTasks() + { + // First we create a dummy task + $dummyTask = new Task('testStartAndReadTasks', 'Mock\Handlers\TestStartAndReadTasksHandler'); + + // Then we write this task to the TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Assert that no PID exists yet + $this->assertNull($dummyTask->attribute('pid')); + + // Then we fire the task + $task = $this->executor->startTask($dummyTask); + + // Assert that the output is the same + $this->assertSame($dummyTask, $task); + + // Also assert that a PID has been added + $this->assertIsInt($task->attribute('pid')); + + // Also assert that the task is currently running + $this->assertTrue($this->executor->getTaskRunning($task)); + } + + /** + * @depends testStartAndReadTasks + */ + public function testGetStats() + { + // First we create a dummy task, using the previous handler since nothing changes + $dummyTask = new Task('testGetStats', 'Mock\Handlers\TestStartAndReadTasksHandler'); + + // Then we write this task to the TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Then we start the task + $dummyTask = $this->executor->startTask($dummyTask); + + // And we fetch some task statistics + $stats = $this->executor->getTaskStats($dummyTask); + + // Assert some assumptions + $this->assertNotNull($stats); + $this->assertIsInt($stats['pid']); + $this->assertIsFloat($stats['cpu']); + $this->assertIsFloat($stats['mem']); + $this->assertIsString($stats['state']); + $this->assertIsString($stats['start']); + } + + /** + * @depends testGetStats + */ + public function testGetStatsNotExist() + { + // First we create a dummy task, using the previous handler since nothing changes + $dummyTask = new Task('testGetStatsNotExist', 'none'); + + // And add a fake PID, since otherwise it will immediately fail + $dummyTask->addAttribute('pid', 1005); + + // Then we fetch the process details + $this->assertNull($this->executor->getTaskStats($dummyTask)); + } + + /** + * @depends testStartAndReadTasks + */ + public function testStopTask() + { + // First we create a dummy task + $dummyTask = new Task('testStopTask', 'Mock\Handlers\TestStopTaskHandler'); + + // Then we write this task to the TaskStorage + $this->taskStorage->addTask($dummyTask); + + // First we start the task and confirm its running + $dummyTask = $this->executor->startTask($dummyTask); + $this->assertTrue($this->executor->getTaskRunning($dummyTask)); + + // But then we try and stop it + $output = $this->executor->stopTask($dummyTask); + + // We check that the output actually is the task + $this->assertSame($dummyTask, $output); + + // And check if the Task has actually stopped now + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + } + +} diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index 1ef5d92..4b021be 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -41,6 +41,9 @@ use FuzeWorks\Async\TaskStorage; use FuzeWorks\Async\TaskStorage\DummyTaskStorage; use PHPUnit\Framework\TestCase; +/** + * Class TaskStorageTest + */ class TaskStorageTest extends TestCase { diff --git a/test/base/TaskTest.php b/test/base/TaskTest.php index d10d7c5..0bbb5f2 100644 --- a/test/base/TaskTest.php +++ b/test/base/TaskTest.php @@ -151,6 +151,14 @@ class TaskTest extends TestCase // Now add it and test if it is there $dummyTask->addAttribute('testKey', 'SomeContent'); $this->assertEquals('SomeContent', $dummyTask->attribute('testKey')); + + // Remove attribute + $dummyTask->removeAttribute('testKey'); + $this->assertNull($dummyTask->attribute('testKey')); + + // Remove attribute not found + $this->expectException(TasksException::class); + $dummyTask->removeAttribute('NotExistant'); } /** diff --git a/test/bootstrap.php b/test/bootstrap.php index 875d2c5..8d389b4 100644 --- a/test/bootstrap.php +++ b/test/bootstrap.php @@ -36,7 +36,7 @@ require_once(dirname(__DIR__) . '/vendor/autoload.php'); -use FuzeWorks\Logger; +use FuzeWorks\Core; use FuzeWorks\Priority; // Open configurator @@ -55,7 +55,8 @@ $configurator->deferComponentClassMethod('libraries', 'addLibraryClass', null, ' // Add test directory so that config.tasks.php can be loaded $configurator->addDirectory(dirname(__FILE__), 'config', Priority::HIGH); +// Add mock directory for tests and other classes +Core::addAutoloadMap('\Mock', dirname(__FILE__) . DIRECTORY_SEPARATOR . 'mock'); + // Create container -$container = $configurator->createContainer(); -Logger::enableScreenLog(); -return $container; \ No newline at end of file +return $configurator->createContainer(); \ No newline at end of file diff --git a/test/mock/Handlers/ArgumentedHandler.php b/test/mock/Handlers/ArgumentedHandler.php new file mode 100644 index 0000000..60c80c4 --- /dev/null +++ b/test/mock/Handlers/ArgumentedHandler.php @@ -0,0 +1,85 @@ +getArguments(); + $this->sleeptime = $arguments[0]; + $this->output = $arguments[1]; + + sleep($this->sleeptime); + return true; + } + + /** + * @inheritDoc + */ + public function getOutput() + { + return $this->output; + } + + /** + * @inheritDoc + */ + public function postHandler(Task $task) + { + sleep($this->sleeptime); + return true; + } + + /** + * @inheritDoc + */ + public function getPostOutput() + { + return $this->output; + } +} \ No newline at end of file diff --git a/test/mock/Handlers/TestStartAndReadTasksHandler.php b/test/mock/Handlers/TestStartAndReadTasksHandler.php new file mode 100644 index 0000000..f852cfe --- /dev/null +++ b/test/mock/Handlers/TestStartAndReadTasksHandler.php @@ -0,0 +1,76 @@ +pid = $pid; + sleep(30); + return true; } /** - * Receive the process Id of this process - * - * @return int + * @inheritDoc */ - public function getPid(): int + public function getOutput() { - return $this->pid; + return "Valid Output"; } + /** + * @inheritDoc + */ + public function postHandler(Task $task) + { + + } + + /** + * @inheritDoc + */ + public function getPostOutput() + { + + } } \ No newline at end of file diff --git a/test/phpunit.xml b/test/phpunit.xml index ac4f16d..cf9d692 100644 --- a/test/phpunit.xml +++ b/test/phpunit.xml @@ -12,6 +12,7 @@ ./base/ + ./system/ diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php new file mode 100644 index 0000000..556b0bd --- /dev/null +++ b/test/system/ParallelSuperVisorTest.php @@ -0,0 +1,464 @@ +taskStorage = $tasks->getTaskStorage(); + $this->taskStorage->reset(); + + // And load the ShellExecutor using the execution settings + $this->executor = new ShellExecutor([ + 'bootstrapFile' => dirname(__DIR__) . DIRECTORY_SEPARATOR, + 'workerFile' => dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'worker' + ]); + + $this->superVisor = new ParallelSuperVisor($this->taskStorage, $this->executor, ['outstream' => null]); + } + + public function testClass() + { + $this->assertInstanceOf('FuzeWorks\Async\Supervisors\ParallelSuperVisor', $this->superVisor); + } + + /* ---------------------------------- Writing and reading tasks ----------------------- */ + + /** + * @depends testClass + */ + public function testEmptyCycle() + { + $this->assertEquals(SuperVisor::FINISHED, $this->superVisor->cycle()); + } + + public function testToRunning() + { + // First create a dummy task + $dummyTask = new Task('testToRunning', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Write the dummy to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Assert that the status is PENDING and not running + $this->assertEquals(Task::PENDING, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it is running for real + $this->assertEquals(Task::RUNNING, $dummyTask->getStatus()); + $this->assertTrue($this->executor->getTaskRunning($dummyTask)); + } + + /** + * @depends testToRunning + */ + public function testConstrainedPending() + { + // First create a dummy task + $dummyTask = new Task('testConstrainedPending', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Add a constraint + $dummyTask->addConstraint(new FixedTimeConstraint(time() + 3600)); + + // Write the dummy to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Assert that the status is PENDING and not running + $this->assertEquals(Task::PENDING, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it is delayed + $this->assertEquals(Task::DELAYED, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + } + + /** + * @depends testToRunning + */ + public function testChangeDelayedToPending() + { + // First create a dummy task + $dummyTask = new Task('testChangeDelayedToPending', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set to delayed and set to NOW + $dummyTask->setStatus(Task::DELAYED); + $dummyTask->setDelayTime(time() - 3600); + + // Write the dummy to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Assert that the status is DELAYED and not running + $this->assertEquals(Task::DELAYED, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it is delayed + $this->assertEquals(Task::PENDING, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + } + + /** + * @depends testToRunning + */ + public function testKeepDelayed() + { + // First create a dummy task + $dummyTask = new Task('testKeepDelayed', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set to delayed and set to NOW + $dummyTask->setStatus(Task::DELAYED); + $dummyTask->setDelayTime(time() + 3600); + + // Write the dummy to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Assert that the status is DELAYED and not running + $this->assertEquals(Task::DELAYED, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it is delayed + $this->assertEquals(Task::DELAYED, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + } + + /** + * @depends testToRunning + */ + public function testFinishedTask() + { + // First create a dummy task + $dummyTask = new Task('testFinishedTask', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set status to running + $dummyTask->setStatus(Task::RUNNING); + $dummyTask->addAttribute('pid', 1005); + + // Write the dummy and some output to TaskStorage + $this->taskStorage->addTask($dummyTask); + $this->taskStorage->writeTaskOutput($dummyTask, 'Some Output', '', Task::SUCCESS); + + // Test if everything is set + $this->assertEquals(Task::RUNNING, $dummyTask->getStatus()); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it is finished indeed + $this->assertEquals(Task::SUCCESS, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + $this->assertEquals('Some Output', $dummyTask->getOutput()); + } + + /** + * @depends testFinishedTask + */ + public function testMissingTask() + { + // First create a dummy task + $dummyTask = new Task('testMissingTask', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set status to running + $dummyTask->setStatus(Task::RUNNING); + $dummyTask->addAttribute('pid', 1006); + + // Write the dummy and no output to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Test if everything is set + $this->assertEquals(Task::RUNNING, $dummyTask->getStatus()); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it has failed indeed + $this->assertEquals(Task::PFAILED, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + } + + /** + * @depends testFinishedTask + */ + public function testFailedTask() + { + // First create a dummy task + $dummyTask = new Task('testFailedTask', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set status to running + $dummyTask->setStatus(Task::RUNNING); + $dummyTask->addAttribute('pid', 1007); + + // Write the dummy and some output to TaskStorage + $this->taskStorage->addTask($dummyTask); + $this->taskStorage->writeTaskOutput($dummyTask, 'Some Output', 'Some Errors', Task::FAILED); + + // Test if everything is set + $this->assertEquals(Task::RUNNING, $dummyTask->getStatus()); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Then re-fetch the Task + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + + // And check that it is finished indeed + $this->assertEquals(Task::FAILED, $dummyTask->getStatus()); + $this->assertFalse($this->executor->getTaskRunning($dummyTask)); + $this->assertEquals('Some Errors', $dummyTask->getErrors()); + } + + /** + * @depends testFailedTask + */ + public function testRetryFailedTask() + { + // First create the dummy tasks + $dummyTaskFailedYes = new Task('testRetryFailedTaskY', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTaskPFailedYes = new Task('testRetryFailedTaskPY', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTaskFailedNo = new Task('testRetryFailedTaskN', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTaskPFailedNo = new Task('testRetryFailedTaskPN', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set statuses + $dummyTaskFailedYes->setStatus(Task::FAILED); + $dummyTaskPFailedYes->setStatus(Task::PFAILED); + $dummyTaskFailedNo->setStatus(Task::FAILED); + $dummyTaskPFailedNo->setStatus(Task::FAILED); + + // Set retry settings + $dummyTaskFailedYes->setRetrySettings(true, 5, true, true,true); + $dummyTaskPFailedYes->setRetrySettings(true, 5, true, true,true); + $dummyTaskFailedNo->setRetrySettings(true, 5, false, false,true); + $dummyTaskPFailedNo->setRetrySettings(true, 5, false, false,true); + + // Save all these tasks + $this->taskStorage->addTask($dummyTaskFailedYes); + $this->taskStorage->addTask($dummyTaskPFailedYes); + $this->taskStorage->addTask($dummyTaskFailedNo); + $this->taskStorage->addTask($dummyTaskPFailedNo); + + // Then cycle the SuperVisor + $this->superVisor->cycle(); + + // Reload all tasks from TaskStorage + $dummyTaskFailedYes = $this->taskStorage->getTaskById($dummyTaskFailedYes->getId()); + $dummyTaskPFailedYes = $this->taskStorage->getTaskById($dummyTaskPFailedYes->getId()); + $dummyTaskFailedNo = $this->taskStorage->getTaskById($dummyTaskFailedNo->getId()); + $dummyTaskPFailedNo = $this->taskStorage->getTaskById($dummyTaskPFailedNo->getId()); + + // Check if the tasks that should retry are running + $this->assertEquals(Task::RUNNING, $dummyTaskFailedYes->getStatus()); + $this->assertEquals(Task::RUNNING, $dummyTaskPFailedYes->getStatus()); + $this->assertTrue($this->executor->getTaskRunning($dummyTaskFailedYes)); + $this->assertTrue($this->executor->getTaskRunning($dummyTaskPFailedYes)); + + // Check if the tasks that shouldn't have been cancelled + $this->assertEquals(Task::CANCELLED, $dummyTaskFailedNo->getStatus()); + $this->assertEquals(Task::CANCELLED, $dummyTaskPFailedNo->getStatus()); + } + + /** + * @depends testFailedTask + */ + public function testExceedMaxRetries() + { + // First create the dummy tasks + $dummyTask = new Task('testExceedMaxRetries', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask2 = new Task('testExceedMaxRetries2', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + + // Set status and retry settings + $dummyTask->setStatus(Task::FAILED); + $dummyTask2->setStatus(Task::FAILED); + $dummyTask->setRetrySettings(true, 2, true, true, true); + $dummyTask2->setRetrySettings(true, 2, true, true, true); + + // Set retries to 2 for the first task, and 1 for the second task + $dummyTask->addRetry(); + $dummyTask->addRetry(); + $dummyTask2->addRetry(); + + // Write the task to TaskStorage + $this->taskStorage->addTask($dummyTask); + $this->taskStorage->addTask($dummyTask2); + + // Cycle the SuperVisor + $this->superVisor->cycle(); + + // And check if the Task has been cancelled + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + $dummyTask2 = $this->taskStorage->getTaskById($dummyTask2->getId()); + $this->assertEquals(Task::CANCELLED, $dummyTask->getStatus()); + $this->assertEquals(Task::RUNNING, $dummyTask2->getStatus()); + } + + /** + * @depends testFailedTask + */ + public function testFailedToPost() + { + // First create the dummy tasks + $dummyTask = new Task('testFailedToPost', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + + // Set status and settings + $dummyTask->setStatus(Task::FAILED); + $dummyTask->addAttribute('pid', 1010); + + // Write the task to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Cycle the SuperVisor + $this->superVisor->cycle(); + + // And check if the Task has been moved to Post + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + $this->assertEquals(Task::POST, $dummyTask->getStatus()); + $this->assertTrue($this->executor->getTaskRunning($dummyTask)); + $this->assertIsInt($dummyTask->attribute('pid')); + $this->assertNotEquals(1010, $dummyTask->attribute('pid')); + } + + /** + * @depends testFinishedTask + */ + public function testSuccessfulTasks() + { + // First create the dummy tasks + $dummyTaskPostNo = new Task('testSuccessfulTasksN', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTaskPostYes = new Task('testSuccessfulTasksY', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + + // Set status and settings + $dummyTaskPostNo->setStatus(Task::SUCCESS); + $dummyTaskPostYes->setStatus(Task::SUCCESS); + + // Write the tasks to TaskStorage + $this->taskStorage->addTask($dummyTaskPostNo); + $this->taskStorage->addTask($dummyTaskPostYes); + + // Cycle the SuperVisor + $this->superVisor->cycle(); + + // And check if the Tasks have been completed or moved to post + $dummyTaskPostNo = $this->taskStorage->getTaskById($dummyTaskPostNo->getId()); + $dummyTaskPostYes = $this->taskStorage->getTaskById($dummyTaskPostYes->getId()); + $this->assertEquals(Task::COMPLETED, $dummyTaskPostNo->getStatus()); + $this->assertEquals(Task::POST, $dummyTaskPostYes->getStatus()); + $this->assertTrue($this->executor->getTaskRunning($dummyTaskPostYes)); + } + + public function testPostTasks() + { + // First create the dummy tasks + $dummyTaskFinished = new Task('testPostTasksFinished', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + $dummyTaskMissing = new Task('testPostTasksMissing', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + + // Set status and settings + $dummyTaskFinished->setStatus(Task::POST); + $dummyTaskMissing->setStatus(Task::POST); + $dummyTaskFinished->addAttribute('pid', 1011); + $dummyTaskMissing->addAttribute('pid', 1012); + + // Write the tasks to TaskStorage + $this->taskStorage->addTask($dummyTaskFinished); + $this->taskStorage->addTask($dummyTaskMissing); + $this->taskStorage->writePostOutput($dummyTaskFinished, 'Post Output', 'Post Errors', Task::COMPLETED); + + // Cycle the SuperVisor + $this->superVisor->cycle(); + + // And check if the Tasks have been completed or failed + $dummyTaskFinished = $this->taskStorage->getTaskById($dummyTaskFinished->getId()); + $dummyTaskMissing = $this->taskStorage->getTaskById($dummyTaskMissing->getId()); + $this->assertEquals(Task::COMPLETED, $dummyTaskFinished->getStatus()); + $this->assertEquals(Task::CANCELLED, $dummyTaskMissing->getStatus()); + $this->assertEquals('Post Output', $dummyTaskFinished->getPostOutput()); + $this->assertNull($dummyTaskMissing->getPostOutput()); + } +} -- 2.40.1 From 8bcebfc1c3736a78fa790c74b5f0a0beeb7e65e8 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 19 May 2020 12:01:48 +0200 Subject: [PATCH 18/33] Made the Docker image Alpine-based. Should work better when running Async in a Cron environment. Also removed compatibility with PHP 7.2. --- Dockerfile | 18 ++++++++++++------ composer.json | 2 +- test/system/ParallelSuperVisorTest.php | 1 - 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c6702d..8aabc53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,17 @@ -FROM php:7.3-cli-buster +FROM php:7.3-alpine -RUN apt-get update &&\ - apt-get install --no-install-recommends --assume-yes --quiet procps ca-certificates curl git unzip &&\ - rm -rf /var/lib/apt/lists/* +# FOR ALPINE +# Install git and bash and procps +RUN apk add git bash procps +RUN apk add --no-cache --update --virtual .phpize-deps $PHPIZE_DEPS -# Install Redis -RUN pecl install redis-5.1.1 && docker-php-ext-enable redis +# FOR DEBIAN/UBUNTU +#RUN apt-get update &&\ +# apt-get install --no-install-recommends --assume-yes --quiet procps ca-certificates curl git unzip &&\ +# rm -rf /var/lib/apt/lists/* + +# Install Redis and XDebug +RUN pecl install redis && docker-php-ext-enable redis RUN pecl install xdebug && docker-php-ext-enable xdebug # Install Composer diff --git a/composer.json b/composer.json index e608d04..ad3c475 100644 --- a/composer.json +++ b/composer.json @@ -13,7 +13,7 @@ } ], "require": { - "php": ">=7.2.0", + "php": ">=7.3.0", "fuzeworks/core": "~1.2.0", "ext-json": "*", "ext-redis": "*" diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index 556b0bd..6897c42 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -64,7 +64,6 @@ class ParallelSuperVisorTest extends TestCase public function setUp(): void { // Load the TaskStorage so temporary tasks can be stored - // Tasks shall NOT be reset between individual tests automatically $tasks = new Tasks(); $this->taskStorage = $tasks->getTaskStorage(); $this->taskStorage->reset(); -- 2.40.1 From 5f369c784b5ef414f3d2ea1d573da833370e9506 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 19 May 2020 22:07:44 +0200 Subject: [PATCH 19/33] Added TaskModifyEvent. Event gets fired when an Event is modified by sending it to TaskStorage::modifyEvent. This allows components to observe changes and report these to the user. Might also be useful to cancel unwanted changes. --- .../Async/Events/TaskModifyEvent.php | 64 ++++++++++++++++++ src/FuzeWorks/Async/TaskStorage.php | 5 ++ .../Async/TaskStorage/DummyTaskStorage.php | 15 +++++ .../Async/TaskStorage/RedisTaskStorage.php | 14 +++- test/base/TaskStorageTest.php | 67 +++++++++++++++++++ 5 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/FuzeWorks/Async/Events/TaskModifyEvent.php diff --git a/src/FuzeWorks/Async/Events/TaskModifyEvent.php b/src/FuzeWorks/Async/Events/TaskModifyEvent.php new file mode 100644 index 0000000..49e8983 --- /dev/null +++ b/src/FuzeWorks/Async/Events/TaskModifyEvent.php @@ -0,0 +1,64 @@ +task = $task; + } + + public function getTask(): Task + { + return $this->task; + } + + public function updateTask(Task $task) + { + $this->task = $task; + } + +} \ No newline at end of file diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index 9dfdee0..1d18ec4 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -79,6 +79,11 @@ interface TaskStorage /** * Modifies a task * + * Should first check if the Task exists in TaskStorage. If not, it should throw a TasksException. + * + * Modifies a Task in TaskStorage. Should fire a TaskModifyEvent and cancel the edit if cancelled by the event. + * If cancelled, should return zero. Preferably be logged as well. + * * @param Task $task * @return bool * @throws TasksException diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index 9056328..bd4cf2b 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -37,9 +37,12 @@ namespace FuzeWorks\Async\TaskStorage; +use FuzeWorks\Async\Events\TaskModifyEvent; use FuzeWorks\Async\Task; use FuzeWorks\Async\TasksException; use FuzeWorks\Async\TaskStorage; +use FuzeWorks\Events; +use FuzeWorks\Logger; /** * Class DummyTaskStorage @@ -126,6 +129,18 @@ class DummyTaskStorage implements TaskStorage */ public function modifyTask(Task $task): bool { + // Fire the TaskModifyEvent + /** @var TaskModifyEvent $event */ + $event = Events::fireEvent(new TaskModifyEvent(), $task); + if ($event->isCancelled()) + { + Logger::log("Did not modify task. Cancelled by taskModifyEvent."); + return false; + } + + // And finally replace Task with the event based one. + $task = $event->getTask(); + $taskId = $task->getId(); for ($i=0;$itasks);$i++) { diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 76aeb55..17402a2 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -35,9 +35,12 @@ */ namespace FuzeWorks\Async\TaskStorage; +use FuzeWorks\Async\Events\TaskModifyEvent; use FuzeWorks\Async\Task; use FuzeWorks\Async\TasksException; use FuzeWorks\Async\TaskStorage; +use FuzeWorks\Events; +use FuzeWorks\Logger; use Redis; use RedisException; @@ -159,8 +162,17 @@ class RedisTaskStorage implements TaskStorage if (!$isMember) throw new TasksException("Could not modify task. Task '$taskId' already exists."); + // Fire the TaskModifyEvent + /** @var TaskModifyEvent $event */ + $event = Events::fireEvent(new TaskModifyEvent(), $task); + if ($event->isCancelled()) + { + Logger::log("Did not modify task. Cancelled by taskModifyEvent."); + return false; + } + // And write the data - $taskData = serialize($task); + $taskData = serialize($event->getTask()); return $this->conn->set($this->key_prefix . $taskId, $taskData); } diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index 4b021be..47f51d7 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -34,11 +34,14 @@ * @version Version 1.0.0 */ +use FuzeWorks\Async\Events\TaskModifyEvent; use FuzeWorks\Async\Task; use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TasksException; use FuzeWorks\Async\TaskStorage; use FuzeWorks\Async\TaskStorage\DummyTaskStorage; +use FuzeWorks\Events; +use FuzeWorks\Priority; use PHPUnit\Framework\TestCase; /** @@ -54,9 +57,13 @@ class TaskStorageTest extends TestCase public function setUp(): void { + // Add TaskStorage $tasks = new Tasks(); $this->taskStorage = $tasks->getTaskStorage(); $this->taskStorage->reset(); + + // Reset events + Events::$listeners = []; } public function testDummyTaskStorageClass() @@ -199,6 +206,66 @@ class TaskStorageTest extends TestCase $this->taskStorage->modifyTask($dummyTask); } + /** + * @depends testModifyTask + */ + public function testModifyTaskEvent() + { + // Prepare a dummy task + $dummyTask = new Task('testModifyTaskEvent', 'none'); + $dummyTask->setStatus(Task::PENDING); + + // Then add the Task + $this->taskStorage->addTask($dummyTask); + + // Now prepare a listener to be fired + Events::addListener(function (TaskModifyEvent $event){ + $task = $event->getTask(); + $this->assertEquals(Task::PENDING, $task->getStatus()); + $task->setStatus(Task::RUNNING); + $event->updateTask($task); + return $event; + }, 'TaskModifyEvent', Priority::NORMAL); + + // Now update the task + $this->assertEquals(Task::PENDING, $dummyTask->getStatus()); + $this->taskStorage->modifyTask($dummyTask); + + // And check whether dummyTask got modified + $this->assertEquals(Task::RUNNING, $dummyTask->getStatus()); + + // And check whether the TaskStorage has this updated version as well + $modifiedTask = $this->taskStorage->getTaskById($dummyTask->getId()); + $this->assertEquals(Task::RUNNING, $modifiedTask->getStatus()); + } + + /** + * @depends testModifyTaskEvent + */ + public function testModifyTaskCancel() + { + // Prepare a dummy task + $dummyTask = new Task('testModifyTaskCancel', 'none'); + $dummyTask->setStatus(Task::PENDING); + + // Then add the Task + $this->taskStorage->addTask($dummyTask); + + // Now prepare a listener to be fired + Events::addListener(function (TaskModifyEvent $event){ + $event->setCancelled(true); + return $event; + }, 'TaskModifyEvent', Priority::NORMAL); + + // Modify the task + $dummyTask->setStatus(Task::SUCCESS); + $this->assertFalse($this->taskStorage->modifyTask($dummyTask)); + + // And check that the task actually hasn't updated + $modifiedTask = $this->taskStorage->getTaskById($dummyTask->getId()); + $this->assertEquals(Task::PENDING, $modifiedTask->getStatus()); + } + /** * @depends testGetTaskById */ -- 2.40.1 From 19f11d9c7e36a4dfa9f1e6324eda4a5151e05447 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 19 May 2020 22:13:47 +0200 Subject: [PATCH 20/33] Fixed DummyTaskStorage persisting outside of the storage. Awkward how that could go wrong... --- src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index bd4cf2b..60bd7c3 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -93,7 +93,7 @@ class DummyTaskStorage implements TaskStorage throw new TasksException("Could not add Task to TaskStorage. Task '$taskId' already exists."); } - $this->tasks[] = $task; + $this->tasks[] = clone $task; return true; } @@ -146,7 +146,7 @@ class DummyTaskStorage implements TaskStorage { if ($this->tasks[$i]->getId() === $taskId) { - $this->tasks[$i] = $task; + $this->tasks[$i] = clone $task; return true; } } -- 2.40.1 From 1dacc29d86f7020731320857a2bfa4c83e9f1d6c Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 19 May 2020 22:19:55 +0200 Subject: [PATCH 21/33] Maybe Events are the problem? --- test/system/ParallelSuperVisorTest.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index 6897c42..70946f9 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -41,6 +41,7 @@ use FuzeWorks\Async\Supervisors\ParallelSuperVisor; use FuzeWorks\Async\Task; use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TaskStorage; +use FuzeWorks\Events; use PHPUnit\Framework\TestCase; class ParallelSuperVisorTest extends TestCase @@ -68,6 +69,9 @@ class ParallelSuperVisorTest extends TestCase $this->taskStorage = $tasks->getTaskStorage(); $this->taskStorage->reset(); + // Clear events + Events::$listeners = []; + // And load the ShellExecutor using the execution settings $this->executor = new ShellExecutor([ 'bootstrapFile' => dirname(__DIR__) . DIRECTORY_SEPARATOR, -- 2.40.1 From 18d702ec26c3d73756d9c6e1152e2a744ecc1184 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 26 May 2020 13:35:07 +0200 Subject: [PATCH 22/33] Try again in the new environment. --- test/config.tasks.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/config.tasks.php b/test/config.tasks.php index 19744d0..81288f7 100644 --- a/test/config.tasks.php +++ b/test/config.tasks.php @@ -36,6 +36,9 @@ use FuzeWorks\Core; +/** + * Contains all the configuration, including docker environment variables. + */ return array( // Add a file lock 'lock' => true, -- 2.40.1 From db08da82138607acb71fb30adec19f55384f9974 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 26 May 2020 13:53:10 +0200 Subject: [PATCH 23/33] Now try while flushing a selected database. --- .../Async/TaskStorage/RedisTaskStorage.php | 15 ++++++--------- test/config.tasks.php | 1 + 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 17402a2..8ebb22d 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -80,6 +80,9 @@ class RedisTaskStorage implements TaskStorage // Otherwise attempt authentication, if needed if (isset($parameters['password']) && !$this->conn->auth($parameters['password'])) throw new TasksException("Could not construct RedisTaskStorage. Authentication failure."); + + // And select the DB index + $this->conn->select($parameters['db_index']); } catch (RedisException $e) { throw new TasksException("Could not construct RedisTaskStorage. RedisException thrown: '" . $e->getMessage() . "'"); } @@ -282,7 +285,7 @@ class RedisTaskStorage implements TaskStorage { // First get the task ID $taskId = $task->getId(); - + // Check if the key already exists if (!$this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt)) return null; @@ -296,16 +299,10 @@ class RedisTaskStorage implements TaskStorage /** * @inheritDoc - * @throws TasksException */ public function reset(): bool { - // First get a list of all tasks - foreach ($this->readTasks() as $task) - $this->deleteTask($task); - - $this->refreshTasks(); - - return true; + // Clear the current db + return $this->conn->flushDB(); } } \ No newline at end of file diff --git a/test/config.tasks.php b/test/config.tasks.php index 81288f7..2ecf95a 100644 --- a/test/config.tasks.php +++ b/test/config.tasks.php @@ -68,6 +68,7 @@ return array( 'password' => Core::getEnv('TASKSTORAGE_REDIS_PASSWORD', null), 'port' => Core::getEnv('TASKSTORAGE_REDIS_PORT', 6379), 'timeout' => Core::getEnv('TASKSTORAGE_REDIS_TIMEOUT', 0), + 'db_index' => Core::getEnv('TASKSTORAGE_REDIS_DBINDEX', 0), ] ], 'Executor' => [ -- 2.40.1 From 5f5718cb72199b52aecc650b508eaa88a2bd6b96 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 26 May 2020 16:16:50 +0200 Subject: [PATCH 24/33] Made many changes. Fixed race-conditions in test code. --- .../Async/Executors/ShellExecutor.php | 20 +++-- test/base/ShellExecutorTest.php | 14 ++++ test/base/ShellWorkerTest.php | 78 +++++++++++++++++++ test/config.tasks.php | 4 +- test/mock/Handlers/ArgumentedHandler.php | 4 + test/system/ParallelSuperVisorTest.php | 38 ++++++++- 6 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 test/base/ShellWorkerTest.php diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index e1e24d6..7e8685b 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -36,7 +36,6 @@ namespace FuzeWorks\Async\Executors; use FuzeWorks\Async\Executor; -use FuzeWorks\Async\Process; use FuzeWorks\Async\Task; use FuzeWorks\Async\TasksException; @@ -52,7 +51,6 @@ class ShellExecutor implements Executor /** * ShellExecutor constructor. * - * @param string $bootstrapFile * @param array $parameters * @throws TasksException */ @@ -70,15 +68,6 @@ class ShellExecutor implements Executor throw new TasksException("Could not construct ShellExecutor. No bootstrap file found."); } - protected function shellExec($format, array $parameters = []) - { - $parameters = array_map("escapeshellarg", $parameters); - array_unshift($parameters, $format); - $command = call_user_func_array("sprintf", $parameters); - exec($command, $output); - return $output; - } - public function startTask(Task $task, bool $post = false): Task { // First prepare the command used to spawn workers @@ -152,4 +141,13 @@ class ShellExecutor implements Executor // Finally, return the Task information return ['pid' => (int) $parts[0], 'cpu' => (float) $parts[1], 'mem' => (float) $parts[2], 'state' => $parts[3], 'start' => $parts[4]]; } + + protected function shellExec($format, array $parameters = []) + { + $parameters = array_map("escapeshellarg", $parameters); + array_unshift($parameters, $format); + $command = call_user_func_array("sprintf", $parameters); + exec($command, $output); + return $output; + } } \ No newline at end of file diff --git a/test/base/ShellExecutorTest.php b/test/base/ShellExecutorTest.php index 831b905..94587e5 100644 --- a/test/base/ShellExecutorTest.php +++ b/test/base/ShellExecutorTest.php @@ -142,6 +142,9 @@ class ShellExecutorTest extends TestCase // Then we fire the task $task = $this->executor->startTask($dummyTask); + // Pause 1/10th of a second + usleep(100000); + // Assert that the output is the same $this->assertSame($dummyTask, $task); @@ -166,6 +169,9 @@ class ShellExecutorTest extends TestCase // Then we start the task $dummyTask = $this->executor->startTask($dummyTask); + // Pause 1/10th of a second + usleep(100000); + // And we fetch some task statistics $stats = $this->executor->getTaskStats($dummyTask); @@ -206,11 +212,19 @@ class ShellExecutorTest extends TestCase // First we start the task and confirm its running $dummyTask = $this->executor->startTask($dummyTask); + + // Pause 1/10th of a second + usleep(100000); + + // Check if the task is running $this->assertTrue($this->executor->getTaskRunning($dummyTask)); // But then we try and stop it $output = $this->executor->stopTask($dummyTask); + // Pause 1/10th of a second + usleep(100000); + // We check that the output actually is the task $this->assertSame($dummyTask, $output); diff --git a/test/base/ShellWorkerTest.php b/test/base/ShellWorkerTest.php new file mode 100644 index 0000000..2d04522 --- /dev/null +++ b/test/base/ShellWorkerTest.php @@ -0,0 +1,78 @@ +taskStorage = $tasks->getTaskStorage(); + $this->taskStorage->reset(); + + // Clear events + Events::$listeners = []; + $this->shellWorker = $tasks->getWorker(); + } + + public function testClass() + { + $this->assertInstanceOf(ShellWorker::class, $this->shellWorker); + } + + /* ---------------------------------- Writing and reading tasks ----------------------- */ + + // @todo Add lots of tests and amend ShellWorker to return results + +} diff --git a/test/config.tasks.php b/test/config.tasks.php index 2ecf95a..510314e 100644 --- a/test/config.tasks.php +++ b/test/config.tasks.php @@ -76,7 +76,9 @@ return array( // For ShellExecutor, first parameter is the file location of the worker script 'parameters' => [ - 'workerFile' => dirname(__FILE__) . DS . 'bin' . DS . 'worker' + 'workerFile' => Core::getEnv('EXECUTOR_SHELL_WORKER', + dirname(__FILE__) . DS . 'bin' . DS . 'worker'), + 'bootstrapFile' => Core::getEnv('EXECUTOR_SHELL_BOOTSTRAP', 'unknown') ] ] ); \ No newline at end of file diff --git a/test/mock/Handlers/ArgumentedHandler.php b/test/mock/Handlers/ArgumentedHandler.php index 60c80c4..61287cb 100644 --- a/test/mock/Handlers/ArgumentedHandler.php +++ b/test/mock/Handlers/ArgumentedHandler.php @@ -71,6 +71,10 @@ class ArgumentedHandler implements Handler */ public function postHandler(Task $task) { + $arguments = $task->getArguments(); + $this->sleeptime = $arguments[0]; + $this->output = $arguments[1]; + sleep($this->sleeptime); return true; } diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index 70946f9..2fe6646 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -74,7 +74,7 @@ class ParallelSuperVisorTest extends TestCase // And load the ShellExecutor using the execution settings $this->executor = new ShellExecutor([ - 'bootstrapFile' => dirname(__DIR__) . DIRECTORY_SEPARATOR, + 'bootstrapFile' => dirname(__DIR__, 1) . DIRECTORY_SEPARATOR . 'bootstrap.php', 'workerFile' => dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'worker' ]); @@ -111,6 +111,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -140,6 +143,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -170,6 +176,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -200,6 +209,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -230,6 +242,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -260,6 +275,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -290,6 +308,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -331,6 +352,9 @@ class ParallelSuperVisorTest extends TestCase // Then cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // Reload all tasks from TaskStorage $dummyTaskFailedYes = $this->taskStorage->getTaskById($dummyTaskFailedYes->getId()); $dummyTaskPFailedYes = $this->taskStorage->getTaskById($dummyTaskPFailedYes->getId()); @@ -375,6 +399,9 @@ class ParallelSuperVisorTest extends TestCase // Cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // And check if the Task has been cancelled $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); $dummyTask2 = $this->taskStorage->getTaskById($dummyTask2->getId()); @@ -400,6 +427,9 @@ class ParallelSuperVisorTest extends TestCase // Cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // And check if the Task has been moved to Post $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); $this->assertEquals(Task::POST, $dummyTask->getStatus()); @@ -428,6 +458,9 @@ class ParallelSuperVisorTest extends TestCase // Cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // And check if the Tasks have been completed or moved to post $dummyTaskPostNo = $this->taskStorage->getTaskById($dummyTaskPostNo->getId()); $dummyTaskPostYes = $this->taskStorage->getTaskById($dummyTaskPostYes->getId()); @@ -456,6 +489,9 @@ class ParallelSuperVisorTest extends TestCase // Cycle the SuperVisor $this->superVisor->cycle(); + // Pause 1/10th of a second + usleep(100000); + // And check if the Tasks have been completed or failed $dummyTaskFinished = $this->taskStorage->getTaskById($dummyTaskFinished->getId()); $dummyTaskMissing = $this->taskStorage->getTaskById($dummyTaskMissing->getId()); -- 2.40.1 From d9da4cfc9551ce9d115ee438bcde425f9ad10c29 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Tue, 26 May 2020 17:56:15 +0200 Subject: [PATCH 25/33] Try with only Redis. --- .drone.yml | 7 --- .../Async/TaskStorage/ArrayTaskStorage.php | 50 +++++++++++++-- src/FuzeWorks/Async/Tasks.php | 47 ++++++++++++-- test/config.tasks.php | 61 +++++++++++-------- .../Handlers/TestStartAndReadTasksHandler.php | 2 +- storage.php => test/temp/storage.php | 0 6 files changed, 120 insertions(+), 47 deletions(-) rename storage.php => test/temp/storage.php (100%) diff --git a/.drone.yml b/.drone.yml index 088a992..1e980ae 100644 --- a/.drone.yml +++ b/.drone.yml @@ -12,13 +12,6 @@ steps: commands: - composer install - - name: basetest - image: phpunit:7.3 - commands: - - vendor/bin/phpunit -c test/phpunit.xml --coverage-php test/temp/covbase.cov - environment: - TASKSTORAGE_TYPE: DummyTaskStorage - - name: redistest image: phpunit:7.3 commands: diff --git a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php index eb6713f..a2a6659 100644 --- a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php @@ -168,10 +168,20 @@ class ArrayTaskStorage implements TaskStorage unset($this->tasks[$i]); $this->commit(); - // Afterwards remove the task output, if it exists - $file = dirname($this->fileName) . DS . 'task_' . md5($taskId) . '_output.json'; - if (file_exists($file)) - unlink($file); + // Remove all task output and post output + $settings = $task->getRetrySettings(); + $maxRetries = $settings['maxRetries']; + for ($j=0;$j<=$maxRetries;$j++) + { + // First remove all possible task output + $outFile = dirname($this->fileName) . DS . 'task_' . md5($taskId) . '_' . $j . '_output.json'; + if (file_exists($outFile)) + unlink($outFile); + + $postFile = dirname($this->fileName) . DS . 'task_' . md5($taskId) . '_' . $j . '_post_output.json'; + if (file_exists($postFile)) + unlink($postFile); + } return true; } @@ -264,11 +274,39 @@ class ArrayTaskStorage implements TaskStorage /** * @inheritDoc + * @throws TasksException */ public function reset(): bool { - // @todo Implement - return false; + // Delete everything + $this->refreshTasks(); + for ($i=0;$itasks);$i++) + { + // Get the task + $task = $this->tasks[$i]; + + // Remove all task output and post output + $settings = $task->getRetrySettings(); + $maxRetries = $settings['maxRetries']; + for ($j=0;$j<=$maxRetries;$j++) + { + // First remove all possible task output + $outFile = dirname($this->fileName) . DS . 'task_' . md5($taskId) . '_' . $j . '_output.json'; + if (file_exists($outFile)) + unlink($outFile); + + $postFile = dirname($this->fileName) . DS . 'task_' . md5($taskId) . '_' . $j . '_post_output.json'; + if (file_exists($postFile)) + unlink($postFile); + } + + // Remove the task from the main storage + unset($this->tasks[$i]); + } + + // And finally commit + $this->commit(); + return true; } private function commit() diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index bd76183..355f009 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -80,13 +80,26 @@ class Tasks implements iLibrary */ public function getSuperVisor(string $bootstrapFile): SuperVisor { + // First get the configuration for SuperVisors $cfg = $this->cfg->get('SuperVisor'); - $class = 'FuzeWorks\Async\Supervisors\\' . $cfg['type']; - $parameters = isset($cfg['parameters']) && is_array($cfg['parameters']) ? $cfg['parameters'] : []; + + // Select the SuperVisor type + $type = $cfg['type']; + + // Load the class of the currently selected type + $class = 'FuzeWorks\Async\Supervisors\\' . $type; + + // Fetch the parameters for the selected SuperVisor + $parameters = isset($cfg[$type]['parameters']) && is_array($cfg[$type]['parameters']) ? $cfg[$type]['parameters'] : []; + + // Then add the TaskStorage and Executor to the parameters array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor($bootstrapFile)); + + // If the type does not exist, throw an exception if (!class_exists($class, true)) throw new TasksException("Could not get SuperVisor. Type of '$class' not found."); + // And load the SuperVisor and test if everything is in order $object = new $class(...$parameters); if (!$object instanceof SuperVisor) throw new TasksException("Could not get SuperVisor. Type of '$class' is not instanceof TaskStorage."); @@ -111,12 +124,23 @@ class Tasks implements iLibrary */ public function getTaskStorage(): TaskStorage { + // First get the configuration for TaskStorage $cfg = $this->cfg->get('TaskStorage'); - $class = 'FuzeWorks\Async\TaskStorage\\' . $cfg['type']; - $parameters = isset($cfg['parameters']) && is_array($cfg['parameters']) ? $cfg['parameters'] : []; + + // Select the TaskStorage type + $type = $cfg['type']; + + // Load the class of the currently selected type + $class = 'FuzeWorks\Async\TaskStorage\\' . $type; + + // Fetch the parameters for the selected type + $parameters = isset($cfg[$type]['parameters']) && is_array($cfg[$type]['parameters']) ? $cfg[$type]['parameters'] : []; + + // If the type does not exist, throw an exception if (!class_exists($class, true)) throw new TasksException("Could not get TaskStorage. Type of '$class' not found."); + // And load the TaskStorage and test if everything is in order $object = new $class($parameters); if (!$object instanceof TaskStorage) throw new TasksException("Could not get TaskStorage. Type '$class' is not instanceof TaskStorage."); @@ -133,12 +157,23 @@ class Tasks implements iLibrary */ protected function getExecutor(string $bootstrapFile): Executor { + // First get the configuration for Executor $cfg = $this->cfg->get('Executor'); - $class = 'FuzeWorks\Async\Executors\\' . $cfg['type']; - $parameters = isset($cfg['parameters']) && is_array($cfg['parameters']) ? $cfg['parameters'] : []; + + // Select the Executor type + $type = $cfg['type']; + + // Load the class of the currently selected type + $class = 'FuzeWorks\Async\Executors\\' . $type; + + // Fetch the parameters for the selected type + $parameters = isset($cfg[$type]['parameters']) && is_array($cfg[$type]['parameters']) ? $cfg[$type]['parameters'] : []; + + // If the type does not exist, throw an exception if (!class_exists($class, true)) throw new TasksException("Could not get Executor. Type of '$class' not found."); + // And load the Executor and test if everything is in order $object = new $class($bootstrapFile, $parameters); if (!$object instanceof Executor) throw new TasksException("Could not get Executor. Type '$class' is not instanceof Executor."); diff --git a/test/config.tasks.php b/test/config.tasks.php index 510314e..393ebe6 100644 --- a/test/config.tasks.php +++ b/test/config.tasks.php @@ -43,42 +43,49 @@ return array( // Add a file lock 'lock' => true, - // Which SuperVisor should be used + // Which SuperVisor should be used, and with what settings 'SuperVisor' => [ 'type' => Core::getEnv('SUPERVISOR_TYPE', 'ParallelSuperVisor'), - 'parameters' => [] + 'ParallelSuperVisor' => ['parameters' => []] ], + + // Which TaskStorage should be used, and with what settings 'TaskStorage' => [ 'type' => Core::getEnv('TASKSTORAGE_TYPE', 'DummyTaskStorage'), - - // For ArrayTaskStorage, first parameter is the file location of the array storage - #'parameters' => [ - # 'filename' => dirname(__FILE__) . DS . 'storage.php' - #], - - // For RedisTaskStorage, parameters are connection properties - 'parameters' => [ - // Type can be 'tcp' or 'unix' - 'socket_type' => Core::getEnv('TASKSTORAGE_REDIS_SOCKET_TYPE', 'tcp'), - // If socket_type == 'unix', set the socket here - 'socket' => Core::getEnv('TASKSTORAGE_REDIS_SOCKET', null), - // If socket_type == 'tcp', set the host here - 'host' => Core::getEnv('TASKSTORAGE_REDIS_HOST', '127.0.0.1'), - // And some standard settings - 'password' => Core::getEnv('TASKSTORAGE_REDIS_PASSWORD', null), - 'port' => Core::getEnv('TASKSTORAGE_REDIS_PORT', 6379), - 'timeout' => Core::getEnv('TASKSTORAGE_REDIS_TIMEOUT', 0), - 'db_index' => Core::getEnv('TASKSTORAGE_REDIS_DBINDEX', 0), - ] + 'DummyTaskStorage' => ['parameters' => []], + 'ArrayTaskStorage' => [ + 'parameters' => [ + 'filename' => Core::getEnv('TASKSTORAGE_ARRAY_FILE', + dirname(__FILE__) . DS . 'temp'. DS . 'storage.php') + ] + ], + 'RedisTaskStorage' => [ + 'parameters' => [ + // Type can be 'tcp' or 'unix' + 'socket_type' => Core::getEnv('TASKSTORAGE_REDIS_SOCKET_TYPE', 'tcp'), + // If socket_type == 'unix', set the socket here + 'socket' => Core::getEnv('TASKSTORAGE_REDIS_SOCKET', null), + // If socket_type == 'tcp', set the host here + 'host' => Core::getEnv('TASKSTORAGE_REDIS_HOST', '127.0.0.1'), + // And some standard settings + 'password' => Core::getEnv('TASKSTORAGE_REDIS_PASSWORD', null), + 'port' => Core::getEnv('TASKSTORAGE_REDIS_PORT', 6379), + 'timeout' => Core::getEnv('TASKSTORAGE_REDIS_TIMEOUT', 0), + 'db_index' => Core::getEnv('TASKSTORAGE_REDIS_DBINDEX', 0), + ] + ], ], + + // Which Executor should be used, and with what settings 'Executor' => [ 'type' => Core::getEnv('EXECUTOR_TYPE', 'ShellExecutor'), - // For ShellExecutor, first parameter is the file location of the worker script - 'parameters' => [ - 'workerFile' => Core::getEnv('EXECUTOR_SHELL_WORKER', - dirname(__FILE__) . DS . 'bin' . DS . 'worker'), - 'bootstrapFile' => Core::getEnv('EXECUTOR_SHELL_BOOTSTRAP', 'unknown') + 'ShellExecutor' => [ + 'parameters' => [ + 'workerFile' => Core::getEnv('EXECUTOR_SHELL_WORKER', + dirname(__FILE__) . DS . 'bin' . DS . 'worker'), + 'bootstrapFile' => Core::getEnv('EXECUTOR_SHELL_BOOTSTRAP', 'unknown') + ] ] ] ); \ No newline at end of file diff --git a/test/mock/Handlers/TestStartAndReadTasksHandler.php b/test/mock/Handlers/TestStartAndReadTasksHandler.php index f852cfe..c162eb0 100644 --- a/test/mock/Handlers/TestStartAndReadTasksHandler.php +++ b/test/mock/Handlers/TestStartAndReadTasksHandler.php @@ -46,7 +46,7 @@ class TestStartAndReadTasksHandler implements Handler */ public function primaryHandler(Task $task): bool { - sleep(2); + sleep(10); return true; } diff --git a/storage.php b/test/temp/storage.php similarity index 100% rename from storage.php rename to test/temp/storage.php -- 2.40.1 From 4555957292210d481a68f10d0ce264c0e3a445d0 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Wed, 3 Jun 2020 11:35:16 +0200 Subject: [PATCH 26/33] Made Handlers an object instead of a string reference. Handlers should now be added as objects, adding some flexibility to the developer. Developers are still cautioned to take great care that Handlers work approriately. Handlers can potentially crash the SuperVisor if not taken good care of. --- src/FuzeWorks/Async/ShellWorker.php | 20 +----- src/FuzeWorks/Async/Task.php | 29 ++++---- .../Async/TaskStorage/ArrayTaskStorage.php | 1 + test/base/ShellExecutorTest.php | 11 +-- test/base/TaskStorageTest.php | 45 ++++++------ test/base/TaskTest.php | 29 ++++---- test/mock/Handlers/ArgumentedHandler.php | 19 +++-- test/mock/Handlers/EmptyHandler.php | 72 +++++++++++++++++++ test/mock/Handlers/TestStopTaskHandler.php | 2 - test/system/ParallelSuperVisorTest.php | 39 +++++----- 10 files changed, 166 insertions(+), 101 deletions(-) create mode 100644 test/mock/Handlers/EmptyHandler.php diff --git a/src/FuzeWorks/Async/ShellWorker.php b/src/FuzeWorks/Async/ShellWorker.php index a614541..7a5ad8b 100644 --- a/src/FuzeWorks/Async/ShellWorker.php +++ b/src/FuzeWorks/Async/ShellWorker.php @@ -91,30 +91,16 @@ class ShellWorker $this->post = $post; // Fetch the callable - $class = $this->task->getHandlerClass(); - if (!class_exists($class, true)) - { - $errors = 'Could not run task. HandlerClass \'' . $class . '\' not found.'; - if (!$post) - $this->taskStorage->writeTaskOutput($this->task, '', $errors, Task::PFAILED, $this->task->getRetries()); - else - $this->taskStorage->writePostOutput($this->task, '', $errors, Task::PFAILED, $this->task->getRetries()); - - throw new TasksException("Could not run task. '$class' not found."); - } - - // Create the handler - /** @var Handler $object */ - $object = new $class(); + $object = $this->task->getHandler(); if (!$object instanceof Handler) { - $errors = "Could not run task. '$class' is not instance of Handler."; + $errors = "Could not run task. '".get_class($object)."' is not instance of Handler."; if (!$post) $this->taskStorage->writeTaskOutput($this->task, '', $errors, Task::PFAILED, $this->task->getRetries()); else $this->taskStorage->writePostOutput($this->task, '', $errors, Task::PFAILED, $this->task->getRetries()); - throw new TasksException("Could not run task. '$class' is not instance of Handler."); + throw new TasksException("Could not run task. '".get_class($object)."' is not instance of Handler."); } // Run postHandler if post mode is requested diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index 4672f46..b8a3d15 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -116,9 +116,9 @@ class Task protected $taskId; /** - * @var string + * @var Handler */ - protected $handlerClass; + protected $handler; /** * @var bool @@ -184,16 +184,20 @@ class Task * * Creates a Task object, which can be added to the TaskQueue. * - * @param string $identifier The unique identifier of this task. Make sure it is always unique! - * @param string $handlerClass The class that shall handle this task - * @param bool $usePostHandler Whether the postHandler on handlerClass should also be used + * @param string $identifier The unique identifier of this task. Make sure it is always unique! + * @param Handler $handler The Handler object which will run the Task in the Worker + * @param bool $usePostHandler Whether the postHandler on Handler should also be used * @param mixed $parameters,... The arguments provided to the method that shall handle this class * @throws TasksException */ - public function __construct(string $identifier, string $handlerClass, bool $usePostHandler = false) + public function __construct(string $identifier, Handler $handler, bool $usePostHandler = false) { + // Check if the provided Handler is serializable + if (!$this->isSerializable($handler)) + throw new TasksException("Could not create Task. Provided Handler is not serializable."); + $this->taskId = $identifier; - $this->handlerClass = $handlerClass; + $this->handler = $handler; $this->usePostHandler = $usePostHandler; if (func_num_args() > 3) $args = array_slice(func_get_args(), 3); @@ -218,17 +222,17 @@ class Task } /** - * Gets the name of the class that shall process this task + * Gets the Handler that shall process this task * - * @return string + * @return Handler */ - public function getHandlerClass(): string + public function getHandler(): Handler { - return $this->handlerClass; + return $this->handler; } /** - * Whether the postHandler on the handlerClass should be invoked after processing the initial task. + * Whether the postHandler on the Handler should be invoked after processing the initial task. * * @return bool */ @@ -470,6 +474,7 @@ class Task * * @param $value * @return bool + * @todo Improve so it is properly tested */ private function isSerializable($value) { diff --git a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php index a2a6659..4e8312c 100644 --- a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php @@ -284,6 +284,7 @@ class ArrayTaskStorage implements TaskStorage { // Get the task $task = $this->tasks[$i]; + $taskId = $task->getId(); // Remove all task output and post output $settings = $task->getRetrySettings(); diff --git a/test/base/ShellExecutorTest.php b/test/base/ShellExecutorTest.php index 94587e5..d88e696 100644 --- a/test/base/ShellExecutorTest.php +++ b/test/base/ShellExecutorTest.php @@ -39,6 +39,9 @@ use FuzeWorks\Async\Task; use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TasksException; use FuzeWorks\Async\TaskStorage; +use Mock\Handlers\EmptyHandler; +use Mock\Handlers\TestStartAndReadTasksHandler; +use Mock\Handlers\TestStopTaskHandler; use PHPUnit\Framework\TestCase; /** @@ -131,7 +134,7 @@ class ShellExecutorTest extends TestCase public function testStartAndReadTasks() { // First we create a dummy task - $dummyTask = new Task('testStartAndReadTasks', 'Mock\Handlers\TestStartAndReadTasksHandler'); + $dummyTask = new Task('testStartAndReadTasks', new TestStartAndReadTasksHandler()); // Then we write this task to the TaskStorage $this->taskStorage->addTask($dummyTask); @@ -161,7 +164,7 @@ class ShellExecutorTest extends TestCase public function testGetStats() { // First we create a dummy task, using the previous handler since nothing changes - $dummyTask = new Task('testGetStats', 'Mock\Handlers\TestStartAndReadTasksHandler'); + $dummyTask = new Task('testGetStats', new TestStartAndReadTasksHandler()); // Then we write this task to the TaskStorage $this->taskStorage->addTask($dummyTask); @@ -190,7 +193,7 @@ class ShellExecutorTest extends TestCase public function testGetStatsNotExist() { // First we create a dummy task, using the previous handler since nothing changes - $dummyTask = new Task('testGetStatsNotExist', 'none'); + $dummyTask = new Task('testGetStatsNotExist', new EmptyHandler()); // And add a fake PID, since otherwise it will immediately fail $dummyTask->addAttribute('pid', 1005); @@ -205,7 +208,7 @@ class ShellExecutorTest extends TestCase public function testStopTask() { // First we create a dummy task - $dummyTask = new Task('testStopTask', 'Mock\Handlers\TestStopTaskHandler'); + $dummyTask = new Task('testStopTask', new TestStopTaskHandler()); // Then we write this task to the TaskStorage $this->taskStorage->addTask($dummyTask); diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index 47f51d7..27bf978 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -42,6 +42,7 @@ use FuzeWorks\Async\TaskStorage; use FuzeWorks\Async\TaskStorage\DummyTaskStorage; use FuzeWorks\Events; use FuzeWorks\Priority; +use Mock\Handlers\EmptyHandler; use PHPUnit\Framework\TestCase; /** @@ -79,7 +80,7 @@ class TaskStorageTest extends TestCase public function testAddAndReadTasks() { // Prepare a dummy task - $dummyTask = new Task('testAddTask', 'none'); + $dummyTask = new Task('testAddTask', new EmptyHandler()); // Nothing is written yet so it should be empty $this->assertEmpty($this->taskStorage->readTasks()); @@ -95,7 +96,7 @@ class TaskStorageTest extends TestCase // Test if the properties match $this->assertEquals('testAddTask', $task->getId()); - $this->assertEquals('none', $task->getHandlerClass()); + $this->assertInstanceOf(EmptyHandler::class, $task->getHandler()); } /** @@ -104,7 +105,7 @@ class TaskStorageTest extends TestCase public function testAddExistingTask() { // Prepare a dummy task - $dummyTask = new Task('testAddExistingTask', 'none'); + $dummyTask = new Task('testAddExistingTask', new EmptyHandler()); // First check that the task storage starts empty $this->assertEmpty($this->taskStorage->readTasks()); @@ -123,8 +124,8 @@ class TaskStorageTest extends TestCase public function testGetTaskById() { // Prepare a dummy task - $dummyTask1 = new Task('testGetTaskById1', 'none'); - $dummyTask2 = new Task('testGetTaskById2', 'none'); + $dummyTask1 = new Task('testGetTaskById1', new EmptyHandler()); + $dummyTask2 = new Task('testGetTaskById2', new EmptyHandler()); // First we add both tasks $this->assertEmpty($this->taskStorage->readTasks()); @@ -155,7 +156,7 @@ class TaskStorageTest extends TestCase public function testGetTaskByIdNotFound() { // Prepare a dummy task - $dummyTask = new Task('testGetTaskByIdNotFound', 'none'); + $dummyTask = new Task('testGetTaskByIdNotFound', new EmptyHandler()); // First we add the task $this->assertEmpty($this->taskStorage->readTasks()); @@ -175,7 +176,7 @@ class TaskStorageTest extends TestCase public function testModifyTask() { // Prepare a dummy task - $dummyTask = new Task('testModifyTask', 'none'); + $dummyTask = new Task('testModifyTask', new EmptyHandler()); $dummyTask->setStatus(Task::RUNNING); // First we add the task @@ -199,7 +200,7 @@ class TaskStorageTest extends TestCase public function testModifyTaskNotFound() { // Prepare a dummy task - $dummyTask = new Task('testModifyTaskNotFound', 'none'); + $dummyTask = new Task('testModifyTaskNotFound', new EmptyHandler()); // Attempt to change this task, which does not exist. $this->expectException(TasksException::class); @@ -212,7 +213,7 @@ class TaskStorageTest extends TestCase public function testModifyTaskEvent() { // Prepare a dummy task - $dummyTask = new Task('testModifyTaskEvent', 'none'); + $dummyTask = new Task('testModifyTaskEvent', new EmptyHandler()); $dummyTask->setStatus(Task::PENDING); // Then add the Task @@ -245,7 +246,7 @@ class TaskStorageTest extends TestCase public function testModifyTaskCancel() { // Prepare a dummy task - $dummyTask = new Task('testModifyTaskCancel', 'none'); + $dummyTask = new Task('testModifyTaskCancel', new EmptyHandler()); $dummyTask->setStatus(Task::PENDING); // Then add the Task @@ -272,7 +273,7 @@ class TaskStorageTest extends TestCase public function testDeleteTask() { // Prepare a dummy task - $dummyTask = new Task('testDeleteTask', 'none'); + $dummyTask = new Task('testDeleteTask', new EmptyHandler()); // Add the task to the storage $this->assertEmpty($this->taskStorage->readTasks()); @@ -295,7 +296,7 @@ class TaskStorageTest extends TestCase public function testDeleteTaskNotFound() { // Prepare a dummy task - $dummyTask = new Task('testDeleteTaskNotFound', 'none'); + $dummyTask = new Task('testDeleteTaskNotFound', new EmptyHandler()); // Attempt to delete this task, which does not exist. $this->expectException(TasksException::class); @@ -310,7 +311,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskOutput() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskOutput', 'none'); + $dummyTask = new Task('testWriteAndReadTaskOutput', new EmptyHandler()); // First write the task output $this->taskStorage->addTask($dummyTask); @@ -329,7 +330,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskOutputTaskNotExist() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskOutputTaskNotExist', 'none'); + $dummyTask = new Task('testWriteAndReadTaskOutputTaskNotExist', new EmptyHandler()); // Write output while the task does not exist yet, expect exception $this->expectException(TasksException::class); @@ -342,7 +343,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskOutputAttempts() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskOutputAttempts', 'none'); + $dummyTask = new Task('testWriteAndReadTaskOutputAttempts', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last @@ -382,7 +383,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskOutputAlreadyExists() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskOutputAlreadyExists', 'none'); + $dummyTask = new Task('testWriteAndReadTaskOutputAlreadyExists', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); // Write a first time @@ -399,7 +400,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskOutputNotExist() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskOutputNotExist', 'none'); + $dummyTask = new Task('testWriteAndReadTaskOutputNotExist', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); $this->assertNull($this->taskStorage->readTaskOutput($dummyTask)); @@ -413,7 +414,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskPostOutput() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskPostOutput', 'none'); + $dummyTask = new Task('testWriteAndReadTaskPostOutput', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); // First write the task output @@ -432,7 +433,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskPostOutputAttempts() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskPostOutputAttempts', 'none'); + $dummyTask = new Task('testWriteAndReadTaskPostOutputAttempts', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last @@ -472,7 +473,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskPostOutputAlreadyExists() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskPostOutputAlreadyExists', 'none'); + $dummyTask = new Task('testWriteAndReadTaskPostOutputAlreadyExists', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); // Write a first time @@ -489,7 +490,7 @@ class TaskStorageTest extends TestCase public function testWriteAndReadTaskPostOutputNotExist() { // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskPostOutputNotExist', 'none'); + $dummyTask = new Task('testWriteAndReadTaskPostOutputNotExist', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); $this->assertNull($this->taskStorage->readPostOutput($dummyTask)); @@ -505,7 +506,7 @@ class TaskStorageTest extends TestCase public function testReset() { // Prepare a dummy task - $dummyTask = new Task('testReset', 'none'); + $dummyTask = new Task('testReset', new EmptyHandler()); // Add the task and some output $this->assertTrue($this->taskStorage->addTask($dummyTask)); diff --git a/test/base/TaskTest.php b/test/base/TaskTest.php index 0bbb5f2..9215f0a 100644 --- a/test/base/TaskTest.php +++ b/test/base/TaskTest.php @@ -37,6 +37,7 @@ use FuzeWorks\Async\Constraint; use FuzeWorks\Async\Task; use FuzeWorks\Async\TasksException; +use Mock\Handlers\EmptyHandler; use PHPUnit\Framework\TestCase; class TaskTest extends TestCase @@ -45,7 +46,7 @@ class TaskTest extends TestCase public function testClass() { // Create dummy task - $dummyTask = new Task('testClass', 'none'); + $dummyTask = new Task('testClass', new EmptyHandler()); // And check the class. A pretty useless but standard test $this->assertInstanceOf('FuzeWorks\Async\Task', $dummyTask); @@ -59,11 +60,11 @@ class TaskTest extends TestCase public function testBaseVariables() { // Create dummy task - $dummyTask = new Task('testBaseVariables', 'someThing', true); + $dummyTask = new Task('testBaseVariables', new EmptyHandler(), true); // test the values $this->assertEquals('testBaseVariables', $dummyTask->getId()); - $this->assertEquals('someThing', $dummyTask->getHandlerClass()); + $this->assertInstanceOf(EmptyHandler::class, $dummyTask->getHandler()); $this->assertTrue($dummyTask->getUsePostHandler()); } @@ -73,11 +74,11 @@ class TaskTest extends TestCase public function testArguments() { // Create task without arguments - $dummyTask1 = new Task('testArguments1', 'none', true); + $dummyTask1 = new Task('testArguments1', new EmptyHandler(), true); $this->assertEmpty($dummyTask1->getArguments()); // Now create a task with some arguments - $dummyTask2 = new Task('testArguments2', 'none', true, 'some', 'arguments'); + $dummyTask2 = new Task('testArguments2', new EmptyHandler(), true, 'some', 'arguments'); $this->assertEquals(['some', 'arguments'], $dummyTask2->getArguments()); } @@ -87,8 +88,8 @@ class TaskTest extends TestCase public function testPostHandler() { // Create dummy tasks - $dummyTask1 = new Task('testPostHandler1', 'someThing', true); - $dummyTask2 = new Task('testPostHandler2', 'someThing', false); + $dummyTask1 = new Task('testPostHandler1', new EmptyHandler(), true); + $dummyTask2 = new Task('testPostHandler2', new EmptyHandler(), false); $this->assertTrue($dummyTask1->getUsePostHandler()); $this->assertFalse($dummyTask2->getUsePostHandler()); @@ -103,7 +104,7 @@ class TaskTest extends TestCase $stub = $this->createMock(Constraint::class); // Then add it to the task - $dummyTask = new Task('testConstraints', 'someThing', false); + $dummyTask = new Task('testConstraints', new EmptyHandler(), false); $dummyTask->addConstraint($stub); // Assert it exists @@ -116,7 +117,7 @@ class TaskTest extends TestCase public function testStatusCodes() { // Create dummy task - $dummyTask = new Task('testStatusCodes', 'someThing', true); + $dummyTask = new Task('testStatusCodes', new EmptyHandler(), true); for ($i = 1; $i <= 9; $i++) { $dummyTask->setStatus($i); @@ -130,7 +131,7 @@ class TaskTest extends TestCase public function testDelayTime() { // Create dummy task - $dummyTask = new Task('testDelayTime', 'someThing', true); + $dummyTask = new Task('testDelayTime', new EmptyHandler(), true); $this->assertEquals(0, $dummyTask->getDelayTime()); $dummyTask->setDelayTime(1000); @@ -143,7 +144,7 @@ class TaskTest extends TestCase public function testAttributes() { // Create dummy task - $dummyTask = new Task('testAttributes', 'someThing', true); + $dummyTask = new Task('testAttributes', new EmptyHandler(), true); // First test a non-existing attribute $this->assertNull($dummyTask->attribute('testKey')); @@ -167,7 +168,7 @@ class TaskTest extends TestCase public function testOutputsAndErrors() { // Create dummy task - $dummyTask = new Task('testOutputsAndErrors', 'someThing', true); + $dummyTask = new Task('testOutputsAndErrors', new EmptyHandler(), true); // Check if non are filled $this->assertNull($dummyTask->getOutput()); @@ -192,7 +193,7 @@ class TaskTest extends TestCase public function testRetrySettings() { // Create dummy task - $dummyTask = new Task('testRetrySettings', 'someThing', true); + $dummyTask = new Task('testRetrySettings', new EmptyHandler(), true); // Test starting position $this->assertEquals([ @@ -222,7 +223,7 @@ class TaskTest extends TestCase public function testRetries() { // Create dummy task - $dummyTask = new Task('testRetries', 'someThing', true); + $dummyTask = new Task('testRetries', new EmptyHandler(), true); // First test the starting position $this->assertEquals(0, $dummyTask->getRetries()); diff --git a/test/mock/Handlers/ArgumentedHandler.php b/test/mock/Handlers/ArgumentedHandler.php index 61287cb..8104e10 100644 --- a/test/mock/Handlers/ArgumentedHandler.php +++ b/test/mock/Handlers/ArgumentedHandler.php @@ -42,19 +42,20 @@ class ArgumentedHandler implements Handler { private $output; + private $sleepTime; - private $sleeptime; + public function __construct(int $sleepTime, string $output) + { + $this->sleepTime = $sleepTime; + $this->output = $output; + } /** * @inheritDoc */ public function primaryHandler(Task $task): bool { - $arguments = $task->getArguments(); - $this->sleeptime = $arguments[0]; - $this->output = $arguments[1]; - - sleep($this->sleeptime); + sleep($this->sleepTime); return true; } @@ -71,11 +72,7 @@ class ArgumentedHandler implements Handler */ public function postHandler(Task $task) { - $arguments = $task->getArguments(); - $this->sleeptime = $arguments[0]; - $this->output = $arguments[1]; - - sleep($this->sleeptime); + sleep($this->sleepTime); return true; } diff --git a/test/mock/Handlers/EmptyHandler.php b/test/mock/Handlers/EmptyHandler.php new file mode 100644 index 0000000..3216365 --- /dev/null +++ b/test/mock/Handlers/EmptyHandler.php @@ -0,0 +1,72 @@ + dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'worker' ]); - $this->superVisor = new ParallelSuperVisor($this->taskStorage, $this->executor, ['outstream' => null]); + $this->superVisor = new ParallelSuperVisor($this->taskStorage, $this->executor); } public function testClass() @@ -99,7 +100,7 @@ class ParallelSuperVisorTest extends TestCase public function testToRunning() { // First create a dummy task - $dummyTask = new Task('testToRunning', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testToRunning', new ArgumentedHandler(10, 'Some Output'), false); // Write the dummy to TaskStorage $this->taskStorage->addTask($dummyTask); @@ -128,7 +129,7 @@ class ParallelSuperVisorTest extends TestCase public function testConstrainedPending() { // First create a dummy task - $dummyTask = new Task('testConstrainedPending', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testConstrainedPending', new ArgumentedHandler(10, 'Some Output'), false); // Add a constraint $dummyTask->addConstraint(new FixedTimeConstraint(time() + 3600)); @@ -160,7 +161,7 @@ class ParallelSuperVisorTest extends TestCase public function testChangeDelayedToPending() { // First create a dummy task - $dummyTask = new Task('testChangeDelayedToPending', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testChangeDelayedToPending', new ArgumentedHandler(10, 'Some Output'), false); // Set to delayed and set to NOW $dummyTask->setStatus(Task::DELAYED); @@ -193,7 +194,7 @@ class ParallelSuperVisorTest extends TestCase public function testKeepDelayed() { // First create a dummy task - $dummyTask = new Task('testKeepDelayed', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testKeepDelayed', new ArgumentedHandler(10, 'Some Output'), false); // Set to delayed and set to NOW $dummyTask->setStatus(Task::DELAYED); @@ -226,7 +227,7 @@ class ParallelSuperVisorTest extends TestCase public function testFinishedTask() { // First create a dummy task - $dummyTask = new Task('testFinishedTask', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testFinishedTask', new ArgumentedHandler(10, 'Some Output'), false); // Set status to running $dummyTask->setStatus(Task::RUNNING); @@ -260,7 +261,7 @@ class ParallelSuperVisorTest extends TestCase public function testMissingTask() { // First create a dummy task - $dummyTask = new Task('testMissingTask', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testMissingTask', new ArgumentedHandler(10, 'Some Output'), false); // Set status to running $dummyTask->setStatus(Task::RUNNING); @@ -292,7 +293,7 @@ class ParallelSuperVisorTest extends TestCase public function testFailedTask() { // First create a dummy task - $dummyTask = new Task('testFailedTask', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testFailedTask', new ArgumentedHandler(10, 'Some Output'), false); // Set status to running $dummyTask->setStatus(Task::RUNNING); @@ -326,10 +327,10 @@ class ParallelSuperVisorTest extends TestCase public function testRetryFailedTask() { // First create the dummy tasks - $dummyTaskFailedYes = new Task('testRetryFailedTaskY', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); - $dummyTaskPFailedYes = new Task('testRetryFailedTaskPY', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); - $dummyTaskFailedNo = new Task('testRetryFailedTaskN', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); - $dummyTaskPFailedNo = new Task('testRetryFailedTaskPN', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTaskFailedYes = new Task('testRetryFailedTaskY', new ArgumentedHandler(10, 'Some Output'), false); + $dummyTaskPFailedYes = new Task('testRetryFailedTaskPY', new ArgumentedHandler(10, 'Some Output'), false); + $dummyTaskFailedNo = new Task('testRetryFailedTaskN', new ArgumentedHandler(10, 'Some Output'), false); + $dummyTaskPFailedNo = new Task('testRetryFailedTaskPN', new ArgumentedHandler(10, 'Some Output'), false); // Set statuses $dummyTaskFailedYes->setStatus(Task::FAILED); @@ -378,8 +379,8 @@ class ParallelSuperVisorTest extends TestCase public function testExceedMaxRetries() { // First create the dummy tasks - $dummyTask = new Task('testExceedMaxRetries', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); - $dummyTask2 = new Task('testExceedMaxRetries2', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); + $dummyTask = new Task('testExceedMaxRetries', new ArgumentedHandler(10, 'Some Output'), false); + $dummyTask2 = new Task('testExceedMaxRetries2', new ArgumentedHandler(10, 'Some Output'), false); // Set status and retry settings $dummyTask->setStatus(Task::FAILED); @@ -415,7 +416,7 @@ class ParallelSuperVisorTest extends TestCase public function testFailedToPost() { // First create the dummy tasks - $dummyTask = new Task('testFailedToPost', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + $dummyTask = new Task('testFailedToPost', new ArgumentedHandler(10, 'Some Output'), true); // Set status and settings $dummyTask->setStatus(Task::FAILED); @@ -444,8 +445,8 @@ class ParallelSuperVisorTest extends TestCase public function testSuccessfulTasks() { // First create the dummy tasks - $dummyTaskPostNo = new Task('testSuccessfulTasksN', 'Mock\Handlers\ArgumentedHandler', false, 10, 'Some Output'); - $dummyTaskPostYes = new Task('testSuccessfulTasksY', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + $dummyTaskPostNo = new Task('testSuccessfulTasksN', new ArgumentedHandler(10, 'Some Output'), false); + $dummyTaskPostYes = new Task('testSuccessfulTasksY', new ArgumentedHandler(10, 'Some Output'), true); // Set status and settings $dummyTaskPostNo->setStatus(Task::SUCCESS); @@ -472,8 +473,8 @@ class ParallelSuperVisorTest extends TestCase public function testPostTasks() { // First create the dummy tasks - $dummyTaskFinished = new Task('testPostTasksFinished', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); - $dummyTaskMissing = new Task('testPostTasksMissing', 'Mock\Handlers\ArgumentedHandler', true, 10, 'Some Output'); + $dummyTaskFinished = new Task('testPostTasksFinished', new ArgumentedHandler(10, 'Some Output'), true); + $dummyTaskMissing = new Task('testPostTasksMissing', new ArgumentedHandler(10, 'Some Output'), true); // Set status and settings $dummyTaskFinished->setStatus(Task::POST); -- 2.40.1 From 902693dbbe2e7ec68bc4b35868adc01a1a8195fc Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Wed, 3 Jun 2020 17:00:44 +0200 Subject: [PATCH 27/33] Implemented Parent Handlers. Parent Handlers can be stacked to run in succession. Output is transfered as input into the child handler which can continue with it. If the parent Handler fails, all Child handlers also fail. --- bin/supervisor | 4 +- bin/worker | 6 +- .../Async/Executors/ShellExecutor.php | 3 + src/FuzeWorks/Async/Handler.php | 21 +++ src/FuzeWorks/Async/ShellWorker.php | 88 +++++---- src/FuzeWorks/Async/Tasks.php | 42 ++++- test/base/ShellExecutorTest.php | 8 +- test/base/ShellWorkerTest.php | 169 +++++++++++++++++- test/mock/Handlers/ArgumentedHandler.php | 22 +++ test/mock/Handlers/EmptyHandler.php | 22 +++ .../Handlers/TestStartAndReadTasksHandler.php | 22 +++ test/mock/Handlers/TestStopTaskHandler.php | 22 +++ test/system/ParallelSuperVisorTest.php | 24 +-- 13 files changed, 386 insertions(+), 67 deletions(-) diff --git a/bin/supervisor b/bin/supervisor index 4ffa970..5abc47b 100644 --- a/bin/supervisor +++ b/bin/supervisor @@ -43,11 +43,11 @@ use FuzeWorks\Exception\LibraryException; use FuzeWorks\Factory; // First perform a PHP version check -if (version_compare('7.1.0', PHP_VERSION, '>')) { +if (version_compare('7.3.0', PHP_VERSION, '>')) { fwrite( STDERR, sprintf( - 'FuzeWorks Async requires PHP 7.1 or higher.' . PHP_EOL . + 'FuzeWorks Async requires PHP 7.3 or higher.' . PHP_EOL . 'You are using PHP %s (%s).' . PHP_EOL, PHP_VERSION, PHP_BINARY diff --git a/bin/worker b/bin/worker index ecac264..c7318a3 100644 --- a/bin/worker +++ b/bin/worker @@ -42,11 +42,11 @@ use FuzeWorks\Factory; // First perform a PHP version check -if (version_compare('7.1.0', PHP_VERSION, '>')) { +if (version_compare('7.3.0', PHP_VERSION, '>')) { fwrite( STDERR, sprintf( - 'FuzeWorks Async requires PHP 7.1 or higher.' . PHP_EOL . + 'FuzeWorks Async requires PHP 7.3 or higher.' . PHP_EOL . 'You are using PHP %s (%s).' . PHP_EOL, PHP_VERSION, PHP_BINARY @@ -119,7 +119,7 @@ $post = isset($arguments['p']); // RUN THE APP $worker = $lib->getWorker(); -$worker->run($taskID, $post); +$worker->runTaskById($taskID, $post); fwrite(STDOUT,'Finished task \'' . $taskID . "'"); ?> \ No newline at end of file diff --git a/src/FuzeWorks/Async/Executors/ShellExecutor.php b/src/FuzeWorks/Async/Executors/ShellExecutor.php index 7e8685b..247adda 100644 --- a/src/FuzeWorks/Async/Executors/ShellExecutor.php +++ b/src/FuzeWorks/Async/Executors/ShellExecutor.php @@ -56,6 +56,9 @@ class ShellExecutor implements Executor */ public function __construct(array $parameters) { + if (!isset($parameters['workerFile']) || !isset($parameters['bootstrapFile'])) + throw new TasksException("Could not construct ShellExecutor. Parameter failure."); + // Fetch workerFile $this->worker = $parameters['workerFile']; if (!file_exists($this->worker)) diff --git a/src/FuzeWorks/Async/Handler.php b/src/FuzeWorks/Async/Handler.php index fe7d4ec..f44c3db 100644 --- a/src/FuzeWorks/Async/Handler.php +++ b/src/FuzeWorks/Async/Handler.php @@ -38,6 +38,27 @@ namespace FuzeWorks\Async; interface Handler { + /** + * Retrieve the parent handler that will first handle this task, before this child Handler + * + * @return Handler|null + */ + public function getParentHandler(): ?Handler; + + /** + * Set the parent handler that will fire before this Handler + * + * @param Handler $parentHandler + */ + public function setParentHandler(Handler $parentHandler): void; + + /** + * Import the parent output into the child + * + * @param mixed $input + */ + public function setParentInput($input): void; + /** * The handler method used to handle this task. * This handler will execute the actual task. diff --git a/src/FuzeWorks/Async/ShellWorker.php b/src/FuzeWorks/Async/ShellWorker.php index 7a5ad8b..4a67902 100644 --- a/src/FuzeWorks/Async/ShellWorker.php +++ b/src/FuzeWorks/Async/ShellWorker.php @@ -67,74 +67,84 @@ class ShellWorker } /** - * @param string $taskID + * Run a task by finding its ID + * + * @param string $taskId * @param bool $post * @throws EventException * @throws TasksException */ - public function run(string $taskID, bool $post = false) + public function runTaskById(string $taskId, bool $post = false) { // First fetch the task try { - $task = $this->taskStorage->getTaskById($taskID); + $task = $this->taskStorage->getTaskById($taskId); } catch (TasksException $e) { throw new TasksException("Could not run worker. Task not found."); } + $this->run($task, $post); + } + + /** + * @param Task $task + * @param bool $post + * @throws EventException + * @throws TasksException + */ + public function run(Task $task, bool $post = false) + { // Fire a taskHandleEvent /** @var TaskHandleEvent $event */ $event = Events::fireEvent(new TaskHandleEvent(), $task); - $task = $event->getTask(); // Set task to this worker - $this->task = $task; + $this->task = $event->getTask(); $this->post = $post; // Fetch the callable - $object = $this->task->getHandler(); - if (!$object instanceof Handler) - { - $errors = "Could not run task. '".get_class($object)."' is not instance of Handler."; - if (!$post) - $this->taskStorage->writeTaskOutput($this->task, '', $errors, Task::PFAILED, $this->task->getRetries()); - else - $this->taskStorage->writePostOutput($this->task, '', $errors, Task::PFAILED, $this->task->getRetries()); + $handler = $this->task->getHandler(); - throw new TasksException("Could not run task. '".get_class($object)."' is not instance of Handler."); - } + // Execute the handler and all its parent handlers + $success = $this->executeHandler($this->task, $handler, $post); - // Run postHandler if post mode is requested - if ($post) - { - $postSuccess = $object->postHandler($this->task); - $postOutput = $object->getPostOutput(); - $postOutput = is_null($postOutput) ? '' : (string) $postOutput; - $postErrors = $this->getErrors(); - - if (!$postSuccess) - $this->taskStorage->writePostOutput($this->task, $postOutput, $postErrors, Task::FAILED, $this->task->getRetries()); - else - $this->taskStorage->writePostOutput($this->task, $postOutput, $postErrors, Task::SUCCESS, $this->task->getRetries()); - - $this->output($postOutput, $postErrors); - return; - } - - // Run primaryHandler if requested - $success = $object->primaryHandler($this->task); - $output = $object->getOutput(); - $output = is_null($output) ? '' : (string) $output; + // Fetch the output and errors + $output = $post ? $handler->getPostOutput() : $handler->getOutput(); + $output = is_null($output) ? '' : $output; $errors = $this->getErrors(); - // And afterwards write the results to the TaskStorage - if (!$success) + // If the task failed, write so to task storage, based on whether this is a post request or not + if (!$success && $post) + $this->taskStorage->writePostOutput($this->task, $output, $errors, Task::FAILED, $this->task->getRetries()); + elseif (!$success && !$post) $this->taskStorage->writeTaskOutput($this->task, $output, $errors, Task::FAILED, $this->task->getRetries()); + elseif ($success && $post) + $this->taskStorage->writePostOutput($this->task, $output, $errors, Task::SUCCESS, $this->task->getRetries()); else $this->taskStorage->writeTaskOutput($this->task, $output, $errors, Task::SUCCESS, $this->task->getRetries()); - $this->output($output, $errors); + // And write the final output + $this->output((string) $output, $errors); } + protected function executeHandler(Task $task, Handler $handler, bool $usePost = false): bool + { + // First check to see if there is a parent handler + $parent = $handler->getParentHandler(); + if (!is_null($parent)) { + // Execute the parent + if ($this->executeHandler($task, $parent, $usePost) === false) + return false; + + // Fetch the output of the parent + $output = $usePost ? $parent->getPostOutput() : $parent->getOutput(); + + // And insert it as input into the child handler + $handler->setParentInput($output); + } + + return $usePost ? $handler->postHandler($task) : $handler->primaryHandler($task); + } /** * In case a fatal error or exception occurs, the errors shall be redirected to stderr * diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index 355f009..1247d2e 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -50,6 +50,26 @@ class Tasks implements iLibrary */ private $cfg; + /** + * @var TaskStorage + */ + private $taskStorage; + + /** + * @var Executor + */ + private $executor; + + /** + * @var SuperVisor + */ + private $supervisor; + + /** + * @var ShellWorker + */ + private $shellWorker; + /** * Tasks constructor. * @@ -80,6 +100,9 @@ class Tasks implements iLibrary */ public function getSuperVisor(string $bootstrapFile): SuperVisor { + if (isset($this->supervisor)) + return $this->supervisor; + // First get the configuration for SuperVisors $cfg = $this->cfg->get('SuperVisor'); @@ -104,6 +127,7 @@ class Tasks implements iLibrary if (!$object instanceof SuperVisor) throw new TasksException("Could not get SuperVisor. Type of '$class' is not instanceof TaskStorage."); + $this->supervisor = $object; return $object; } @@ -113,7 +137,11 @@ class Tasks implements iLibrary */ public function getWorker(): ShellWorker { - return new ShellWorker($this->getTaskStorage()); + if (isset($this->shellWorker)) + return $this->shellWorker; + + $this->shellWorker = new ShellWorker($this->getTaskStorage()); + return $this->shellWorker; } /** @@ -124,6 +152,9 @@ class Tasks implements iLibrary */ public function getTaskStorage(): TaskStorage { + if (isset($this->taskStorage)) + return $this->taskStorage; + // First get the configuration for TaskStorage $cfg = $this->cfg->get('TaskStorage'); @@ -145,7 +176,8 @@ class Tasks implements iLibrary if (!$object instanceof TaskStorage) throw new TasksException("Could not get TaskStorage. Type '$class' is not instanceof TaskStorage."); - return $object; + $this->taskStorage = $object; + return $this->taskStorage; } /** @@ -157,6 +189,9 @@ class Tasks implements iLibrary */ protected function getExecutor(string $bootstrapFile): Executor { + if (isset($this->executor)) + return $this->executor; + // First get the configuration for Executor $cfg = $this->cfg->get('Executor'); @@ -178,7 +213,8 @@ class Tasks implements iLibrary if (!$object instanceof Executor) throw new TasksException("Could not get Executor. Type '$class' is not instanceof Executor."); - return $object; + $this->executor = $object; + return $this->executor; } /** diff --git a/test/base/ShellExecutorTest.php b/test/base/ShellExecutorTest.php index d88e696..eeb8f89 100644 --- a/test/base/ShellExecutorTest.php +++ b/test/base/ShellExecutorTest.php @@ -146,7 +146,7 @@ class ShellExecutorTest extends TestCase $task = $this->executor->startTask($dummyTask); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Assert that the output is the same $this->assertSame($dummyTask, $task); @@ -173,7 +173,7 @@ class ShellExecutorTest extends TestCase $dummyTask = $this->executor->startTask($dummyTask); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // And we fetch some task statistics $stats = $this->executor->getTaskStats($dummyTask); @@ -217,7 +217,7 @@ class ShellExecutorTest extends TestCase $dummyTask = $this->executor->startTask($dummyTask); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Check if the task is running $this->assertTrue($this->executor->getTaskRunning($dummyTask)); @@ -226,7 +226,7 @@ class ShellExecutorTest extends TestCase $output = $this->executor->stopTask($dummyTask); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // We check that the output actually is the task $this->assertSame($dummyTask, $output); diff --git a/test/base/ShellWorkerTest.php b/test/base/ShellWorkerTest.php index 2d04522..2817cb2 100644 --- a/test/base/ShellWorkerTest.php +++ b/test/base/ShellWorkerTest.php @@ -34,9 +34,9 @@ * @version Version 1.0.0 */ +use FuzeWorks\Async\Handler; use FuzeWorks\Async\ShellWorker; use FuzeWorks\Async\Task; -use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TaskStorage; use FuzeWorks\Events; use PHPUnit\Framework\TestCase; @@ -57,13 +57,13 @@ class ShellWorkerTest extends TestCase public function setUp(): void { // Load the TaskStorage so temporary tasks can be stored - $tasks = new Tasks(); - $this->taskStorage = $tasks->getTaskStorage(); + // This system uses DummyTaskStorage because Redis can't serialize the Mock Handlers. Doesn't matter much anyway. + $this->taskStorage = new TaskStorage\DummyTaskStorage([]); + $this->shellWorker = new ShellWorker($this->taskStorage); $this->taskStorage->reset(); // Clear events Events::$listeners = []; - $this->shellWorker = $tasks->getWorker(); } public function testClass() @@ -75,4 +75,165 @@ class ShellWorkerTest extends TestCase // @todo Add lots of tests and amend ShellWorker to return results + /** + * @depends testClass + */ + public function testUseHandler() + { + // First prepare a Mock Handler + $mockHandler = $this->createMock(Handler::class); + $mockHandler->expects($this->exactly(2))->method('getParentHandler')->willReturn(null); + $mockHandler->expects($this->once()) + ->method('primaryHandler') + ->with($this->callback(function($subject){return $subject instanceof Task;})) + ->willReturn(true); + $mockHandler->expects($this->once()) + ->method('postHandler') + ->with($this->callback(function($subject){return $subject instanceof Task;})) + ->willReturn(true); + $mockHandler->expects($this->once())->method('getOutput')->willReturn('Some Output!'); + $mockHandler->expects($this->once())->method('getPostOutput')->willReturn('Post Output!'); + + // Create a Dummy task + $dummyTask = new Task('testUseHandler', $mockHandler); + $this->taskStorage->addTask($dummyTask); + + // Run the task in ShellWorker + $this->shellWorker->run($dummyTask, false); + + // And verify if the Output is correctly set + $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals('Some Output!', $output['output']); + $this->assertEquals(Task::SUCCESS, $output['statusCode']); + + // And run the post handler + $this->shellWorker->run($dummyTask, true); + $output = $this->taskStorage->readPostOutput($dummyTask); + $this->assertEquals('Post Output!', $output['output']); + $this->assertEquals(Task::SUCCESS, $output['statusCode']); + } + + /** + * @depends testUseHandler + */ + public function testFailingHandlers() + { + $mockHandler = $this->createMock(Handler::class); + $mockHandler->expects($this->once()) + ->method('primaryHandler') + ->with($this->callback(function($subject){return $subject instanceof Task;})) + ->willReturn(false); + $mockHandler->expects($this->once()) + ->method('postHandler') + ->with($this->callback(function($subject){return $subject instanceof Task;})) + ->willReturn(false); + + // Create a Dummy task + $dummyTask = new Task('testFailingHandlers1', $mockHandler); + $this->taskStorage->addTask($dummyTask); + + // Run the task in ShellWorker + $this->shellWorker->run($dummyTask, false); + + // And verify if the Output is correctly set + $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals('', $output['output']); + $this->assertEquals(Task::FAILED, $output['statusCode']); + + // And run a post failure + $this->shellWorker->run($dummyTask, true); + $output = $this->taskStorage->readPostOutput($dummyTask); + $this->assertEquals('', $output['output']); + $this->assertEquals(Task::FAILED, $output['statusCode']); + } + + /** + * @depends testUseHandler + */ + public function testParentHandlers() + { + // First create the Handlers + $parentHandler = $this->createMock(Handler::class); + $childHandler = $this->createMock(Handler::class); + + // Prepare parent handler output + $parentHandler->expects($this->once()) + ->method('primaryHandler') + ->with($this->callback(function($subject){return $subject instanceof Task;})) + ->willReturn(true); + $parentHandler->expects($this->once()) + ->method('getOutput') + ->willReturn('Parent Output'); + + // Prepare the child handler + $childHandler->expects($this->once()) + ->method('getParentHandler') + ->willReturn($parentHandler); + $childHandler->expects($this->once()) + ->method('setParentInput') + ->with($this->equalTo('Parent Output')); + $childHandler->expects($this->once()) + ->method('primaryHandler') + ->with($this->callback(function($subject){return $subject instanceof Task;})) + ->willReturn(true); + $childHandler->expects($this->once()) + ->method('getOutput') + ->willReturn('Child Output'); + + // Set the relation + $childHandler->setParentHandler($parentHandler); + + // Create the Dummy Task + $dummyTask = new Task('testParentHandlers', $childHandler); + $this->taskStorage->addTask($dummyTask); + + // Run the task in ShellWorker + $this->shellWorker->run($dummyTask, false); + + // And verify if the Output is correctly set + $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals('Child Output', $output['output']); + $this->assertEquals(Task::SUCCESS, $output['statusCode']); + } + + /** + * @depends testParentHandlers + */ + public function testCascadingParentFailure() + { + // First create the Handlers + $parentHandler = $this->createMock(Handler::class); + $childHandler = $this->createMock(Handler::class); + + // Set the relation + $childHandler->setParentHandler($parentHandler); + $childHandler->expects($this->once()) + ->method('getParentHandler') + ->willReturn($parentHandler); + + // Set the results + $parentHandler->expects($this->once()) + ->method('primaryHandler') + ->willReturn(false); + $childHandler->expects($this->never()) + ->method('primaryHandler') + ->willReturn(true); + + // And some methods which shall be called in the end + $childHandler->expects($this->once()) + ->method('getOutput') + ->willReturn('Task failed successfully'); + + // Create the task to run this + $dummyTask = new Task('testCascadingParentFailure', $childHandler); + $this->taskStorage->addTask($dummyTask); + + // Run the task in ShellWorker + $this->shellWorker->run($dummyTask, false); + + // And verify whether the task has indeed failed + $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals('Task failed successfully', $output['output']); + $this->assertEquals(Task::FAILED, $output['statusCode']); + } } diff --git a/test/mock/Handlers/ArgumentedHandler.php b/test/mock/Handlers/ArgumentedHandler.php index 8104e10..dd6d458 100644 --- a/test/mock/Handlers/ArgumentedHandler.php +++ b/test/mock/Handlers/ArgumentedHandler.php @@ -83,4 +83,26 @@ class ArgumentedHandler implements Handler { return $this->output; } + + /** + * @inheritDoc + */ + public function getParentHandler(): ?Handler + { + return null; + } + + /** + * @inheritDoc + */ + public function setParentInput($input): void + { + } + + /** + * @inheritDoc + */ + public function setParentHandler(Handler $parentHandler): void + { + } } \ No newline at end of file diff --git a/test/mock/Handlers/EmptyHandler.php b/test/mock/Handlers/EmptyHandler.php index 3216365..71cf04d 100644 --- a/test/mock/Handlers/EmptyHandler.php +++ b/test/mock/Handlers/EmptyHandler.php @@ -69,4 +69,26 @@ class EmptyHandler implements Handler public function getPostOutput() { } + + /** + * @inheritDoc + */ + public function getParentHandler(): ?Handler + { + return null; + } + + /** + * @inheritDoc + */ + public function setParentInput($input): void + { + } + + /** + * @inheritDoc + */ + public function setParentHandler(Handler $parentHandler): void + { + } } \ No newline at end of file diff --git a/test/mock/Handlers/TestStartAndReadTasksHandler.php b/test/mock/Handlers/TestStartAndReadTasksHandler.php index c162eb0..eff209c 100644 --- a/test/mock/Handlers/TestStartAndReadTasksHandler.php +++ b/test/mock/Handlers/TestStartAndReadTasksHandler.php @@ -73,4 +73,26 @@ class TestStartAndReadTasksHandler implements Handler { } + + /** + * @inheritDoc + */ + public function getParentHandler(): ?Handler + { + return null; + } + + /** + * @inheritDoc + */ + public function setParentInput($input): void + { + } + + /** + * @inheritDoc + */ + public function setParentHandler(Handler $parentHandler): void + { + } } \ No newline at end of file diff --git a/test/mock/Handlers/TestStopTaskHandler.php b/test/mock/Handlers/TestStopTaskHandler.php index 0ee2f11..8182f77 100644 --- a/test/mock/Handlers/TestStopTaskHandler.php +++ b/test/mock/Handlers/TestStopTaskHandler.php @@ -71,4 +71,26 @@ class TestStopTaskHandler implements Handler public function getPostOutput() { } + + /** + * @inheritDoc + */ + public function getParentHandler(): ?Handler + { + return null; + } + + /** + * @inheritDoc + */ + public function setParentInput($input): void + { + } + + /** + * @inheritDoc + */ + public function setParentHandler(Handler $parentHandler): void + { + } } \ No newline at end of file diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index 79d7172..db44e4f 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -113,7 +113,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -145,7 +145,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -178,7 +178,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -211,7 +211,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -244,7 +244,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -277,7 +277,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -310,7 +310,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Then re-fetch the Task $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -354,7 +354,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // Reload all tasks from TaskStorage $dummyTaskFailedYes = $this->taskStorage->getTaskById($dummyTaskFailedYes->getId()); @@ -401,7 +401,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // And check if the Task has been cancelled $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -429,7 +429,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // And check if the Task has been moved to Post $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); @@ -460,7 +460,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // And check if the Tasks have been completed or moved to post $dummyTaskPostNo = $this->taskStorage->getTaskById($dummyTaskPostNo->getId()); @@ -491,7 +491,7 @@ class ParallelSuperVisorTest extends TestCase $this->superVisor->cycle(); // Pause 1/10th of a second - usleep(100000); + usleep(500000); // And check if the Tasks have been completed or failed $dummyTaskFinished = $this->taskStorage->getTaskById($dummyTaskFinished->getId()); -- 2.40.1 From 4f39b0bec3c9133e16836c17efe2632c7b3a93a2 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Thu, 4 Jun 2020 21:29:37 +0200 Subject: [PATCH 28/33] Changed the way task output and post output is saved. Redis now saves all output for a task within a hash. This hash contains individual tasks and all its output. Attempts also start at 1, since that makes most sense for this context. When output is written, the TaskStorage must figure out at which attempt the Task is. --- src/FuzeWorks/Async/ShellWorker.php | 12 +- .../Async/Supervisors/ParallelSuperVisor.php | 4 +- src/FuzeWorks/Async/TaskStorage.php | 8 +- .../Async/TaskStorage/DummyTaskStorage.php | 26 +++- .../Async/TaskStorage/RedisTaskStorage.php | 126 +++++++++--------- test/base/TaskStorageTest.php | 73 +++------- 6 files changed, 118 insertions(+), 131 deletions(-) diff --git a/src/FuzeWorks/Async/ShellWorker.php b/src/FuzeWorks/Async/ShellWorker.php index 4a67902..6d8b452 100644 --- a/src/FuzeWorks/Async/ShellWorker.php +++ b/src/FuzeWorks/Async/ShellWorker.php @@ -115,13 +115,13 @@ class ShellWorker // If the task failed, write so to task storage, based on whether this is a post request or not if (!$success && $post) - $this->taskStorage->writePostOutput($this->task, $output, $errors, Task::FAILED, $this->task->getRetries()); + $this->taskStorage->writePostOutput($this->task, $output, $errors, Task::FAILED); elseif (!$success && !$post) - $this->taskStorage->writeTaskOutput($this->task, $output, $errors, Task::FAILED, $this->task->getRetries()); + $this->taskStorage->writeTaskOutput($this->task, $output, $errors, Task::FAILED); elseif ($success && $post) - $this->taskStorage->writePostOutput($this->task, $output, $errors, Task::SUCCESS, $this->task->getRetries()); + $this->taskStorage->writePostOutput($this->task, $output, $errors, Task::SUCCESS); else - $this->taskStorage->writeTaskOutput($this->task, $output, $errors, Task::SUCCESS, $this->task->getRetries()); + $this->taskStorage->writeTaskOutput($this->task, $output, $errors, Task::SUCCESS); // And write the final output $this->output((string) $output, $errors); @@ -166,9 +166,9 @@ class ShellWorker try { // Write to TaskStorage if (!$this->post) - $this->taskStorage->writeTaskOutput($this->task, '', $errors, Task::FAILED, $this->task->getRetries()); + $this->taskStorage->writeTaskOutput($this->task, '', $errors, Task::FAILED); else - $this->taskStorage->writePostOutput($this->task, '', $errors, Task::FAILED, $this->task->getRetries()); + $this->taskStorage->writePostOutput($this->task, '', $errors, Task::FAILED); } catch (TasksException $e) { // Ignore } diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index f3a36d0..2d0425f 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -118,7 +118,7 @@ class ParallelSuperVisor implements SuperVisor elseif ($task->getStatus() === Task::RUNNING) { $isRunning = $this->executor->getTaskRunning($task); - $output = $this->taskStorage->readTaskOutput($task, $task->getRetries()); + $output = $this->taskStorage->readTaskOutput($task); $hasOutput = !is_null($output); // If nothing is found, the process has crashed and status PFAILED should be set @@ -204,7 +204,7 @@ class ParallelSuperVisor implements SuperVisor elseif ($task->getStatus() === Task::POST) { $isRunning = $this->executor->getTaskRunning($task); - $output = $this->taskStorage->readPostOutput($task, $task->getRetries()); + $output = $this->taskStorage->readPostOutput($task); $hasOutput = !is_null($output); // If a task is not running and has no output, an error has occurred diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index 1d18ec4..47db727 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -111,7 +111,7 @@ interface TaskStorage * @return bool * @throws TasksException */ - public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool; + public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode): bool; /** * Write the output of the postHandler into TaskStorage @@ -126,7 +126,7 @@ interface TaskStorage * @return bool * @throws TasksException */ - public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool; + public function writePostOutput(Task $task, string $output, string $errors, int $statusCode): bool; /** * Read the task output from taskStorage. @@ -144,7 +144,7 @@ interface TaskStorage * @param int $attempt * @return array|null */ - public function readTaskOutput(Task $task, int $attempt = 0): ?array; + public function readTaskOutput(Task $task, int $attempt = 1): ?array; /** * Read the output from the postHandler @@ -162,7 +162,7 @@ interface TaskStorage * @param int $attempt * @return array|null */ - public function readPostOutput(Task $task, int $attempt = 0): ?array; + public function readPostOutput(Task $task, int $attempt = 1): ?array; /** * Reset the TaskStorage. diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index 60bd7c3..9d6e87d 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -177,11 +177,20 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode): bool { // First check if the task exists $task = $this->getTaskById($task->getId()); + // Set the attempt number + if (!isset($this->taskOutput[$task->getId()]['outAttempts'])) + { + $this->taskOutput[$task->getId()]['outAttempts'] = 1; + $attempt = $this->taskOutput[$task->getId()]['outAttempts']; + } + else + $attempt = $this->taskOutput[$task->getId()]['outAttempts']++; + if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) throw new TasksException("Could not write task output. Output already written."); @@ -197,11 +206,20 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + public function writePostOutput(Task $task, string $output, string $errors, int $statusCode): bool { // First check if the task exists $task = $this->getTaskById($task->getId()); + // Set the attempt number + if (!isset($this->taskOutput[$task->getId()]['postAttempts'])) + { + $this->taskOutput[$task->getId()]['postAttempts'] = 1; + $attempt = $this->taskOutput[$task->getId()]['postAttempts']; + } + else + $attempt = $this->taskOutput[$task->getId()]['postAttempts']++; + if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) throw new TasksException("Could not write task post output. Output already written."); @@ -217,7 +235,7 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readTaskOutput(Task $task, int $attempt = 0): ?array + public function readTaskOutput(Task $task, int $attempt = 1): ?array { if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) return $this->taskOutput[$task->getId()]['task'][$attempt]; @@ -228,7 +246,7 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readPostOutput(Task $task, int $attempt = 0): ?array + public function readPostOutput(Task $task, int $attempt = 1): ?array { if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) return $this->taskOutput[$task->getId()]['post'][$attempt]; diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index 8ebb22d..a24ca82 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -104,9 +104,14 @@ class RedisTaskStorage implements TaskStorage // Serialize the task and save it $taskData = serialize($task); - $this->conn->set($this->key_prefix . $taskId, $taskData); + + // Register the task $this->conn->sAdd($this->indexSet, $taskId); + // And create a hash for it + if ($this->conn->hSet($this->key_prefix . $taskId, 'data', $taskData) === FALSE) + return false; + return true; } @@ -129,7 +134,7 @@ class RedisTaskStorage implements TaskStorage // Go over each taskId and fetch the specific task $tasks = []; foreach ($taskList as $taskId) - $tasks[] = unserialize($this->conn->get($this->key_prefix . $taskId)); + $tasks[] = unserialize($this->conn->hGet($this->key_prefix . $taskId, 'data')); return $tasks; } @@ -146,7 +151,7 @@ class RedisTaskStorage implements TaskStorage // Fetch the task /** @var Task $task */ - $task = unserialize($this->conn->get($this->key_prefix . $identifier)); + $task = unserialize($this->conn->hGet($this->key_prefix . $identifier, 'data')); // Return the task return $task; @@ -163,7 +168,7 @@ class RedisTaskStorage implements TaskStorage // Check if it exists $isMember = $this->conn->sIsMember($this->indexSet, $taskId); if (!$isMember) - throw new TasksException("Could not modify task. Task '$taskId' already exists."); + throw new TasksException("Could not modify task. Task '$taskId' does not exists."); // Fire the TaskModifyEvent /** @var TaskModifyEvent $event */ @@ -176,7 +181,10 @@ class RedisTaskStorage implements TaskStorage // And write the data $taskData = serialize($event->getTask()); - return $this->conn->set($this->key_prefix . $taskId, $taskData); + if ($this->conn->hSet($this->key_prefix . $taskId, 'data', $taskData) === FALSE) + return false; + + return true; } /** @@ -191,24 +199,40 @@ class RedisTaskStorage implements TaskStorage // Check if it exists $isMember = $this->conn->sIsMember($this->indexSet, $taskId); if (!$isMember) - throw new TasksException("Could not modify task. Task '$taskId' already exists."); + throw new TasksException("Could not delete task. Task '$taskId' does not exists."); - // Delete the key - $this->conn->del($this->key_prefix . $taskId); + // Delete the task from the index $this->conn->sRem($this->indexSet, $taskId); - // Remove all task output and post output - $settings = $task->getRetrySettings(); - $maxRetries = $settings['maxRetries']; - for ($i=0;$i<=$maxRetries;$i++) - { - // First remove all possible task output - if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $i)) - $this->conn->del($this->key_prefix . $taskId . '_output_' . $i); + // Delete the task itself + if ($this->conn->del($this->key_prefix . $taskId) > 0) + return true; - if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $i)) - $this->conn->del($this->key_prefix . $taskId . '_post_' . $i); - } + return false; + } + + /** + * @inheritDoc + */ + public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode): bool + { + // First get the task ID + $taskId = $task->getId(); + + // Check if the task exists + $isMember = $this->conn->sIsMember($this->indexSet, $taskId); + if (!$isMember) + throw new TasksException("Could not write task output. Task '$taskId' not found."); + + // Prepare the data + $contents = ['taskId' => $taskId, 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode]; + + // Determine the attempt number + $attempt = $this->conn->hIncrBy($this->key_prefix . $taskId, 'taskOutputAttempts', 1); + + // Then write this output + if ($this->conn->hSet($this->key_prefix . $taskId, 'output' . $attempt, serialize($contents)) === FALSE) + return false; return true; } @@ -216,63 +240,43 @@ class RedisTaskStorage implements TaskStorage /** * @inheritDoc */ - public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + public function writePostOutput(Task $task, string $output, string $errors, int $statusCode): bool { // First get the task ID $taskId = $task->getId(); // Check if the task exists - $task = $this->getTaskById($taskId); + $isMember = $this->conn->sIsMember($this->indexSet, $taskId); + if (!$isMember) + throw new TasksException("Could not write post output. Task '$taskId' not found."); - // Check if the key already exists - if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt)) - throw new TasksException("Could not write task output. Output already written."); + // Prepare the data + $contents = ['taskId' => $taskId, 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode]; - // Prepare contents - $contents = ['taskId' => $task->getId(), 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode]; - $data = serialize($contents); + // Determine the attempt number + $attempt = $this->conn->hIncrBy($this->key_prefix . $taskId, 'taskPostAttempts', 1); - // Write contents - return $this->conn->set($this->key_prefix . $taskId . '_output_' . $attempt, $data); + // Then write this output + if ($this->conn->hSet($this->key_prefix . $taskId, 'postOutput' . $attempt, serialize($contents)) === FALSE) + return false; + + return true; } /** * @inheritDoc */ - public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool + public function readTaskOutput(Task $task, int $attempt = 1): ?array { // First get the task ID $taskId = $task->getId(); - // Check if the task exists - $task = $this->getTaskById($taskId); - - // Check if the key already exists - if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt)) - throw new TasksException("Could not write post output. Output already written."); - - // Prepare contents - $contents = ['taskId' => $task->getId(), 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode]; - $data = serialize($contents); - - // Write contents - return $this->conn->set($this->key_prefix . $taskId . '_post_' . $attempt, $data); - } - - /** - * @inheritDoc - */ - public function readTaskOutput(Task $task, int $attempt = 0): ?array - { - // First get the task ID - $taskId = $task->getId(); - - // Check if the key already exists - if (!$this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt)) + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'output' . $attempt)) return null; // Load and convert the data - $data = $this->conn->get($this->key_prefix . $taskId . '_output_' . $attempt); + $data = $this->conn->hGet($this->key_prefix . $taskId, 'output' . $attempt); $data = unserialize($data); return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; @@ -281,17 +285,17 @@ class RedisTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readPostOutput(Task $task, int $attempt = 0): ?array + public function readPostOutput(Task $task, int $attempt = 1): ?array { // First get the task ID $taskId = $task->getId(); - - // Check if the key already exists - if (!$this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt)) + + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'postOutput' . $attempt)) return null; // Load and convert the data - $data = $this->conn->get($this->key_prefix . $taskId . '_post_' . $attempt); + $data = $this->conn->hGet($this->key_prefix . $taskId, 'postOutput' . $attempt); $data = unserialize($data); return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index 27bf978..bcaf999 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -315,10 +315,10 @@ class TaskStorageTest extends TestCase // First write the task output $this->taskStorage->addTask($dummyTask); - $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0, 0)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0)); // Then try to read the output - $output = $this->taskStorage->readTaskOutput($dummyTask, 0); + $output = $this->taskStorage->readTaskOutput($dummyTask, 1); $this->assertEquals('output', $output['output']); $this->assertEquals('errors', $output['errors']); $this->assertEquals(0, $output['statusCode']); @@ -346,26 +346,25 @@ class TaskStorageTest extends TestCase $dummyTask = new Task('testWriteAndReadTaskOutputAttempts', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); - // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last - // to make sure the default is not selected by accident by the TaskStorage - $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output2', 'errors2', 102, 2)); - $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output0', 'errors0', 100, 0)); - $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output1', 'errors1', 101, 1)); + // Write the different outputs. + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output0', 'errors0', 100)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output1', 'errors1', 101)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output2', 'errors2', 102)); // Attempt to load the first output - $output0 = $this->taskStorage->readTaskOutput($dummyTask, 0); + $output0 = $this->taskStorage->readTaskOutput($dummyTask, 1); $this->assertEquals('output0', $output0['output']); $this->assertEquals('errors0', $output0['errors']); $this->assertEquals(100, $output0['statusCode']); // Attempt to load the second output - $output1 = $this->taskStorage->readTaskOutput($dummyTask, 1); + $output1 = $this->taskStorage->readTaskOutput($dummyTask, 2); $this->assertEquals('output1', $output1['output']); $this->assertEquals('errors1', $output1['errors']); $this->assertEquals(101, $output1['statusCode']); // Attempt to load the third output - $output2 = $this->taskStorage->readTaskOutput($dummyTask, 2); + $output2 = $this->taskStorage->readTaskOutput($dummyTask, 3); $this->assertEquals('output2', $output2['output']); $this->assertEquals('errors2', $output2['errors']); $this->assertEquals(102, $output2['statusCode']); @@ -377,23 +376,6 @@ class TaskStorageTest extends TestCase $this->assertEquals(100, $output['statusCode']); } - /** - * @depends testWriteAndReadTaskOutput - */ - public function testWriteAndReadTaskOutputAlreadyExists() - { - // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskOutputAlreadyExists', new EmptyHandler()); - $this->taskStorage->addTask($dummyTask); - - // Write a first time - $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 100, 0)); - - // And write it a second time - $this->expectException(TasksException::class); - $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 100, 0)); - } - /** * @depends testWriteAndReadTaskOutput */ @@ -418,10 +400,10 @@ class TaskStorageTest extends TestCase $this->taskStorage->addTask($dummyTask); // First write the task output - $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'postOutput', 'errors', 0, 0)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'postOutput', 'errors', 0)); // Then try to read the output - $output = $this->taskStorage->readPostOutput($dummyTask, 0); + $output = $this->taskStorage->readPostOutput($dummyTask, 1); $this->assertEquals('postOutput', $output['output']); $this->assertEquals('errors', $output['errors']); $this->assertEquals(0, $output['statusCode']); @@ -436,26 +418,26 @@ class TaskStorageTest extends TestCase $dummyTask = new Task('testWriteAndReadTaskPostOutputAttempts', new EmptyHandler()); $this->taskStorage->addTask($dummyTask); - // Write the different outputs. Done in a weird order to make sure the default is inserted not first or last - // to make sure the default is not selected by accident by the TaskStorage - $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output2', 'errors2', 102, 2)); - $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output0', 'errors0', 100, 0)); - $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output1', 'errors1', 101, 1)); + // Write the different outputs + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output0', 'errors0', 100)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output1', 'errors1', 101)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output2', 'errors2', 102)); + // Attempt to load the first output - $output0 = $this->taskStorage->readPostOutput($dummyTask, 0); + $output0 = $this->taskStorage->readPostOutput($dummyTask, 1); $this->assertEquals('output0', $output0['output']); $this->assertEquals('errors0', $output0['errors']); $this->assertEquals(100, $output0['statusCode']); // Attempt to load the second output - $output1 = $this->taskStorage->readPostOutput($dummyTask, 1); + $output1 = $this->taskStorage->readPostOutput($dummyTask, 2); $this->assertEquals('output1', $output1['output']); $this->assertEquals('errors1', $output1['errors']); $this->assertEquals(101, $output1['statusCode']); // Attempt to load the third output - $output2 = $this->taskStorage->readPostOutput($dummyTask, 2); + $output2 = $this->taskStorage->readPostOutput($dummyTask, 3); $this->assertEquals('output2', $output2['output']); $this->assertEquals('errors2', $output2['errors']); $this->assertEquals(102, $output2['statusCode']); @@ -467,23 +449,6 @@ class TaskStorageTest extends TestCase $this->assertEquals(100, $output['statusCode']); } - /** - * @depends testWriteAndReadTaskPostOutput - */ - public function testWriteAndReadTaskPostOutputAlreadyExists() - { - // Prepare a dummy task - $dummyTask = new Task('testWriteAndReadTaskPostOutputAlreadyExists', new EmptyHandler()); - $this->taskStorage->addTask($dummyTask); - - // Write a first time - $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output', 'errors', 100, 0)); - - // And write it a second time - $this->expectException(TasksException::class); - $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output', 'errors', 100, 0)); - } - /** * @depends testWriteAndReadTaskPostOutput */ -- 2.40.1 From fc83a931aebb5e3e4e983b0bb0fe549046b83e58 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Thu, 4 Jun 2020 21:58:22 +0200 Subject: [PATCH 29/33] Return all output when providing attempt = 0. When providing $attempt = 0 at readTaskOutput and readPostOutput, all output shall be returned. This is the default return. Hence, a lot of tests had to be altered slightly. --- .../Async/Supervisors/ParallelSuperVisor.php | 6 +- src/FuzeWorks/Async/TaskStorage.php | 10 ++- .../Async/TaskStorage/DummyTaskStorage.php | 28 ++++-- .../Async/TaskStorage/RedisTaskStorage.php | 88 +++++++++++++++---- test/base/ShellWorkerTest.php | 12 +-- test/base/TaskStorageTest.php | 60 +++++++++++-- 6 files changed, 161 insertions(+), 43 deletions(-) diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index 2d0425f..44d3c35 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -117,8 +117,9 @@ class ParallelSuperVisor implements SuperVisor // RUNNING: check if task is still running. If not, set result based on output elseif ($task->getStatus() === Task::RUNNING) { + // @todo Find a way to use the latest output $isRunning = $this->executor->getTaskRunning($task); - $output = $this->taskStorage->readTaskOutput($task); + $output = $this->taskStorage->readTaskOutput($task, 1); $hasOutput = !is_null($output); // If nothing is found, the process has crashed and status PFAILED should be set @@ -203,8 +204,9 @@ class ParallelSuperVisor implements SuperVisor // POST: when a task is currently running in it's postHandler elseif ($task->getStatus() === Task::POST) { + // @todo Find a way to use the latest output $isRunning = $this->executor->getTaskRunning($task); - $output = $this->taskStorage->readPostOutput($task); + $output = $this->taskStorage->readPostOutput($task, 1); $hasOutput = !is_null($output); // If a task is not running and has no output, an error has occurred diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index 47db727..f6be8c6 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -135,7 +135,8 @@ interface TaskStorage * array('output' => string $output, 'errors' => string $errors, 'status' => $code) * OR null if not found (yet) * - * $attempt refers to $task->getRetries(). If 0, it is the initial attempt. If > 0, it seeks a retry output. + * Attempt refers to the attempt that the task has ran and that individual output. + * If > 0, the output as mentioned above will be returned. If = 0, an array of such outputs will be returned. * * Returns null because that is a very valid response. Oftentimes output will need to be checked and its undesirable * to always throw an exception for expected behaviour. @@ -144,7 +145,7 @@ interface TaskStorage * @param int $attempt * @return array|null */ - public function readTaskOutput(Task $task, int $attempt = 1): ?array; + public function readTaskOutput(Task $task, int $attempt = 0): ?array; /** * Read the output from the postHandler @@ -153,7 +154,8 @@ interface TaskStorage * array('output' => string $output, 'errors' => string $errors, 'status' => $code) * OR null if not found (yet) * - * $attempt refers to $task->getRetries(). If 0, it is the initial attempt. If > 0, it seeks a retry output. + * Attempt refers to the attempt that the task has ran and that individual output. + * If > 0, the output as mentioned above will be returned. If = 0, an array of such outputs will be returned. * * Returns null because that is a very valid response. Oftentimes output will need to be checked and its undesirable * to always throw an exception for expected behaviour. @@ -162,7 +164,7 @@ interface TaskStorage * @param int $attempt * @return array|null */ - public function readPostOutput(Task $task, int $attempt = 1): ?array; + public function readPostOutput(Task $task, int $attempt = 0): ?array; /** * Reset the TaskStorage. diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index 9d6e87d..f0a47ca 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -235,10 +235,18 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readTaskOutput(Task $task, int $attempt = 1): ?array + public function readTaskOutput(Task $task, int $attempt = 0): ?array { - if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) - return $this->taskOutput[$task->getId()]['task'][$attempt]; + if ($attempt !== 0) + { + if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) + return $this->taskOutput[$task->getId()]['task'][$attempt]; + } + else + { + if (isset($this->taskOutput[$task->getId()]['task'])) + return $this->taskOutput[$task->getId()]['task']; + } return null; } @@ -246,10 +254,18 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readPostOutput(Task $task, int $attempt = 1): ?array + public function readPostOutput(Task $task, int $attempt = 0): ?array { - if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) - return $this->taskOutput[$task->getId()]['post'][$attempt]; + if ($attempt !== 0) + { + if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) + return $this->taskOutput[$task->getId()]['post'][$attempt]; + } + else + { + if (isset($this->taskOutput[$task->getId()]['post'])) + return $this->taskOutput[$task->getId()]['post']; + } return null; } diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index a24ca82..deab7c0 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -266,39 +266,91 @@ class RedisTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readTaskOutput(Task $task, int $attempt = 1): ?array + public function readTaskOutput(Task $task, int $attempt = 0): ?array { // First get the task ID $taskId = $task->getId(); - // Check if this output exists - if (!$this->conn->hExists($this->key_prefix . $taskId, 'output' . $attempt)) + if ($attempt !== 0) + { + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'output' . $attempt)) + return null; + + // Load and convert the data + $data = $this->conn->hGet($this->key_prefix . $taskId, 'output' . $attempt); + $data = unserialize($data); + + return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + else + { + // Get amount of attempts + $totalAttempts = $this->conn->hGet($this->key_prefix . $taskId, 'taskOutputAttempts'); + $output = []; + for ($i=1;$i<=$totalAttempts;$i++) + { + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'output' . $i)) + $output[] = null; + + // Load and convert the data + $data = $this->conn->hGet($this->key_prefix . $taskId, 'output' . $i); + $data = unserialize($data); + + $output[] = ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + + if (!empty($output)) + return $output; + return null; - - // Load and convert the data - $data = $this->conn->hGet($this->key_prefix . $taskId, 'output' . $attempt); - $data = unserialize($data); - - return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } } /** * @inheritDoc */ - public function readPostOutput(Task $task, int $attempt = 1): ?array + public function readPostOutput(Task $task, int $attempt = 0): ?array { // First get the task ID $taskId = $task->getId(); - // Check if this output exists - if (!$this->conn->hExists($this->key_prefix . $taskId, 'postOutput' . $attempt)) + if ($attempt !== 0) + { + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'postOutput' . $attempt)) + return null; + + // Load and convert the data + $data = $this->conn->hGet($this->key_prefix . $taskId, 'postOutput' . $attempt); + $data = unserialize($data); + + return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + else + { + // Get amount of attempts + $totalAttempts = $this->conn->hGet($this->key_prefix . $taskId, 'taskPostAttempts'); + $output = []; + for ($i=1;$i<=$totalAttempts;$i++) + { + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'postOutput' . $i)) + $output[] = null; + + // Load and convert the data + $data = $this->conn->hGet($this->key_prefix . $taskId, 'postOutput' . $i); + $data = unserialize($data); + + $output[] = ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + + if (!empty($output)) + return $output; + return null; - - // Load and convert the data - $data = $this->conn->hGet($this->key_prefix . $taskId, 'postOutput' . $attempt); - $data = unserialize($data); - - return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } } /** diff --git a/test/base/ShellWorkerTest.php b/test/base/ShellWorkerTest.php index 2817cb2..c56cb40 100644 --- a/test/base/ShellWorkerTest.php +++ b/test/base/ShellWorkerTest.php @@ -102,13 +102,13 @@ class ShellWorkerTest extends TestCase $this->shellWorker->run($dummyTask, false); // And verify if the Output is correctly set - $output = $this->taskStorage->readTaskOutput($dummyTask); + $output = $this->taskStorage->readTaskOutput($dummyTask, 1); $this->assertEquals('Some Output!', $output['output']); $this->assertEquals(Task::SUCCESS, $output['statusCode']); // And run the post handler $this->shellWorker->run($dummyTask, true); - $output = $this->taskStorage->readPostOutput($dummyTask); + $output = $this->taskStorage->readPostOutput($dummyTask, 1); $this->assertEquals('Post Output!', $output['output']); $this->assertEquals(Task::SUCCESS, $output['statusCode']); } @@ -136,13 +136,13 @@ class ShellWorkerTest extends TestCase $this->shellWorker->run($dummyTask, false); // And verify if the Output is correctly set - $output = $this->taskStorage->readTaskOutput($dummyTask); + $output = $this->taskStorage->readTaskOutput($dummyTask, 1); $this->assertEquals('', $output['output']); $this->assertEquals(Task::FAILED, $output['statusCode']); // And run a post failure $this->shellWorker->run($dummyTask, true); - $output = $this->taskStorage->readPostOutput($dummyTask); + $output = $this->taskStorage->readPostOutput($dummyTask, 1); $this->assertEquals('', $output['output']); $this->assertEquals(Task::FAILED, $output['statusCode']); } @@ -191,7 +191,7 @@ class ShellWorkerTest extends TestCase $this->shellWorker->run($dummyTask, false); // And verify if the Output is correctly set - $output = $this->taskStorage->readTaskOutput($dummyTask); + $output = $this->taskStorage->readTaskOutput($dummyTask, 1); $this->assertEquals('Child Output', $output['output']); $this->assertEquals(Task::SUCCESS, $output['statusCode']); } @@ -232,7 +232,7 @@ class ShellWorkerTest extends TestCase $this->shellWorker->run($dummyTask, false); // And verify whether the task has indeed failed - $output = $this->taskStorage->readTaskOutput($dummyTask); + $output = $this->taskStorage->readTaskOutput($dummyTask, 1); $this->assertEquals('Task failed successfully', $output['output']); $this->assertEquals(Task::FAILED, $output['statusCode']); } diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index bcaf999..dd54e7d 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -324,6 +324,27 @@ class TaskStorageTest extends TestCase $this->assertEquals(0, $output['statusCode']); } + /** + * @depends testWriteAndReadTaskOutput + */ + public function testWriteAndReadMultipleOutput() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadMultipleOutput', new EmptyHandler()); + + // Write some task output + $this->taskStorage->addTask($dummyTask); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output1', 'errors1', 0)); + $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output2', 'errors2', 0)); + + // Then try to read all the output + $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals([ + ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 0], + ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 0] + ], $output); + } + /** * @depends testWriteAndReadTaskOutput */ @@ -334,7 +355,7 @@ class TaskStorageTest extends TestCase // Write output while the task does not exist yet, expect exception $this->expectException(TasksException::class); - $this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0, 0); + $this->taskStorage->writeTaskOutput($dummyTask, 'output', 'errors', 0); } /** @@ -371,9 +392,11 @@ class TaskStorageTest extends TestCase // Attempt to load the default output $output = $this->taskStorage->readTaskOutput($dummyTask); - $this->assertEquals('output0', $output['output']); - $this->assertEquals('errors0', $output['errors']); - $this->assertEquals(100, $output['statusCode']); + $this->assertEquals([ + ['output' => 'output0', 'errors' => 'errors0', 'statusCode' => 100], + ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 101], + ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 102] + ], $output); } /** @@ -409,6 +432,27 @@ class TaskStorageTest extends TestCase $this->assertEquals(0, $output['statusCode']); } + /** + * @depends testWriteAndReadTaskOutput + */ + public function testWriteAndReadMultiplePostOutput() + { + // Prepare a dummy task + $dummyTask = new Task('testWriteAndReadMultiplePostOutput', new EmptyHandler()); + + // Write some task output + $this->taskStorage->addTask($dummyTask); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output1', 'errors1', 0)); + $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output2', 'errors2', 0)); + + // Then try to read all the output + $output = $this->taskStorage->readPostOutput($dummyTask); + $this->assertEquals([ + ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 0], + ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 0] + ], $output); + } + /** * @depends testWriteAndReadTaskPostOutput */ @@ -444,9 +488,11 @@ class TaskStorageTest extends TestCase // Attempt to load the default output $output = $this->taskStorage->readPostOutput($dummyTask); - $this->assertEquals('output0', $output['output']); - $this->assertEquals('errors0', $output['errors']); - $this->assertEquals(100, $output['statusCode']); + $this->assertEquals([ + ['output' => 'output0', 'errors' => 'errors0', 'statusCode' => 100], + ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 101], + ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 102] + ], $output); } /** -- 2.40.1 From d42e7f23ef5535391c025fd2a76c9c17cf42787b Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 5 Jun 2020 15:23:21 +0200 Subject: [PATCH 30/33] Implemented proper dependencies. Dependencies can now pass output to each other using the DependentTaskHandler. Also fixed some general problems in the Tasks class, for instance the Executor not starting correctly because of problematic parameters. Also, SuperVisor now sets the output of a Task using the last output of the task, and not the first. --- .../Async/Constraint/DependencyConstraint.php | 13 +- src/FuzeWorks/Async/Handler.php | 21 +- .../Async/Handler/DependentTaskHandler.php | 250 ++++++++++ .../Async/Supervisors/ParallelSuperVisor.php | 6 +- src/FuzeWorks/Async/Task.php | 3 + .../Async/TaskStorage/DummyTaskStorage.php | 30 +- .../Async/TaskStorage/RedisTaskStorage.php | 68 +-- src/FuzeWorks/Async/Tasks.php | 10 +- test/base/DependenciesTest.php | 453 ++++++++++++++++++ test/base/TaskStorageTest.php | 23 +- test/base/TaskTest.php | 14 + test/config.tasks.php | 5 +- test/mock/Handlers/ArgumentedHandler.php | 14 +- test/mock/Handlers/EmptyHandler.php | 13 +- .../Handlers/TestStartAndReadTasksHandler.php | 13 +- test/mock/Handlers/TestStopTaskHandler.php | 13 +- test/system/ParallelSuperVisorTest.php | 5 + 17 files changed, 878 insertions(+), 76 deletions(-) create mode 100644 src/FuzeWorks/Async/Handler/DependentTaskHandler.php create mode 100644 test/base/DependenciesTest.php diff --git a/src/FuzeWorks/Async/Constraint/DependencyConstraint.php b/src/FuzeWorks/Async/Constraint/DependencyConstraint.php index 426b1ee..ff78586 100644 --- a/src/FuzeWorks/Async/Constraint/DependencyConstraint.php +++ b/src/FuzeWorks/Async/Constraint/DependencyConstraint.php @@ -56,7 +56,7 @@ use FuzeWorks\Logger; class DependencyConstraint implements Constraint { - public $dependencies = []; + protected $dependencies = []; protected $delayTimes = 3; @@ -100,7 +100,6 @@ class DependencyConstraint implements Constraint // Get dependency $dependencyTask = $taskStorage->getTaskById($dependency); - // If the dependency task is completed, ignore it and continue to next dependency if ($dependencyTask->getStatus() === Task::COMPLETED) continue; @@ -146,6 +145,16 @@ class DependencyConstraint implements Constraint return time() + $this->delayTimes; } + /** + * Return a list of dependencies + * + * @return array + */ + public function getDependencies() + { + return $this->dependencies; + } + /** * Load the tasks library, so that dependencies can get scanned later * diff --git a/src/FuzeWorks/Async/Handler.php b/src/FuzeWorks/Async/Handler.php index f44c3db..6e3c570 100644 --- a/src/FuzeWorks/Async/Handler.php +++ b/src/FuzeWorks/Async/Handler.php @@ -38,6 +38,15 @@ namespace FuzeWorks\Async; interface Handler { + /** + * Gets invoked upon being added to the Task + * + * @param Task $task + * @return mixed + * @throws TasksException + */ + public function init(Task $task); + /** * Retrieve the parent handler that will first handle this task, before this child Handler * @@ -55,9 +64,9 @@ interface Handler /** * Import the parent output into the child * - * @param mixed $input + * @param string $input */ - public function setParentInput($input): void; + public function setParentInput(string $input): void; /** * The handler method used to handle this task. @@ -73,9 +82,9 @@ interface Handler /** * Any output generated by primaryHandler should be returned here. * - * @return mixed + * @return string */ - public function getOutput(); + public function getOutput(): string; /** * The handler method used after the primaryHandler if so requested @@ -90,8 +99,8 @@ interface Handler /** * Any output generated by postHandler should be returned here * - * @return mixed + * @return string */ - public function getPostOutput(); + public function getPostOutput(): string; } \ No newline at end of file diff --git a/src/FuzeWorks/Async/Handler/DependentTaskHandler.php b/src/FuzeWorks/Async/Handler/DependentTaskHandler.php new file mode 100644 index 0000000..e93e9fa --- /dev/null +++ b/src/FuzeWorks/Async/Handler/DependentTaskHandler.php @@ -0,0 +1,250 @@ +dependencyList = $dependencyList; + $this->delayTimes = $delayTimes; + } + + /** + * @inheritDoc + */ + public function init(Task $task) + { + if (!empty($this->dependencyList)) + $task->addConstraint(new DependencyConstraint($this->dependencyList, $this->delayTimes)); + } + + /** + * @inheritDoc + */ + public function primaryHandler(Task $task): bool + { + // First find all the dependencies + try { + $dependencies = $this->fetchDependencies($task); + $this->output = json_encode($dependencies); + return true; + } catch (TasksException $e) { + $this->output = 'Failed to fetch dependencies. TasksException: ' . $e->getMessage(); + return false; + } + } + + /** + * @inheritDoc + */ + public function getOutput(): string + { + return $this->output; + } + + /** + * @inheritDoc + */ + public function postHandler(Task $task) + { + // First find all the dependencies + try { + $dependencies = $this->fetchDependencies($task); + $this->output = json_encode($dependencies); + return true; + } catch (TasksException $e) { + $this->output = 'Failed to fetch dependencies. TasksException: ' . $e->getMessage(); + return false; + } + } + + /** + * @inheritDoc + */ + public function getPostOutput(): string + { + return $this->output; + } + + /** + * @inheritDoc + */ + public function getParentHandler(): ?Handler + { + return $this->parentHandler; + } + + /** + * @inheritDoc + */ + public function setParentHandler(Handler $parentHandler): void + { + $this->parentHandler = $parentHandler; + } + + /** + * @inheritDoc + */ + public function setParentInput(string $input): void + { + // Parent output gets set at this handler's output. + // Only if this class has something to intervene it will override the parent output + // Which should be always... but alas. + $this->output = $input; + } + + /** + * @param Task $task + * @return array + * @throws TasksException + */ + protected function fetchDependencies(Task $task): array + { + // When it receives the task, all dependencies should already be handled + // the primary handler will therefore connect with the DependencyConstraint and fetch dependencies + $constraints = $task->getConstraints(); + + // First prepare a list of dependencies + $dependencies = []; + $dependencyConstraints = []; + foreach ($constraints as $constraint) { + if ($constraint instanceof Constraint) + $dependencyConstraints[] = $constraint; + } + + // If no dependencies found, throw exception + if (empty($dependencyConstraints)) + return $dependencies; + + // Afterwards build a list of dependencies + /** @var DependencyConstraint $constraint */ + foreach ($dependencyConstraints as $constraint) { + foreach ($constraint->getDependencies() as $dependency) { + if (!isset($dependencies[$dependency])) + $dependencies[$dependency] = []; + } + } + + // Now that all dependencies are determined, fetch all the output + $tasks = $this->loadTasksLib(); + $taskStorage = $tasks->getTaskStorage(); + foreach ($dependencies as $dependency => $data) + { + // Fetch the task + try { + $dependencyTask = $taskStorage->getTaskById($dependency); + + // Then fetch all output + $dependencies[$dependency]['status'] = $dependencyTask->getStatus(); + $dependencies[$dependency]['output'] = $dependencyTask->getOutput(); + $dependencies[$dependency]['errors'] = $dependencyTask->getErrors(); + $dependencies[$dependency]['post'] = $dependencyTask->getPostOutput(); + $dependencies[$dependency]['postErrors'] = $dependencyTask->getPostErrors(); + } catch (TasksException $e) { + $dependencies[$dependency]['status'] = Task::FAILED; + $dependencies[$dependency]['output'] = null; + $dependencies[$dependency]['errors'] = 'Task not found.'; + $dependencies[$dependency]['post'] = null; + $dependencies[$dependency]['postErrors'] = null; + } + } + + return $dependencies; + } + + /** + * Load the tasks library, so that dependencies can get scanned later + * + * @return Tasks + * @throws TasksException + */ + private function loadTasksLib(): Tasks + { + try { + /** @var Libraries $libraries */ + $libraries = Factory::getInstance('libraries'); + + /** @var Tasks $tasks */ + $tasks = $libraries->get('async'); + + return $tasks; + } catch (FactoryException | LibraryException $e) { + throw new TasksException("Could not constrain task. Async library could not be loaded."); + } + } +} \ No newline at end of file diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index 44d3c35..2d0425f 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -117,9 +117,8 @@ class ParallelSuperVisor implements SuperVisor // RUNNING: check if task is still running. If not, set result based on output elseif ($task->getStatus() === Task::RUNNING) { - // @todo Find a way to use the latest output $isRunning = $this->executor->getTaskRunning($task); - $output = $this->taskStorage->readTaskOutput($task, 1); + $output = $this->taskStorage->readTaskOutput($task); $hasOutput = !is_null($output); // If nothing is found, the process has crashed and status PFAILED should be set @@ -204,9 +203,8 @@ class ParallelSuperVisor implements SuperVisor // POST: when a task is currently running in it's postHandler elseif ($task->getStatus() === Task::POST) { - // @todo Find a way to use the latest output $isRunning = $this->executor->getTaskRunning($task); - $output = $this->taskStorage->readPostOutput($task, 1); + $output = $this->taskStorage->readPostOutput($task); $hasOutput = !is_null($output); // If a task is not running and has no output, an error has occurred diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index b8a3d15..be800ca 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -209,6 +209,9 @@ class Task throw new TasksException("Could not create Task. Provided arguments are not serializable."); $this->arguments = $args; + + // Init the handler + $this->handler->init($this); } /** diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index f0a47ca..d99aa2d 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -237,16 +237,19 @@ class DummyTaskStorage implements TaskStorage */ public function readTaskOutput(Task $task, int $attempt = 0): ?array { - if ($attempt !== 0) + if (!isset($this->taskOutput[$task->getId()]['task'])) + return null; + + if ($attempt === 0) + $attempt = count($this->taskOutput[$task->getId()]['task']); + + if ($attempt === -1) + return $this->taskOutput[$task->getId()]['task']; + else { if (isset($this->taskOutput[$task->getId()]['task'][$attempt])) return $this->taskOutput[$task->getId()]['task'][$attempt]; } - else - { - if (isset($this->taskOutput[$task->getId()]['task'])) - return $this->taskOutput[$task->getId()]['task']; - } return null; } @@ -256,16 +259,19 @@ class DummyTaskStorage implements TaskStorage */ public function readPostOutput(Task $task, int $attempt = 0): ?array { - if ($attempt !== 0) + if (!isset($this->taskOutput[$task->getId()]['post'])) + return null; + + if ($attempt === 0) + $attempt = count($this->taskOutput[$task->getId()]['post']); + + if ($attempt === -1) + return $this->taskOutput[$task->getId()]['post']; + else { if (isset($this->taskOutput[$task->getId()]['post'][$attempt])) return $this->taskOutput[$task->getId()]['post'][$attempt]; } - else - { - if (isset($this->taskOutput[$task->getId()]['post'])) - return $this->taskOutput[$task->getId()]['post']; - } return null; } diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index deab7c0..e63dd89 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -271,19 +271,12 @@ class RedisTaskStorage implements TaskStorage // First get the task ID $taskId = $task->getId(); - if ($attempt !== 0) - { - // Check if this output exists - if (!$this->conn->hExists($this->key_prefix . $taskId, 'output' . $attempt)) - return null; + // If a nothing in particular is requested, fetch the latest and select that as the attempt + if ($attempt === 0) + $attempt = $this->conn->hGet($this->key_prefix . $taskId, 'taskOutputAttempts'); - // Load and convert the data - $data = $this->conn->hGet($this->key_prefix . $taskId, 'output' . $attempt); - $data = unserialize($data); - - return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; - } - else + // If -1 is requested, fetch all + if ($attempt === -1) { // Get amount of attempts $totalAttempts = $this->conn->hGet($this->key_prefix . $taskId, 'taskOutputAttempts'); @@ -303,9 +296,22 @@ class RedisTaskStorage implements TaskStorage if (!empty($output)) return $output; - - return null; } + // If a specific one is requested, fetch that one + else + { + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'output' . $attempt)) + return null; + + // Load and convert the data + $data = $this->conn->hGet($this->key_prefix . $taskId, 'output' . $attempt); + $data = unserialize($data); + + return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + + return null; } /** @@ -316,19 +322,12 @@ class RedisTaskStorage implements TaskStorage // First get the task ID $taskId = $task->getId(); - if ($attempt !== 0) - { - // Check if this output exists - if (!$this->conn->hExists($this->key_prefix . $taskId, 'postOutput' . $attempt)) - return null; + // If a nothing in particular is requested, fetch the latest and select that as the attempt + if ($attempt === 0) + $attempt = $this->conn->hGet($this->key_prefix . $taskId, 'taskPostAttempts'); - // Load and convert the data - $data = $this->conn->hGet($this->key_prefix . $taskId, 'postOutput' . $attempt); - $data = unserialize($data); - - return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; - } - else + // If -1 is requested, fetch all + if ($attempt === -1) { // Get amount of attempts $totalAttempts = $this->conn->hGet($this->key_prefix . $taskId, 'taskPostAttempts'); @@ -348,9 +347,22 @@ class RedisTaskStorage implements TaskStorage if (!empty($output)) return $output; - - return null; } + // If a specific one is requested, fetch that one + else + { + // Check if this output exists + if (!$this->conn->hExists($this->key_prefix . $taskId, 'postOutput' . $attempt)) + return null; + + // Load and convert the data + $data = $this->conn->hGet($this->key_prefix . $taskId, 'postOutput' . $attempt); + $data = unserialize($data); + + return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']]; + } + + return null; } /** diff --git a/src/FuzeWorks/Async/Tasks.php b/src/FuzeWorks/Async/Tasks.php index 1247d2e..9875d12 100644 --- a/src/FuzeWorks/Async/Tasks.php +++ b/src/FuzeWorks/Async/Tasks.php @@ -94,11 +94,10 @@ class Tasks implements iLibrary } /** - * @param string $bootstrapFile * @return SuperVisor * @throws TasksException */ - public function getSuperVisor(string $bootstrapFile): SuperVisor + public function getSuperVisor(): SuperVisor { if (isset($this->supervisor)) return $this->supervisor; @@ -116,7 +115,7 @@ class Tasks implements iLibrary $parameters = isset($cfg[$type]['parameters']) && is_array($cfg[$type]['parameters']) ? $cfg[$type]['parameters'] : []; // Then add the TaskStorage and Executor to the parameters - array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor($bootstrapFile)); + array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor()); // If the type does not exist, throw an exception if (!class_exists($class, true)) @@ -183,11 +182,10 @@ class Tasks implements iLibrary /** * Fetch the Executor based on the configured type * - * @param string $bootstrapFile * @return Executor * @throws TasksException */ - protected function getExecutor(string $bootstrapFile): Executor + protected function getExecutor(): Executor { if (isset($this->executor)) return $this->executor; @@ -209,7 +207,7 @@ class Tasks implements iLibrary throw new TasksException("Could not get Executor. Type of '$class' not found."); // And load the Executor and test if everything is in order - $object = new $class($bootstrapFile, $parameters); + $object = new $class($parameters); if (!$object instanceof Executor) throw new TasksException("Could not get Executor. Type '$class' is not instanceof Executor."); diff --git a/test/base/DependenciesTest.php b/test/base/DependenciesTest.php new file mode 100644 index 0000000..6cd00d7 --- /dev/null +++ b/test/base/DependenciesTest.php @@ -0,0 +1,453 @@ +tasks = Factory::getInstance('libraries')->get('async'); + $this->taskStorage = $this->tasks->getTaskStorage(); + $this->taskStorage->reset(); + + // Reset events + Events::$listeners = []; + } + + /* ---------------------------------- Test the dependency constraint ------------------ */ + + public function testHasConstrainedDeps() + { + // Create the dependent tasks + $depTask1 = new Task('depTask1', new EmptyHandler()); + $depTask2 = new Task('depTask2', new EmptyHandler()); + + // Write those dependencies to TaskStorage + $this->taskStorage->addTask($depTask1); + $this->taskStorage->addTask($depTask2); + + // Create the constraint + $constraint = new DependencyConstraint(['depTask1', 'depTask2']); + + // And a dummyTask to accompany + $dummyTask = new Task('dependentTask', new EmptyHandler()); + $dummyTask->addConstraint($constraint); + + // Test that the constraint is the same + $this->assertSame([$constraint], $dummyTask->getConstraints()); + + // And test the intervention + $this->assertTrue($constraint->intervene($dummyTask)); + $this->assertEquals(Task::DELAYED, $constraint->blockCode()); + $this->assertEquals(time() + 3, $constraint->delayTime()); + } + + /** + * @depends testHasConstrainedDeps + */ + public function testDelayTimes() + { + // Create the dependent tasks + $depTask1 = new Task('depTask1', new EmptyHandler()); + $depTask2 = new Task('depTask2', new EmptyHandler()); + + // Write those dependencies to TaskStorage + $this->taskStorage->addTask($depTask1); + $this->taskStorage->addTask($depTask2); + + // Create some useless dummy task + $dummyTask = new Task('dependentTask', new EmptyHandler()); + + // Create the constraints + // Default time + $constraintDef = new DependencyConstraint(['depTask1', 'depTask2']); + + // Modified time (30) + $constraintMod1 = new DependencyConstraint(['depTask1', 'depTask2'], 30); + + // And another (60) + $constraintMod2 = new DependencyConstraint(['depTask1', 'depTask2'], 60); + + // And intervene all of them + $this->assertTrue($constraintDef->intervene($dummyTask)); + $this->assertTrue($constraintMod1->intervene($dummyTask)); + $this->assertTrue($constraintMod2->intervene($dummyTask)); + + // And check the results + $this->assertEquals(Task::DELAYED, $constraintDef->blockCode()); + $this->assertEquals(Task::DELAYED, $constraintMod1->blockCode()); + $this->assertEquals(Task::DELAYED, $constraintMod2->blockCode()); + $this->assertEquals(time() + 3, $constraintDef->delayTime()); + $this->assertEquals(time() + 30, $constraintMod1->delayTime()); + $this->assertEquals(time() + 60, $constraintMod2->delayTime()); + } + + public function testHasFailedDeps() + { + // Create the dependent tasks + $depTask1 = new Task('depTask1', new EmptyHandler()); + $depTask2 = new Task('depTask2', new EmptyHandler()); + + // And set the first as completed, and second as failed + $depTask1->setStatus(Task::COMPLETED); + $depTask2->setStatus(Task::CANCELLED); + + // Write those dependencies to TaskStorage + $this->taskStorage->addTask($depTask1); + $this->taskStorage->addTask($depTask2); + + // Create the constraint + $constraint = new DependencyConstraint(['depTask1', 'depTask2']); + + // And a dummyTask to accompany + $dummyTask = new Task('dependentTask', new EmptyHandler()); + $dummyTask->addConstraint($constraint); + + // Test that the constraint is the same + $this->assertSame([$constraint], $dummyTask->getConstraints()); + + // And test the intervention + $this->assertTrue($constraint->intervene($dummyTask)); + $this->assertEquals(Task::CANCELLED, $constraint->blockCode()); + $this->assertEquals('Task cancelled due to failed dependency.', $dummyTask->getErrors()); + } + + public function testHasCompletedDeps() + { + // Create the dependent tasks + $depTask1 = new Task('depTask1', new EmptyHandler()); + $depTask2 = new Task('depTask2', new EmptyHandler()); + + // And set the first as completed, and second as failed + $depTask1->setStatus(Task::COMPLETED); + $depTask2->setStatus(Task::COMPLETED); + + // Write those dependencies to TaskStorage + $this->taskStorage->addTask($depTask1); + $this->taskStorage->addTask($depTask2); + + // Create the constraint + $constraint = new DependencyConstraint(['depTask1', 'depTask2']); + + // And a dummyTask to accompany + $dummyTask = new Task('dependentTask', new EmptyHandler()); + $dummyTask->addConstraint($constraint); + + // Test that the constraint is the same + $this->assertSame([$constraint], $dummyTask->getConstraints()); + + // And test the intervention + $this->assertFalse($constraint->intervene($dummyTask)); + } + + public function testGetDependencies() + { + $constraint = new DependencyConstraint(['someTask1', 'someTask2']); + $this->assertEquals(['someTask1', 'someTask2'], $constraint->getDependencies()); + } + + /* ---------------------------------- Test the dependent task handler ----------------- */ + + public function testAddedDependencies() + { + $handler = new DependentTaskHandler(['someTask1', 'someTask2']); + $dummyTask = new Task('someTask', $handler); + + // Check that the constraints match expectations + /** @var DependencyConstraint[] $constraints */ + $constraints = $dummyTask->getConstraints(); + $this->assertInstanceOf(DependencyConstraint::class, $constraints[0]); + + // And that the dependencies match + $this->assertEquals(['someTask1', 'someTask2'], $constraints[0]->getDependencies()); + } + + public function testPassingOutput() + { + // Create the dependent tasks + $depTask1 = new Task('someTask', new EmptyHandler()); + $depTask2 = new Task('someTask2', new EmptyHandler()); + + // Give the dependencies some output + $depTask1->setOutput('First Output', ''); + $depTask2->setOutput('Second Output', ''); + + // Write those to TaskStorage + $this->taskStorage->addTask($depTask1); + $this->taskStorage->addTask($depTask2); + + // Create the task + $handler = new DependentTaskHandler(['someTask', 'someTask2']); + + // Create a dummy Task + $dummyTask = new Task('someTask3', $handler); + + // Assert that all is well + $this->assertTrue($handler->primaryHandler($dummyTask)); + + // And test the handler's output + $this->assertEquals(json_encode([ + 'someTask' => [ + 'status' => Task::PENDING, + 'output' => 'First Output', + 'errors' => '', + 'post' => null, + 'postErrors' => null + ], + 'someTask2' => [ + 'status' => Task::PENDING, + 'output' => 'Second Output', + 'errors' => '', + 'post' => null, + 'postErrors' => null + ] + ]), $handler->getOutput()); + + // And test the post handler + $this->assertTrue($handler->postHandler($dummyTask)); + $this->assertEquals(json_encode([ + 'someTask' => [ + 'status' => Task::PENDING, + 'output' => 'First Output', + 'errors' => '', + 'post' => null, + 'postErrors' => null + ], + 'someTask2' => [ + 'status' => Task::PENDING, + 'output' => 'Second Output', + 'errors' => '', + 'post' => null, + 'postErrors' => null + ] + ]), $handler->getPostOutput()); + } + + /** + * @depends testPassingOutput + */ + public function testMissingDependency() + { + // Create the task + $handler = new DependentTaskHandler(['someTask']); + + // Create a dummy Task + $dummyTask = new Task('someTask2', $handler); + + // Assert that all is well + $this->assertTrue($handler->primaryHandler($dummyTask)); + + // And test the handler's output + $this->assertEquals(json_encode([ + 'someTask' => [ + 'status' => Task::FAILED, + 'output' => null, + 'errors' => 'Task not found.', + 'post' => null, + 'postErrors' => null + ], + ]), $handler->getOutput()); + + // And test the post handler + $this->assertTrue($handler->postHandler($dummyTask)); + $this->assertEquals(json_encode([ + 'someTask' => [ + 'status' => Task::FAILED, + 'output' => null, + 'errors' => 'Task not found.', + 'post' => null, + 'postErrors' => null + ], + ]), $handler->getPostOutput()); + } + + /** + * @depends testPassingOutput + */ + public function testNoDepedencies() + { + // Create the task + $handler = new DependentTaskHandler([]); + + // Create a dummy Task + $dummyTask = new Task('someTask', $handler); + + // Assert that all is well + $this->assertTrue($handler->primaryHandler($dummyTask)); + $this->assertEquals(json_encode([]), $handler->getOutput()); + + // And test the post handler + $this->assertTrue($handler->postHandler($dummyTask)); + $this->assertEquals(json_encode([]), $handler->getPostOutput()); + } + + public function testParentHandler() + { + // Test pass output + $handler = new DependentTaskHandler([]); + $handler->setParentInput('Passed Input'); + $this->assertEquals('Passed Input', $handler->getOutput()); + + // Test passing a handler + $handler = new DependentTaskHandler([]); + $parentHandler = $this->createMock(Handler::class); + $handler->setParentHandler($parentHandler); + $this->assertSame($parentHandler, $handler->getParentHandler()); + } + + public function testPassDependencyOutput() + { + // Build all systems for this test + $superVisor = $this->tasks->getSuperVisor(); + + // Create the dependency + $dependency = new Task('dependency', new ArgumentedHandler(0, 'Prepared Output')); + + // Write the task to TaskStorage + $this->taskStorage->addTask($dependency); + + // Now create the dependent task + $dependent = new Task('dependent', new DependentTaskHandler(['dependency'], 2)); + + // And write that task to TaskStorage + $this->taskStorage->addTask($dependent); + + // Now we make the SuperVisor cycle, to start the dependency and set the dependent to WAIT + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + + // Assert that everything is running + $this->assertEquals(Task::RUNNING, + $this->taskStorage->getTaskById($dependency->getId())->getStatus() + ); + $this->assertEquals(Task::DELAYED, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + + // Give the task some time to finish + usleep(500000); + + // And re-run the SuperVisor + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + + // Now check the tasks again. Dependency should be finished and have output, + // whereas dependent should still be delayed + $this->assertEquals(Task::SUCCESS, + $this->taskStorage->getTaskById($dependency->getId())->getStatus() + ); + $this->assertEquals(Task::DELAYED, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + + // Cycle again and see the dependency be completed, and dependent still delayed + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::COMPLETED, + $this->taskStorage->getTaskById($dependency->getId())->getStatus() + ); + $this->assertEquals(Task::DELAYED, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + + // Also check that output is correct + $this->assertEquals('Prepared Output', + $this->taskStorage->getTaskById($dependency->getId())->getOutput() + ); + + // Now wait long enough for the delay to be finished + usleep(2500000); + + // Now cycle again, and expect the task to be pending + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::PENDING, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + + // Cycle again and expect it to be running + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::RUNNING, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + + // Give the task some time to finish + usleep(500000); + + // And cycle again and expect the task to have succeeded + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::SUCCESS, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + + // Cycle again and expect the task to have completed, and test its output + $this->assertEquals(SuperVisor::FINISHED, $superVisor->cycle()); + $this->assertEquals(Task::COMPLETED, + $this->taskStorage->getTaskById($dependent->getId())->getStatus() + ); + $this->assertEquals( + json_encode(['dependency' => [ + 'status' => Task::COMPLETED, + 'output' => 'Prepared Output', + 'errors' => '', + 'post' => null, + 'postErrors' => null + ]]), + $this->taskStorage->getTaskById($dependent->getId())->getOutput() + ); + } +} diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index dd54e7d..7bc3502 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -338,11 +338,14 @@ class TaskStorageTest extends TestCase $this->assertTrue($this->taskStorage->writeTaskOutput($dummyTask, 'output2', 'errors2', 0)); // Then try to read all the output - $output = $this->taskStorage->readTaskOutput($dummyTask); + $output = $this->taskStorage->readTaskOutput($dummyTask, -1); $this->assertEquals([ ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 0], ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 0] ], $output); + + // Then try and read the latest + $this->assertEquals(['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 0], $this->taskStorage->readTaskOutput($dummyTask)); } /** @@ -392,11 +395,16 @@ class TaskStorageTest extends TestCase // Attempt to load the default output $output = $this->taskStorage->readTaskOutput($dummyTask); + $this->assertEquals('output2', $output['output']); + $this->assertEquals('errors2', $output['errors']); + $this->assertEquals(102, $output['statusCode']); + + // And to load all $this->assertEquals([ ['output' => 'output0', 'errors' => 'errors0', 'statusCode' => 100], ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 101], ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 102] - ], $output); + ], $this->taskStorage->readTaskOutput($dummyTask, -1)); } /** @@ -446,11 +454,13 @@ class TaskStorageTest extends TestCase $this->assertTrue($this->taskStorage->writePostOutput($dummyTask, 'output2', 'errors2', 0)); // Then try to read all the output - $output = $this->taskStorage->readPostOutput($dummyTask); + $output = $this->taskStorage->readPostOutput($dummyTask, -1); $this->assertEquals([ ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 0], ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 0] ], $output); + + $this->assertEquals(['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 0], $this->taskStorage->readPostOutput($dummyTask)); } /** @@ -488,11 +498,16 @@ class TaskStorageTest extends TestCase // Attempt to load the default output $output = $this->taskStorage->readPostOutput($dummyTask); + $this->assertEquals('output2', $output['output']); + $this->assertEquals('errors2', $output['errors']); + $this->assertEquals(102, $output['statusCode']); + + // And to load all $this->assertEquals([ ['output' => 'output0', 'errors' => 'errors0', 'statusCode' => 100], ['output' => 'output1', 'errors' => 'errors1', 'statusCode' => 101], ['output' => 'output2', 'errors' => 'errors2', 'statusCode' => 102] - ], $output); + ], $this->taskStorage->readPostOutput($dummyTask, -1)); } /** diff --git a/test/base/TaskTest.php b/test/base/TaskTest.php index 9215f0a..cebada8 100644 --- a/test/base/TaskTest.php +++ b/test/base/TaskTest.php @@ -35,6 +35,7 @@ */ use FuzeWorks\Async\Constraint; +use FuzeWorks\Async\Handler; use FuzeWorks\Async\Task; use FuzeWorks\Async\TasksException; use Mock\Handlers\EmptyHandler; @@ -111,6 +112,19 @@ class TaskTest extends TestCase $this->assertEquals([$stub], $dummyTask->getConstraints()); } + /** + * @depends testBaseVariables + */ + public function testInitHandler() + { + $mockHandler = $this->createMock(Handler::class); + $mockHandler->expects($this->once())->method('init') + ->with($this->callback(function($subject){return $subject instanceof Task;})); + + // Then create a class + new Task('testInitHandler', $mockHandler); + } + /** * @depends testBaseVariables */ diff --git a/test/config.tasks.php b/test/config.tasks.php index 393ebe6..e3190f1 100644 --- a/test/config.tasks.php +++ b/test/config.tasks.php @@ -83,8 +83,9 @@ return array( 'ShellExecutor' => [ 'parameters' => [ 'workerFile' => Core::getEnv('EXECUTOR_SHELL_WORKER', - dirname(__FILE__) . DS . 'bin' . DS . 'worker'), - 'bootstrapFile' => Core::getEnv('EXECUTOR_SHELL_BOOTSTRAP', 'unknown') + dirname(__FILE__, 2) . DS . 'bin' . DS . 'worker'), + 'bootstrapFile' => Core::getEnv('EXECUTOR_SHELL_BOOTSTRAP', + dirname(__FILE__) . DS . 'bootstrap.php') ] ] ] diff --git a/test/mock/Handlers/ArgumentedHandler.php b/test/mock/Handlers/ArgumentedHandler.php index dd6d458..3b0ae46 100644 --- a/test/mock/Handlers/ArgumentedHandler.php +++ b/test/mock/Handlers/ArgumentedHandler.php @@ -37,6 +37,7 @@ namespace Mock\Handlers; use FuzeWorks\Async\Handler; use FuzeWorks\Async\Task; +use FuzeWorks\Async\TasksException; class ArgumentedHandler implements Handler { @@ -50,6 +51,13 @@ class ArgumentedHandler implements Handler $this->output = $output; } + /** + * @inheritDoc + */ + public function init(Task $task) + { + } + /** * @inheritDoc */ @@ -62,7 +70,7 @@ class ArgumentedHandler implements Handler /** * @inheritDoc */ - public function getOutput() + public function getOutput(): string { return $this->output; } @@ -79,7 +87,7 @@ class ArgumentedHandler implements Handler /** * @inheritDoc */ - public function getPostOutput() + public function getPostOutput(): string { return $this->output; } @@ -95,7 +103,7 @@ class ArgumentedHandler implements Handler /** * @inheritDoc */ - public function setParentInput($input): void + public function setParentInput(string $input): void { } diff --git a/test/mock/Handlers/EmptyHandler.php b/test/mock/Handlers/EmptyHandler.php index 71cf04d..30c5636 100644 --- a/test/mock/Handlers/EmptyHandler.php +++ b/test/mock/Handlers/EmptyHandler.php @@ -41,6 +41,13 @@ use FuzeWorks\Async\Task; class EmptyHandler implements Handler { + /** + * @inheritDoc + */ + public function init(Task $task) + { + } + /** * @inheritDoc */ @@ -52,7 +59,7 @@ class EmptyHandler implements Handler /** * @inheritDoc */ - public function getOutput() + public function getOutput(): string { } @@ -66,7 +73,7 @@ class EmptyHandler implements Handler /** * @inheritDoc */ - public function getPostOutput() + public function getPostOutput(): string { } @@ -81,7 +88,7 @@ class EmptyHandler implements Handler /** * @inheritDoc */ - public function setParentInput($input): void + public function setParentInput(string $input): void { } diff --git a/test/mock/Handlers/TestStartAndReadTasksHandler.php b/test/mock/Handlers/TestStartAndReadTasksHandler.php index eff209c..540d4db 100644 --- a/test/mock/Handlers/TestStartAndReadTasksHandler.php +++ b/test/mock/Handlers/TestStartAndReadTasksHandler.php @@ -41,6 +41,13 @@ use FuzeWorks\Async\Task; class TestStartAndReadTasksHandler implements Handler { + /** + * @inheritDoc + */ + public function init(Task $task) + { + } + /** * @inheritDoc */ @@ -53,7 +60,7 @@ class TestStartAndReadTasksHandler implements Handler /** * @inheritDoc */ - public function getOutput() + public function getOutput(): string { return "Valid Output"; } @@ -69,7 +76,7 @@ class TestStartAndReadTasksHandler implements Handler /** * @inheritDoc */ - public function getPostOutput() + public function getPostOutput(): string { } @@ -85,7 +92,7 @@ class TestStartAndReadTasksHandler implements Handler /** * @inheritDoc */ - public function setParentInput($input): void + public function setParentInput(string $input): void { } diff --git a/test/mock/Handlers/TestStopTaskHandler.php b/test/mock/Handlers/TestStopTaskHandler.php index 8182f77..cebe04a 100644 --- a/test/mock/Handlers/TestStopTaskHandler.php +++ b/test/mock/Handlers/TestStopTaskHandler.php @@ -41,6 +41,13 @@ use FuzeWorks\Async\Task; class TestStopTaskHandler implements Handler { + /** + * @inheritDoc + */ + public function init(Task $task) + { + } + /** * @inheritDoc */ @@ -53,7 +60,7 @@ class TestStopTaskHandler implements Handler /** * @inheritDoc */ - public function getOutput() + public function getOutput(): string { return "Valid Output"; } @@ -68,7 +75,7 @@ class TestStopTaskHandler implements Handler /** * @inheritDoc */ - public function getPostOutput() + public function getPostOutput(): string { } @@ -83,7 +90,7 @@ class TestStopTaskHandler implements Handler /** * @inheritDoc */ - public function setParentInput($input): void + public function setParentInput(string $input): void { } diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index db44e4f..e90613e 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -45,6 +45,11 @@ use FuzeWorks\Events; use Mock\Handlers\ArgumentedHandler; use PHPUnit\Framework\TestCase; +/** + * Class ParallelSuperVisorTest + * + * @todo Add test that latest output is added to Task, and not just 'any' output + */ class ParallelSuperVisorTest extends TestCase { -- 2.40.1 From 20994674a4eee09a8c396c997c3820a9cf09adb4 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Fri, 5 Jun 2020 16:35:15 +0200 Subject: [PATCH 31/33] Finished tasks are no longer loaded in SuperVisor. By adding a parameter in TaskStorage, it is now possible to distinguish between finished and unfinished tasks. Finished tasks are those tasks that have a status of Task::COMPLETED or Task::CANCELLED. Unfinished tasks are all others. This allows the SuperVisor to not bother with the mountain of tasks that will be saved during large projects. --- .../Async/Supervisors/ParallelSuperVisor.php | 3 +- src/FuzeWorks/Async/Task.php | 21 +++++++------- src/FuzeWorks/Async/TaskStorage.php | 12 +++----- .../Async/TaskStorage/DummyTaskStorage.php | 19 +++++++------ .../Async/TaskStorage/RedisTaskStorage.php | 28 ++++++++++++------- test/base/TaskStorageTest.php | 27 +++++++++++++++++- 6 files changed, 70 insertions(+), 40 deletions(-) diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index 2d0425f..c518883 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -70,8 +70,7 @@ class ParallelSuperVisor implements SuperVisor public function cycle(): int { // First: if there are no tasks, load them - $this->taskStorage->refreshTasks(); - $this->tasks = $this->taskStorage->readTasks(); + $this->tasks = $this->taskStorage->readTasks(true); // If there are still no tasks, nothing is queued, so this cycle can end. if (empty($this->tasks)) diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index be800ca..d86664f 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -115,6 +115,16 @@ class Task */ protected $taskId; + /** + * @var int + */ + protected $status = Task::PENDING; + + /** + * @var array + */ + protected $arguments; + /** * @var Handler */ @@ -125,16 +135,6 @@ class Task */ protected $usePostHandler = false; - /** - * @var array - */ - protected $arguments; - - /** - * @var int - */ - protected $status = Task::PENDING; - /** * @var Constraint[] */ @@ -389,7 +389,6 @@ class Task return $this->postErrors; } - /** * @param string $output * @param string $errors diff --git a/src/FuzeWorks/Async/TaskStorage.php b/src/FuzeWorks/Async/TaskStorage.php index f6be8c6..f0ffafe 100644 --- a/src/FuzeWorks/Async/TaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage.php @@ -58,14 +58,12 @@ interface TaskStorage /** * Retrieves a list of all tasks logged in the system * + * When using $noIncludeDone, Tasks that are Completed and Cancelled will not be returned. + * + * @param bool $noIncludeDone * @return Task[] */ - public function readTasks(): array; - - /** - * Reload the list of all tasks. - */ - public function refreshTasks(); + public function readTasks(bool $noIncludeDone = false): array; /** * Retrieves an individual task by its identifier @@ -107,7 +105,6 @@ interface TaskStorage * @param string $output * @param string $errors * @param int $statusCode - * @param int $attempt * @return bool * @throws TasksException */ @@ -122,7 +119,6 @@ interface TaskStorage * @param string $output * @param string $errors * @param int $statusCode - * @param int $attempt * @return bool * @throws TasksException */ diff --git a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php index d99aa2d..460b860 100644 --- a/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/DummyTaskStorage.php @@ -100,16 +100,19 @@ class DummyTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readTasks(): array + public function readTasks(bool $noIncludeDone = false): array { - return $this->tasks; - } + if ($noIncludeDone === false) + return $this->tasks; - /** - * @inheritDoc - */ - public function refreshTasks() - {// Ignore + $tasks = []; + foreach ($this->tasks as $task) + { + if ($task->getStatus() !== Task::COMPLETED && $task->getStatus() !== Task::CANCELLED) + $tasks[] = $task; + } + + return $tasks; } /** diff --git a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php index e63dd89..1f6d3b3 100644 --- a/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php @@ -53,6 +53,7 @@ class RedisTaskStorage implements TaskStorage protected $conn; protected $indexSet = 'async_index'; + protected $unfinishedSet = 'async_index_unfinished'; protected $key_prefix = 'async_task_'; /** @@ -107,6 +108,8 @@ class RedisTaskStorage implements TaskStorage // Register the task $this->conn->sAdd($this->indexSet, $taskId); + if ($task->getStatus() !== Task::COMPLETED && $task->getStatus() !== Task::CANCELLED) + $this->conn->sAdd($this->unfinishedSet, $taskId); // And create a hash for it if ($this->conn->hSet($this->key_prefix . $taskId, 'data', $taskData) === FALSE) @@ -118,18 +121,13 @@ class RedisTaskStorage implements TaskStorage /** * @inheritDoc */ - public function readTasks(): array - { - return $this->refreshTasks(); - } - - /** - * @inheritDoc - */ - public function refreshTasks() + public function readTasks(bool $noIncludeDone = false): array { // First fetch an array of all tasks in the set - $taskList = $this->conn->sMembers($this->indexSet); + if ($noIncludeDone) + $taskList = $this->conn->sMembers($this->unfinishedSet); + else + $taskList = $this->conn->sMembers($this->indexSet); // Go over each taskId and fetch the specific task $tasks = []; @@ -184,6 +182,12 @@ class RedisTaskStorage implements TaskStorage if ($this->conn->hSet($this->key_prefix . $taskId, 'data', $taskData) === FALSE) return false; + // Modify the unfinished set + if ($this->conn->sIsMember($this->unfinishedSet, $taskId) && ($task->getStatus() === Task::COMPLETED || $task->getStatus() === Task::CANCELLED )) + $this->conn->sRem($this->unfinishedSet, $taskId); + elseif (!$this->conn->sIsMember($this->unfinishedSet, $taskId) && $task->getStatus() !== Task::COMPLETED && $task->getStatus() !== Task::CANCELLED) + $this->conn->sAdd($this->unfinishedSet, $taskId); + return true; } @@ -204,6 +208,10 @@ class RedisTaskStorage implements TaskStorage // Delete the task from the index $this->conn->sRem($this->indexSet, $taskId); + // And remove the task from the unfinishedSet + if ($this->conn->sIsMember($this->unfinishedSet, $taskId)) + $this->conn->sRem($this->unfinishedSet, $taskId); + // Delete the task itself if ($this->conn->del($this->key_prefix . $taskId) > 0) return true; diff --git a/test/base/TaskStorageTest.php b/test/base/TaskStorageTest.php index 7bc3502..d3e4b6d 100644 --- a/test/base/TaskStorageTest.php +++ b/test/base/TaskStorageTest.php @@ -39,7 +39,6 @@ use FuzeWorks\Async\Task; use FuzeWorks\Async\Tasks; use FuzeWorks\Async\TasksException; use FuzeWorks\Async\TaskStorage; -use FuzeWorks\Async\TaskStorage\DummyTaskStorage; use FuzeWorks\Events; use FuzeWorks\Priority; use Mock\Handlers\EmptyHandler; @@ -99,6 +98,32 @@ class TaskStorageTest extends TestCase $this->assertInstanceOf(EmptyHandler::class, $task->getHandler()); } + /** + * @depends testAddAndReadTasks + */ + public function testReadUnfinishedTasks() + { + // Add the tasks + $finishedTask = new Task('finishedTask', new EmptyHandler()); + $finishedTask->setStatus(Task::COMPLETED); + $unfinishedTask1 = new Task('unfinishedTask1', new EmptyHandler()); + $unfinishedTask2 = new Task('unfinishedTask2', new EmptyHandler()); + + // Nothing is written yet so it should be empty + $this->assertEmpty($this->taskStorage->readTasks()); + + // Write the tasks to TaskStorage + $this->taskStorage->addTask($finishedTask); + $this->taskStorage->addTask($unfinishedTask1); + $this->taskStorage->addTask($unfinishedTask2); + + // And check whether they get properly read + $this->assertCount(3, $this->taskStorage->readTasks()); + + // And whether the finished task gets omitted in the unfinished list + $this->assertCount(2, $this->taskStorage->readTasks(true)); + } + /** * @depends testAddAndReadTasks */ -- 2.40.1 From ee5312fa1b9414443a43aee722dc2d7066aa23b3 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Sat, 6 Jun 2020 00:11:27 +0200 Subject: [PATCH 32/33] Started work on making tasks forcefully quit after a maximum time has expired. --- bin/supervisor | 2 +- .../Async/Supervisors/ParallelSuperVisor.php | 16 ++-- src/FuzeWorks/Async/Task.php | 79 +++++++++++++++++-- .../Async/TaskStorage/ArrayTaskStorage.php | 4 +- test/base/TaskTest.php | 12 +-- test/system/ParallelSuperVisorTest.php | 12 +-- 6 files changed, 95 insertions(+), 30 deletions(-) diff --git a/bin/supervisor b/bin/supervisor index 5abc47b..0542413 100644 --- a/bin/supervisor +++ b/bin/supervisor @@ -109,7 +109,7 @@ try { // And finally, run the supervisor try { $supervisor = $lib->getSuperVisor($bootstrap); - while ($supervisor->cycle() === SuperVisor::RUNNING) { + while ($supervisor->cycle() !== SuperVisor::RUNNING) { usleep(250000); } diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index c518883..e043976 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -93,6 +93,7 @@ class ParallelSuperVisor implements SuperVisor // Start the process using the executor service $task = $this->executor->startTask($task); $task->setStatus(Task::RUNNING); + $task->startTaskTime(); // Modify the task in TaskStorage $this->taskStorage->modifyTask($task); @@ -107,12 +108,6 @@ class ParallelSuperVisor implements SuperVisor fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } - // CANCELLED/COMPLETED: remove the task if requested to do so - elseif ($task->getStatus() === Task::COMPLETED || $task->getStatus() === Task::CANCELLED) - { - // @todo Remove old tasks automatically - } - // RUNNING: check if task is still running. If not, set result based on output elseif ($task->getStatus() === Task::RUNNING) { @@ -140,6 +135,7 @@ class ParallelSuperVisor implements SuperVisor continue; // If any changes have been made, they should be written to TaskStorage + $task->endTaskTime(); $this->taskStorage->modifyTask($task); fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } @@ -148,7 +144,7 @@ class ParallelSuperVisor implements SuperVisor elseif ($task->getStatus() === Task::PFAILED || $task->getStatus() === Task::FAILED) { // First fetch retry conditions - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); // First test if any retries should be tried at all if ($settings['retryOnFail'] === true && $task->getRetries() < $settings['maxRetries']) @@ -163,6 +159,7 @@ class ParallelSuperVisor implements SuperVisor $task->addRetry(); $task = $this->executor->startTask($task); $task->setStatus(Task::RUNNING); + $task->startTaskTime(); $this->taskStorage->modifyTask($task); fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); continue; @@ -174,6 +171,7 @@ class ParallelSuperVisor implements SuperVisor { $task->resetRetries(); $task = $this->executor->startTask($task, true); + $task->startPostTime(); $task->setStatus(Task::POST); } else @@ -190,6 +188,7 @@ class ParallelSuperVisor implements SuperVisor { $task->resetRetries(); $task = $this->executor->startTask($task, true); + $task->startPostTime(); $task->setStatus(Task::POST); } else @@ -210,7 +209,7 @@ class ParallelSuperVisor implements SuperVisor if (!$isRunning && !$hasOutput) { // Test if a retry should be attempted - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); if ($settings['retryOnFail'] === true && $settings['retryPostFailures'] === true && $settings['maxRetries'] > $task->getRetries()) { $task->addRetry(); @@ -233,6 +232,7 @@ class ParallelSuperVisor implements SuperVisor continue; // If any changes have been made, they should be written to TaskStorage + $task->endPostTime(); $this->taskStorage->modifyTask($task); fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index d86664f..2da3945 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -178,6 +178,7 @@ class Task protected $retryRFailures = true; protected $retryPostFailures = true; protected $retries = 0; + protected $maxTime = 30; /** * Task constructor. @@ -316,6 +317,8 @@ class Task return $this->delayTime; } + /* ---------------------------------- Attributes setters and getters ------------------ */ + /** * Fetch an attribute of this task * @@ -359,6 +362,8 @@ class Task unset($this->attributes[$key]); } + /* ---------------------------------- Output setters and getters ---------------------- */ + /** * Return the output of this task execution * @@ -411,22 +416,26 @@ class Task $this->postErrors = $errors; } + /* ---------------------------------- Failure settings and criteria ------------------- */ + /** * Set whether this task should retry after a failure, and how many times * - * @param bool $retryOnFail - * @param int $maxRetries - * @param bool $retryRegularFailures - * @param bool $retryProcessFailures - * @param bool $retryPostFailures + * @param bool $retryOnFail Whether this task should be retried if failing + * @param int $maxRetries How many times the task should be retried + * @param int $maxTime How long a task may run before it shall be forcefully shut down + * @param bool $retryRegularFailures Whether regular Task::FAILED should be retried + * @param bool $retryProcessFailures Whether process based Task::PFAILED should be retried + * @param bool $retryPostFailures Whether failures during the Task::POST phase should be retried. */ - public function setRetrySettings(bool $retryOnFail, int $maxRetries = 2, bool $retryRegularFailures = true, bool $retryProcessFailures = true, bool $retryPostFailures = true) + public function setSettings(bool $retryOnFail, int $maxRetries = 2, int $maxTime = 30, bool $retryRegularFailures = true, bool $retryProcessFailures = true, bool $retryPostFailures = true) { $this->retryOnFail = $retryOnFail; $this->maxRetries = $maxRetries; $this->retryPFailures = $retryProcessFailures; $this->retryRFailures = $retryRegularFailures; $this->retryPostFailures = $retryPostFailures; + $this->maxTime = $maxTime; } /** @@ -434,17 +443,20 @@ class Task * * @return array */ - public function getRetrySettings(): array + public function getSettings(): array { return [ 'retryOnFail' => $this->retryOnFail, 'maxRetries' => $this->maxRetries, 'retryPFailures' => $this->retryPFailures, 'retryRFailures' => $this->retryRFailures, - 'retryPostFailures' => $this->retryPostFailures + 'retryPostFailures' => $this->retryPostFailures, + 'maxTime' => $this->maxTime ]; } + /* ---------------------------------- Retries and attempts registers ------------------ */ + /** * Add a retry to the retry counter */ @@ -471,6 +483,57 @@ class Task return $this->retries; } + /* ---------------------------------- Runtime data getters and setters----------------- */ + + protected $taskStartTime; + protected $taskEndTime; + protected $postStartTime; + protected $postEndTime; + + public function startTaskTime() + { + $this->taskEndTime = null; + $this->taskStartTime = time(); + } + + public function endTaskTime() + { + $this->taskEndTime = time(); + } + + public function getTaskTime(): ?int + { + if (is_null($this->taskStartTime)) + return null; + + if (is_null($this->taskEndTime)) + return time() - $this->taskStartTime; + + return $this->taskEndTime - $this->taskStartTime; + } + + public function startPostTime() + { + $this->postEndTime = null; + $this->postStartTime = time(); + } + + public function endPostTime() + { + $this->postEndTime = time(); + } + + public function getPostTime(): ?int + { + if (is_null($this->postStartTime)) + return null; + + if (is_null($this->postEndTime)) + return time() - $this->postStartTime; + + return $this->postEndTime - $this->postStartTime; + } + /** * Checks whether an object can be serialized * diff --git a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php index 4e8312c..6946379 100644 --- a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php @@ -169,7 +169,7 @@ class ArrayTaskStorage implements TaskStorage $this->commit(); // Remove all task output and post output - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); $maxRetries = $settings['maxRetries']; for ($j=0;$j<=$maxRetries;$j++) { @@ -287,7 +287,7 @@ class ArrayTaskStorage implements TaskStorage $taskId = $task->getId(); // Remove all task output and post output - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); $maxRetries = $settings['maxRetries']; for ($j=0;$j<=$maxRetries;$j++) { diff --git a/test/base/TaskTest.php b/test/base/TaskTest.php index cebada8..4f21316 100644 --- a/test/base/TaskTest.php +++ b/test/base/TaskTest.php @@ -215,11 +215,12 @@ class TaskTest extends TestCase 'maxRetries' => 2, 'retryPFailures' => true, 'retryRFailures' => true, - 'retryPostFailures' => true - ], $dummyTask->getRetrySettings()); + 'retryPostFailures' => true, + 'maxTime' => 30 + ], $dummyTask->getSettings()); // Then change the settings - $dummyTask->setRetrySettings(true, 30, false, false, false); + $dummyTask->setSettings(true, 30, 60, false, false, false); // And test the new positions $this->assertEquals([ @@ -227,8 +228,9 @@ class TaskTest extends TestCase 'maxRetries' => 30, 'retryPFailures' => false, 'retryRFailures' => false, - 'retryPostFailures' => false - ], $dummyTask->getRetrySettings()); + 'retryPostFailures' => false, + 'maxTime' => 60 + ], $dummyTask->getSettings()); } /** diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index e90613e..e3a194a 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -344,10 +344,10 @@ class ParallelSuperVisorTest extends TestCase $dummyTaskPFailedNo->setStatus(Task::FAILED); // Set retry settings - $dummyTaskFailedYes->setRetrySettings(true, 5, true, true,true); - $dummyTaskPFailedYes->setRetrySettings(true, 5, true, true,true); - $dummyTaskFailedNo->setRetrySettings(true, 5, false, false,true); - $dummyTaskPFailedNo->setRetrySettings(true, 5, false, false,true); + $dummyTaskFailedYes->setSettings(true, 5, 30, true, true,true); + $dummyTaskPFailedYes->setSettings(true, 5, 30, true, true,true); + $dummyTaskFailedNo->setSettings(true, 5, 30, false, false,true); + $dummyTaskPFailedNo->setSettings(true, 5, 30, false, false,true); // Save all these tasks $this->taskStorage->addTask($dummyTaskFailedYes); @@ -390,8 +390,8 @@ class ParallelSuperVisorTest extends TestCase // Set status and retry settings $dummyTask->setStatus(Task::FAILED); $dummyTask2->setStatus(Task::FAILED); - $dummyTask->setRetrySettings(true, 2, true, true, true); - $dummyTask2->setRetrySettings(true, 2, true, true, true); + $dummyTask->setSettings(true, 2, 30, true, true, true); + $dummyTask2->setSettings(true, 2, 30, true, true, true); // Set retries to 2 for the first task, and 1 for the second task $dummyTask->addRetry(); -- 2.40.1 From 164e7d877c67028a7c43f8fb97dece83e1bf51b4 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Sun, 7 Jun 2020 15:47:51 +0200 Subject: [PATCH 33/33] Finished ControllerHandler into a working state. --- .../Async/Handler/ControllerHandler.php | 140 +++++++++---- .../Async/Supervisors/ParallelSuperVisor.php | 5 +- test/base/ControllerHandlerTest.php | 195 ++++++++++++++++++ test/mock/Controllers/EmptyController.php | 78 +++++++ test/mock/Controllers/FailingController.php | 83 ++++++++ test/system/ParallelSuperVisorTest.php | 2 +- 6 files changed, 466 insertions(+), 37 deletions(-) create mode 100644 test/base/ControllerHandlerTest.php create mode 100644 test/mock/Controllers/EmptyController.php create mode 100644 test/mock/Controllers/FailingController.php diff --git a/src/FuzeWorks/Async/Handler/ControllerHandler.php b/src/FuzeWorks/Async/Handler/ControllerHandler.php index ae8f1fd..15f88d8 100644 --- a/src/FuzeWorks/Async/Handler/ControllerHandler.php +++ b/src/FuzeWorks/Async/Handler/ControllerHandler.php @@ -58,14 +58,60 @@ class ControllerHandler implements Handler */ protected $controllerMethod; + /** + * The namespace to use to load the controller + * + * @var string + */ + protected $controllerNamespace; + /** * @var string|null The method used to handle the post phase; if requested */ protected $postMethod = null; + /** + * @var string The output to be returned to ShellWorker + */ protected $output; + + /** + * @var string The postOutput to be returned to ShellWorker + */ protected $postOutput; + /** + * Input imported from the parent handler + * + * @var string|null + */ + protected $parentInput; + + /** + * ControllerHandler constructor. + * + * Provides all information of which controller to use. Requests get redirected to that controller. + * + * @param string $controllerName The name of the controller to use + * @param string $controllerMethod The method to use for the task execution + * @param string|null $postMethod The method to use for the post execution + * @param string $controllerNamespace A potential custom namespace for the controller + */ + public function __construct(string $controllerName, string $controllerMethod, string $postMethod = null, string $controllerNamespace = '\Application\Controller\\') + { + $this->controllerName = $controllerName; + $this->controllerMethod = $controllerMethod; + $this->postMethod = $postMethod; + $this->controllerNamespace = $controllerNamespace; + } + + /** + * @inheritDoc + */ + public function init(Task $task) + { + } + /** * @inheritDoc * @throws TasksException @@ -73,24 +119,35 @@ class ControllerHandler implements Handler public function primaryHandler(Task $task): bool { // Set the arguments - $args = $this->setArguments($task); + $args = $task->getArguments(); + array_unshift($args, $task); // First we fetch the controller - $controller = $this->getController($this->controllerName); + $controller = $this->getController(); // Check if method exists if (!method_exists($controller, $this->controllerMethod)) throw new TasksException("Could not handle task. Method '$this->controllerMethod' not found on controller."); + if (!method_exists($controller, 'getTaskStatus')) + throw new TasksException("Could not handle task. Method 'getTaskStatus()' not found on controller, which is required."); + + if ($this->parentInput !== null && method_exists($controller, 'setInput')) + $controller->setInput($this->parentInput); + // Call method and collect output $this->output = call_user_func_array([$controller, $this->controllerMethod], $args); - return true; + $success = $controller->getTaskStatus(); + if (!is_bool($success)) + throw new TasksException("Could not determine whether task has succeeded. getTaskStatus() returned non-bool."); + + return $success; } /** * @inheritDoc */ - public function getOutput() + public function getOutput(): string { return $this->output; } @@ -101,61 +158,45 @@ class ControllerHandler implements Handler */ public function postHandler(Task $task) { - // Set the arguments - $this->setArguments($task); - // Abort if no postMethod exists if (is_null($this->postMethod)) throw new TasksException("Could not handle task. No post method provided."); // First we fetch the controller - $controller = $this->getController($this->controllerName); + $controller = $this->getController(); // Check if method exists if (!method_exists($controller, $this->postMethod)) throw new TasksException("Could not handle task. Post method '$this->postMethod' not found on controller."); + if (!method_exists($controller, 'getTaskStatus')) + throw new TasksException("Could not handle task. Method 'getTaskStatus()' not found on controller, which is required."); + + if ($this->parentInput !== null && method_exists($controller, 'setInput')) + $controller->setInput($this->parentInput); + // Call method and collect output $this->postOutput = call_user_func_array([$controller, $this->postMethod], [$task]); - return true; + $success = $controller->getTaskStatus(); + if (!is_bool($success)) + throw new TasksException("Could not determine whether task has succeeded. getTaskStatus() returned non-bool."); + + return $success; } /** * @inheritDoc */ - public function getPostOutput() + public function getPostOutput(): string { return $this->postOutput; } /** - * Set the arguments of this handler using the provided task - * - * @param Task $task - * @return array - * @throws TasksException - */ - public function setArguments(Task $task): array - { - // Direct arguments - $args = $task->getArguments(); - if (count($args) < 2) - throw new TasksException("Could not handle task. Not enough arguments provided."); - - // First argument: controllerName - $this->controllerName = $args[0]; - $this->controllerMethod = $args[1]; - $this->postMethod = isset($args[2]) ? $args[2] : null; - - return !array_key_exists(2, $args) ? [] : array_slice($args, 3); - } - - /** - * @param string $controllerName * @return Controller * @throws TasksException */ - private function getController(string $controllerName): Controller + private function getController(): Controller { // First load the controllers component try { @@ -163,7 +204,7 @@ class ControllerHandler implements Handler $controllers = Factory::getInstance('controllers'); // Load the requested controller - return $controllers->get($controllerName); + return $controllers->get($this->controllerName, [], $this->controllerNamespace); } catch (FactoryException $e) { throw new TasksException("Could not get controller. FuzeWorks\MVCR is not installed!"); } catch (ControllerException $e) { @@ -172,4 +213,33 @@ class ControllerHandler implements Handler throw new TasksException("Could not get controller. Controller was not found."); } } + + /** + * @var Handler + */ + private $parentHandler; + + /** + * @inheritDoc + */ + public function getParentHandler(): ?Handler + { + return $this->parentHandler; + } + + /** + * @inheritDoc + */ + public function setParentHandler(Handler $parentHandler): void + { + $this->parentHandler = $parentHandler; + } + + /** + * @inheritDoc + */ + public function setParentInput(string $input): void + { + $this->parentInput = $input; + } } \ No newline at end of file diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index e043976..ee6277a 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -225,7 +225,10 @@ class ParallelSuperVisor implements SuperVisor elseif (!$isRunning && $hasOutput) { $task->setPostOutput($output['output'], $output['errors']); - $task->setStatus(Task::COMPLETED); + if ($output['statusCode'] === Task::SUCCESS) + $task->setStatus(Task::COMPLETED); + else + $task->setStatus(Task::CANCELLED); } // If the task is still running, leave it be else diff --git a/test/base/ControllerHandlerTest.php b/test/base/ControllerHandlerTest.php new file mode 100644 index 0000000..f296f42 --- /dev/null +++ b/test/base/ControllerHandlerTest.php @@ -0,0 +1,195 @@ +tasks = Factory::getInstance('libraries')->get('async'); + $this->taskStorage = $this->tasks->getTaskStorage(); + $this->taskStorage->reset(); + + // Reset events + Events::$listeners = []; + } + + /* ---------------------------------- Test the class itself --------------------------- */ + + public function testParametersAndClass() + { + // Create a test class + $handler = new ControllerHandler('TestController', 'testMethod', 'testPostMethod', '\Test\Namespace\\'); + $parentHandler = new EmptyHandler(); + + // Set some parameters + $handler->setParentHandler($parentHandler); + $handler->setParentInput('Some Parent Input'); + + // And test return values + $this->assertInstanceOf(ControllerHandler::class, $handler); + $this->assertSame($parentHandler, $handler->getParentHandler()); + } + + public function testEmptyController() + { + // Create the handler + $handler = new ControllerHandler('empty', 'primary', 'post', '\Mock\Controllers\\'); + $handler->setParentInput('Some input'); + + // And create the dummy Task + $dummyTask = new Task('testEmptyController', $handler, true, 'para1', 'para2'); + + // Write the task to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Get the SuperVisor and start running the build + $superVisor = $this->tasks->getSuperVisor(); + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::RUNNING, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Give the task some time to finish + usleep(750000); + + // Assert that the task is now waiting in POST + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::SUCCESS, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Cycle again so it goes into POST mode + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::POST, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Give the task some extra time to finish + usleep(750000); + + // Cycle again so it goes into POST mode + $this->assertEquals(SuperVisor::FINISHED, $superVisor->cycle()); + $this->assertEquals(Task::COMPLETED, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Now that the task is finished, let's see if the results match expectations + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + $this->assertEquals('Primary success: para1 + para2 + Some input', $dummyTask->getOutput()); + $this->assertEquals('Post success: testEmptyController', $dummyTask->getPostOutput()); + } + + /** + * @depends testEmptyController + */ + public function testFailingController() + { + // Create the handler + $handler = new ControllerHandler('failing', 'primary', 'post', '\Mock\Controllers\\'); + + // And create the dummy Task + $dummyTask = new Task('testFailingController', $handler, true, 'para1', 'para2'); + + // Set this task to not retry + $dummyTask->setSettings(false); + + // Write the task to TaskStorage + $this->taskStorage->addTask($dummyTask); + + // Get the SuperVisor and start running the build + $superVisor = $this->tasks->getSuperVisor(); + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::RUNNING, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Give the task some time to finish + usleep(750000); + + // Assert that the task is now waiting in POST + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::FAILED, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // The task should not retry and move to POST now + $this->assertEquals(SuperVisor::RUNNING, $superVisor->cycle()); + $this->assertEquals(Task::POST, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Give the task some extra time to finish + usleep(750000); + + // The task should not retry and move to POST now + $this->assertEquals(SuperVisor::FINISHED, $superVisor->cycle()); + $this->assertEquals(Task::CANCELLED, + $this->taskStorage->getTaskById($dummyTask->getId())->getStatus() + ); + + // Now that the task is finished, let's see if the results match expectations + $dummyTask = $this->taskStorage->getTaskById($dummyTask->getId()); + $this->assertEquals('Primary success: para1 + para2', $dummyTask->getOutput()); + $this->assertEquals('Post success: testFailingController', $dummyTask->getPostOutput()); + $this->assertEquals('ERROR \'Logged some task error!\'', $dummyTask->getErrors()); + $this->assertEquals('ERROR \'Logged some post error!\'', $dummyTask->getPostErrors()); + } + + +} diff --git a/test/mock/Controllers/EmptyController.php b/test/mock/Controllers/EmptyController.php new file mode 100644 index 0000000..1793d01 --- /dev/null +++ b/test/mock/Controllers/EmptyController.php @@ -0,0 +1,78 @@ +input = $input; + } + + public function primary(Task $task, $param1, $param2) + { + $this->task = $task; + $this->param1 = $param1; + $this->param2 = $param2; + + return "Primary success: " . $param1 . ' + ' . $param2 . ' + ' . $this->input; + } + + public function post(Task $task) + { + $this->postTask = $task; + + return "Post success: " . $task->getId(); + } + + public function getTaskStatus(): bool + { + return true; + } + +} \ No newline at end of file diff --git a/test/mock/Controllers/FailingController.php b/test/mock/Controllers/FailingController.php new file mode 100644 index 0000000..eb8ac04 --- /dev/null +++ b/test/mock/Controllers/FailingController.php @@ -0,0 +1,83 @@ +input = $input; + } + + public function primary(Task $task, $param1, $param2) + { + $this->task = $task; + $this->param1 = $param1; + $this->param2 = $param2; + + // Also log some errors + Logger::logError('Logged some task error!'); + + return "Primary success: " . $param1 . ' + ' . $param2; + } + + public function post(Task $task) + { + $this->postTask = $task; + + // Also log some errors + Logger::logError('Logged some post error!'); + + return "Post success: " . $task->getId(); + } + + public function getTaskStatus(): bool + { + return false; + } +} \ No newline at end of file diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index e3a194a..f069a81 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -490,7 +490,7 @@ class ParallelSuperVisorTest extends TestCase // Write the tasks to TaskStorage $this->taskStorage->addTask($dummyTaskFinished); $this->taskStorage->addTask($dummyTaskMissing); - $this->taskStorage->writePostOutput($dummyTaskFinished, 'Post Output', 'Post Errors', Task::COMPLETED); + $this->taskStorage->writePostOutput($dummyTaskFinished, 'Post Output', 'Post Errors', Task::SUCCESS); // Cycle the SuperVisor $this->superVisor->cycle(); -- 2.40.1