From ee5312fa1b9414443a43aee722dc2d7066aa23b3 Mon Sep 17 00:00:00 2001 From: Abel Hoogeveen Date: Sat, 6 Jun 2020 00:11:27 +0200 Subject: [PATCH] Started work on making tasks forcefully quit after a maximum time has expired. --- bin/supervisor | 2 +- .../Async/Supervisors/ParallelSuperVisor.php | 16 ++-- src/FuzeWorks/Async/Task.php | 79 +++++++++++++++++-- .../Async/TaskStorage/ArrayTaskStorage.php | 4 +- test/base/TaskTest.php | 12 +-- test/system/ParallelSuperVisorTest.php | 12 +-- 6 files changed, 95 insertions(+), 30 deletions(-) diff --git a/bin/supervisor b/bin/supervisor index 5abc47b..0542413 100644 --- a/bin/supervisor +++ b/bin/supervisor @@ -109,7 +109,7 @@ try { // And finally, run the supervisor try { $supervisor = $lib->getSuperVisor($bootstrap); - while ($supervisor->cycle() === SuperVisor::RUNNING) { + while ($supervisor->cycle() !== SuperVisor::RUNNING) { usleep(250000); } diff --git a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php index c518883..e043976 100644 --- a/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php +++ b/src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php @@ -93,6 +93,7 @@ class ParallelSuperVisor implements SuperVisor // Start the process using the executor service $task = $this->executor->startTask($task); $task->setStatus(Task::RUNNING); + $task->startTaskTime(); // Modify the task in TaskStorage $this->taskStorage->modifyTask($task); @@ -107,12 +108,6 @@ class ParallelSuperVisor implements SuperVisor fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } - // CANCELLED/COMPLETED: remove the task if requested to do so - elseif ($task->getStatus() === Task::COMPLETED || $task->getStatus() === Task::CANCELLED) - { - // @todo Remove old tasks automatically - } - // RUNNING: check if task is still running. If not, set result based on output elseif ($task->getStatus() === Task::RUNNING) { @@ -140,6 +135,7 @@ class ParallelSuperVisor implements SuperVisor continue; // If any changes have been made, they should be written to TaskStorage + $task->endTaskTime(); $this->taskStorage->modifyTask($task); fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } @@ -148,7 +144,7 @@ class ParallelSuperVisor implements SuperVisor elseif ($task->getStatus() === Task::PFAILED || $task->getStatus() === Task::FAILED) { // First fetch retry conditions - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); // First test if any retries should be tried at all if ($settings['retryOnFail'] === true && $task->getRetries() < $settings['maxRetries']) @@ -163,6 +159,7 @@ class ParallelSuperVisor implements SuperVisor $task->addRetry(); $task = $this->executor->startTask($task); $task->setStatus(Task::RUNNING); + $task->startTaskTime(); $this->taskStorage->modifyTask($task); fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); continue; @@ -174,6 +171,7 @@ class ParallelSuperVisor implements SuperVisor { $task->resetRetries(); $task = $this->executor->startTask($task, true); + $task->startPostTime(); $task->setStatus(Task::POST); } else @@ -190,6 +188,7 @@ class ParallelSuperVisor implements SuperVisor { $task->resetRetries(); $task = $this->executor->startTask($task, true); + $task->startPostTime(); $task->setStatus(Task::POST); } else @@ -210,7 +209,7 @@ class ParallelSuperVisor implements SuperVisor if (!$isRunning && !$hasOutput) { // Test if a retry should be attempted - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); if ($settings['retryOnFail'] === true && $settings['retryPostFailures'] === true && $settings['maxRetries'] > $task->getRetries()) { $task->addRetry(); @@ -233,6 +232,7 @@ class ParallelSuperVisor implements SuperVisor continue; // If any changes have been made, they should be written to TaskStorage + $task->endPostTime(); $this->taskStorage->modifyTask($task); fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus())); } diff --git a/src/FuzeWorks/Async/Task.php b/src/FuzeWorks/Async/Task.php index d86664f..2da3945 100644 --- a/src/FuzeWorks/Async/Task.php +++ b/src/FuzeWorks/Async/Task.php @@ -178,6 +178,7 @@ class Task protected $retryRFailures = true; protected $retryPostFailures = true; protected $retries = 0; + protected $maxTime = 30; /** * Task constructor. @@ -316,6 +317,8 @@ class Task return $this->delayTime; } + /* ---------------------------------- Attributes setters and getters ------------------ */ + /** * Fetch an attribute of this task * @@ -359,6 +362,8 @@ class Task unset($this->attributes[$key]); } + /* ---------------------------------- Output setters and getters ---------------------- */ + /** * Return the output of this task execution * @@ -411,22 +416,26 @@ class Task $this->postErrors = $errors; } + /* ---------------------------------- Failure settings and criteria ------------------- */ + /** * Set whether this task should retry after a failure, and how many times * - * @param bool $retryOnFail - * @param int $maxRetries - * @param bool $retryRegularFailures - * @param bool $retryProcessFailures - * @param bool $retryPostFailures + * @param bool $retryOnFail Whether this task should be retried if failing + * @param int $maxRetries How many times the task should be retried + * @param int $maxTime How long a task may run before it shall be forcefully shut down + * @param bool $retryRegularFailures Whether regular Task::FAILED should be retried + * @param bool $retryProcessFailures Whether process based Task::PFAILED should be retried + * @param bool $retryPostFailures Whether failures during the Task::POST phase should be retried. */ - public function setRetrySettings(bool $retryOnFail, int $maxRetries = 2, bool $retryRegularFailures = true, bool $retryProcessFailures = true, bool $retryPostFailures = true) + public function setSettings(bool $retryOnFail, int $maxRetries = 2, int $maxTime = 30, bool $retryRegularFailures = true, bool $retryProcessFailures = true, bool $retryPostFailures = true) { $this->retryOnFail = $retryOnFail; $this->maxRetries = $maxRetries; $this->retryPFailures = $retryProcessFailures; $this->retryRFailures = $retryRegularFailures; $this->retryPostFailures = $retryPostFailures; + $this->maxTime = $maxTime; } /** @@ -434,17 +443,20 @@ class Task * * @return array */ - public function getRetrySettings(): array + public function getSettings(): array { return [ 'retryOnFail' => $this->retryOnFail, 'maxRetries' => $this->maxRetries, 'retryPFailures' => $this->retryPFailures, 'retryRFailures' => $this->retryRFailures, - 'retryPostFailures' => $this->retryPostFailures + 'retryPostFailures' => $this->retryPostFailures, + 'maxTime' => $this->maxTime ]; } + /* ---------------------------------- Retries and attempts registers ------------------ */ + /** * Add a retry to the retry counter */ @@ -471,6 +483,57 @@ class Task return $this->retries; } + /* ---------------------------------- Runtime data getters and setters----------------- */ + + protected $taskStartTime; + protected $taskEndTime; + protected $postStartTime; + protected $postEndTime; + + public function startTaskTime() + { + $this->taskEndTime = null; + $this->taskStartTime = time(); + } + + public function endTaskTime() + { + $this->taskEndTime = time(); + } + + public function getTaskTime(): ?int + { + if (is_null($this->taskStartTime)) + return null; + + if (is_null($this->taskEndTime)) + return time() - $this->taskStartTime; + + return $this->taskEndTime - $this->taskStartTime; + } + + public function startPostTime() + { + $this->postEndTime = null; + $this->postStartTime = time(); + } + + public function endPostTime() + { + $this->postEndTime = time(); + } + + public function getPostTime(): ?int + { + if (is_null($this->postStartTime)) + return null; + + if (is_null($this->postEndTime)) + return time() - $this->postStartTime; + + return $this->postEndTime - $this->postStartTime; + } + /** * Checks whether an object can be serialized * diff --git a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php index 4e8312c..6946379 100644 --- a/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php +++ b/src/FuzeWorks/Async/TaskStorage/ArrayTaskStorage.php @@ -169,7 +169,7 @@ class ArrayTaskStorage implements TaskStorage $this->commit(); // Remove all task output and post output - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); $maxRetries = $settings['maxRetries']; for ($j=0;$j<=$maxRetries;$j++) { @@ -287,7 +287,7 @@ class ArrayTaskStorage implements TaskStorage $taskId = $task->getId(); // Remove all task output and post output - $settings = $task->getRetrySettings(); + $settings = $task->getSettings(); $maxRetries = $settings['maxRetries']; for ($j=0;$j<=$maxRetries;$j++) { diff --git a/test/base/TaskTest.php b/test/base/TaskTest.php index cebada8..4f21316 100644 --- a/test/base/TaskTest.php +++ b/test/base/TaskTest.php @@ -215,11 +215,12 @@ class TaskTest extends TestCase 'maxRetries' => 2, 'retryPFailures' => true, 'retryRFailures' => true, - 'retryPostFailures' => true - ], $dummyTask->getRetrySettings()); + 'retryPostFailures' => true, + 'maxTime' => 30 + ], $dummyTask->getSettings()); // Then change the settings - $dummyTask->setRetrySettings(true, 30, false, false, false); + $dummyTask->setSettings(true, 30, 60, false, false, false); // And test the new positions $this->assertEquals([ @@ -227,8 +228,9 @@ class TaskTest extends TestCase 'maxRetries' => 30, 'retryPFailures' => false, 'retryRFailures' => false, - 'retryPostFailures' => false - ], $dummyTask->getRetrySettings()); + 'retryPostFailures' => false, + 'maxTime' => 60 + ], $dummyTask->getSettings()); } /** diff --git a/test/system/ParallelSuperVisorTest.php b/test/system/ParallelSuperVisorTest.php index e90613e..e3a194a 100644 --- a/test/system/ParallelSuperVisorTest.php +++ b/test/system/ParallelSuperVisorTest.php @@ -344,10 +344,10 @@ class ParallelSuperVisorTest extends TestCase $dummyTaskPFailedNo->setStatus(Task::FAILED); // Set retry settings - $dummyTaskFailedYes->setRetrySettings(true, 5, true, true,true); - $dummyTaskPFailedYes->setRetrySettings(true, 5, true, true,true); - $dummyTaskFailedNo->setRetrySettings(true, 5, false, false,true); - $dummyTaskPFailedNo->setRetrySettings(true, 5, false, false,true); + $dummyTaskFailedYes->setSettings(true, 5, 30, true, true,true); + $dummyTaskPFailedYes->setSettings(true, 5, 30, true, true,true); + $dummyTaskFailedNo->setSettings(true, 5, 30, false, false,true); + $dummyTaskPFailedNo->setSettings(true, 5, 30, false, false,true); // Save all these tasks $this->taskStorage->addTask($dummyTaskFailedYes); @@ -390,8 +390,8 @@ class ParallelSuperVisorTest extends TestCase // Set status and retry settings $dummyTask->setStatus(Task::FAILED); $dummyTask2->setStatus(Task::FAILED); - $dummyTask->setRetrySettings(true, 2, true, true, true); - $dummyTask2->setRetrySettings(true, 2, true, true, true); + $dummyTask->setSettings(true, 2, 30, true, true, true); + $dummyTask2->setSettings(true, 2, 30, true, true, true); // Set retries to 2 for the first task, and 1 for the second task $dummyTask->addRetry();