FuzeWorks/Async
FuzeWorks/Async
1
0
Fork 0
Code Issues 5 Pull Requests Releases 1 Activity

Implemented basic RedisTaskStorage.

Browse Source 
- Fixed bug where worker is not provided with bootstrap by ShellExecutor.
- Added composer and Redis to Dockerfile
- Added more output to ParallelSuperVisor
This commit is contained in:
Abel Hoogeveen 2020-02-14 13:49:27 +01:00
parent ab198e9ef1
commit f229be0305
No known key found for this signature in database
GPG Key ID: 96C2234920BF4292
7 changed files with 314 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile
View File

@ -3,5 +3,9 @@ FROM php:7.3-cli-buster
RUN apt-get update &&\
apt-get install --no-install-recommends --assume-yes --quiet procps ca-certificates curl git &&\
rm -rf /var/lib/apt/lists/*
# PDO
RUN docker-php-ext-install pdo_mysql
# Install Redis
RUN pecl install redis-5.1.1 && docker-php-ext-enable redis
# Install Composer
RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer

8
bin/supervisor
View File

@ -77,8 +77,8 @@ if (!isset($arguments['bootstrap']) || empty($arguments['bootstrap']))
// Load the file. If it doesn't exist, fail.
$file = $arguments['bootstrap'];
if (!file_exists($file))
$bootstrap = $arguments['bootstrap'];
if (!file_exists($bootstrap))
{
fwrite(STDERR, "Could not load supervisor. Provided bootstrap doesn't exist.");
die(1);
@ -86,7 +86,7 @@ if (!file_exists($file))
// Load the bootstrap
/** @var Factory $container */
$container = require($file);
$container = require($bootstrap);
// Check if container is a Factory
if (!$container instanceof Factory)
@ -108,7 +108,7 @@ try {
// And finally, run the supervisor
try {
$supervisor = $lib->getSuperVisor();
$supervisor = $lib->getSuperVisor($bootstrap);
while ($supervisor->cycle() === SuperVisor::RUNNING) {
usleep(250000);
}

6
composer.json
View File

@ -15,7 +15,11 @@
"require": {
"php": ">=7.2.0",
"fuzeworks/core": "~1.2.0",
"ext-json": "*"
"ext-json": "*",
"ext-redis": "*"
},
"require-dev": {
"fuzeworks/tracycomponent": "~1.2.0"
},
"autoload": {
"psr-4": {

7
src/FuzeWorks/Async/Executors/ShellExecutor.php
View File

@ -45,22 +45,25 @@ class ShellExecutor implements Executor
private $binary;
private $worker;
private $bootstrapFile;
private $stdout = "> /dev/null";
private $stderr = "2> /dev/null";
/**
* ShellExecutor constructor.
*
* @param string $bootstrapFile
* @param array $parameters
* @throws TasksException
*/
public function __construct(array $parameters)
public function __construct(string $bootstrapFile, array $parameters)
{
// Fetch workerFile
$workerFile = $parameters['workerFile'];
// First determine the PHP binary
$this->binary = PHP_BINDIR . DS . 'php';
$this->bootstrapFile = $bootstrapFile;
if (!file_exists($workerFile))
throw new TasksException("Could not construct ShellExecutor. ShellWorker script does not exist.");
@ -80,7 +83,7 @@ class ShellExecutor implements Executor
public function startTask(Task $task, bool $post = false): Task
{
// First prepare the command used to spawn workers
$commandString = "$this->binary $this->worker -t %s ".($post ? 'p' : '')." $this->stdout $this->stderr & echo $!";
$commandString = "$this->binary $this->worker --bootstrap=".$this->bootstrapFile." -t %s ".($post ? 'p' : '')." $this->stdout $this->stderr & echo $!";
// Then execute the command using the base64_encoded string of the taskID
$output = $this->shellExec($commandString, [base64_encode($task->getId())]);

10
src/FuzeWorks/Async/Supervisors/ParallelSuperVisor.php
View File

@ -80,7 +80,7 @@ class ParallelSuperVisor implements SuperVisor
for ($i=0;$i<count($this->tasks);$i++)
{
$task = $this->tasks[$i];
// PENDING: should start if not constrained
if ($task->getStatus() === Task::PENDING)
{
@ -97,6 +97,7 @@ class ParallelSuperVisor implements SuperVisor
// Modify the task in TaskStorage
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
// DELAYED: If task is delayed, and enough time has passed, change the status back to pending
@ -104,6 +105,7 @@ class ParallelSuperVisor implements SuperVisor
{
$task->setStatus(Task::PENDING);
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
// CANCELLED/COMPLETED: remove the task if requested to do so
@ -139,6 +141,7 @@ class ParallelSuperVisor implements SuperVisor
// If any changes have been made, they should be written to TaskStorage
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
// FAILED: if a process has failed, attempt to rety if requested to do so
@ -161,6 +164,7 @@ class ParallelSuperVisor implements SuperVisor
$task = $this->executor->startTask($task);
$task->setStatus(Task::RUNNING);
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
continue;
}
}
@ -176,6 +180,7 @@ class ParallelSuperVisor implements SuperVisor
$task->setStatus(Task::CANCELLED);
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
// SUCCESS: if a task has succeeded, see if it needs a postHandler
@ -191,6 +196,7 @@ class ParallelSuperVisor implements SuperVisor
$task->setStatus(Task::COMPLETED);
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
// POST: when a task is currently running in it's postHandler
@ -226,6 +232,7 @@ class ParallelSuperVisor implements SuperVisor
// If any changes have been made, they should be written to TaskStorage
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
}
@ -263,6 +270,7 @@ class ParallelSuperVisor implements SuperVisor
// Save changes to TaskStorage
$this->taskStorage->modifyTask($task);
fwrite(STDOUT, "\nChanged status of task '".$task->getId()."' to status " . Task::getStatusType($task->getStatus()));
}
}

278
src/FuzeWorks/Async/TaskStorage/RedisTaskStorage.php Normal file
View File

@ -0,0 +1,278 @@
<?php
/**
* FuzeWorks Async Library
*
* The FuzeWorks PHP FrameWork
*
* Copyright (C) 2013-2020 TechFuze
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* @author TechFuze
* @copyright Copyright (c) 2013 - 2020, TechFuze. (http://techfuze.net)
* @license https://opensource.org/licenses/MIT MIT License
*
* @link http://techfuze.net/fuzeworks
* @since Version 1.0.0
*
* @version Version 1.0.0
*/
namespace FuzeWorks\Async\TaskStorage;
use FuzeWorks\Async\Task;
use FuzeWorks\Async\TasksException;
use FuzeWorks\Async\TaskStorage;
use Redis;
use RedisException;
class RedisTaskStorage implements TaskStorage
{
/**
* @var Redis
*/
protected $conn;
protected $indexSet = 'async_index';
protected $key_prefix = 'async_task_';
/**
* @inheritDoc
*/
public function __construct(array $parameters)
{
// Attempt to connect to Redis
try {
$this->conn = new Redis();
// Afterwards connect to server
$socketType = $parameters['socket_type'];
if ($socketType == 'unix')
$success = $this->conn->connect($parameters['socket']);
elseif ($socketType == 'tcp')
$success = $this->conn->connect($parameters['host'], $parameters['port'], $parameters['timeout']);
else
$success = false;
// If unsuccessful, return false
if (!$success)
throw new TasksException("Could not construct RedisTaskStorage. Failed to connect.");
// Otherwise attempt authentication, if needed
if (isset($parameters['password']) && !$this->conn->auth($parameters['password']))
throw new TasksException("Could not construct RedisTaskStorage. Authentication failure.");
} catch (RedisException $e) {
throw new TasksException("Could not construct RedisTaskStorage. RedisException thrown: '" . $e->getMessage() . "'");
}
}
/**
* @inheritDoc
* @throws TasksException
*/
public function addTask(Task $task): bool
{
// Check if the task doesn't exist yet
$taskId = $task->getId();
// Query the index
$isMember = $this->conn->sIsMember($this->indexSet, $taskId);
if ($isMember)
throw new TasksException("Could not add Task to TaskStorage. Task '$taskId' already exists.");
// Serialize the task and save it
$taskData = serialize($task);
$this->conn->set($this->key_prefix . $taskId, $taskData);
$this->conn->sAdd($this->indexSet, $taskId);
return true;
}
/**
* @inheritDoc
*/
public function readTasks(): array
{
return $this->refreshTasks();
}
/**
* @inheritDoc
*/
public function refreshTasks()
{
// First fetch an array of all tasks in the set
$taskList = $this->conn->sMembers($this->indexSet);
// Go over each taskId and fetch the specific task
$tasks = [];
foreach ($taskList as $taskId)
$tasks[] = unserialize($this->conn->get($this->key_prefix . $taskId));
return $tasks;
}
/**
* @inheritDoc
*/
public function getTaskById(string $identifier): Task
{
// Query the index
$isMember = $this->conn->sIsMember($this->indexSet, $identifier);
if (!$isMember)
throw new TasksException("Could not get task by ID. Task not found.");
// Fetch the task
/** @var Task $task */
$task = unserialize($this->conn->get($this->key_prefix . $identifier));
// Return the task
return $task;
}
/**
* @inheritDoc
*/
public function modifyTask(Task $task): bool
{
// First get the task ID
$taskId = $task->getId();
// Check if it exists
$isMember = $this->conn->sIsMember($this->indexSet, $taskId);
if (!$isMember)
throw new TasksException("Could not modify task. Task '$taskId' already exists.");
// And write the data
$taskData = serialize($task);
return $this->conn->set($this->key_prefix . $taskId, $taskData);
}
/**
* @inheritDoc
* @throws TasksException
*/
public function deleteTask(Task $task): bool
{
// First get the task ID
$taskId = $task->getId();
// Check if it exists
$isMember = $this->conn->sIsMember($this->indexSet, $taskId);
if (!$isMember)
throw new TasksException("Could not modify task. Task '$taskId' already exists.");
// Delete the key
$this->conn->del($this->key_prefix . $taskId);
$this->conn->sRem($this->indexSet, $taskId);
// Remove all task output and post output
$settings = $task->getRetrySettings();
$maxRetries = $settings['maxRetries'];
for ($i=0;$i<=$maxRetries;$i++)
{
// First remove all possible task output
if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $i))
$this->conn->del($this->key_prefix . $taskId . '_output_' . $i);
if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $i))
$this->conn->del($this->key_prefix . $taskId . '_post_' . $i);
}
return true;
}
/**
* @inheritDoc
*/
public function writeTaskOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool
{
// First get the task ID
$taskId = $task->getId();
// Check if the key already exists
if ($this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt))
throw new TasksException("Could not write task output. Output already written.");
// Prepare contents
$contents = ['taskId' => $task->getId(), 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode];
$data = serialize($contents);
// Write contents
return $this->conn->set($this->key_prefix . $taskId . '_output_' . $attempt, $data);
}
/**
* @inheritDoc
*/
public function writePostOutput(Task $task, string $output, string $errors, int $statusCode, int $attempt = 0): bool
{
// First get the task ID
$taskId = $task->getId();
// Check if the key already exists
if ($this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt))
throw new TasksException("Could not write post output. Output already written.");
// Prepare contents
$contents = ['taskId' => $task->getId(), 'output' => $output, 'errors' => $errors, 'statusCode' => $statusCode];
$data = serialize($contents);
// Write contents
return $this->conn->set($this->key_prefix . $taskId . '_post_' . $attempt, $data);
}
/**
* @inheritDoc
*/
public function readTaskOutput(Task $task, int $attempt = 0): ?array
{
// First get the task ID
$taskId = $task->getId();
// Check if the key already exists
if (!$this->conn->exists($this->key_prefix . $taskId . '_output_' . $attempt))
return null;
// Load and convert the data
$data = $this->conn->get($this->key_prefix . $taskId . '_output_' . $attempt);
$data = unserialize($data);
return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']];
}
/**
* @inheritDoc
*/
public function readPostOutput(Task $task, int $attempt = 0): ?array
{
// First get the task ID
$taskId = $task->getId();
// Check if the key already exists
if (!$this->conn->exists($this->key_prefix . $taskId . '_post_' . $attempt))
return null;
// Load and convert the data
$data = $this->conn->get($this->key_prefix . $taskId . '_post_' . $attempt);
$data = unserialize($data);
return ['output' => $data['output'], 'errors' => $data['errors'], 'statusCode' => $data['statusCode']];
}
}

11
src/FuzeWorks/Async/Tasks.php
View File

@ -75,14 +75,16 @@ class Tasks implements iLibrary
}
/**
* @param string $bootstrapFile
* @return SuperVisor
* @throws TasksException
*/
public function getSuperVisor(): SuperVisor
public function getSuperVisor(string $bootstrapFile): SuperVisor
{
$cfg = $this->cfg->get('SuperVisor');
$class = 'FuzeWorks\Async\Supervisors\\' . $cfg['type'];
$parameters = isset($cfg['parameters']) && is_array($cfg['parameters']) ? $cfg['parameters'] : [];
array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor());
array_unshift($parameters, $this->getTaskStorage(), $this->getExecutor($bootstrapFile));
if (!class_exists($class, true))
throw new TasksException("Could not get SuperVisor. Type of '$class' not found.");
@ -126,10 +128,11 @@ class Tasks implements iLibrary
/**
* Fetch the Executor based on the configured type
*
* @param string $bootstrapFile
* @return Executor
* @throws TasksException
*/
protected function getExecutor(): Executor
protected function getExecutor(string $bootstrapFile): Executor
{
$cfg = $this->cfg->get('Executor');
$class = 'FuzeWorks\Async\Executors\\' . $cfg['type'];
@ -137,7 +140,7 @@ class Tasks implements iLibrary
if (!class_exists($class, true))
throw new TasksException("Could not get Executor. Type of '$class' not found.");
$object = new $class($parameters);
$object = new $class($bootstrapFile, $parameters);
if (!$object instanceof Executor)
throw new TasksException("Could not get Executor. Type '$class' is not instanceof Executor.");