ProVM/emails
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Jobs setup

Browse Source
This commit is contained in:
Aldarien
2023-06-12 21:14:07 -04:00
parent 03c1dac2f2
commit 88f91c4bd5
60 changed files with 965 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
cli/common/Command/Jobs/Check.php
View File

@ -1,14 +1,12 @@
<?php
namespace ProVM\Command\Jobs;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use ProVM\Service\Communicator;
use function Safe\json_decode;
use ProVM\Service\Jobs;
#[AsCommand(
name: 'jobs:check',
@ -17,35 +15,11 @@ use function Safe\json_decode;
)]
class Check extends Command
{
public function __construct(protected Communicator $communicator, protected LoggerInterface $logger, string $name = null)
public function __construct(protected Jobs $service, string $name = null)
{
parent::__construct($name);
}
protected function getPendingJobs(): array
{
$this->logger->notice('Grabbing pending jobs.');
$response = $this->communicator->get('/jobs/pending');
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return [];
}
return json_decode($body)->jobs;
}
protected function runJob($job): bool
{
$base_command = '/app/bin/emails';
$cmd = [$base_command, $job->command];
if ($job->arguments !== '') {
$cmd []= $job->arguments;
}
$cmd = implode(' ', $cmd);
$this->logger->notice("Running '{$cmd}'");
$response = shell_exec($cmd);
$this->logger->info("Result: {$response}");
return $response !== false;
}
public function execute(InputInterface $input, OutputInterface $output)
{
$section1 = $output->section();
@ -53,17 +27,16 @@ class Check extends Command
$io1 = new SymfonyStyle($input, $section1);
$io2 = new SymfonyStyle($input, $section2);
$io1->title('Checking Pending Jobs');
$pending_jobs = $this->getPendingJobs();
$pending_jobs = $this->service->getPending();
$notice = 'Found ' . count($pending_jobs) . ' jobs';
$io1->text($notice);
$this->logger->info($notice);
if (count($pending_jobs) > 0) {
$io1->section('Running Jobs');
$io1->progressStart(count($pending_jobs));
foreach ($pending_jobs as $job) {
$section2->clear();
$io2->text("Running {$job->command}");
if ($this->runJob($job)) {
if ($this->service->run($job)) {
$io2->success('Success');
} else {
$io2->error('Failure');

41
cli/common/Command/Jobs/Execute.php Normal file
View File

@ -0,0 +1,41 @@
<?php
namespace ProVM\Command\Jobs;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use ProVM\Service\Jobs;
#[AsCommand(
name: 'jobs:execute',
description: 'Execute job by job_id',
hidden: false
)]
class Execute extends Command
{
public function __construct(protected Jobs $service, string $name = null)
{
parent::__construct($name);
}
protected function configure()
{
$this->addArgument('job_id', InputArgument::REQUIRED, 'Job ID to be executed');
}
public function execute(InputInterface $input, OutputInterface $output)
{
$io = new SymfonyStyle($input, $output);
$job_id = $input->getArgument('job_id');
$job = $this->service->get($job_id);
if ($this->service->run($job)) {
$io->success('Success');
} else {
$io->error('Failed');
}
return Command::SUCCESS;
}
}

31
cli/common/Command/Mailboxes/Check.php
View File

@ -5,9 +5,8 @@ use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use ProVM\Service\Communicator;
use Symfony\Component\Console\Style\SymfonyStyle;
use function Safe\json_decode;
use ProVM\Service\Mailboxes;
#[AsCommand(
name: 'mailboxes:check',
@ -16,33 +15,11 @@ use function Safe\json_decode;
)]
class Check extends Command
{
public function __construct(protected Communicator $communicator, protected int $min_check_days, string $name = null)
public function __construct(protected Mailboxes $service, string $name = null)
{
parent::__construct($name);
}
protected function getMailboxes(): array
{
$response = $this->communicator->get('/mailboxes/registered');
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return [];
}
return json_decode($body)->mailboxes;
}
protected function checkMailbox($mailbox): bool
{
if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) {
return true;
}
$response = $this->communicator->get("/mailbox/{$mailbox->id}/check");
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return true;
}
return json_decode($body)->status;
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$section1 = $output->section();
@ -50,7 +27,7 @@ class Check extends Command
$io1 = new SymfonyStyle($input, $section1);
$io2 = new SymfonyStyle($input, $section2);
$io1->title('Checking for New Messages');
$mailboxes = $this->getMailboxes();
$mailboxes = $this->service->getAll();
$notice = 'Found ' . count($mailboxes) . ' mailboxes';
$io1->text($notice);
if (count($mailboxes) > 0) {
@ -59,7 +36,7 @@ class Check extends Command
foreach ($mailboxes as $mailbox) {
$section2->clear();
$io2->text("Checking {$mailbox->name}");
if ($this->checkMailbox($mailbox)) {
if ($this->service->check($mailbox)) {
$io2->success("Found new emails in {$mailbox->name}");
} else {
$io2->info("No new emails in {$mailbox->name}");

31
cli/common/Command/Messages/Grab.php
View File

@ -7,19 +7,17 @@ use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use ProVM\Service\Communicator;
use function Safe\json_decode;
use ProVM\Service\Mailboxes;
#[AsCommand(
name: 'messages:grab',
description: 'Run grab messages job for registered mailboxes',
description: 'Run grab messages job for mailbox',
hidden: false
)]
class Grab extends Command
{
public function __construct(Communicator $communicator, string $name = null)
public function __construct(protected Mailboxes $service, string $name = null)
{
$this->setCommunicator($communicator);
parent::__construct($name);
}
protected function configure()
@ -27,27 +25,6 @@ class Grab extends Command
$this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails');
}
protected Communicator $communicator;
public function getCommunicator(): Communicator
{
return $this->communicator;
}
public function setCommunicator(Communicator $communicator): Grab
{
$this->communicator = $communicator;
return $this;
}
protected function grabMessages(int $mailbox_id): int
{
$response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab");
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return 0;
}
return json_decode($body)->messages->count;
}
public function execute(InputInterface $input, OutputInterface $output)
{
$io = new SymfonyStyle($input, $output);
@ -55,7 +32,7 @@ class Grab extends Command
$mailbox_id = $input->getArgument('mailbox_id');
$io->title("Grabbing Messages for Mailbox ID {$mailbox_id}");
$io->section('Grabbing Messages');
$count = $this->grabMessages($mailbox_id);
$count = $this->service->grabMessages($mailbox_id);
$io->info("Found {$count} messages");
$io->success('Done.');

16
cli/common/Exception/Response/EmptyResponse.php Normal file
View File

@ -0,0 +1,16 @@
<?php
namespace ProVM\Exception\Response;
use Exception;
use Throwable;
class EmptyResponse extends Exception
{
public function __construct(string $uri, string $method = 'get', ?Throwable $previous = null)
{
$method = strtoupper($method);
$message = "Received empty response for request {$method} '{$uri}'";
$code = 410;
parent::__construct($message, $code, $previous);
}
}

15
cli/common/Exception/Response/MissingResponse.php Normal file
View File

@ -0,0 +1,15 @@
<?php
namespace ProVM\Exception\Response;
use Exception;
use Throwable;
class MissingResponse extends Exception
{
public function __construct(string $expected, ?Throwable $previous = null)
{
$message = "Response is missing parameter(s) {$expected}";
$code = 406;
parent::__construct($message, $code, $previous);
}
}

38
cli/common/Middleware/Logging.php
View File

@ -1,38 +0,0 @@
<?php
namespace ProVM\Middleware;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Server\RequestHandlerInterface;
use Psr\Log\LoggerInterface;
class Logging
{
public function __construct(LoggerInterface $logger) {
$this->setLogger($logger);
}
protected LoggerInterface $logger;
public function getLogger(): LoggerInterface
{
return $this->logger;
}
public function setLogger(LoggerInterface $logger): Logging
{
$this->logger = $logger;
return $this;
}
public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
{
$response = $handler->handle($request);
$output = [
'uri' => var_export($request->getUri(), true),
'body' => $request->getBody()->getContents()
];
$this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
return $response;
}
}

114
cli/common/Service/Attachments.php Normal file
View File

@ -0,0 +1,114 @@
<?php
namespace ProVM\Service;
use ProVM\Exception\Response\EmptyResponse;
use ProVM\Exception\Response\MissingResponse;
use Psr\Log\LoggerInterface;
use function Safe\json_decode;
class Attachments
{
public function __construct(protected Communicator $communicator, protected LoggerInterface $logger, protected array $passwords, protected string $base_command = 'qpdf') {}
protected array $attachments;
public function findAll(): \Generator
{
$this->logger->info('Finding all downloaded attachment files');
$folder = '/attachments';
$files = new \FilesystemIterator($folder);
foreach ($files as $file) {
if ($file->isDir()) {
continue;
}
yield $file->getRealPath();
}
}
public function getAll(): array
{
if (!isset($this->attachments)) {
$this->logger->info('Grabbing all attachments');
$response = $this->communicator->get('/attachments');
$body = $response->getBody()->getContents();
if (trim($body) === '') {
$this->attachments = [];
return $this->attachments;
}
$this->attachments = json_decode($body)->attachments;
}
return $this->attachments;
}
public function get(int $attachment_id): object
{
$this->logger->info("Getting attachment {$attachment_id}");
$uri = "/attachment/{$attachment_id}";
$response = $this->communicator->get($uri);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
throw new EmptyResponse($uri);
}
$json = json_decode($body);
if (!isset($json->attachment)) {
throw new MissingResponse('attachment');
}
return $json->attachment;
}
public function find(string $filename): int
{
$this->logger->info("Finding attachment {$filename}");
foreach ($this->getAll() as $attachment) {
if ($attachment->fullfilename === $filename) {
return $attachment->id;
}
}
throw new \Exception("{$filename} is not in the database");
}
public function isEncrypted(string $filename): bool
{
if (!file_exists($filename)) {
throw new \InvalidArgumentException("File not found {$filename}");
}
$escaped_filename = escapeshellarg($filename);
$cmd = "{$this->base_command} --is-encrypted {$escaped_filename}";
exec($cmd, $output, $retcode);
return $retcode == 0;
}
public function scheduleDecrypt(int $attachment_id): bool
{
$this->logger->info("Scheduling decryption of attachment {$attachment_id}");
$uri = "/attachment/{$attachment_id}/decrypt";
$response = $this->communicator->get($uri);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
throw new EmptyResponse($uri);
}
$json = json_decode($body);
if (!isset($json->status)) {
throw new MissingResponse('status');
}
return $json->status;
}
public function decrypt(string $basename): bool
{
$this->logger->info("Decrypting {$basename}");
$in_filename = implode('/', ['attachments', $basename]);
$out_filename = implode('/', ['attachments', 'decrypted', $basename]);
if (file_exists($out_filename)) {
throw new \Exception("{$basename} already decrypted");
}
foreach ($this->passwords as $password) {
$cmd = $this->base_command . ' -password=' . escapeshellarg($password) . ' -decrypt ' . escapeshellarg($in_filename) . ' ' . escapeshellarg($out_filename);
exec($cmd, $output, $retcode);
$success = $retcode == 0;
if ($success) {
return true;
}
if (file_exists($out_filename)) {
unlink($out_filename);
}
unset($output);
}
return false;
}
}

21
cli/common/Service/Communicator.php
View File

@ -4,29 +4,12 @@ namespace ProVM\Service;
use HttpResponseException;
use Psr\Http\Client\ClientInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Log\LoggerInterface;
use Safe\Exceptions\JsonException;
use function Safe\json_encode;
class Communicator
{
public function __construct(ClientInterface $client)
{
$this->setClient($client);
}
protected ClientInterface $client;
public function getClient(): ClientInterface
{
return $this->client;
}
public function setClient(ClientInterface $client): Communicator
{
$this->client = $client;
return $this;
}
public function __construct(protected ClientInterface $client) {}
/**
* @throws HttpResponseException
@ -52,7 +35,7 @@ class Communicator
];
$options['body'] = json_encode($body);
}
return $this->handleResponse($this->getClient()->request($method, $uri, $options));
return $this->handleResponse($this->client->request($method, $uri, $options));
}
/**

71
cli/common/Service/Jobs.php Normal file
View File

@ -0,0 +1,71 @@
<?php
namespace ProVM\Service;
use Psr\Log\LoggerInterface;
use ProVM\Exception\Response\{EmptyResponse, MissingResponse};
use function Safe\json_decode;
class Jobs
{
public function __construct(protected Communicator $communicator, protected LoggerInterface $logger) {}
public function getPending(): array
{
$this->logger->info('Getting pending jobs');
$response = $this->communicator->get('/jobs/pending');
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return [];
}
$json = json_decode($body);
if (!isset($json->jobs)) {
return [];
}
return $json->jobs;
}
public function get(int $job_id): object
{
$this->logger->info("Getting Job {$job_id}");
$uri = "/job/{$job_id}";
return $this->send($uri, 'job');
}
public function run(object $job): bool
{
$this->logger->debug("Running Job {$job->id}");
$base_command = '/app/bin/emails';
$cmd = [$base_command, $job->command];
if ($job->arguments !== '') {
$cmd []= $job->arguments;
}
$cmd = implode(' ', $cmd);
$response = shell_exec($cmd);
if ($response !== false) {
return $this->finished($job->id);
}
return $this->failure($job->id);
}
protected function finished(int $job_id): bool
{
$uri = "/job/{$job_id}/finish";
return $this->send($uri, 'status');
}
protected function failure(int $job_id): bool
{
$uri = "/job/{$job_id}/failed";
return $this->send($uri, 'status');
}
protected function send(string $uri, string $param): mixed
{
$response = $this->communicator->get($uri);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
throw new EmptyResponse($uri);
}
$json = json_decode($body);
if (!isset($json->{$param})) {
throw new MissingResponse($param);
}
return $json->{$param};
}
}

61
cli/common/Service/Mailboxes.php Normal file
View File

@ -0,0 +1,61 @@
<?php
namespace ProVM\Service;
use DateTimeImmutable;
use ProVM\Exception\Response\EmptyResponse;
use ProVM\Exception\Response\MissingResponse;
use Psr\Log\LoggerInterface;
use function Safe\json_decode;
class Mailboxes
{
public function __construct(protected Communicator $communicator, protected LoggerInterface $logger, protected int $min_check_days) {}
public function getAll(): array
{
$this->logger->info('Getting all registered mailboxes');
$response = $this->communicator->get('/mailboxes/registered');
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return [];
}
$json = json_decode($body);
if (!isset($json->mailboxes)) {
return [];
}
return $json->mailboxes;
}
public function check(object $mailbox): bool
{
$this->logger->info("Checking mailbox {$mailbox->id}");
if ((new DateTimeImmutable())->diff(new DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) {
return true;
}
$uri = "/mailbox/{$mailbox->id}/check";
$response = $this->communicator->get($uri);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
throw new EmptyResponse($uri);
}
$json = json_decode($body);
if (!isset($json->status)) {
throw new MissingResponse('status');
}
return $json->status;
}
public function grabMessages(int $mailbox_id): int
{
$this->logger->info("Grabbing messages for {$mailbox_id}");
$uri = "/mailbox/{$mailbox_id}/messages/grab";
$response = $this->communicator->get($uri);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return 0;
}
$json = json_decode($body);
if (!isset($json->count)) {
return 0;
}
return $json->count;
}
}

43
cli/common/Service/Messages.php Normal file
View File

@ -0,0 +1,43 @@
<?php
namespace ProVM\Service;
use ProVM\Exception\Response\EmptyResponse;
use ProVM\Exception\Response\MissingResponse;
use Psr\Log\LoggerInterface;
use function Safe\json_decode;
class Messages
{
public function __construct(protected Communicator $communicator, protected LoggerInterface $logger) {}
public function get(int $message_id): object
{
$this->logger->info("Getting message {$message_id}");
$uri = "/message/{$message_id}";
$response = $this->communicator->get($uri);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
throw new EmptyResponse($uri);
}
$json = json_decode($body);
if (!isset($json->message)) {
throw new MissingResponse('message');
}
return $json->message;
}
public function grabAttachments(string $message_uid): int
{
$this->logger->info("Grabbing attachments for message UID {$message_uid}");
$uri = '/attachments/grab';
$response = $this->communicator->put($uri, ['messages' => [$message_uid]]);
$body = $response->getBody()->getContents();
if (trim($body) === '') {
return 0;
}
$json = json_decode($body);
if (!isset($json->total)) {
return 0;
}
return $json->total;
}
}

1
cli/common/Wrapper/Application.php
View File

@ -3,7 +3,6 @@ namespace ProVM\Wrapper;
use Psr\Container\ContainerInterface;
use Symfony\Component\Console\Application as Base;
class Application extends Base
{
public function __construct(ContainerInterface $container, string $name = 'UNKNOWN', string $version = 'UNKNOWN')

10
cli/crontab
View File

@ -1,10 +1,10 @@
# minutes hour day_of_month month day_of_week command
#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log
#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log
#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log
#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log
# Pending jobs every minute
1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log
* * * * * /app/bin/emails jobs:check >> /logs/jobs.log
# Check mailboxes for new emails every weekday
0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log
0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log
# Check attachments every weekday
0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log
0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log

1
cli/docker-compose.yml
View File

@ -12,5 +12,6 @@ services:
- .key.env
volumes:
- ${CLI_PATH:-.}/:/app
- ${CLI_PATH}/crontab:/var/spool/cron/crontabs/root
- ${LOGS_PATH}/cli:/logs
- ${ATT_PATH}:/attachments

1
cli/resources/commands/03_jobs.php
View File

@ -1,2 +1,3 @@
<?php
$app->add($app->getContainer()->get(ProVM\Command\Jobs\Check::class));
$app->add($app->getContainer()->get(ProVM\Command\Jobs\Execute::class));

2
cli/setup/middleware/98_log.php
View File

@ -1,2 +0,0 @@
<?php
//$app->add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class));

2
cli/setup/settings/01_env.php
View File

@ -2,7 +2,9 @@
return [
'api_uri' => $_ENV['API_URI'],
'api_key' => sha1($_ENV['API_KEY']),
'base_command' => 'qpdf',
'passwords' => function() {
return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? '');
},
'min_check_days' => 1
];

17
cli/setup/setups/04_commands.php
View File

@ -3,18 +3,19 @@
use Psr\Container\ContainerInterface;
return [
ProVM\Command\Attachments\DecryptPdf::class => function(ContainerInterface $container) {
return new ProVM\Command\Attachments\DecryptPdf(
ProVM\Service\Mailboxes::class => function(ContainerInterface $container) {
return new ProVM\Service\Mailboxes(
$container->get(ProVM\Service\Communicator::class),
$container->get(Psr\Log\LoggerInterface::class),
'qpdf',
$container->get('passwords')
$container->get('min_check_days')
);
},
ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) {
return new ProVM\Command\Mailboxes\Check(
ProVM\Service\Attachments::class => function(ContainerInterface $container) {
return new ProVM\Service\Attachments(
$container->get(ProVM\Service\Communicator::class),
1
$container->get(Psr\Log\LoggerInterface::class),
$container->get('passwords'),
$container->get('base_command')
);
}
},
];

3
cli/setup/setups/98_log.php
View File

@ -10,8 +10,7 @@ return [
];
},
'request_log_handler' => function(ContainerInterface $container) {
return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log'])))
->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true));
return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log'])));
},
'request_logger' => function(ContainerInterface $container) {
return new Monolog\Logger(