From 03c1dac2f2bfb107b41a92c9f02cc18e41a4c191 Mon Sep 17 00:00:00 2001 From: Aldarien Date: Fri, 9 Jun 2023 00:54:34 -0400 Subject: [PATCH] Cleanup of cli --- NOTES.md | 33 ++--- api/common/Exception/Job/Stateless.php | 15 +++ api/common/Exception/Mailbox/EmptyMailbox.php | 5 +- api/common/Exception/Mailbox/Invalid.php | 3 +- api/common/Exception/Mailbox/Stateless.php | 3 +- api/resources/routes/01_mailboxes.php | 1 + api/resources/routes/02_attachments.php | 3 +- api/setup/setups/98_log.php | 3 +- api/src/Model/Job.php | 65 ++++++++-- api/src/Model/State/Job.php | 65 ++++++++++ api/src/Repository/Job.php | 67 +++++------ api/src/Repository/State/Job.php | 113 ++++++++++++++++++ cli/Dockerfile | 2 +- cli/common/Command/DecryptPdf.php | 68 ----------- cli/common/Command/GrabAttachments.php | 66 ---------- cli/common/Command/Jobs/Check.php | 79 ++++++++++++ cli/common/Command/Mailboxes/Check.php | 76 ++++++++++++ .../{Messages.php => Messages/Grab.php} | 43 ++++--- cli/common/Middleware/Logging.php | 2 +- cli/common/Service/Communicator.php | 33 ++++- cli/common/Wrapper/Application.php | 4 +- cli/composer.json | 2 +- cli/crontab | 11 +- cli/docker-compose.yml | 3 +- cli/public/index.php | 9 +- cli/resources/commands/01_mailboxes.php | 2 + cli/resources/commands/01_messages.php | 2 +- cli/resources/commands/02_attachments.php | 5 +- cli/resources/commands/03_jobs.php | 2 + cli/setup/app.php | 4 +- cli/setup/middleware/98_log.php | 2 +- cli/setup/settings/01_env.php | 5 +- cli/setup/setups/02_api.php | 6 +- cli/setup/setups/03_middleware.php | 4 +- cli/setup/setups/04_commands.php | 20 ++++ cli/setup/setups/98_log.php | 60 +++++++--- 36 files changed, 614 insertions(+), 272 deletions(-) create mode 100644 api/common/Exception/Job/Stateless.php create mode 100644 api/src/Model/State/Job.php create mode 100644 api/src/Repository/State/Job.php delete mode 100644 cli/common/Command/DecryptPdf.php delete mode 100644 cli/common/Command/GrabAttachments.php create mode 100644 cli/common/Command/Jobs/Check.php create mode 100644 cli/common/Command/Mailboxes/Check.php rename cli/common/Command/{Messages.php => Messages/Grab.php} (54%) create mode 100644 cli/resources/commands/01_mailboxes.php create mode 100644 cli/resources/commands/03_jobs.php create mode 100644 cli/setup/setups/04_commands.php diff --git a/NOTES.md b/NOTES.md index ec9035b..b00fae6 100644 --- a/NOTES.md +++ b/NOTES.md @@ -7,8 +7,14 @@ * [ ] Download `attachments` (*encrypted* & *decrypted*). ## CLI -* [x] Get `mailboxes` from **[API]** then run `grab messages` job in **[API]** for each one. -* [x] Get `pending attachments` jobs from **[API]** and run. +#### Automatic +* [x] `mailboxes:check`: Get *registered* `mailboxes` and schedule `messages:grab` for `mailbox_id`. +* [x] `attachments:check`: Check *saved* `attachments` and schedule `attachments:decrypt` for `attachment_id`. +* [x] `jobs:check`: Get *pending* `jobs` and run them. +#### Scheduled +* [x] `messages:grab`: Grab `messages` for `mailbox`. Arguments: `mailbox_id`. +* [x] `attachments:grab`: Grab `attachments` for `message`. Arguments: `message_id`. +* [x] `attachments:decrypt`: Decrypt `attachment`. Arguments: `attachment_id`. ## API * [x] Grab all `mailboxes` from `Email Provider`, identifying those that are registered. @@ -21,18 +27,17 @@ ## Workflow -* **[User]** Choose `mailboxes` to register or unregister - -> **[API]** Register selected `mailboxes` and get `messages` for recently registered. -* **[Cron]** Get registered `mailboxes` -> **[API]** Get `messages` -* **[User]** Check messages found -> **[API]** Schedule `attachments` -* **[Cron]** Get `attachment download` jobs -> **[API]** grab `attachments` +* **[User]** Choose `mailboxes` to register or unregister. + -> **[API]** Register selected `mailboxes`, register new `messages:grab` job. +* **[Cron]** Get `jobs`, run `jobs`. +* **[User]** Check messages found -> **[API]** Schedule `attachments`. -### Jobs +## Jobs #### Automatic -* [ ] Check registered `mailboxes` for new `messages`, logging last check. -* [ ] Check if `attachments` are `encrypted`. -* [ ] Check for new scheduled jobs. +* [x] Check *registered* `mailboxes` for new `messages`. Every weekday. +* [x] Check if `attachments` are *encrypted*. Every weekday. +* [x] Check for new *scheduled* `jobs`. Every minute. #### Scheduled -* [ ] Grab `messages`. -* [ ] Grab `attachments`. -* [ ] Decrypt `attachments`. +* [ ] Grab `messages` for `mailbox` id. +* [ ] Grab `attachments` for `message` id. +* [ ] Decrypt `attachment`. diff --git a/api/common/Exception/Job/Stateless.php b/api/common/Exception/Job/Stateless.php new file mode 100644 index 0000000..7f2d7f5 --- /dev/null +++ b/api/common/Exception/Job/Stateless.php @@ -0,0 +1,15 @@ +group('/mailboxes', function($app) { $app->group('/mailbox/{mailbox_id}', function($app) { $app->group('/messages', function($app) { + $app->get('/grab[/]', [Messages::class, 'grab']); $app->get('/valid[/]', [Messages::class, 'valid']); $app->get('[/]', Messages::class); }); diff --git a/api/resources/routes/02_attachments.php b/api/resources/routes/02_attachments.php index 62969b7..9b45b9d 100644 --- a/api/resources/routes/02_attachments.php +++ b/api/resources/routes/02_attachments.php @@ -3,9 +3,10 @@ use ProVM\Common\Controller\Attachments; $app->group('/attachments', function($app) { $app->put('/grab', [Attachments::class, 'grab']); + $app->get('/pending', [Attachments::class, 'pending']); $app->post('/decrypt', [Attachments::class, 'decrypt']); $app->get('[/]', Attachments::class); }); $app->group('/attachment/{attachment_id}', function($app) { $app->get('[/]', [Attachments::class, 'get']); -}); \ No newline at end of file +}); diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index 4c6a66e..0ebc4cb 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -6,6 +6,7 @@ return [ return [ $container->get(Monolog\Processor\PsrLogMessageProcessor::class), $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\WebProcessor::class), $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), ]; }, @@ -36,7 +37,7 @@ return [ ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - return $container->get('elk_logger'); + return $container->get('file_logger'); }, 'file_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index b4a9cc5..5aa26ee 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -3,29 +3,31 @@ namespace ProVM\Emails\Model; use DateTimeInterface; use ProVM\Common\Define\Model; +use ProVM\Common\Exception\Database\BlankResult; +use ProVM\Common\Exception\Job\Stateless; +use ProVM\Emails; class Job implements Model { protected int $id; - protected Message $message; - protected DateTimeInterface $dateTime; - protected bool $executed; + protected string $command; + protected string $arguments; public function getId(): int { return $this->id; } - public function getMessage(): Message + public function getCommand(): string { - return $this->message; + return $this->command; } - public function getDateTime(): DateTimeInterface + public function getArguments(): string { - return $this->dateTime; + return $this->arguments ?? ''; } public function isExecuted(): bool { - return $this->executed ?? false; + return $this->lastState()->getStatus() === State\Job::Executed; } public function setId(int $id): Job @@ -33,22 +35,61 @@ class Job implements Model $this->id = $id; return $this; } - public function setMessage(Message $message): Job + public function setCommand(string $message): Job { $this->message = $message; return $this; } - public function setDateTime(DateTimeInterface $dateTime): Job + public function setArguments(string $dateTime): Job { $this->dateTime = $dateTime; return $this; } - public function wasExecuted(): Job + + protected Emails\Repository\State\Job $stateRepository; + public function getStateRepository(): Emails\Repository\State\Job { - $this->executed = true; + return $this->stateRepository; + } + public function setStateRepository(Emails\Repository\State\Job $repository): Job + { + $this->stateRepository = $repository; return $this; } + protected array $states; + public function getStates(): array + { + if (!isset($this->states)) { + try { + $this->setStates($this->getStateRepository()->fetchByJob($this->getId())); + } catch (BlankResult $e) { + return []; + } + } + return $this->states; + } + public function addState(State\Job $state): Job + { + $this->states []= $state; + return $this; + } + public function setStates(array $states): Job + { + foreach ($states as $state) { + $this->addState($state); + } + return $this; + } + + public function lastState(): State\Job + { + if (count($this->getStates()) === 0) { + throw new Stateless($this); + } + return $this->getStates()[array_key_last($this->getStates())]; + } + public function toArray(): array { return [ diff --git a/api/src/Model/State/Job.php b/api/src/Model/State/Job.php new file mode 100644 index 0000000..2826908 --- /dev/null +++ b/api/src/Model/State/Job.php @@ -0,0 +1,65 @@ +id; + } + public function getJob(): Emails\Model\Job + { + return $this->job; + } + public function getDateTime(): \DateTimeInterface + { + return $this->dateTime; + } + public function getStatus(): int + { + return $this->status; + } + + public function setId(int $id): Job + { + $this->id = $id; + return $this; + } + public function setJob(Emails\Model\Job $job): Job + { + $this->job = $job; + return $this; + } + public function setDateTime(\DateTimeInterface $dateTime): Job + { + $this->dateTime = $dateTime; + return $this; + } + public function setStatus(int $status): Job + { + $this->status = $status; + return $this; + } + + public function jsonSerialize(): mixed + { + return [ + 'id' => $this->getId(), + 'job' => $this->getJob(), + 'date' => $this->getDateTime(), + 'status' => $this->getStatus() + ]; + } +} diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index 0f47ee3..e8f6be9 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -2,28 +2,28 @@ namespace ProVM\Emails\Repository; use PDO; -use ProVM\Common\Factory\Model; +use ProVM\Common\Factory; use Psr\Log\LoggerInterface; -use Safe\DateTimeImmutable; use ProVM\Common\Define\Model as ModelInterface; use ProVM\Common\Implement\Repository; use ProVM\Emails\Model\Job as BaseModel; +use ProVM\Emails; class Job extends Repository { - public function __construct(PDO $connection, LoggerInterface $logger, Model $factory) + public function __construct(PDO $connection, LoggerInterface $logger, Factory\Model $factory) { parent::__construct($connection, $logger); $this->setFactory($factory) ->setTable('attachments_jobs'); } - protected \ProVM\Common\Factory\Model $factory; - public function getFactory(): \ProVM\Common\Factory\Model + protected Factory\Model $factory; + public function getFactory(): Factory\Model { return $this->factory; } - public function setFactory(\ProVM\Common\Factory\Model $factory): Job + public function setFactory(Factory\Model $factory): Job { $this->factory = $factory; return $this; @@ -34,12 +34,9 @@ class Job extends Repository $query = " CREATE TABLE {$this->getTable()} ( `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, - `message_id` INT UNSIGNED NOT NULL, - `date_time` DATETIME NOT NULL, - `executed` INT(1) UNSIGNED DEFAULT 0, - PRIMARY KEY (`id`), - FOREIGN KEY `fk_messages_{$this->getTable()}` (`message_id`) - REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + `command` VARCHAR(100) NOT NULL, + `arguments` TEXT NOT NULL, + PRIMARY KEY (`id`) )"; $this->getConnection()->query($query); } @@ -55,65 +52,61 @@ CREATE TABLE {$this->getTable()} ( protected function fieldsForInsert(): array { return [ - 'message_id', - 'date_time', - 'executed' + 'command', + 'arguments', ]; } protected function valuesForInsert(ModelInterface $model): array { return [ - $model->getMessage()->getId(), - $model->getDateTime()->format('Y-m-d H:i:s'), - $model->isExecuted() ? 1 : 0 + $model->getCommand(), + $model->getArguments(), ]; } protected function defaultFind(ModelInterface $model): ModelInterface { - return $this->fetchByMessageAndDate($model->getMessage()->getId(), $model->getDateTime()->format('Y-m-d H:i:s')); + return $this->fetchByCommandAndArguments($model->getCommand(), $model->getArguments()); } protected function fieldsForCreate(): array { return [ - 'message_id', - 'date_time', - 'executed' + 'command', + 'arguments', ]; } protected function valuesForCreate(array $data): array { return [ - $data['message_id'], - $data['date_time'], - $data['executed'] ?? 0 + $data['command'], + $data['arguments'], ]; } protected function defaultSearch(array $data): ModelInterface { - return $this->fetchByMessageAndDate($data['message_id'], $data['date_time']); + return $this->fetchByCommandAndArguments($data['command'], $data['arguments']); } public function load(array $row): ModelInterface { - $model = (new BaseModel()) + return (new BaseModel()) ->setId($row['id']) - ->setMessage($this->getFactory()->find(\ProVM\Emails\Model\Message::class)->fetchById($row['message_id'])) - ->setDateTime(new DateTimeImmutable($row['date_time'])); - if ($row['executed'] ?? 0 === 1) { - $model->wasExecuted(); - } - return $model; + ->setCommand($row['command']) + ->setArguments($row['arguments']) + ->setStateRepository($this->getFactory()->find(Emails\Model\State\Job::class)); } public function fetchAllPending(): array { - $query = "SELECT * FROM {$this->getTable()} WHERE `executed` = 0"; + $query = "SELECT a.* +FROM {$this->getTable()} a + JOIN `jobs_states` b ON b.job_id = a.id +WHERE b.`status` = ?"; return $this->fetchMany($query); } - public function fetchByMessageAndDate(int $message_id, string $date_time): \ProVM\Emails\Model\Job + public function fetchByCommandAndArguments(string $command, string $arguments): \ProVM\Emails\Model\Job { - $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `date_time` = ?"; - return $this->fetchOne($query, [$message_id, $date_time]); + $query = "SELECT * FROM {$this->getTable()} WHERE `command` = ? AND `arguments` = ?"; + return $this->fetchOne($query, [$command, $arguments]); } public function fetchPendingByMessage(int $message_id): \ProVM\Emails\Model\Job { diff --git a/api/src/Repository/State/Job.php b/api/src/Repository/State/Job.php new file mode 100644 index 0000000..e7f9d57 --- /dev/null +++ b/api/src/Repository/State/Job.php @@ -0,0 +1,113 @@ +setTable('jobs_states') + ->setFactory($factory); + } + + protected Factory\Model $factory; + public function getFactory(): Factory\Model + { + return $this->factory; + } + public function setFactory(Factory\Model $factory): Job + { + $this->factory = $factory; + return $this; + } + + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `job_id` INT UNSIGNED NOT NULL, + `date_time` DATETIME NOT NULL, + `status` INT NOT NULL DEFAULT 1, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_jobs_{$this->getTable()}` (`job_id`) + REFERENCES `jobs` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +)"; + $this->getConnection()->query($query); + } + + public function load(array $row): Define\Model + { + return (new Emails\Model\State\Job()) + ->setId($row['id']) + ->setJob($this->getFactory()->find(Emails\Model\Job::class)->fetchById($row['job_id'])) + ->setDateTime(new \DateTimeImmutable($row['date_time'])) + ->setStatus($row['status']); + } + + public function fetchByJob(int $job_id): array + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ?"; + return $this->fetchMany($query, [$job_id]); + } + public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\Job + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ? AND `status` = ?"; + return $this->fetchOne($query, [$job_id, $status]); + } + + protected function fieldsForInsert(): array + { + return [ + 'job_id', + 'date_time', + 'status' + ]; + } + protected function valuesForInsert(Define\Model $model): array + { + return [ + $model->getJob()->getId(), + $model->getDateTime()->format('Y-m-d H:i:s'), + $model->getStatus() + ]; + } + + protected function fieldsForUpdate(): array + { + return $this->fieldsForInsert(); + } + protected function fieldsForCreate(): array + { + return $this->fieldsForInsert(); + } + + protected function valuesForUpdate(Define\Model $model): array + { + return $this->valuesForInsert($model); + } + protected function valuesForCreate(array $data): array + { + return [ + $data['job_id'], + $data['date_time'], + $data['status'] + ]; + } + + protected function defaultFind(Define\Model $model): Define\Model + { + return $this->fetchByJobAndStatus($model->getJob()->getId(), $model->getStatus()); + } + protected function defaultSearch(array $data): Define\Model + { + return $this->fetchByJobAndStatus($data['job_id'], $data['status']); + } +} diff --git a/cli/Dockerfile b/cli/Dockerfile index 2ed89eb..d596ae4 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -3,7 +3,7 @@ FROM php:8-cli ENV PATH ${PATH}:/app/bin RUN apt-get update \ - && apt-get install -y cron git libzip-dev unzip \ + && apt-get install -y cron git libzip-dev unzip qpdf \ && rm -r /var/lib/apt/lists/* \ && docker-php-ext-install zip diff --git a/cli/common/Command/DecryptPdf.php b/cli/common/Command/DecryptPdf.php deleted file mode 100644 index e868843..0000000 --- a/cli/common/Command/DecryptPdf.php +++ /dev/null @@ -1,68 +0,0 @@ -setCommunicator($communicator); - parent::__construct($name); - } - - protected Communicator $communicator; - public function getCommunicator(): Communicator - { - return $this->communicator; - } - public function setCommunicator(Communicator $communicator): DecryptPdf - { - $this->communicator = $communicator; - return $this; - } - - protected function getAttachments(): array - { - $response = $this->getCommunicator()->get('/attachments/pending'); - return json_decode($response->getBody()->getContents())->attachments; - } - protected function decrypt(string $attachment): bool - { - $response = $this->getCommunicator()->put('/attachments/decrypt', ['attachments' => [$attachment]]); - return json_decode($response->getBody()->getContents())->status; - } - - public function execute(InputInterface $input, OutputInterface $output) - { - $io = new SymfonyStyle($input, $output); - $io->title('Decrypt Attachments'); - - $io->section('Grabbing Attachments'); - $attachments = $this->getAttachments(); - $io->text('Found ' . count($attachments) . ' attachments.'); - $io->section('Decrypting Attachments'); - foreach ($attachments as $attachment) { - $status = $this->decrypt($attachment); - if ($status) { - $io->success("{$attachment} decrypted correctly."); - } else { - $io->error("Problem decrypting {$attachment}."); - } - } - $io->success('Done.'); - - return Command::SUCCESS; - } -} diff --git a/cli/common/Command/GrabAttachments.php b/cli/common/Command/GrabAttachments.php deleted file mode 100644 index 23295e1..0000000 --- a/cli/common/Command/GrabAttachments.php +++ /dev/null @@ -1,66 +0,0 @@ -setCommunicator($communicator); - parent::__construct($name); - } - - protected Communicator $service; - public function getCommunicator(): Communicator - { - return $this->service; - } - public function setCommunicator(Communicator $service): GrabAttachments - { - $this->service = $service; - return $this; - } - - protected function getMessages(): array - { - $response = $this->getCommunicator()->get('/messages/pending'); - return json_decode($response->getBody()->getContents())->messages; - } - protected function grabAttachments(int $message_uid): int - { - $response = $this->getCommunicator()->put('/attachments/grab', ['messages' => [$message_uid]]); - return json_decode($response->getBody()->getContents())->attachment_count; - } - - public function execute(InputInterface $input, OutputInterface $output): int - { - $io = new SymfonyStyle($input, $output); - $io->title('Grab Attachments'); - - $io->section('Grabbing Messages'); - $messages = $this->getMessages(); - $io->text('Found ' . count($messages) . ' messages.'); - $io->section('Grabbing Attachments'); - foreach ($messages as $job) { - $message = $job->message; - $attachments = $this->grabAttachments($message->uid); - $io->text("Found {$attachments} attachments for message UID:{$message->uid}."); - } - $io->success('Done.'); - - return Command::SUCCESS; - } -} diff --git a/cli/common/Command/Jobs/Check.php b/cli/common/Command/Jobs/Check.php new file mode 100644 index 0000000..d3c9b23 --- /dev/null +++ b/cli/common/Command/Jobs/Check.php @@ -0,0 +1,79 @@ +logger->notice('Grabbing pending jobs.'); + $response = $this->communicator->get('/jobs/pending'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + return json_decode($body)->jobs; + } + protected function runJob($job): bool + { + $base_command = '/app/bin/emails'; + $cmd = [$base_command, $job->command]; + if ($job->arguments !== '') { + $cmd []= $job->arguments; + } + $cmd = implode(' ', $cmd); + $this->logger->notice("Running '{$cmd}'"); + $response = shell_exec($cmd); + $this->logger->info("Result: {$response}"); + return $response !== false; + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $section1 = $output->section(); + $section2 = $output->section(); + $io1 = new SymfonyStyle($input, $section1); + $io2 = new SymfonyStyle($input, $section2); + $io1->title('Checking Pending Jobs'); + $pending_jobs = $this->getPendingJobs(); + $notice = 'Found ' . count($pending_jobs) . ' jobs'; + $io1->text($notice); + $this->logger->info($notice); + if (count($pending_jobs) > 0) { + $io1->section('Running Jobs'); + $io1->progressStart(count($pending_jobs)); + foreach ($pending_jobs as $job) { + $section2->clear(); + $io2->text("Running {$job->command}"); + if ($this->runJob($job)) { + $io2->success('Success'); + } else { + $io2->error('Failure'); + } + $io1->progressAdvance(); + } + } + $section2->clear(); + $io2->success('Done'); + + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Mailboxes/Check.php b/cli/common/Command/Mailboxes/Check.php new file mode 100644 index 0000000..c0fa6eb --- /dev/null +++ b/cli/common/Command/Mailboxes/Check.php @@ -0,0 +1,76 @@ +communicator->get('/mailboxes/registered'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + return json_decode($body)->mailboxes; + } + protected function checkMailbox($mailbox): bool + { + if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { + return true; + } + $response = $this->communicator->get("/mailbox/{$mailbox->id}/check"); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return true; + } + return json_decode($body)->status; + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + $section1 = $output->section(); + $section2 = $output->section(); + $io1 = new SymfonyStyle($input, $section1); + $io2 = new SymfonyStyle($input, $section2); + $io1->title('Checking for New Messages'); + $mailboxes = $this->getMailboxes(); + $notice = 'Found ' . count($mailboxes) . ' mailboxes'; + $io1->text($notice); + if (count($mailboxes) > 0) { + $io1->section('Checking for new messages'); + $io1->progressStart(count($mailboxes)); + foreach ($mailboxes as $mailbox) { + $section2->clear(); + $io2->text("Checking {$mailbox->name}"); + if ($this->checkMailbox($mailbox)) { + $io2->success("Found new emails in {$mailbox->name}"); + } else { + $io2->info("No new emails in {$mailbox->name}"); + } + $io1->progressAdvance(); + } + $io1->progressFinish(); + } + $section2->clear(); + $io2->success('Done'); + + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Messages.php b/cli/common/Command/Messages/Grab.php similarity index 54% rename from cli/common/Command/Messages.php rename to cli/common/Command/Messages/Grab.php index 848436e..89a85f9 100644 --- a/cli/common/Command/Messages.php +++ b/cli/common/Command/Messages/Grab.php @@ -1,65 +1,62 @@ setCommunicator($communicator); parent::__construct($name); } + protected function configure() + { + $this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails'); + } protected Communicator $communicator; public function getCommunicator(): Communicator { return $this->communicator; } - public function setCommunicator(Communicator $communicator): Messages + public function setCommunicator(Communicator $communicator): Grab { $this->communicator = $communicator; return $this; } - protected function getMailboxes(): array + protected function grabMessages(int $mailbox_id): int { - $response = $this->getCommunicator()->get('/mailboxes/registered'); - return json_decode($response->getBody()->getContents())->mailboxes; - } - protected function grabMessages(string $mailbox): int - { - $response = $this->getCommunicator()->put('/messages/grab', ['mailboxes' => [$mailbox]]); - $body = json_decode($response->getBody()->getContents()); - return $body->message_count; + $response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab"); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + return json_decode($body)->messages->count; } public function execute(InputInterface $input, OutputInterface $output) { $io = new SymfonyStyle($input, $output); - $io->title('Messages'); - $io->section('Grabbing Registered Mailboxes'); - - $mailboxes = $this->getMailboxes(); - $io->text('Found ' . count($mailboxes) . ' registered mailboxes.'); + $mailbox_id = $input->getArgument('mailbox_id'); + $io->title("Grabbing Messages for Mailbox ID {$mailbox_id}"); $io->section('Grabbing Messages'); - foreach ($mailboxes as $mailbox) { - $message_count = $this->grabMessages($mailbox->name); - $io->text("Found {$message_count} messages in {$mailbox->name}."); - } + $count = $this->grabMessages($mailbox_id); + $io->info("Found {$count} messages"); $io->success('Done.'); return Command::SUCCESS; diff --git a/cli/common/Middleware/Logging.php b/cli/common/Middleware/Logging.php index a95c37b..25021c5 100644 --- a/cli/common/Middleware/Logging.php +++ b/cli/common/Middleware/Logging.php @@ -1,5 +1,5 @@ getStatusCode() < 200 or $response->getStatusCode() >= 300) { @@ -33,6 +38,11 @@ class Communicator } return $response; } + + /** + * @throws HttpResponseException + * @throws JsonException + */ protected function request(string $method, string $uri, ?array $body = null): ResponseInterface { $options = []; @@ -45,20 +55,39 @@ class Communicator return $this->handleResponse($this->getClient()->request($method, $uri, $options)); } + /** + * @throws HttpResponseException + * @throws JsonException + */ public function get(string $uri): ResponseInterface { return $this->request('get', $uri); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function post(string $uri, array $data): ResponseInterface { return $this->request('post', $uri, $data); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function put(string $uri, array $data): ResponseInterface { return $this->request('put', $uri, $data); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function delete(string $uri, array $data): ResponseInterface { return $this->request('delete', $uri, $data); } -} \ No newline at end of file +} diff --git a/cli/common/Wrapper/Application.php b/cli/common/Wrapper/Application.php index 8d8e4aa..c27dbf4 100644 --- a/cli/common/Wrapper/Application.php +++ b/cli/common/Wrapper/Application.php @@ -1,5 +1,5 @@ container = $container; return $this; } -} \ No newline at end of file +} diff --git a/cli/composer.json b/cli/composer.json index 2c1d0c1..4365fcb 100644 --- a/cli/composer.json +++ b/cli/composer.json @@ -15,7 +15,7 @@ }, "autoload": { "psr-4": { - "ProVM\\Common\\": "common/" + "ProVM\\": "common/" } }, "authors": [ diff --git a/cli/crontab b/cli/crontab index 49d2722..fc8ab81 100644 --- a/cli/crontab +++ b/cli/crontab @@ -1,3 +1,10 @@ # minutes hour day_of_month month day_of_week command -0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log -0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log +#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log +#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log + +# Pending jobs every minute +1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log +# Check mailboxes for new emails every weekday +0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log +# Check attachments every weekday +0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log diff --git a/cli/docker-compose.yml b/cli/docker-compose.yml index 38d19e3..ea1ec32 100644 --- a/cli/docker-compose.yml +++ b/cli/docker-compose.yml @@ -12,4 +12,5 @@ services: - .key.env volumes: - ${CLI_PATH:-.}/:/app - - ./logs/cli:/logs + - ${LOGS_PATH}/cli:/logs + - ${ATT_PATH}:/attachments diff --git a/cli/public/index.php b/cli/public/index.php index 63341a5..f8b2dac 100644 --- a/cli/public/index.php +++ b/cli/public/index.php @@ -7,9 +7,8 @@ $app = require_once implode(DIRECTORY_SEPARATOR, [ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface::class)); try { $app->run(); -} catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; +} catch (Error $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); +} catch (Exception $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->warning($e); } diff --git a/cli/resources/commands/01_mailboxes.php b/cli/resources/commands/01_mailboxes.php new file mode 100644 index 0000000..843e8e4 --- /dev/null +++ b/cli/resources/commands/01_mailboxes.php @@ -0,0 +1,2 @@ +add($app->getContainer()->get(ProVM\Command\Mailboxes\Check::class)); diff --git a/cli/resources/commands/01_messages.php b/cli/resources/commands/01_messages.php index c034ee1..4313973 100644 --- a/cli/resources/commands/01_messages.php +++ b/cli/resources/commands/01_messages.php @@ -1,2 +1,2 @@ add($app->getContainer()->get(\ProVM\Common\Command\Messages::class)); +$app->add($app->getContainer()->get(ProVM\Command\Messages\Grab::class)); diff --git a/cli/resources/commands/02_attachments.php b/cli/resources/commands/02_attachments.php index 85793e1..d46dbc1 100644 --- a/cli/resources/commands/02_attachments.php +++ b/cli/resources/commands/02_attachments.php @@ -1,3 +1,4 @@ add($app->getContainer()->get(\ProVM\Common\Command\GrabAttachments::class)); -$app->add($app->getContainer()->get(\ProVM\Common\Command\DecryptPdf::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Check::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Grab::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Decrypt::class)); diff --git a/cli/resources/commands/03_jobs.php b/cli/resources/commands/03_jobs.php new file mode 100644 index 0000000..bd6eed1 --- /dev/null +++ b/cli/resources/commands/03_jobs.php @@ -0,0 +1,2 @@ +add($app->getContainer()->get(ProVM\Command\Jobs\Check::class)); diff --git a/cli/setup/app.php b/cli/setup/app.php index 16c1e0f..d00dbc6 100644 --- a/cli/setup/app.php +++ b/cli/setup/app.php @@ -1,7 +1,7 @@ build()); +$app = new ProVM\Wrapper\Application($builder->build()); $folder = implode(DIRECTORY_SEPARATOR, [ __DIR__, diff --git a/cli/setup/middleware/98_log.php b/cli/setup/middleware/98_log.php index 4df6f6c..5628448 100644 --- a/cli/setup/middleware/98_log.php +++ b/cli/setup/middleware/98_log.php @@ -1,2 +1,2 @@ add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); +//$app->add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); diff --git a/cli/setup/settings/01_env.php b/cli/setup/settings/01_env.php index 58b2a65..107718a 100644 --- a/cli/setup/settings/01_env.php +++ b/cli/setup/settings/01_env.php @@ -1,5 +1,8 @@ $_ENV['API_URI'], - 'api_key' => sha1($_ENV['API_KEY']) + 'api_key' => sha1($_ENV['API_KEY']), + 'passwords' => function() { + return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? ''); + }, ]; diff --git a/cli/setup/setups/02_api.php b/cli/setup/setups/02_api.php index b123345..bf102d3 100644 --- a/cli/setup/setups/02_api.php +++ b/cli/setup/setups/02_api.php @@ -2,12 +2,12 @@ use Psr\Container\ContainerInterface; return [ - \Psr\Http\Client\ClientInterface::class => function(ContainerInterface $container) { - return new \GuzzleHttp\Client([ + Psr\Http\Client\ClientInterface::class => function(ContainerInterface $container) { + return new GuzzleHttp\Client([ 'base_uri' => $container->get('api_uri'), 'headers' => [ 'Authorization' => "Bearer {$container->get('api_key')}" ] ]); } -]; \ No newline at end of file +]; diff --git a/cli/setup/setups/03_middleware.php b/cli/setup/setups/03_middleware.php index 1d6cc44..7d872ab 100644 --- a/cli/setup/setups/03_middleware.php +++ b/cli/setup/setups/03_middleware.php @@ -2,7 +2,7 @@ use Psr\Container\ContainerInterface; return [ - ProVM\Common\Middleware\Logging::class => function(ContainerInterface $container) { - return new ProVM\Common\Middleware\Logging($container->get('request_logger')); + ProVM\Middleware\Logging::class => function(ContainerInterface $container) { + return new ProVM\Middleware\Logging($container->get('request_logger')); } ]; diff --git a/cli/setup/setups/04_commands.php b/cli/setup/setups/04_commands.php new file mode 100644 index 0000000..50ae9d1 --- /dev/null +++ b/cli/setup/setups/04_commands.php @@ -0,0 +1,20 @@ + function(ContainerInterface $container) { + return new ProVM\Command\Attachments\DecryptPdf( + $container->get(ProVM\Service\Communicator::class), + $container->get(Psr\Log\LoggerInterface::class), + 'qpdf', + $container->get('passwords') + ); + }, + ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) { + return new ProVM\Command\Mailboxes\Check( + $container->get(ProVM\Service\Communicator::class), + 1 + ); + } +]; diff --git a/cli/setup/setups/98_log.php b/cli/setup/setups/98_log.php index b33a5ce..7f6e30b 100644 --- a/cli/setup/setups/98_log.php +++ b/cli/setup/setups/98_log.php @@ -2,28 +2,50 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { - $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); - $handler->setFormatter($container->get(Monolog\Formatter\LineFormatter::class)); - return $handler; + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), + ]; + }, + 'request_log_handler' => function(ContainerInterface $container) { + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger( + 'request_logger', + [$container->get('request_log_handler')], + $container->get('log_processors') + ); + }, + 'file_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ); + }, + 'debug_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return $container->get('file_logger'); + }, + 'file_logger' => function(ContainerInterface $container) { + return new Monolog\Logger( + 'file', + [ + $container->get('file_log_handler'), + $container->get('debug_log_handler') + ], + $container->get('log_processors') + ); }, ];