From 03c1dac2f2bfb107b41a92c9f02cc18e41a4c191 Mon Sep 17 00:00:00 2001 From: Aldarien Date: Fri, 9 Jun 2023 00:54:34 -0400 Subject: [PATCH 1/2] Cleanup of cli --- NOTES.md | 33 ++--- api/common/Exception/Job/Stateless.php | 15 +++ api/common/Exception/Mailbox/EmptyMailbox.php | 5 +- api/common/Exception/Mailbox/Invalid.php | 3 +- api/common/Exception/Mailbox/Stateless.php | 3 +- api/resources/routes/01_mailboxes.php | 1 + api/resources/routes/02_attachments.php | 3 +- api/setup/setups/98_log.php | 3 +- api/src/Model/Job.php | 65 ++++++++-- api/src/Model/State/Job.php | 65 ++++++++++ api/src/Repository/Job.php | 67 +++++------ api/src/Repository/State/Job.php | 113 ++++++++++++++++++ cli/Dockerfile | 2 +- cli/common/Command/DecryptPdf.php | 68 ----------- cli/common/Command/GrabAttachments.php | 66 ---------- cli/common/Command/Jobs/Check.php | 79 ++++++++++++ cli/common/Command/Mailboxes/Check.php | 76 ++++++++++++ .../{Messages.php => Messages/Grab.php} | 43 ++++--- cli/common/Middleware/Logging.php | 2 +- cli/common/Service/Communicator.php | 33 ++++- cli/common/Wrapper/Application.php | 4 +- cli/composer.json | 2 +- cli/crontab | 11 +- cli/docker-compose.yml | 3 +- cli/public/index.php | 9 +- cli/resources/commands/01_mailboxes.php | 2 + cli/resources/commands/01_messages.php | 2 +- cli/resources/commands/02_attachments.php | 5 +- cli/resources/commands/03_jobs.php | 2 + cli/setup/app.php | 4 +- cli/setup/middleware/98_log.php | 2 +- cli/setup/settings/01_env.php | 5 +- cli/setup/setups/02_api.php | 6 +- cli/setup/setups/03_middleware.php | 4 +- cli/setup/setups/04_commands.php | 20 ++++ cli/setup/setups/98_log.php | 60 +++++++--- 36 files changed, 614 insertions(+), 272 deletions(-) create mode 100644 api/common/Exception/Job/Stateless.php create mode 100644 api/src/Model/State/Job.php create mode 100644 api/src/Repository/State/Job.php delete mode 100644 cli/common/Command/DecryptPdf.php delete mode 100644 cli/common/Command/GrabAttachments.php create mode 100644 cli/common/Command/Jobs/Check.php create mode 100644 cli/common/Command/Mailboxes/Check.php rename cli/common/Command/{Messages.php => Messages/Grab.php} (54%) create mode 100644 cli/resources/commands/01_mailboxes.php create mode 100644 cli/resources/commands/03_jobs.php create mode 100644 cli/setup/setups/04_commands.php diff --git a/NOTES.md b/NOTES.md index ec9035b..b00fae6 100644 --- a/NOTES.md +++ b/NOTES.md @@ -7,8 +7,14 @@ * [ ] Download `attachments` (*encrypted* & *decrypted*). ## CLI -* [x] Get `mailboxes` from **[API]** then run `grab messages` job in **[API]** for each one. -* [x] Get `pending attachments` jobs from **[API]** and run. +#### Automatic +* [x] `mailboxes:check`: Get *registered* `mailboxes` and schedule `messages:grab` for `mailbox_id`. +* [x] `attachments:check`: Check *saved* `attachments` and schedule `attachments:decrypt` for `attachment_id`. +* [x] `jobs:check`: Get *pending* `jobs` and run them. +#### Scheduled +* [x] `messages:grab`: Grab `messages` for `mailbox`. Arguments: `mailbox_id`. +* [x] `attachments:grab`: Grab `attachments` for `message`. Arguments: `message_id`. +* [x] `attachments:decrypt`: Decrypt `attachment`. Arguments: `attachment_id`. ## API * [x] Grab all `mailboxes` from `Email Provider`, identifying those that are registered. @@ -21,18 +27,17 @@ ## Workflow -* **[User]** Choose `mailboxes` to register or unregister - -> **[API]** Register selected `mailboxes` and get `messages` for recently registered. -* **[Cron]** Get registered `mailboxes` -> **[API]** Get `messages` -* **[User]** Check messages found -> **[API]** Schedule `attachments` -* **[Cron]** Get `attachment download` jobs -> **[API]** grab `attachments` +* **[User]** Choose `mailboxes` to register or unregister. + -> **[API]** Register selected `mailboxes`, register new `messages:grab` job. +* **[Cron]** Get `jobs`, run `jobs`. +* **[User]** Check messages found -> **[API]** Schedule `attachments`. -### Jobs +## Jobs #### Automatic -* [ ] Check registered `mailboxes` for new `messages`, logging last check. -* [ ] Check if `attachments` are `encrypted`. -* [ ] Check for new scheduled jobs. +* [x] Check *registered* `mailboxes` for new `messages`. Every weekday. +* [x] Check if `attachments` are *encrypted*. Every weekday. +* [x] Check for new *scheduled* `jobs`. Every minute. #### Scheduled -* [ ] Grab `messages`. -* [ ] Grab `attachments`. -* [ ] Decrypt `attachments`. +* [ ] Grab `messages` for `mailbox` id. +* [ ] Grab `attachments` for `message` id. +* [ ] Decrypt `attachment`. diff --git a/api/common/Exception/Job/Stateless.php b/api/common/Exception/Job/Stateless.php new file mode 100644 index 0000000..7f2d7f5 --- /dev/null +++ b/api/common/Exception/Job/Stateless.php @@ -0,0 +1,15 @@ +group('/mailboxes', function($app) { $app->group('/mailbox/{mailbox_id}', function($app) { $app->group('/messages', function($app) { + $app->get('/grab[/]', [Messages::class, 'grab']); $app->get('/valid[/]', [Messages::class, 'valid']); $app->get('[/]', Messages::class); }); diff --git a/api/resources/routes/02_attachments.php b/api/resources/routes/02_attachments.php index 62969b7..9b45b9d 100644 --- a/api/resources/routes/02_attachments.php +++ b/api/resources/routes/02_attachments.php @@ -3,9 +3,10 @@ use ProVM\Common\Controller\Attachments; $app->group('/attachments', function($app) { $app->put('/grab', [Attachments::class, 'grab']); + $app->get('/pending', [Attachments::class, 'pending']); $app->post('/decrypt', [Attachments::class, 'decrypt']); $app->get('[/]', Attachments::class); }); $app->group('/attachment/{attachment_id}', function($app) { $app->get('[/]', [Attachments::class, 'get']); -}); \ No newline at end of file +}); diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index 4c6a66e..0ebc4cb 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -6,6 +6,7 @@ return [ return [ $container->get(Monolog\Processor\PsrLogMessageProcessor::class), $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\WebProcessor::class), $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), ]; }, @@ -36,7 +37,7 @@ return [ ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - return $container->get('elk_logger'); + return $container->get('file_logger'); }, 'file_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index b4a9cc5..5aa26ee 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -3,29 +3,31 @@ namespace ProVM\Emails\Model; use DateTimeInterface; use ProVM\Common\Define\Model; +use ProVM\Common\Exception\Database\BlankResult; +use ProVM\Common\Exception\Job\Stateless; +use ProVM\Emails; class Job implements Model { protected int $id; - protected Message $message; - protected DateTimeInterface $dateTime; - protected bool $executed; + protected string $command; + protected string $arguments; public function getId(): int { return $this->id; } - public function getMessage(): Message + public function getCommand(): string { - return $this->message; + return $this->command; } - public function getDateTime(): DateTimeInterface + public function getArguments(): string { - return $this->dateTime; + return $this->arguments ?? ''; } public function isExecuted(): bool { - return $this->executed ?? false; + return $this->lastState()->getStatus() === State\Job::Executed; } public function setId(int $id): Job @@ -33,22 +35,61 @@ class Job implements Model $this->id = $id; return $this; } - public function setMessage(Message $message): Job + public function setCommand(string $message): Job { $this->message = $message; return $this; } - public function setDateTime(DateTimeInterface $dateTime): Job + public function setArguments(string $dateTime): Job { $this->dateTime = $dateTime; return $this; } - public function wasExecuted(): Job + + protected Emails\Repository\State\Job $stateRepository; + public function getStateRepository(): Emails\Repository\State\Job { - $this->executed = true; + return $this->stateRepository; + } + public function setStateRepository(Emails\Repository\State\Job $repository): Job + { + $this->stateRepository = $repository; return $this; } + protected array $states; + public function getStates(): array + { + if (!isset($this->states)) { + try { + $this->setStates($this->getStateRepository()->fetchByJob($this->getId())); + } catch (BlankResult $e) { + return []; + } + } + return $this->states; + } + public function addState(State\Job $state): Job + { + $this->states []= $state; + return $this; + } + public function setStates(array $states): Job + { + foreach ($states as $state) { + $this->addState($state); + } + return $this; + } + + public function lastState(): State\Job + { + if (count($this->getStates()) === 0) { + throw new Stateless($this); + } + return $this->getStates()[array_key_last($this->getStates())]; + } + public function toArray(): array { return [ diff --git a/api/src/Model/State/Job.php b/api/src/Model/State/Job.php new file mode 100644 index 0000000..2826908 --- /dev/null +++ b/api/src/Model/State/Job.php @@ -0,0 +1,65 @@ +id; + } + public function getJob(): Emails\Model\Job + { + return $this->job; + } + public function getDateTime(): \DateTimeInterface + { + return $this->dateTime; + } + public function getStatus(): int + { + return $this->status; + } + + public function setId(int $id): Job + { + $this->id = $id; + return $this; + } + public function setJob(Emails\Model\Job $job): Job + { + $this->job = $job; + return $this; + } + public function setDateTime(\DateTimeInterface $dateTime): Job + { + $this->dateTime = $dateTime; + return $this; + } + public function setStatus(int $status): Job + { + $this->status = $status; + return $this; + } + + public function jsonSerialize(): mixed + { + return [ + 'id' => $this->getId(), + 'job' => $this->getJob(), + 'date' => $this->getDateTime(), + 'status' => $this->getStatus() + ]; + } +} diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index 0f47ee3..e8f6be9 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -2,28 +2,28 @@ namespace ProVM\Emails\Repository; use PDO; -use ProVM\Common\Factory\Model; +use ProVM\Common\Factory; use Psr\Log\LoggerInterface; -use Safe\DateTimeImmutable; use ProVM\Common\Define\Model as ModelInterface; use ProVM\Common\Implement\Repository; use ProVM\Emails\Model\Job as BaseModel; +use ProVM\Emails; class Job extends Repository { - public function __construct(PDO $connection, LoggerInterface $logger, Model $factory) + public function __construct(PDO $connection, LoggerInterface $logger, Factory\Model $factory) { parent::__construct($connection, $logger); $this->setFactory($factory) ->setTable('attachments_jobs'); } - protected \ProVM\Common\Factory\Model $factory; - public function getFactory(): \ProVM\Common\Factory\Model + protected Factory\Model $factory; + public function getFactory(): Factory\Model { return $this->factory; } - public function setFactory(\ProVM\Common\Factory\Model $factory): Job + public function setFactory(Factory\Model $factory): Job { $this->factory = $factory; return $this; @@ -34,12 +34,9 @@ class Job extends Repository $query = " CREATE TABLE {$this->getTable()} ( `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, - `message_id` INT UNSIGNED NOT NULL, - `date_time` DATETIME NOT NULL, - `executed` INT(1) UNSIGNED DEFAULT 0, - PRIMARY KEY (`id`), - FOREIGN KEY `fk_messages_{$this->getTable()}` (`message_id`) - REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + `command` VARCHAR(100) NOT NULL, + `arguments` TEXT NOT NULL, + PRIMARY KEY (`id`) )"; $this->getConnection()->query($query); } @@ -55,65 +52,61 @@ CREATE TABLE {$this->getTable()} ( protected function fieldsForInsert(): array { return [ - 'message_id', - 'date_time', - 'executed' + 'command', + 'arguments', ]; } protected function valuesForInsert(ModelInterface $model): array { return [ - $model->getMessage()->getId(), - $model->getDateTime()->format('Y-m-d H:i:s'), - $model->isExecuted() ? 1 : 0 + $model->getCommand(), + $model->getArguments(), ]; } protected function defaultFind(ModelInterface $model): ModelInterface { - return $this->fetchByMessageAndDate($model->getMessage()->getId(), $model->getDateTime()->format('Y-m-d H:i:s')); + return $this->fetchByCommandAndArguments($model->getCommand(), $model->getArguments()); } protected function fieldsForCreate(): array { return [ - 'message_id', - 'date_time', - 'executed' + 'command', + 'arguments', ]; } protected function valuesForCreate(array $data): array { return [ - $data['message_id'], - $data['date_time'], - $data['executed'] ?? 0 + $data['command'], + $data['arguments'], ]; } protected function defaultSearch(array $data): ModelInterface { - return $this->fetchByMessageAndDate($data['message_id'], $data['date_time']); + return $this->fetchByCommandAndArguments($data['command'], $data['arguments']); } public function load(array $row): ModelInterface { - $model = (new BaseModel()) + return (new BaseModel()) ->setId($row['id']) - ->setMessage($this->getFactory()->find(\ProVM\Emails\Model\Message::class)->fetchById($row['message_id'])) - ->setDateTime(new DateTimeImmutable($row['date_time'])); - if ($row['executed'] ?? 0 === 1) { - $model->wasExecuted(); - } - return $model; + ->setCommand($row['command']) + ->setArguments($row['arguments']) + ->setStateRepository($this->getFactory()->find(Emails\Model\State\Job::class)); } public function fetchAllPending(): array { - $query = "SELECT * FROM {$this->getTable()} WHERE `executed` = 0"; + $query = "SELECT a.* +FROM {$this->getTable()} a + JOIN `jobs_states` b ON b.job_id = a.id +WHERE b.`status` = ?"; return $this->fetchMany($query); } - public function fetchByMessageAndDate(int $message_id, string $date_time): \ProVM\Emails\Model\Job + public function fetchByCommandAndArguments(string $command, string $arguments): \ProVM\Emails\Model\Job { - $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `date_time` = ?"; - return $this->fetchOne($query, [$message_id, $date_time]); + $query = "SELECT * FROM {$this->getTable()} WHERE `command` = ? AND `arguments` = ?"; + return $this->fetchOne($query, [$command, $arguments]); } public function fetchPendingByMessage(int $message_id): \ProVM\Emails\Model\Job { diff --git a/api/src/Repository/State/Job.php b/api/src/Repository/State/Job.php new file mode 100644 index 0000000..e7f9d57 --- /dev/null +++ b/api/src/Repository/State/Job.php @@ -0,0 +1,113 @@ +setTable('jobs_states') + ->setFactory($factory); + } + + protected Factory\Model $factory; + public function getFactory(): Factory\Model + { + return $this->factory; + } + public function setFactory(Factory\Model $factory): Job + { + $this->factory = $factory; + return $this; + } + + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `job_id` INT UNSIGNED NOT NULL, + `date_time` DATETIME NOT NULL, + `status` INT NOT NULL DEFAULT 1, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_jobs_{$this->getTable()}` (`job_id`) + REFERENCES `jobs` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +)"; + $this->getConnection()->query($query); + } + + public function load(array $row): Define\Model + { + return (new Emails\Model\State\Job()) + ->setId($row['id']) + ->setJob($this->getFactory()->find(Emails\Model\Job::class)->fetchById($row['job_id'])) + ->setDateTime(new \DateTimeImmutable($row['date_time'])) + ->setStatus($row['status']); + } + + public function fetchByJob(int $job_id): array + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ?"; + return $this->fetchMany($query, [$job_id]); + } + public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\Job + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ? AND `status` = ?"; + return $this->fetchOne($query, [$job_id, $status]); + } + + protected function fieldsForInsert(): array + { + return [ + 'job_id', + 'date_time', + 'status' + ]; + } + protected function valuesForInsert(Define\Model $model): array + { + return [ + $model->getJob()->getId(), + $model->getDateTime()->format('Y-m-d H:i:s'), + $model->getStatus() + ]; + } + + protected function fieldsForUpdate(): array + { + return $this->fieldsForInsert(); + } + protected function fieldsForCreate(): array + { + return $this->fieldsForInsert(); + } + + protected function valuesForUpdate(Define\Model $model): array + { + return $this->valuesForInsert($model); + } + protected function valuesForCreate(array $data): array + { + return [ + $data['job_id'], + $data['date_time'], + $data['status'] + ]; + } + + protected function defaultFind(Define\Model $model): Define\Model + { + return $this->fetchByJobAndStatus($model->getJob()->getId(), $model->getStatus()); + } + protected function defaultSearch(array $data): Define\Model + { + return $this->fetchByJobAndStatus($data['job_id'], $data['status']); + } +} diff --git a/cli/Dockerfile b/cli/Dockerfile index 2ed89eb..d596ae4 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -3,7 +3,7 @@ FROM php:8-cli ENV PATH ${PATH}:/app/bin RUN apt-get update \ - && apt-get install -y cron git libzip-dev unzip \ + && apt-get install -y cron git libzip-dev unzip qpdf \ && rm -r /var/lib/apt/lists/* \ && docker-php-ext-install zip diff --git a/cli/common/Command/DecryptPdf.php b/cli/common/Command/DecryptPdf.php deleted file mode 100644 index e868843..0000000 --- a/cli/common/Command/DecryptPdf.php +++ /dev/null @@ -1,68 +0,0 @@ -setCommunicator($communicator); - parent::__construct($name); - } - - protected Communicator $communicator; - public function getCommunicator(): Communicator - { - return $this->communicator; - } - public function setCommunicator(Communicator $communicator): DecryptPdf - { - $this->communicator = $communicator; - return $this; - } - - protected function getAttachments(): array - { - $response = $this->getCommunicator()->get('/attachments/pending'); - return json_decode($response->getBody()->getContents())->attachments; - } - protected function decrypt(string $attachment): bool - { - $response = $this->getCommunicator()->put('/attachments/decrypt', ['attachments' => [$attachment]]); - return json_decode($response->getBody()->getContents())->status; - } - - public function execute(InputInterface $input, OutputInterface $output) - { - $io = new SymfonyStyle($input, $output); - $io->title('Decrypt Attachments'); - - $io->section('Grabbing Attachments'); - $attachments = $this->getAttachments(); - $io->text('Found ' . count($attachments) . ' attachments.'); - $io->section('Decrypting Attachments'); - foreach ($attachments as $attachment) { - $status = $this->decrypt($attachment); - if ($status) { - $io->success("{$attachment} decrypted correctly."); - } else { - $io->error("Problem decrypting {$attachment}."); - } - } - $io->success('Done.'); - - return Command::SUCCESS; - } -} diff --git a/cli/common/Command/GrabAttachments.php b/cli/common/Command/GrabAttachments.php deleted file mode 100644 index 23295e1..0000000 --- a/cli/common/Command/GrabAttachments.php +++ /dev/null @@ -1,66 +0,0 @@ -setCommunicator($communicator); - parent::__construct($name); - } - - protected Communicator $service; - public function getCommunicator(): Communicator - { - return $this->service; - } - public function setCommunicator(Communicator $service): GrabAttachments - { - $this->service = $service; - return $this; - } - - protected function getMessages(): array - { - $response = $this->getCommunicator()->get('/messages/pending'); - return json_decode($response->getBody()->getContents())->messages; - } - protected function grabAttachments(int $message_uid): int - { - $response = $this->getCommunicator()->put('/attachments/grab', ['messages' => [$message_uid]]); - return json_decode($response->getBody()->getContents())->attachment_count; - } - - public function execute(InputInterface $input, OutputInterface $output): int - { - $io = new SymfonyStyle($input, $output); - $io->title('Grab Attachments'); - - $io->section('Grabbing Messages'); - $messages = $this->getMessages(); - $io->text('Found ' . count($messages) . ' messages.'); - $io->section('Grabbing Attachments'); - foreach ($messages as $job) { - $message = $job->message; - $attachments = $this->grabAttachments($message->uid); - $io->text("Found {$attachments} attachments for message UID:{$message->uid}."); - } - $io->success('Done.'); - - return Command::SUCCESS; - } -} diff --git a/cli/common/Command/Jobs/Check.php b/cli/common/Command/Jobs/Check.php new file mode 100644 index 0000000..d3c9b23 --- /dev/null +++ b/cli/common/Command/Jobs/Check.php @@ -0,0 +1,79 @@ +logger->notice('Grabbing pending jobs.'); + $response = $this->communicator->get('/jobs/pending'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + return json_decode($body)->jobs; + } + protected function runJob($job): bool + { + $base_command = '/app/bin/emails'; + $cmd = [$base_command, $job->command]; + if ($job->arguments !== '') { + $cmd []= $job->arguments; + } + $cmd = implode(' ', $cmd); + $this->logger->notice("Running '{$cmd}'"); + $response = shell_exec($cmd); + $this->logger->info("Result: {$response}"); + return $response !== false; + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $section1 = $output->section(); + $section2 = $output->section(); + $io1 = new SymfonyStyle($input, $section1); + $io2 = new SymfonyStyle($input, $section2); + $io1->title('Checking Pending Jobs'); + $pending_jobs = $this->getPendingJobs(); + $notice = 'Found ' . count($pending_jobs) . ' jobs'; + $io1->text($notice); + $this->logger->info($notice); + if (count($pending_jobs) > 0) { + $io1->section('Running Jobs'); + $io1->progressStart(count($pending_jobs)); + foreach ($pending_jobs as $job) { + $section2->clear(); + $io2->text("Running {$job->command}"); + if ($this->runJob($job)) { + $io2->success('Success'); + } else { + $io2->error('Failure'); + } + $io1->progressAdvance(); + } + } + $section2->clear(); + $io2->success('Done'); + + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Mailboxes/Check.php b/cli/common/Command/Mailboxes/Check.php new file mode 100644 index 0000000..c0fa6eb --- /dev/null +++ b/cli/common/Command/Mailboxes/Check.php @@ -0,0 +1,76 @@ +communicator->get('/mailboxes/registered'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + return json_decode($body)->mailboxes; + } + protected function checkMailbox($mailbox): bool + { + if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { + return true; + } + $response = $this->communicator->get("/mailbox/{$mailbox->id}/check"); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return true; + } + return json_decode($body)->status; + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + $section1 = $output->section(); + $section2 = $output->section(); + $io1 = new SymfonyStyle($input, $section1); + $io2 = new SymfonyStyle($input, $section2); + $io1->title('Checking for New Messages'); + $mailboxes = $this->getMailboxes(); + $notice = 'Found ' . count($mailboxes) . ' mailboxes'; + $io1->text($notice); + if (count($mailboxes) > 0) { + $io1->section('Checking for new messages'); + $io1->progressStart(count($mailboxes)); + foreach ($mailboxes as $mailbox) { + $section2->clear(); + $io2->text("Checking {$mailbox->name}"); + if ($this->checkMailbox($mailbox)) { + $io2->success("Found new emails in {$mailbox->name}"); + } else { + $io2->info("No new emails in {$mailbox->name}"); + } + $io1->progressAdvance(); + } + $io1->progressFinish(); + } + $section2->clear(); + $io2->success('Done'); + + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Messages.php b/cli/common/Command/Messages/Grab.php similarity index 54% rename from cli/common/Command/Messages.php rename to cli/common/Command/Messages/Grab.php index 848436e..89a85f9 100644 --- a/cli/common/Command/Messages.php +++ b/cli/common/Command/Messages/Grab.php @@ -1,65 +1,62 @@ setCommunicator($communicator); parent::__construct($name); } + protected function configure() + { + $this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails'); + } protected Communicator $communicator; public function getCommunicator(): Communicator { return $this->communicator; } - public function setCommunicator(Communicator $communicator): Messages + public function setCommunicator(Communicator $communicator): Grab { $this->communicator = $communicator; return $this; } - protected function getMailboxes(): array + protected function grabMessages(int $mailbox_id): int { - $response = $this->getCommunicator()->get('/mailboxes/registered'); - return json_decode($response->getBody()->getContents())->mailboxes; - } - protected function grabMessages(string $mailbox): int - { - $response = $this->getCommunicator()->put('/messages/grab', ['mailboxes' => [$mailbox]]); - $body = json_decode($response->getBody()->getContents()); - return $body->message_count; + $response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab"); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + return json_decode($body)->messages->count; } public function execute(InputInterface $input, OutputInterface $output) { $io = new SymfonyStyle($input, $output); - $io->title('Messages'); - $io->section('Grabbing Registered Mailboxes'); - - $mailboxes = $this->getMailboxes(); - $io->text('Found ' . count($mailboxes) . ' registered mailboxes.'); + $mailbox_id = $input->getArgument('mailbox_id'); + $io->title("Grabbing Messages for Mailbox ID {$mailbox_id}"); $io->section('Grabbing Messages'); - foreach ($mailboxes as $mailbox) { - $message_count = $this->grabMessages($mailbox->name); - $io->text("Found {$message_count} messages in {$mailbox->name}."); - } + $count = $this->grabMessages($mailbox_id); + $io->info("Found {$count} messages"); $io->success('Done.'); return Command::SUCCESS; diff --git a/cli/common/Middleware/Logging.php b/cli/common/Middleware/Logging.php index a95c37b..25021c5 100644 --- a/cli/common/Middleware/Logging.php +++ b/cli/common/Middleware/Logging.php @@ -1,5 +1,5 @@ getStatusCode() < 200 or $response->getStatusCode() >= 300) { @@ -33,6 +38,11 @@ class Communicator } return $response; } + + /** + * @throws HttpResponseException + * @throws JsonException + */ protected function request(string $method, string $uri, ?array $body = null): ResponseInterface { $options = []; @@ -45,20 +55,39 @@ class Communicator return $this->handleResponse($this->getClient()->request($method, $uri, $options)); } + /** + * @throws HttpResponseException + * @throws JsonException + */ public function get(string $uri): ResponseInterface { return $this->request('get', $uri); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function post(string $uri, array $data): ResponseInterface { return $this->request('post', $uri, $data); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function put(string $uri, array $data): ResponseInterface { return $this->request('put', $uri, $data); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function delete(string $uri, array $data): ResponseInterface { return $this->request('delete', $uri, $data); } -} \ No newline at end of file +} diff --git a/cli/common/Wrapper/Application.php b/cli/common/Wrapper/Application.php index 8d8e4aa..c27dbf4 100644 --- a/cli/common/Wrapper/Application.php +++ b/cli/common/Wrapper/Application.php @@ -1,5 +1,5 @@ container = $container; return $this; } -} \ No newline at end of file +} diff --git a/cli/composer.json b/cli/composer.json index 2c1d0c1..4365fcb 100644 --- a/cli/composer.json +++ b/cli/composer.json @@ -15,7 +15,7 @@ }, "autoload": { "psr-4": { - "ProVM\\Common\\": "common/" + "ProVM\\": "common/" } }, "authors": [ diff --git a/cli/crontab b/cli/crontab index 49d2722..fc8ab81 100644 --- a/cli/crontab +++ b/cli/crontab @@ -1,3 +1,10 @@ # minutes hour day_of_month month day_of_week command -0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log -0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log +#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log +#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log + +# Pending jobs every minute +1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log +# Check mailboxes for new emails every weekday +0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log +# Check attachments every weekday +0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log diff --git a/cli/docker-compose.yml b/cli/docker-compose.yml index 38d19e3..ea1ec32 100644 --- a/cli/docker-compose.yml +++ b/cli/docker-compose.yml @@ -12,4 +12,5 @@ services: - .key.env volumes: - ${CLI_PATH:-.}/:/app - - ./logs/cli:/logs + - ${LOGS_PATH}/cli:/logs + - ${ATT_PATH}:/attachments diff --git a/cli/public/index.php b/cli/public/index.php index 63341a5..f8b2dac 100644 --- a/cli/public/index.php +++ b/cli/public/index.php @@ -7,9 +7,8 @@ $app = require_once implode(DIRECTORY_SEPARATOR, [ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface::class)); try { $app->run(); -} catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; +} catch (Error $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); +} catch (Exception $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->warning($e); } diff --git a/cli/resources/commands/01_mailboxes.php b/cli/resources/commands/01_mailboxes.php new file mode 100644 index 0000000..843e8e4 --- /dev/null +++ b/cli/resources/commands/01_mailboxes.php @@ -0,0 +1,2 @@ +add($app->getContainer()->get(ProVM\Command\Mailboxes\Check::class)); diff --git a/cli/resources/commands/01_messages.php b/cli/resources/commands/01_messages.php index c034ee1..4313973 100644 --- a/cli/resources/commands/01_messages.php +++ b/cli/resources/commands/01_messages.php @@ -1,2 +1,2 @@ add($app->getContainer()->get(\ProVM\Common\Command\Messages::class)); +$app->add($app->getContainer()->get(ProVM\Command\Messages\Grab::class)); diff --git a/cli/resources/commands/02_attachments.php b/cli/resources/commands/02_attachments.php index 85793e1..d46dbc1 100644 --- a/cli/resources/commands/02_attachments.php +++ b/cli/resources/commands/02_attachments.php @@ -1,3 +1,4 @@ add($app->getContainer()->get(\ProVM\Common\Command\GrabAttachments::class)); -$app->add($app->getContainer()->get(\ProVM\Common\Command\DecryptPdf::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Check::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Grab::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Decrypt::class)); diff --git a/cli/resources/commands/03_jobs.php b/cli/resources/commands/03_jobs.php new file mode 100644 index 0000000..bd6eed1 --- /dev/null +++ b/cli/resources/commands/03_jobs.php @@ -0,0 +1,2 @@ +add($app->getContainer()->get(ProVM\Command\Jobs\Check::class)); diff --git a/cli/setup/app.php b/cli/setup/app.php index 16c1e0f..d00dbc6 100644 --- a/cli/setup/app.php +++ b/cli/setup/app.php @@ -1,7 +1,7 @@ build()); +$app = new ProVM\Wrapper\Application($builder->build()); $folder = implode(DIRECTORY_SEPARATOR, [ __DIR__, diff --git a/cli/setup/middleware/98_log.php b/cli/setup/middleware/98_log.php index 4df6f6c..5628448 100644 --- a/cli/setup/middleware/98_log.php +++ b/cli/setup/middleware/98_log.php @@ -1,2 +1,2 @@ add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); +//$app->add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); diff --git a/cli/setup/settings/01_env.php b/cli/setup/settings/01_env.php index 58b2a65..107718a 100644 --- a/cli/setup/settings/01_env.php +++ b/cli/setup/settings/01_env.php @@ -1,5 +1,8 @@ $_ENV['API_URI'], - 'api_key' => sha1($_ENV['API_KEY']) + 'api_key' => sha1($_ENV['API_KEY']), + 'passwords' => function() { + return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? ''); + }, ]; diff --git a/cli/setup/setups/02_api.php b/cli/setup/setups/02_api.php index b123345..bf102d3 100644 --- a/cli/setup/setups/02_api.php +++ b/cli/setup/setups/02_api.php @@ -2,12 +2,12 @@ use Psr\Container\ContainerInterface; return [ - \Psr\Http\Client\ClientInterface::class => function(ContainerInterface $container) { - return new \GuzzleHttp\Client([ + Psr\Http\Client\ClientInterface::class => function(ContainerInterface $container) { + return new GuzzleHttp\Client([ 'base_uri' => $container->get('api_uri'), 'headers' => [ 'Authorization' => "Bearer {$container->get('api_key')}" ] ]); } -]; \ No newline at end of file +]; diff --git a/cli/setup/setups/03_middleware.php b/cli/setup/setups/03_middleware.php index 1d6cc44..7d872ab 100644 --- a/cli/setup/setups/03_middleware.php +++ b/cli/setup/setups/03_middleware.php @@ -2,7 +2,7 @@ use Psr\Container\ContainerInterface; return [ - ProVM\Common\Middleware\Logging::class => function(ContainerInterface $container) { - return new ProVM\Common\Middleware\Logging($container->get('request_logger')); + ProVM\Middleware\Logging::class => function(ContainerInterface $container) { + return new ProVM\Middleware\Logging($container->get('request_logger')); } ]; diff --git a/cli/setup/setups/04_commands.php b/cli/setup/setups/04_commands.php new file mode 100644 index 0000000..50ae9d1 --- /dev/null +++ b/cli/setup/setups/04_commands.php @@ -0,0 +1,20 @@ + function(ContainerInterface $container) { + return new ProVM\Command\Attachments\DecryptPdf( + $container->get(ProVM\Service\Communicator::class), + $container->get(Psr\Log\LoggerInterface::class), + 'qpdf', + $container->get('passwords') + ); + }, + ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) { + return new ProVM\Command\Mailboxes\Check( + $container->get(ProVM\Service\Communicator::class), + 1 + ); + } +]; diff --git a/cli/setup/setups/98_log.php b/cli/setup/setups/98_log.php index b33a5ce..7f6e30b 100644 --- a/cli/setup/setups/98_log.php +++ b/cli/setup/setups/98_log.php @@ -2,28 +2,50 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { - $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); - $handler->setFormatter($container->get(Monolog\Formatter\LineFormatter::class)); - return $handler; + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), + ]; + }, + 'request_log_handler' => function(ContainerInterface $container) { + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger( + 'request_logger', + [$container->get('request_log_handler')], + $container->get('log_processors') + ); + }, + 'file_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ); + }, + 'debug_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return $container->get('file_logger'); + }, + 'file_logger' => function(ContainerInterface $container) { + return new Monolog\Logger( + 'file', + [ + $container->get('file_log_handler'), + $container->get('debug_log_handler') + ], + $container->get('log_processors') + ); }, ]; From 88f91c4bd52b8b945b7b6f3266fbf59d1e62549f Mon Sep 17 00:00:00 2001 From: Aldarien Date: Mon, 12 Jun 2023 21:14:07 -0400 Subject: [PATCH 2/2] Jobs setup --- api/common/Controller/Attachments.php | 64 +++---- api/common/Controller/Base.php | 5 +- api/common/Controller/Jobs.php | 66 +++++-- api/common/Controller/Messages.php | 56 +++--- .../Exception/Request/Auth/Forbidden.php | 15 ++ .../Exception/Request/Auth/Unauthorized.php | 15 ++ api/common/Middleware/Auth.php | 35 ++-- api/common/Middleware/Messages.php | 24 --- api/common/Service/Attachments.php | 2 +- api/common/Service/Auth.php | 33 ++++ api/common/Service/Jobs.php | 65 ++++--- api/common/Service/Messages.php | 16 +- api/docker-compose.yml | 2 +- api/nginx.conf | 15 +- api/resources/routes/02_attachments.php | 5 +- api/resources/routes/02_messages.php | 6 +- api/resources/routes/03_jobs.php | 15 ++ api/setup/app.php | 7 +- api/setup/middleware/04_db.php | 1 - api/setup/setups/02_services.php | 3 + api/setup/setups/03_factories.php | 4 +- api/setup/setups/97_auth.php | 2 +- api/setup/setups/98_log.php | 3 +- api/src/Model/Attachment.php | 4 +- api/src/Model/Job.php | 24 +-- api/src/Model/State/Job.php | 2 +- api/src/Repository/Job.php | 19 +- api/src/Repository/State/Job.php | 2 +- cli/common/Command/Jobs/Check.php | 35 +--- cli/common/Command/Jobs/Execute.php | 41 +++++ cli/common/Command/Mailboxes/Check.php | 31 +--- cli/common/Command/Messages/Grab.php | 31 +--- .../Exception/Response/EmptyResponse.php | 16 ++ .../Exception/Response/MissingResponse.php | 15 ++ cli/common/Middleware/Logging.php | 38 ---- cli/common/Service/Attachments.php | 114 ++++++++++++ cli/common/Service/Communicator.php | 21 +-- cli/common/Service/Jobs.php | 71 ++++++++ cli/common/Service/Mailboxes.php | 61 +++++++ cli/common/Service/Messages.php | 43 +++++ cli/common/Wrapper/Application.php | 1 - cli/crontab | 10 +- cli/docker-compose.yml | 1 + cli/resources/commands/03_jobs.php | 1 + cli/setup/middleware/98_log.php | 2 - cli/setup/settings/01_env.php | 2 + cli/setup/setups/04_commands.php | 17 +- cli/setup/setups/98_log.php | 3 +- emails-2022-11-30-03-59-49.sql | 94 ---------- ui/common/Controller/Jobs.php | 14 ++ ui/common/Middleware/Logging.php | 2 +- ui/nginx.conf | 5 +- ui/public/index.php | 8 +- ui/resources/routes/03_jobs.php | 4 + ui/resources/views/emails/messages.blade.php | 43 +++-- ui/resources/views/home.blade.php | 6 +- ui/resources/views/jobs/base.blade.php | 14 ++ ui/resources/views/jobs/list.blade.php | 165 ++++++++++++++++++ .../views/layout/body/header/navbar.blade.php | 1 + ui/setup/setups/98_log.php | 40 +++-- 60 files changed, 965 insertions(+), 495 deletions(-) create mode 100644 api/common/Exception/Request/Auth/Forbidden.php create mode 100644 api/common/Exception/Request/Auth/Unauthorized.php delete mode 100644 api/common/Middleware/Messages.php create mode 100644 api/common/Service/Auth.php create mode 100644 api/resources/routes/03_jobs.php create mode 100644 cli/common/Command/Jobs/Execute.php create mode 100644 cli/common/Exception/Response/EmptyResponse.php create mode 100644 cli/common/Exception/Response/MissingResponse.php delete mode 100644 cli/common/Middleware/Logging.php create mode 100644 cli/common/Service/Attachments.php create mode 100644 cli/common/Service/Jobs.php create mode 100644 cli/common/Service/Mailboxes.php create mode 100644 cli/common/Service/Messages.php delete mode 100644 cli/setup/middleware/98_log.php delete mode 100644 emails-2022-11-30-03-59-49.sql create mode 100644 ui/common/Controller/Jobs.php create mode 100644 ui/resources/routes/03_jobs.php create mode 100644 ui/resources/views/jobs/base.blade.php create mode 100644 ui/resources/views/jobs/list.blade.php diff --git a/api/common/Controller/Attachments.php b/api/common/Controller/Attachments.php index 68d3dd6..d95d028 100644 --- a/api/common/Controller/Attachments.php +++ b/api/common/Controller/Attachments.php @@ -1,19 +1,20 @@ toArray(); @@ -24,35 +25,7 @@ class Attachments ]; return $this->withJson($response, $output); } - public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Jobs $jobsService): ResponseInterface - { - $body = $request->getBody(); - $json = \Safe\json_decode($body->getContents()); - if (!isset($json->messages)) { - throw new MissingArgument('messages', 'array', 'message UIDs'); - } - $output = [ - 'messages' => $json->messages, - 'total' => count($json->messages), - 'saved' => [ - 'attachments' => [], - 'total' => 0 - ] - ]; - foreach ($json->messages as $message_id) { - if (!$jobsService->isPending($message_id)) { - continue; - } - if ($service->grab($message_id)) { - $job = $jobsService->find($message_id); - $jobsService->execute($job->getId()); - $output['saved']['attachments'] []= $job->toArray(); - $output['saved']['total'] ++; - } - } - return $this->withJson($response, $output); - } - public function get(ServerRequestInterface $request, ResponseInterface $response, Service $service, LoggerInterface $logger, int $attachment_id): ResponseInterface + public function get(ServerRequestInterface $request, ResponseInterface $response, Service\Attachments $service, LoggerInterface $logger, int $attachment_id): ResponseInterface { $attachment = $service->getRepository()->fetchById($attachment_id); @@ -61,4 +34,25 @@ class Attachments $response->getBody()->write($service->getFile($attachment_id)); return $response; } -} \ No newline at end of file + public function grab(ServerRequestInterface $request, ResponseInterface $response, Service\Attachments $service, Service\Messages $messagesService): ResponseInterface + { + $body = $request->getBody(); + $json = json_decode($body->getContents()); + if (!isset($json->messages)) { + throw new MissingArgument('messages', 'array', 'Messages UIDs'); + } + $output = [ + 'messages' => $json->messages, + 'attachments' => [], + 'total' => 0 + ]; + foreach ($json->messages as $message_uid) { + $message = $messagesService->getLocalMessage($message_uid); + $attachments = $service->grab($message->getId()); + $output['attachments'] = array_merge($output['attachments'], $attachments); + $output['total'] += count($attachments); + } + + return $this->withJson($response, $output); + } +} diff --git a/api/common/Controller/Base.php b/api/common/Controller/Base.php index 8611788..4b8013f 100644 --- a/api/common/Controller/Base.php +++ b/api/common/Controller/Base.php @@ -12,7 +12,8 @@ class Base public function __invoke(ServerRequestInterface $request, ResponseInterface $response): ResponseInterface { return $this->withJson($response, [ - 'version' => '1.0.0' + 'version' => '1.0.0', + 'app' => 'emails' ]); } -} \ No newline at end of file +} diff --git a/api/common/Controller/Jobs.php b/api/common/Controller/Jobs.php index 437555f..5979fdc 100644 --- a/api/common/Controller/Jobs.php +++ b/api/common/Controller/Jobs.php @@ -1,7 +1,6 @@ getRepository()->fetchAll(); + return $this->withJson($response, compact('jobs')); + } + public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface { $body = $request->getBody(); $json = json_decode($body->getContents()); - if (!isset($json->messages)) { - throw new MissingArgument('messages', 'array', 'messages ids'); + if (!isset($json->jobs)) { + throw new MissingArgument('jobs', 'array', 'job commands with arguments'); } $output = [ - 'messages' => $json->messages, - 'total' => count($json->messages), + 'jobs' => $json->jobs, + 'total' => count($json->jobs), 'scheduled' => 0 ]; - foreach ($json->messages as $message_id) { - if ($service->schedule($message_id)) { - $message = $messagesService->getRepository()->fetchById($message_id); - $message->doesHaveScheduledDownloads(); - $messagesService->getRepository()->save($message); + foreach ($json->jobs as $job) { + if ($service->queue($job->command, $job->arguments)) { $output['scheduled'] ++; } } @@ -37,12 +38,47 @@ class Jobs } public function pending(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface { - $pending = array_map(function(Job $job) { - return $job->toArray(); - }, $service->getPending()); + $pending = $service->getPending(); $output = [ 'total' => count($pending), - 'pending' => $pending + 'jobs' => $pending + ]; + return $this->withJson($response, $output); + } + public function pendingCommands(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface + { + $body = $response->getBody(); + $json = json_decode($body->getContents()); + if (!isset($json->commands)) { + throw new MissingArgument('commands', 'array', 'job commands'); + } + $output = [ + 'commands' => $json->commands, + 'total' => count($json->commands), + 'pending' => [] + ]; + foreach ($json->commands as $command) { + $pending = $service->getPendingByCommand($command); + if (count($pending) === 0) { + continue; + } + $output['pending'][$command] = $pending; + } + return $this->withJson($response, $output); + } + public function finish(ServerRequestInterface $request, ResponseInterface $response, Service $service, $job_id): ResponseInterface + { + $output = [ + 'job_id' => $job_id, + 'status' => $service->finish($job_id) + ]; + return $this->withJson($response, $output); + } + public function failed(ServerRequestInterface $request, ResponseInterface $response, Service $service, $job_id): ResponseInterface + { + $output = [ + 'job_id' => $job_id, + 'status' => $service->failed($job_id) ]; return $this->withJson($response, $output); } diff --git a/api/common/Controller/Messages.php b/api/common/Controller/Messages.php index db86073..7b0d387 100644 --- a/api/common/Controller/Messages.php +++ b/api/common/Controller/Messages.php @@ -1,11 +1,11 @@ getMailboxes()->get($mailbox_id); $messages = array_map(function(Message $message) { @@ -37,7 +37,7 @@ class Messages ]; return $this->withJson($response, $output); } - public function valid(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachments, int $mailbox_id): ResponseInterface + public function valid(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, Service\Attachments $attachments, int $mailbox_id): ResponseInterface { $mailbox = $service->getMailboxes()->get($mailbox_id); $messages = array_values(array_filter(array_map(function(Message $message) use ($service, $attachments) { @@ -63,32 +63,48 @@ class Messages ]; return $this->withJson($response, $output); } - public function get(ServerRequestInterface $request, ResponseInterface $response, Service $service, int $message_id): ResponseInterface + public function get(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, int $message_id): ResponseInterface { $message = $service->getRepository()->fetchById($message_id); return $this->withJson($response, ['message' => $message->toArray()]); } - public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachmentsService): ResponseInterface + public function grab(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, \ProVM\Common\Service\Attachments $attachmentsService, $mailbox_id): ResponseInterface + { + $output = [ + 'mailbox_id' => $mailbox_id, + 'messages' => [], + 'count' => 0 + ]; + $mailbox = $service->getMailboxes()->get($mailbox_id); + $messages = $service->grab($mailbox->getName()); + foreach ($messages as $message_id) { + $message = $service->getLocalMessage($message_id); + if ($message->hasValidAttachments()) { + $attachmentsService->create($message->getId()); + } + } + $output['messages'] = $messages; + $output['count'] = count($messages); + return $this->withJson($response, $output); + } + public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, Service\Jobs $jobsService): ResponseInterface { $body = $request->getBody(); - $json = json_decode($body->getContents()); - if (!isset($json->mailboxes)) { - throw new MissingArgument('mailboxes', 'array', 'mailboxes names'); + $json = json_decode($body); + if (!isset($json->messages)) { + throw new MissingArgument('messages', 'array', 'messages IDs'); } $output = [ - 'mailboxes' => $json->mailboxes, - 'messages' => [], - 'message_count' => 0 + 'messages' => $json->messages, + 'scheduled' => 0 ]; - foreach ($json->mailboxes as $mailbox_name) { - $messages = $service->grab($mailbox_name); - foreach ($messages as $message) { - if ($message->hasValidAttachments()) { - $attachmentsService->create($message); - } + foreach ($json->messages as $message_id) { + if ($jobsService->queue('attachments:grab', [$message_id])) { + $message = $service->getRepository()->fetchById($message_id); + $message->doesHaveScheduledDownloads(); + $service->getRepository()->save($message); + $output['scheduled'] ++; } - $output['messages'] = array_merge($output['messages'], $messages); - $output['message_count'] += count($messages); } return $this->withJson($response, $output); } diff --git a/api/common/Exception/Request/Auth/Forbidden.php b/api/common/Exception/Request/Auth/Forbidden.php new file mode 100644 index 0000000..9c24549 --- /dev/null +++ b/api/common/Exception/Request/Auth/Forbidden.php @@ -0,0 +1,15 @@ +setResponseFactory($factory); $this->setLogger($logger); - $this->setAPIKey($api_key); } protected ResponseFactoryInterface $factory; protected LoggerInterface $logger; - protected string $api_key; public function getResponseFactory(): ResponseFactoryInterface { @@ -28,10 +28,6 @@ class Auth { return $this->logger; } - public function getAPIKey(): string - { - return $this->api_key; - } public function setResponseFactory(ResponseFactoryInterface $factory): Auth { @@ -43,30 +39,23 @@ class Auth $this->logger = $logger; return $this; } - public function setAPIKey(string $key): Auth - { - $this->api_key = $key; - return $this; - } public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface { if ($request->getMethod() === 'OPTIONS') { return $handler->handle($request); } - $auths = $request->getHeader('Authorization'); - foreach ($auths as $auth) { - if (str_contains($auth, 'Bearer')) { - $key = str_replace('Bearer ', '', $auth); - if (sha1($this->getAPIKey()) === $key) { - return $handler->handle($request); - } + try { + if ($this->service->validate($request)) { + return $handler->handle($request); } + } catch (Unauthorized $e) { + $response = $this->getResponseFactory()->createResponse($e->getCode()); + $response->getBody()->write(json_encode(['error' => $e->getCode(), 'message' => $e->getMessage()])); } - $this->getLogger()->debug(sha1($this->getAPIKey())); - $response = $this->getResponseFactory()->createResponse(401); - $response->getBody()->write(\Safe\json_encode(['error' => 401, 'message' => 'Incorrect token'])); + $response = $this->getResponseFactory()->createResponse(413); + $response->getBody()->write(\Safe\json_encode(['error' => 413, 'message' => 'Incorrect token'])); return $response ->withHeader('Content-Type', 'application/json'); } -} \ No newline at end of file +} diff --git a/api/common/Middleware/Messages.php b/api/common/Middleware/Messages.php deleted file mode 100644 index 13c13cb..0000000 --- a/api/common/Middleware/Messages.php +++ /dev/null @@ -1,24 +0,0 @@ -service->checkSchedule(); - } catch (BlankResult $e) { - $this->logger->notice($e); - } - return $handler->handle($request); - } -} diff --git a/api/common/Service/Attachments.php b/api/common/Service/Attachments.php index 1487b77..c5b6db9 100644 --- a/api/common/Service/Attachments.php +++ b/api/common/Service/Attachments.php @@ -117,7 +117,7 @@ class Attachments extends Base continue; } $name = $file->getBasename(".{$file->getExtension()}"); - list($subject, $date, $filename) = explode(' - ', $name); + list($date, $subject, $filename) = explode(' - ', $name); try { $message = $this->getMessages()->find($subject, $date)[0]; $filename = "{$filename}.{$file->getExtension()}"; diff --git a/api/common/Service/Auth.php b/api/common/Service/Auth.php new file mode 100644 index 0000000..7442981 --- /dev/null +++ b/api/common/Service/Auth.php @@ -0,0 +1,33 @@ +getHeaderKey($request); + if (sha1($this->api_key) === $key) { + return true; + } + return false; + } + + protected function getHeaderKey(ServerRequestInterface $request): string + { + if (!$request->hasHeader('Authorization')) { + throw new Unauthorized(); + } + $auths = $request->getHeader('Authorization'); + foreach ($auths as $auth) { + if (str_contains($auth, 'Bearer')) { + return str_replace('Bearer ', '', $auth); + } + } + throw new Unauthorized(); + } +} diff --git a/api/common/Service/Jobs.php b/api/common/Service/Jobs.php index 13ef3d8..f9822c4 100644 --- a/api/common/Service/Jobs.php +++ b/api/common/Service/Jobs.php @@ -1,40 +1,49 @@ setRepository($repository); } - protected Job $repository; + protected Repository\Job $repository; - public function getRepository(): Job + public function getRepository(): Repository\Job { return $this->repository; } - public function setRepository(Job $repository): Jobs + public function setRepository(Repository\Job $repository): Jobs { $this->repository = $repository; return $this; } - public function schedule(int $message_id): bool + public function queue(string $command, ?array $arguments = null): bool { $data = [ - 'message_id' => $message_id, - 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s') + 'command' => $command, + 'arguments' => implode(' ', $arguments) ]; try { $job = $this->getRepository()->create($data); $this->getRepository()->save($job); + $data = [ + 'job_id' => $job->getId(), + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Pending + ]; + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); return true; } catch (PDOException $e) { return false; @@ -44,28 +53,42 @@ class Jobs extends Base { return $this->getRepository()->fetchAllPending(); } - public function isPending(int $message_id): bool + public function getPendingByCommand(string $command): array { try { - $this->getRepository()->fetchPendingByMessage($message_id); - return true; + return $this->getRepository()->fetchAllPendingByCommand($command); } catch (BlankResult $e) { - return false; + return []; } } - public function find(int $message_id): \ProVM\Emails\Model\Job - { - return $this->getRepository()->fetchPendingByMessage($message_id); - } - public function execute(int $job_id): bool + public function finish(int $job_id): bool { + $data = [ + 'job_id' => $job_id, + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Executed + ]; try { - $job = $this->getRepository()->fetchById($job_id); - $job->wasExecuted(); - $this->getRepository()->save($job); + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); return true; } catch (PDOException $e) { return false; } } -} \ No newline at end of file + public function failed(int $job_id): bool + { + $data = [ + 'job_id' => $job_id, + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Failure + ]; + try { + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); + return true; + } catch (PDOException $e) { + return false; + } + } +} diff --git a/api/common/Service/Messages.php b/api/common/Service/Messages.php index a0cd5e0..ac52463 100644 --- a/api/common/Service/Messages.php +++ b/api/common/Service/Messages.php @@ -154,7 +154,6 @@ class Messages extends Base $message->doesHaveValidAttachments(); } } - error_log(json_encode(compact('message')).PHP_EOL,3,'/logs/debug'); $this->getRepository()->save($message); return true; } catch (PDOException $e) { @@ -196,20 +195,7 @@ class Messages extends Base $registered = $this->getMailboxes()->getRegistered(); foreach ($registered as $mailbox) { if (!$this->getMailboxes()->isUpdated($mailbox)) { - $this->logger->info("Updating messages from {$mailbox->getName()}"); - $this->grab($mailbox->getName()); - } - } - } - public function checkSchedule(): void - { - $messages = $this->getRepository()->fetchAll(); - foreach ($messages as $message) { - if ($message->hasAttachments() and $message->hasValidAttachments() and !$message->hasDownloadedAttachments() and !$message->hasScheduledDownloads()) { - if ($this->getJobsService()->schedule($message->getId())) { - $message->doesHaveDownloadedAttachments(); - $this->getRepository()->save($message); - } + $this->getJobsService()->queue('messages:grab', [$mailbox->getId()]); } } } diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 173b1be..0f71511 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -7,7 +7,7 @@ services: - ${API_PATH:-.}:/app/api - ${API_PATH:-.}/nginx.conf:/etc/nginx/conf.d/api.conf ports: - - "${API_PORT:-8080}:8080" + - "${API_PORT:-8080}:81" api: profiles: - api diff --git a/api/nginx.conf b/api/nginx.conf index 6d22e8e..6a50944 100644 --- a/api/nginx.conf +++ b/api/nginx.conf @@ -1,15 +1,15 @@ server { - listen 0.0.0.0:8080; + listen 81; root /app/api/public; index index.php index.html index.htm; - access_log /var/logs/nginx/access.log; - error_log /var/logs/nginx/error.log; + access_log /var/logs/nginx/api.access.log; + error_log /var/logs/nginx/api.error.log; location / { - try_files $uri $uri/ /index.php?$query_string; + try_files $uri /index.php$is_args$args; } - location ~ \.php$ { + location ~ \.php { if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Headers' 'Authorization,Accept,Origin,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; @@ -26,11 +26,12 @@ server { try_files $uri =404; fastcgi_split_path_info ^(.+\.php)(/.+)$; + include fastcgi_params; fastcgi_pass api:9000; fastcgi_index index.php; - include fastcgi_params; fastcgi_param REQUEST_URI $request_uri; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param SCRIPT_NAME $fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; } -} \ No newline at end of file +} diff --git a/api/resources/routes/02_attachments.php b/api/resources/routes/02_attachments.php index 9b45b9d..8c53514 100644 --- a/api/resources/routes/02_attachments.php +++ b/api/resources/routes/02_attachments.php @@ -2,9 +2,8 @@ use ProVM\Common\Controller\Attachments; $app->group('/attachments', function($app) { - $app->put('/grab', [Attachments::class, 'grab']); - $app->get('/pending', [Attachments::class, 'pending']); - $app->post('/decrypt', [Attachments::class, 'decrypt']); + $app->put('/grab[/]', [Attachments::class, 'grab']); + $app->get('/pending[/]', [Attachments::class, 'pending']); $app->get('[/]', Attachments::class); }); $app->group('/attachment/{attachment_id}', function($app) { diff --git a/api/resources/routes/02_messages.php b/api/resources/routes/02_messages.php index ed080d5..4d10259 100644 --- a/api/resources/routes/02_messages.php +++ b/api/resources/routes/02_messages.php @@ -4,9 +4,9 @@ use ProVM\Common\Controller\Jobs; $app->group('/messages', function($app) { $app->put('/grab', [Messages::class, 'grab']); - $app->put('/schedule', [Jobs::class, 'schedule']); - $app->get('/pending', [Jobs::class, 'pending']); + $app->put('/schedule', [Messages::class, 'schedule']); + //$app->get('/pending', [Jobs::class, 'pending']); }); $app->group('/message/{message_id}', function($app) { $app->get('[/]', [Messages::class, 'get']); -}); \ No newline at end of file +}); diff --git a/api/resources/routes/03_jobs.php b/api/resources/routes/03_jobs.php new file mode 100644 index 0000000..1753d96 --- /dev/null +++ b/api/resources/routes/03_jobs.php @@ -0,0 +1,15 @@ +group('/jobs', function($app) { + $app->group('/pending', function($app) { + $app->put('/command[/]', [Jobs::class, 'pendingCommands']); + $app->get('[/]', [Jobs::class, 'pending']); + }); + $app->post('/schedule[/]', [Jobs::class, 'schedule']); + $app->get('[/]', Jobs::class); +}); +$app->group('/job/{job_id}', function($app) { + $app->get('/finish[/]', [Jobs::class, 'finish']); + $app->get('/failed[/]', [Jobs::class, 'failed']); +}); diff --git a/api/setup/app.php b/api/setup/app.php index 7367296..0ae6f29 100644 --- a/api/setup/app.php +++ b/api/setup/app.php @@ -1,7 +1,10 @@ addDefinitions($file->getRealPath()); } } -$app = \DI\Bridge\Slim\Bridge::create($builder->build()); +$app = Bridge::create($builder->build()); $folder = implode(DIRECTORY_SEPARATOR, [ __DIR__, diff --git a/api/setup/middleware/04_db.php b/api/setup/middleware/04_db.php index 9d831e8..b71467b 100644 --- a/api/setup/middleware/04_db.php +++ b/api/setup/middleware/04_db.php @@ -1,5 +1,4 @@ add($app->getContainer()->get(ProVM\Common\Middleware\Attachments::class)); -$app->add($app->getContainer()->get(ProVM\Common\Middleware\Messages::class)); $app->add($app->getContainer()->get(ProVM\Common\Middleware\Mailboxes::class)); $app->add($app->getContainer()->get(ProVM\Common\Middleware\Install::class)); diff --git a/api/setup/setups/02_services.php b/api/setup/setups/02_services.php index 2a09318..4348151 100644 --- a/api/setup/setups/02_services.php +++ b/api/setup/setups/02_services.php @@ -33,5 +33,8 @@ return [ $container->get(ProVM\Common\Factory\Model::class), $container->get('model_list') ); + }, + ProVM\Common\Service\Auth::class => function(ContainerInterface $container) { + return new ProVM\Common\Service\Auth($container->get('api_key')); } ]; diff --git a/api/setup/setups/03_factories.php b/api/setup/setups/03_factories.php index 281eafa..49b605e 100644 --- a/api/setup/setups/03_factories.php +++ b/api/setup/setups/03_factories.php @@ -8,9 +8,11 @@ return [ 'Mailbox' => ProVM\Emails\Repository\Mailbox::class, 'Message' => ProVM\Emails\Repository\Message::class, 'Attachment' => ProVM\Emails\Repository\Attachment::class, + 'Job' => ProVM\Emails\Repository\Job::class, "State\\Mailbox" => ProVM\Emails\Repository\State\Mailbox::class, "State\\Message" => ProVM\Emails\Repository\State\Message::class, - "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class + "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class, + "State\\Job" => ProVM\Emails\Repository\State\Job::class, ]); } ]; diff --git a/api/setup/setups/97_auth.php b/api/setup/setups/97_auth.php index 03fa0f3..9476f57 100644 --- a/api/setup/setups/97_auth.php +++ b/api/setup/setups/97_auth.php @@ -6,7 +6,7 @@ return [ return new ProVM\Common\Middleware\Auth( $container->get(Nyholm\Psr7\Factory\Psr17Factory::class), $container->get(Psr\Log\LoggerInterface::class), - $container->get('api_key') + $container->get(ProVM\Common\Service\Auth::class) ); } ]; diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index 0ebc4cb..39788a4 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -11,8 +11,7 @@ return [ ]; }, 'request_log_handler' => function(ContainerInterface $container) { - return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) - ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))); }, 'request_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/api/src/Model/Attachment.php b/api/src/Model/Attachment.php index 912a2e8..9cc8683 100644 --- a/api/src/Model/Attachment.php +++ b/api/src/Model/Attachment.php @@ -95,9 +95,9 @@ class Attachment implements Model public function getFullFilename(): string { return implode(' - ', [ - $this->getMessage()->getSubject(), $this->getMessage()->getDateTime()->format('Y-m-d His'), - $this->getFilename() + $this->getMessage()->getSubject(), + $this->getFilename(), ]); } diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index 5aa26ee..7b69d93 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -25,24 +25,20 @@ class Job implements Model { return $this->arguments ?? ''; } - public function isExecuted(): bool - { - return $this->lastState()->getStatus() === State\Job::Executed; - } public function setId(int $id): Job { $this->id = $id; return $this; } - public function setCommand(string $message): Job + public function setCommand(string $command): Job { - $this->message = $message; + $this->command = $command; return $this; } - public function setArguments(string $dateTime): Job + public function setArguments(string $arguments): Job { - $this->dateTime = $dateTime; + $this->arguments = $arguments; return $this; } @@ -82,10 +78,14 @@ class Job implements Model return $this; } + public function isExecuted(): bool + { + return $this->lastState()->getStatus() === State\Job::Executed; + } public function lastState(): State\Job { if (count($this->getStates()) === 0) { - throw new Stateless($this); + throw new Stateless($this->getId()); } return $this->getStates()[array_key_last($this->getStates())]; } @@ -94,9 +94,9 @@ class Job implements Model { return [ 'id' => $this->getId(), - 'message' => $this->getMessage()->toArray(), - 'date_time' => $this->getDateTime()->format('Y-m-d H:i:s'), - 'executed' => $this->isExecuted() + 'command' => $this->getCommand(), + 'arguments' => $this->getArguments(), + 'states' => $this->getStates() ]; } public function jsonSerialize(): mixed diff --git a/api/src/Model/State/Job.php b/api/src/Model/State/Job.php index 2826908..1d1e9af 100644 --- a/api/src/Model/State/Job.php +++ b/api/src/Model/State/Job.php @@ -57,7 +57,7 @@ class Job implements Model { return [ 'id' => $this->getId(), - 'job' => $this->getJob(), + 'job_id' => $this->getJob()->getId(), 'date' => $this->getDateTime(), 'status' => $this->getStatus() ]; diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index e8f6be9..17c39df 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -15,7 +15,7 @@ class Job extends Repository { parent::__construct($connection, $logger); $this->setFactory($factory) - ->setTable('attachments_jobs'); + ->setTable('jobs'); } protected Factory\Model $factory; @@ -98,19 +98,22 @@ CREATE TABLE {$this->getTable()} ( public function fetchAllPending(): array { $query = "SELECT a.* -FROM {$this->getTable()} a - JOIN `jobs_states` b ON b.job_id = a.id +FROM `{$this->getTable()}` a + JOIN (SELECT s1.* FROM `jobs_states` s1 JOIN (SELECT MAX(id) AS id, job_id FROM `jobs_states` GROUP BY job_id) s2 ON s2.id = s1.id) b ON b.`job_id` = a.`id` WHERE b.`status` = ?"; - return $this->fetchMany($query); + return $this->fetchMany($query, [Emails\Model\State\Job::Pending]); } - public function fetchByCommandAndArguments(string $command, string $arguments): \ProVM\Emails\Model\Job + public function fetchByCommandAndArguments(string $command, string $arguments): Emails\Model\Job { $query = "SELECT * FROM {$this->getTable()} WHERE `command` = ? AND `arguments` = ?"; return $this->fetchOne($query, [$command, $arguments]); } - public function fetchPendingByMessage(int $message_id): \ProVM\Emails\Model\Job + public function fetchAllPendingByCommand(string $command): array { - $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `executed` = 0"; - return $this->fetchOne($query, [$message_id]); + $query = "SELECT a.* +FROM `{$this->getTable()}` a + JOIN (SELECT s1.* FROM `jobs_states` s1 JOIN (SELECT MAX(id) AS id, job_id FROM `jobs_states` GROUP BY job_id) s2 ON s2.id = s1.id) b ON b.`job_id` = a.`id` +WHERE a.`command` = ? AND b.`status` = ?"; + return $this->fetchMany($query, [$command, Emails\Model\State\Job::Pending]); } } diff --git a/api/src/Repository/State/Job.php b/api/src/Repository/State/Job.php index e7f9d57..11a6699 100644 --- a/api/src/Repository/State/Job.php +++ b/api/src/Repository/State/Job.php @@ -57,7 +57,7 @@ CREATE TABLE {$this->getTable()} ( $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ?"; return $this->fetchMany($query, [$job_id]); } - public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\Job + public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\State\Job { $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ? AND `status` = ?"; return $this->fetchOne($query, [$job_id, $status]); diff --git a/cli/common/Command/Jobs/Check.php b/cli/common/Command/Jobs/Check.php index d3c9b23..5a5ecc6 100644 --- a/cli/common/Command/Jobs/Check.php +++ b/cli/common/Command/Jobs/Check.php @@ -1,14 +1,12 @@ logger->notice('Grabbing pending jobs.'); - $response = $this->communicator->get('/jobs/pending'); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return []; - } - return json_decode($body)->jobs; - } - protected function runJob($job): bool - { - $base_command = '/app/bin/emails'; - $cmd = [$base_command, $job->command]; - if ($job->arguments !== '') { - $cmd []= $job->arguments; - } - $cmd = implode(' ', $cmd); - $this->logger->notice("Running '{$cmd}'"); - $response = shell_exec($cmd); - $this->logger->info("Result: {$response}"); - return $response !== false; - } - public function execute(InputInterface $input, OutputInterface $output) { $section1 = $output->section(); @@ -53,17 +27,16 @@ class Check extends Command $io1 = new SymfonyStyle($input, $section1); $io2 = new SymfonyStyle($input, $section2); $io1->title('Checking Pending Jobs'); - $pending_jobs = $this->getPendingJobs(); + $pending_jobs = $this->service->getPending(); $notice = 'Found ' . count($pending_jobs) . ' jobs'; $io1->text($notice); - $this->logger->info($notice); if (count($pending_jobs) > 0) { $io1->section('Running Jobs'); $io1->progressStart(count($pending_jobs)); foreach ($pending_jobs as $job) { $section2->clear(); $io2->text("Running {$job->command}"); - if ($this->runJob($job)) { + if ($this->service->run($job)) { $io2->success('Success'); } else { $io2->error('Failure'); diff --git a/cli/common/Command/Jobs/Execute.php b/cli/common/Command/Jobs/Execute.php new file mode 100644 index 0000000..0c41a4e --- /dev/null +++ b/cli/common/Command/Jobs/Execute.php @@ -0,0 +1,41 @@ +addArgument('job_id', InputArgument::REQUIRED, 'Job ID to be executed'); + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $io = new SymfonyStyle($input, $output); + $job_id = $input->getArgument('job_id'); + $job = $this->service->get($job_id); + if ($this->service->run($job)) { + $io->success('Success'); + } else { + $io->error('Failed'); + } + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Mailboxes/Check.php b/cli/common/Command/Mailboxes/Check.php index c0fa6eb..eb2f94c 100644 --- a/cli/common/Command/Mailboxes/Check.php +++ b/cli/common/Command/Mailboxes/Check.php @@ -5,9 +5,8 @@ use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use ProVM\Service\Communicator; use Symfony\Component\Console\Style\SymfonyStyle; -use function Safe\json_decode; +use ProVM\Service\Mailboxes; #[AsCommand( name: 'mailboxes:check', @@ -16,33 +15,11 @@ use function Safe\json_decode; )] class Check extends Command { - public function __construct(protected Communicator $communicator, protected int $min_check_days, string $name = null) + public function __construct(protected Mailboxes $service, string $name = null) { parent::__construct($name); } - protected function getMailboxes(): array - { - $response = $this->communicator->get('/mailboxes/registered'); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return []; - } - return json_decode($body)->mailboxes; - } - protected function checkMailbox($mailbox): bool - { - if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { - return true; - } - $response = $this->communicator->get("/mailbox/{$mailbox->id}/check"); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return true; - } - return json_decode($body)->status; - } - public function execute(InputInterface $input, OutputInterface $output): int { $section1 = $output->section(); @@ -50,7 +27,7 @@ class Check extends Command $io1 = new SymfonyStyle($input, $section1); $io2 = new SymfonyStyle($input, $section2); $io1->title('Checking for New Messages'); - $mailboxes = $this->getMailboxes(); + $mailboxes = $this->service->getAll(); $notice = 'Found ' . count($mailboxes) . ' mailboxes'; $io1->text($notice); if (count($mailboxes) > 0) { @@ -59,7 +36,7 @@ class Check extends Command foreach ($mailboxes as $mailbox) { $section2->clear(); $io2->text("Checking {$mailbox->name}"); - if ($this->checkMailbox($mailbox)) { + if ($this->service->check($mailbox)) { $io2->success("Found new emails in {$mailbox->name}"); } else { $io2->info("No new emails in {$mailbox->name}"); diff --git a/cli/common/Command/Messages/Grab.php b/cli/common/Command/Messages/Grab.php index 89a85f9..1be0fd0 100644 --- a/cli/common/Command/Messages/Grab.php +++ b/cli/common/Command/Messages/Grab.php @@ -7,19 +7,17 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; -use ProVM\Service\Communicator; -use function Safe\json_decode; +use ProVM\Service\Mailboxes; #[AsCommand( name: 'messages:grab', - description: 'Run grab messages job for registered mailboxes', + description: 'Run grab messages job for mailbox', hidden: false )] class Grab extends Command { - public function __construct(Communicator $communicator, string $name = null) + public function __construct(protected Mailboxes $service, string $name = null) { - $this->setCommunicator($communicator); parent::__construct($name); } protected function configure() @@ -27,27 +25,6 @@ class Grab extends Command $this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails'); } - protected Communicator $communicator; - public function getCommunicator(): Communicator - { - return $this->communicator; - } - public function setCommunicator(Communicator $communicator): Grab - { - $this->communicator = $communicator; - return $this; - } - - protected function grabMessages(int $mailbox_id): int - { - $response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab"); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return 0; - } - return json_decode($body)->messages->count; - } - public function execute(InputInterface $input, OutputInterface $output) { $io = new SymfonyStyle($input, $output); @@ -55,7 +32,7 @@ class Grab extends Command $mailbox_id = $input->getArgument('mailbox_id'); $io->title("Grabbing Messages for Mailbox ID {$mailbox_id}"); $io->section('Grabbing Messages'); - $count = $this->grabMessages($mailbox_id); + $count = $this->service->grabMessages($mailbox_id); $io->info("Found {$count} messages"); $io->success('Done.'); diff --git a/cli/common/Exception/Response/EmptyResponse.php b/cli/common/Exception/Response/EmptyResponse.php new file mode 100644 index 0000000..5fff71f --- /dev/null +++ b/cli/common/Exception/Response/EmptyResponse.php @@ -0,0 +1,16 @@ +setLogger($logger); - } - - protected LoggerInterface $logger; - - public function getLogger(): LoggerInterface - { - return $this->logger; - } - - public function setLogger(LoggerInterface $logger): Logging - { - $this->logger = $logger; - return $this; - } - - public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface - { - $response = $handler->handle($request); - $output = [ - 'uri' => var_export($request->getUri(), true), - 'body' => $request->getBody()->getContents() - ]; - $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); - return $response; - } -} diff --git a/cli/common/Service/Attachments.php b/cli/common/Service/Attachments.php new file mode 100644 index 0000000..681b291 --- /dev/null +++ b/cli/common/Service/Attachments.php @@ -0,0 +1,114 @@ +logger->info('Finding all downloaded attachment files'); + $folder = '/attachments'; + $files = new \FilesystemIterator($folder); + foreach ($files as $file) { + if ($file->isDir()) { + continue; + } + yield $file->getRealPath(); + } + } + public function getAll(): array + { + if (!isset($this->attachments)) { + $this->logger->info('Grabbing all attachments'); + $response = $this->communicator->get('/attachments'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + $this->attachments = []; + return $this->attachments; + } + $this->attachments = json_decode($body)->attachments; + } + return $this->attachments; + } + public function get(int $attachment_id): object + { + $this->logger->info("Getting attachment {$attachment_id}"); + $uri = "/attachment/{$attachment_id}"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->attachment)) { + throw new MissingResponse('attachment'); + } + return $json->attachment; + } + public function find(string $filename): int + { + $this->logger->info("Finding attachment {$filename}"); + foreach ($this->getAll() as $attachment) { + if ($attachment->fullfilename === $filename) { + return $attachment->id; + } + } + throw new \Exception("{$filename} is not in the database"); + } + public function isEncrypted(string $filename): bool + { + if (!file_exists($filename)) { + throw new \InvalidArgumentException("File not found {$filename}"); + } + $escaped_filename = escapeshellarg($filename); + $cmd = "{$this->base_command} --is-encrypted {$escaped_filename}"; + exec($cmd, $output, $retcode); + return $retcode == 0; + } + public function scheduleDecrypt(int $attachment_id): bool + { + $this->logger->info("Scheduling decryption of attachment {$attachment_id}"); + $uri = "/attachment/{$attachment_id}/decrypt"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->status)) { + throw new MissingResponse('status'); + } + return $json->status; + } + public function decrypt(string $basename): bool + { + $this->logger->info("Decrypting {$basename}"); + $in_filename = implode('/', ['attachments', $basename]); + $out_filename = implode('/', ['attachments', 'decrypted', $basename]); + if (file_exists($out_filename)) { + throw new \Exception("{$basename} already decrypted"); + } + foreach ($this->passwords as $password) { + $cmd = $this->base_command . ' -password=' . escapeshellarg($password) . ' -decrypt ' . escapeshellarg($in_filename) . ' ' . escapeshellarg($out_filename); + exec($cmd, $output, $retcode); + $success = $retcode == 0; + if ($success) { + return true; + } + if (file_exists($out_filename)) { + unlink($out_filename); + } + unset($output); + } + return false; + } + +} diff --git a/cli/common/Service/Communicator.php b/cli/common/Service/Communicator.php index b22589c..c655079 100644 --- a/cli/common/Service/Communicator.php +++ b/cli/common/Service/Communicator.php @@ -4,29 +4,12 @@ namespace ProVM\Service; use HttpResponseException; use Psr\Http\Client\ClientInterface; use Psr\Http\Message\ResponseInterface; -use Psr\Log\LoggerInterface; use Safe\Exceptions\JsonException; use function Safe\json_encode; class Communicator { - public function __construct(ClientInterface $client) - { - $this->setClient($client); - } - - protected ClientInterface $client; - - public function getClient(): ClientInterface - { - return $this->client; - } - - public function setClient(ClientInterface $client): Communicator - { - $this->client = $client; - return $this; - } + public function __construct(protected ClientInterface $client) {} /** * @throws HttpResponseException @@ -52,7 +35,7 @@ class Communicator ]; $options['body'] = json_encode($body); } - return $this->handleResponse($this->getClient()->request($method, $uri, $options)); + return $this->handleResponse($this->client->request($method, $uri, $options)); } /** diff --git a/cli/common/Service/Jobs.php b/cli/common/Service/Jobs.php new file mode 100644 index 0000000..d3543ef --- /dev/null +++ b/cli/common/Service/Jobs.php @@ -0,0 +1,71 @@ +logger->info('Getting pending jobs'); + $response = $this->communicator->get('/jobs/pending'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + $json = json_decode($body); + if (!isset($json->jobs)) { + return []; + } + return $json->jobs; + } + public function get(int $job_id): object + { + $this->logger->info("Getting Job {$job_id}"); + $uri = "/job/{$job_id}"; + return $this->send($uri, 'job'); + } + public function run(object $job): bool + { + $this->logger->debug("Running Job {$job->id}"); + $base_command = '/app/bin/emails'; + $cmd = [$base_command, $job->command]; + if ($job->arguments !== '') { + $cmd []= $job->arguments; + } + $cmd = implode(' ', $cmd); + $response = shell_exec($cmd); + if ($response !== false) { + return $this->finished($job->id); + } + return $this->failure($job->id); + } + + protected function finished(int $job_id): bool + { + $uri = "/job/{$job_id}/finish"; + return $this->send($uri, 'status'); + } + protected function failure(int $job_id): bool + { + $uri = "/job/{$job_id}/failed"; + return $this->send($uri, 'status'); + } + protected function send(string $uri, string $param): mixed + { + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->{$param})) { + throw new MissingResponse($param); + } + return $json->{$param}; + } +} diff --git a/cli/common/Service/Mailboxes.php b/cli/common/Service/Mailboxes.php new file mode 100644 index 0000000..4d8eabf --- /dev/null +++ b/cli/common/Service/Mailboxes.php @@ -0,0 +1,61 @@ +logger->info('Getting all registered mailboxes'); + $response = $this->communicator->get('/mailboxes/registered'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + $json = json_decode($body); + if (!isset($json->mailboxes)) { + return []; + } + return $json->mailboxes; + } + public function check(object $mailbox): bool + { + $this->logger->info("Checking mailbox {$mailbox->id}"); + if ((new DateTimeImmutable())->diff(new DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { + return true; + } + $uri = "/mailbox/{$mailbox->id}/check"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->status)) { + throw new MissingResponse('status'); + } + return $json->status; + } + public function grabMessages(int $mailbox_id): int + { + $this->logger->info("Grabbing messages for {$mailbox_id}"); + $uri = "/mailbox/{$mailbox_id}/messages/grab"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + $json = json_decode($body); + if (!isset($json->count)) { + return 0; + } + return $json->count; + } +} diff --git a/cli/common/Service/Messages.php b/cli/common/Service/Messages.php new file mode 100644 index 0000000..3854dbd --- /dev/null +++ b/cli/common/Service/Messages.php @@ -0,0 +1,43 @@ +logger->info("Getting message {$message_id}"); + $uri = "/message/{$message_id}"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->message)) { + throw new MissingResponse('message'); + } + return $json->message; + } + public function grabAttachments(string $message_uid): int + { + $this->logger->info("Grabbing attachments for message UID {$message_uid}"); + $uri = '/attachments/grab'; + $response = $this->communicator->put($uri, ['messages' => [$message_uid]]); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + $json = json_decode($body); + if (!isset($json->total)) { + return 0; + } + return $json->total; + } +} diff --git a/cli/common/Wrapper/Application.php b/cli/common/Wrapper/Application.php index c27dbf4..7ffac08 100644 --- a/cli/common/Wrapper/Application.php +++ b/cli/common/Wrapper/Application.php @@ -3,7 +3,6 @@ namespace ProVM\Wrapper; use Psr\Container\ContainerInterface; use Symfony\Component\Console\Application as Base; - class Application extends Base { public function __construct(ContainerInterface $container, string $name = 'UNKNOWN', string $version = 'UNKNOWN') diff --git a/cli/crontab b/cli/crontab index fc8ab81..b567b58 100644 --- a/cli/crontab +++ b/cli/crontab @@ -1,10 +1,10 @@ # minutes hour day_of_month month day_of_week command -#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log -#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log +#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log +#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log # Pending jobs every minute -1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log +* * * * * /app/bin/emails jobs:check >> /logs/jobs.log # Check mailboxes for new emails every weekday -0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log +0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log # Check attachments every weekday -0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log +0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log diff --git a/cli/docker-compose.yml b/cli/docker-compose.yml index ea1ec32..e86b638 100644 --- a/cli/docker-compose.yml +++ b/cli/docker-compose.yml @@ -12,5 +12,6 @@ services: - .key.env volumes: - ${CLI_PATH:-.}/:/app + - ${CLI_PATH}/crontab:/var/spool/cron/crontabs/root - ${LOGS_PATH}/cli:/logs - ${ATT_PATH}:/attachments diff --git a/cli/resources/commands/03_jobs.php b/cli/resources/commands/03_jobs.php index bd6eed1..ea36967 100644 --- a/cli/resources/commands/03_jobs.php +++ b/cli/resources/commands/03_jobs.php @@ -1,2 +1,3 @@ add($app->getContainer()->get(ProVM\Command\Jobs\Check::class)); +$app->add($app->getContainer()->get(ProVM\Command\Jobs\Execute::class)); diff --git a/cli/setup/middleware/98_log.php b/cli/setup/middleware/98_log.php deleted file mode 100644 index 5628448..0000000 --- a/cli/setup/middleware/98_log.php +++ /dev/null @@ -1,2 +0,0 @@ -add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); diff --git a/cli/setup/settings/01_env.php b/cli/setup/settings/01_env.php index 107718a..76b4051 100644 --- a/cli/setup/settings/01_env.php +++ b/cli/setup/settings/01_env.php @@ -2,7 +2,9 @@ return [ 'api_uri' => $_ENV['API_URI'], 'api_key' => sha1($_ENV['API_KEY']), + 'base_command' => 'qpdf', 'passwords' => function() { return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? ''); }, + 'min_check_days' => 1 ]; diff --git a/cli/setup/setups/04_commands.php b/cli/setup/setups/04_commands.php index 50ae9d1..23fd362 100644 --- a/cli/setup/setups/04_commands.php +++ b/cli/setup/setups/04_commands.php @@ -3,18 +3,19 @@ use Psr\Container\ContainerInterface; return [ - ProVM\Command\Attachments\DecryptPdf::class => function(ContainerInterface $container) { - return new ProVM\Command\Attachments\DecryptPdf( + ProVM\Service\Mailboxes::class => function(ContainerInterface $container) { + return new ProVM\Service\Mailboxes( $container->get(ProVM\Service\Communicator::class), $container->get(Psr\Log\LoggerInterface::class), - 'qpdf', - $container->get('passwords') + $container->get('min_check_days') ); }, - ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) { - return new ProVM\Command\Mailboxes\Check( + ProVM\Service\Attachments::class => function(ContainerInterface $container) { + return new ProVM\Service\Attachments( $container->get(ProVM\Service\Communicator::class), - 1 + $container->get(Psr\Log\LoggerInterface::class), + $container->get('passwords'), + $container->get('base_command') ); - } + }, ]; diff --git a/cli/setup/setups/98_log.php b/cli/setup/setups/98_log.php index 7f6e30b..ef4a697 100644 --- a/cli/setup/setups/98_log.php +++ b/cli/setup/setups/98_log.php @@ -10,8 +10,7 @@ return [ ]; }, 'request_log_handler' => function(ContainerInterface $container) { - return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) - ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))); }, 'request_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/emails-2022-11-30-03-59-49.sql b/emails-2022-11-30-03-59-49.sql deleted file mode 100644 index 3fd22d9..0000000 --- a/emails-2022-11-30-03-59-49.sql +++ /dev/null @@ -1,94 +0,0 @@ --- Adminer 4.8.1 MySQL 5.5.5-10.9.3-MariaDB-1:10.9.3+maria~ubu2204 dump - -SET NAMES utf8; -SET time_zone = '+00:00'; -SET foreign_key_checks = 0; -SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; - -SET NAMES utf8mb4; - -DROP TABLE IF EXISTS `attachments`; -CREATE TABLE `attachments` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `filename` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `attachments_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `attachments_jobs`; -CREATE TABLE `attachments_jobs` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `date_time` datetime NOT NULL, - `executed` int(1) DEFAULT 0, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `attachments_jobs_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `attachments_states`; -CREATE TABLE `attachments_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `attachment_id` int(10) unsigned NOT NULL, - `name` varchar(100) NOT NULL, - `value` int(1) unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `attachment_id` (`attachment_id`), - CONSTRAINT `attachments_states_ibfk_1` FOREIGN KEY (`attachment_id`) REFERENCES `attachments` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `mailboxes`; -CREATE TABLE `mailboxes` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(100) NOT NULL, - `validity` int(11) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `mailboxes_states`; -CREATE TABLE `mailboxes_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `mailbox_id` int(10) unsigned NOT NULL, - `date_time` datetime NOT NULL, - `count` int(10) unsigned NOT NULL, - `uids` text NOT NULL, - PRIMARY KEY (`id`), - KEY `mailbox_id` (`mailbox_id`), - CONSTRAINT `mailboxes_states_ibfk_2` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `messages`; -CREATE TABLE `messages` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `mailbox_id` int(10) unsigned NOT NULL, - `position` int(10) unsigned NOT NULL, - `uid` varchar(255) NOT NULL, - `subject` varchar(255) NOT NULL, - `from` varchar(100) NOT NULL, - `date_time` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `mailbox_id` (`mailbox_id`), - CONSTRAINT `messages_ibfk_1` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `messages_states`; -CREATE TABLE `messages_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `name` varchar(100) NOT NULL, - `value` int(1) unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `messages_states_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - --- 2022-11-30 03:59:49 diff --git a/ui/common/Controller/Jobs.php b/ui/common/Controller/Jobs.php new file mode 100644 index 0000000..091539f --- /dev/null +++ b/ui/common/Controller/Jobs.php @@ -0,0 +1,14 @@ +render($response, 'jobs.list'); + } +} diff --git a/ui/common/Middleware/Logging.php b/ui/common/Middleware/Logging.php index c60465d..a1e97a0 100644 --- a/ui/common/Middleware/Logging.php +++ b/ui/common/Middleware/Logging.php @@ -29,7 +29,7 @@ class Logging { $response = $handler->handle($request); $output = [ - 'uri' => var_export($request->getUri(), true), + 'uri' => print_r($request->getUri(), true), 'body' => $request->getParsedBody() ]; $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); diff --git a/ui/nginx.conf b/ui/nginx.conf index 893e62a..43895cc 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -3,6 +3,9 @@ server { root /app/ui/public; index index.php index.html index.htm; + access_log /var/logs/nginx/ui.access.log; + error_log /var/logs/nginx/ui.error.log; + location / { try_files $uri $uri/ /index.php?$query_string; } @@ -16,4 +19,4 @@ server { fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; } -} \ No newline at end of file +} diff --git a/ui/public/index.php b/ui/public/index.php index 9e2bc0e..3408a32 100644 --- a/ui/public/index.php +++ b/ui/public/index.php @@ -8,11 +8,5 @@ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface try { $app->run(); } catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - if (isset($_REQUEST)) { - $logger->debug(Safe\json_encode(compact('_REQUEST'))); - } - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); } diff --git a/ui/resources/routes/03_jobs.php b/ui/resources/routes/03_jobs.php new file mode 100644 index 0000000..f9d8f0a --- /dev/null +++ b/ui/resources/routes/03_jobs.php @@ -0,0 +1,4 @@ +get('/jobs', Jobs::class); diff --git a/ui/resources/views/emails/messages.blade.php b/ui/resources/views/emails/messages.blade.php index 4af8bb1..089eaee 100644 --- a/ui/resources/views/emails/messages.blade.php +++ b/ui/resources/views/emails/messages.blade.php @@ -5,7 +5,12 @@
@endsection +@push('page_styles') + +@endpush + @push('page_scripts') + + +@endpush diff --git a/ui/resources/views/layout/body/header/navbar.blade.php b/ui/resources/views/layout/body/header/navbar.blade.php index 92fa22b..a0a622f 100644 --- a/ui/resources/views/layout/body/header/navbar.blade.php +++ b/ui/resources/views/layout/body/header/navbar.blade.php @@ -1,5 +1,6 @@
diff --git a/ui/setup/setups/98_log.php b/ui/setup/setups/98_log.php index ef8b9d6..f12fc26 100644 --- a/ui/setup/setups/98_log.php +++ b/ui/setup/setups/98_log.php @@ -2,8 +2,13 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\DeduplicationHandler::class => function(ContainerInterface $container) { - return new Monolog\Handler\DeduplicationHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\WebProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class) + ]; }, Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); @@ -11,22 +16,23 @@ return [ return $handler; }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger('request_logger', [ + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'requests.log']))), + ], $container->get('log_processors')); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\DeduplicationHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger('file', [ + new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ), + new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ) + ], $container->get('log_processors')); }, ];