From 806d1be9cf6f86b6b780bb1b03b9e203bbc53bca Mon Sep 17 00:00:00 2001
From: Aldarien <aldarien85@gmail.com>
Date: Fri, 19 May 2023 11:16:50 -0400
Subject: [PATCH] Added multiline parsing

---
 app/common/Define/Log.php                     |  23 +++
 app/common/Define/Parser.php                  |  29 +++-
 app/common/Implement/Parser.php               |  34 +++-
 app/common/Service/Logs.php                   |   3 +-
 app/src/Exception/Parse/EmptyException.php    |  14 ++
 .../Exception/Parse/EmptyLineException.php    |  14 ++
 app/src/Log/File.php                          |  28 ++--
 app/src/Parser/Access.php                     |  15 +-
 app/src/Parser/Debug.php                      | 104 +++++++++++++
 app/src/Parser/Monolog.php                    |  55 ++++++-
 app/src/Parser/Monolog/Multiline.php          | 145 ++++++++++++++++++
 app/src/Parser/PHPDefault.php                 |   7 +-
 12 files changed, 440 insertions(+), 31 deletions(-)
 create mode 100644 app/src/Exception/Parse/EmptyException.php
 create mode 100644 app/src/Exception/Parse/EmptyLineException.php
 create mode 100644 app/src/Parser/Debug.php
 create mode 100644 app/src/Parser/Monolog/Multiline.php

diff --git a/app/common/Define/Log.php b/app/common/Define/Log.php
index 5e59620..3bc1a65 100644
--- a/app/common/Define/Log.php
+++ b/app/common/Define/Log.php
@@ -1,7 +1,30 @@
 <?php
 namespace ProVM\Common\Define;
 
+use DateTimeInterface;
+
 interface Log
 {
+    public function getOriginal(): string;
+    public function getDate(): DateTimeInterface;
+    public function getChannel(): string;
+    public function getSeverity(): string;
+    public function getMessage(): string;
+    public function getStack(): array;
+    public function getContext(): string;
+    public function getExtra(): string;
 
+    public function setOriginal(string $original): Log;
+    public function setDate(DateTimeInterface $dateTime): Log;
+    public function setChannel(string $channel): Log;
+    public function setSeverity(string $severity): Log;
+    public function setMessage(string $message): Log;
+    public function setStack(array $stack): Log;
+    public function setContext(string $context): Log;
+    public function setExtra(string $extra): Log;
+
+    public function parsed(): bool;
+
+    public function hasStack(): bool;
+    public function hasContext(): bool;
 }
diff --git a/app/common/Define/Parser.php b/app/common/Define/Parser.php
index 7dd9fdb..c278595 100644
--- a/app/common/Define/Parser.php
+++ b/app/common/Define/Parser.php
@@ -3,5 +3,32 @@ namespace ProVM\Common\Define;
 
 interface Parser
 {
-    public function parse(string $content): Log;
+    /**
+     * Determine if file is multiline
+     * @param string $filename
+     * @return bool
+     */
+    public function isMultiline(string $filename): bool;
+
+    /**
+     * Get the total amount of errors
+     * @param string $filename
+     * @return int
+     */
+    public function total(string $filename): int;
+
+    /**
+     * Parse line(s)
+     * @param mixed &$file_handler
+     * @return Log
+     */
+    public function parse(mixed &$file_handler): Log;
+
+    /**
+     * Advance $offset errors
+     * @param mixed $file_handler
+     * @param int $offset
+     * @return void
+     */
+    public function advance(mixed &$file_handler, int $offset): void;
 }
diff --git a/app/common/Implement/Parser.php b/app/common/Implement/Parser.php
index 316f501..367997a 100644
--- a/app/common/Implement/Parser.php
+++ b/app/common/Implement/Parser.php
@@ -3,12 +3,19 @@ namespace ProVM\Common\Implement;
 
 use Exception;
 use ProVM\Common\Define\Log;
+use ProVM\Logview\Exception\Parse\EmptyException;
+use ProVM\Logview\Exception\Parse\EmptyLineException;
 use ProVM\Logview\Log as LogContent;
 use ProVM\Common\Define\Parser as Definition;
-use function Safe\fopen;
+use function Safe\{fopen, fclose};
 
 abstract class Parser implements Definition
 {
+    public function isMultiline(string $filename): bool
+    {
+        return false;
+    }
+
     public function total(string $filename): int
     {
         try {
@@ -24,8 +31,31 @@ abstract class Parser implements Definition
             return 0;
         }
     }
-    public function parse(string $content): Log
+    public function parse(mixed &$file_handler): Log
     {
+        $content = fgets($file_handler);
+        if (!$content) {
+            $meta_data = stream_get_meta_data($file_handler);
+            throw new EmptyException($meta_data['uri'], ftell($file_handler));
+        }
+        if (trim($content) === '') {
+            $meta_data = stream_get_meta_data($file_handler);
+            throw new EmptyLineException($meta_data['uri'], ftell($file_handler));
+        }
         return new LogContent($content);
     }
+    public function advance(mixed &$file_handler, int $offset): void
+    {
+        if ($offset === 0) {
+            return;
+        }
+        $cnt = 0;
+        while(!feof($file_handler)) {
+            fgets($file_handler);
+            $cnt ++;
+            if ($cnt >= $offset) {
+                break;
+            }
+        }
+    }
 }
diff --git a/app/common/Service/Logs.php b/app/common/Service/Logs.php
index 1d0fa31..4a4b43c 100644
--- a/app/common/Service/Logs.php
+++ b/app/common/Service/Logs.php
@@ -60,8 +60,9 @@ class Logs
         $map = [
             Parsers\Access::class => '/(access.log)/',
             Parsers\Error::class => '/(error.log)/',
+            Parsers\Debug::class => '/(debug.log)/',
             Parsers\Monolog::class => '/(-\d{4}-\d{2}-\d{2}.log)/',
-            Parsers\PHPDefault::class => '/(php_errors.log)/'
+            Parsers\PHPDefault::class => '/(php_errors.log)/',
         ];
         foreach ($map as $class => $regex) {
             if (preg_match($regex, $filename) === 1) {
diff --git a/app/src/Exception/Parse/EmptyException.php b/app/src/Exception/Parse/EmptyException.php
new file mode 100644
index 0000000..dd6ddcb
--- /dev/null
+++ b/app/src/Exception/Parse/EmptyException.php
@@ -0,0 +1,14 @@
+<?php
+namespace ProVM\Logview\Exception\Parse;
+
+use Exception;
+
+class EmptyException extends Exception
+{
+    public function __construct(string $filename, int $line_number, ?Throwable $previous = null)
+    {
+        $message = "No content in {$filename} line {$line_number}";
+        $code = '1001';
+        parent::__construct($message, $code, $previous);
+    }
+}
diff --git a/app/src/Exception/Parse/EmptyLineException.php b/app/src/Exception/Parse/EmptyLineException.php
new file mode 100644
index 0000000..3af3b00
--- /dev/null
+++ b/app/src/Exception/Parse/EmptyLineException.php
@@ -0,0 +1,14 @@
+<?php
+namespace ProVM\Logview\Exception\Parse;
+
+use Exception;
+
+class EmptyLineException extends Exception
+{
+    public function __construct(string $filename, int $line_number, ?Throwable $previous = null)
+    {
+        $message = "Empty line in {$filename} line {$line_number}";
+        $code = 1002;
+        parent::__construct($message, $code, $previous);
+    }
+}
diff --git a/app/src/Log/File.php b/app/src/Log/File.php
index 7e4e8d8..449b890 100644
--- a/app/src/Log/File.php
+++ b/app/src/Log/File.php
@@ -3,9 +3,13 @@ namespace ProVM\Logview\Log;
 
 use Generator;
 use DateTimeInterface;
+use ProVM\Logview\Exception\Parse\EmptyException;
+use ProVM\Logview\Exception\Parse\EmptyLineException;
 use Psr\Log\LoggerInterface;
 use ProVM\Common\Define\Parser;
 
+use function Safe\{fopen, fclose};
+
 class File
 {
     protected LoggerInterface $logger;
@@ -77,26 +81,24 @@ class File
             $i = 0;
         }
 
-        $cnt = 1;
-        $fh = \Safe\fopen($this->getFullname(), 'r');
+        $debug = compact('i', 'f');
+        $cnt = $i;
+        $fh = fopen($this->getFullname(), 'r');
+        $debug []= ftell($fh);
+        $this->getParser()->advance($fh, $i);
+        $debug []= ftell($fh);
+        \Safe\error_log(var_export($debug,true).PHP_EOL,3,'/logs/debug');
         while (!feof($fh)) {
-            $line = fgets($fh);
-            if ($cnt < $i) {
-                $cnt ++;
+            try {
+                yield $this->getParser()->parse($fh);
+            } catch (EmptyException | EmptyLineException $e) {
                 continue;
             }
-            if (!$line) {
-                continue;
-            }
-            if (trim($line) === '') {
-                continue;
-            }
-            yield $this->getParser()->parse(trim($line));
             $cnt ++;
             if ($cnt > $f) {
                 break;
             }
         }
-        \Safe\fclose($fh);
+        fclose($fh);
     }
 }
diff --git a/app/src/Parser/Access.php b/app/src/Parser/Access.php
index 6c8c057..9914c34 100644
--- a/app/src/Parser/Access.php
+++ b/app/src/Parser/Access.php
@@ -1,18 +1,25 @@
 <?php
 namespace ProVM\Logview\Parser;
 
+use Exception;
 use Safe\DateTimeImmutable;
+use Safe\Exceptions\DatetimeException;
 use ProVM\Common\Define\Log;
 use ProVM\Common\Implement\Parser;
-use Safe\Exceptions\DatetimeException;
+use function Safe\{json_encode, preg_match};
 
 class Access extends Parser
 {
-    public function parse(string $content): Log
+    public function parse(mixed &$file_handler): Log
     {
-        $log = parent::parse($content);
+        $log = parent::parse($file_handler);
+        $content = $log->getOriginal();
         $regex = "/(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?<date>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} [\+|\-]\d{4})\] (?<message>.*)/";
-        preg_match($regex, $content, $matches);
+        try {
+            preg_match($regex, $content, $matches);
+        } catch (Exception $e) {
+            return $log;
+        }
         try {
             $log->setDate(DateTimeImmutable::createFromFormat('d/M/Y:H:i:s P', $matches['date']));
         } catch (DatetimeException $e) {
diff --git a/app/src/Parser/Debug.php b/app/src/Parser/Debug.php
new file mode 100644
index 0000000..8330b50
--- /dev/null
+++ b/app/src/Parser/Debug.php
@@ -0,0 +1,104 @@
+<?php
+namespace ProVM\Logview\Parser;
+
+use Exception;
+use ProVM\Common\Define\Log;
+use ProVM\Common\Implement\Parser;
+
+use function Safe\{error_log, fclose, fopen, preg_match};
+
+class Debug extends Parser
+{
+    public function isMultiline(string $filename): bool
+    {
+        return true;
+    }
+
+    public function total(string $filename): int
+    {
+        try {
+            $fh = fopen($filename, 'r');
+            $cnt = 0;
+            while(!feof($fh)) {
+                $line = fgets($fh);
+                if (str_starts_with($line, 'Error')) {
+                    $cnt ++;
+                }
+            }
+            fclose($fh);
+            return parent::total($filename);
+        } catch (Exception $e) {
+            error_log($e . PHP_EOL, 3, '/logs/total.log');
+            return 0;
+        }
+    }
+
+    public function parse(mixed &$file_handler): Log
+    {
+        $log = parent::parse($file_handler);
+        $content = [$log->getOriginal()];
+        $this->parseError($log, $content[0]);
+        while(!feof($file_handler)) {
+            $line_number = ftell($file_handler);
+            $line = fgets($file_handler);
+            if (!$line or trim($line) === '') {
+                continue;
+            }
+            if (str_contains($line, 'Error')) {
+                fseek($file_handler, $line_number);
+                break;
+            }
+            $content []= $line;
+            if (str_starts_with($line, '#')) {
+                $this->parseStack($log, $line);
+                continue;
+            }
+            $this->parseLine($log, $line);
+        }
+        $log->setOriginal(implode(PHP_EOL, $content));
+        return $log;
+    }
+
+    public function advance(mixed &$file_handler, int $offset): void
+    {
+        if ($offset === 0) {
+            return;
+        }
+        $cnt = 0;
+        while(!feof($file_handler)) {
+            $line = fgets($file_handler);
+            if (str_starts_with($line, 'Error')) {
+                $cnt ++;
+            }
+            if ($cnt >= $offset) {
+                break;
+            }
+        }
+    }
+
+    protected function parseError(Log &$log, string $line): void
+    {
+        $regex = '/(?<level>Error): (?<message>\w|\s|\"|\\*)(?<filename>\/\w\.*):(?<line>\d*)/';
+        try {
+            preg_match($regex, $line, $matches);
+        } catch (Exception $e) {
+            return;
+        }
+        $log->setSeverity($matches['level']);
+        $log->setChannel($matches['level']);
+        $log->setMessage("{$matches['message']} {$matches['filename']}:{$matches['line']}");
+        $log->setContext("{$matches['filename']} ({$matches['line']})");
+    }
+    protected function parseStack(Log &$log, string $line): void
+    {
+        $stack = $log->getStack();
+        $stack []= $line;
+        $log->setStack($stack);
+    }
+    protected function parseLine(Log &$log, string $line): void
+    {
+        $extra = explode(PHP_EOL, $log->getExtra()) ?? [];
+        $extra []= $line;
+        $log->setExtra(implode(PHP_EOL, $extra));
+    }
+}
diff --git a/app/src/Parser/Monolog.php b/app/src/Parser/Monolog.php
index 460f209..9f0a9ab 100644
--- a/app/src/Parser/Monolog.php
+++ b/app/src/Parser/Monolog.php
@@ -4,14 +4,27 @@ namespace ProVM\Logview\Parser;
 use Exception;
 use Error;
 use Safe\DateTimeImmutable;
-use function Safe\{fopen, error_log, json_encode, preg_match, preg_match_all};
 use ProVM\Common\Define\Log;
 use ProVM\Common\Implement\Parser;
 
+use function Safe\{fopen, fclose, error_log, json_encode, preg_match, preg_match_all, filesize};
+
 class Monolog extends Parser
 {
+    public function isMultiline(string $filename): bool
+    {
+        $multiline = new Monolog\Multiline();
+        if ($multiline->total($filename) !== $this->getLines($filename)) {
+            return true;
+        }
+        return false;
+    }
+
     public function total(string $filename): int
     {
+        if ($this->isMultiline($filename)) {
+            return (new Monolog\Multiline())->total($filename);
+        }
         try {
             $regex = "/\[(?P<date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}(?:\+|-)\d{2}:\d{2})\]/";
             $fh = fopen($filename, 'r');
@@ -23,13 +36,22 @@ class Monolog extends Parser
             fclose($fh);
             return $sum;
         } catch (Exception $e) {
-            error_log($e . PHP_EOL, 3, '/logs/total.log');
             return 0;
         }
     }
-    public function parse(string $content): Log
+    public function parse(mixed &$file_handler): Log
     {
-        $log = parent::parse($content);
+        $reset_line = ftell($file_handler);
+        $log = parent::parse($file_handler);
+        $content = $log->getOriginal();
+
+        $line_number = ftell($file_handler);
+        $line = fgets($file_handler);
+        fseek($file_handler, $line_number);
+        if (str_starts_with($line, 'Stack trace')) {
+            fseek($file_handler, $reset_line);
+            return (new Monolog\Multiline())->parse($file_handler);
+        }
 
         $regex = [
             "\[(?P<date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}(?:\+|-)\d{2}:\d{2})\]",
@@ -43,9 +65,7 @@ class Monolog extends Parser
         try {
             preg_match("/{$regex}/", $content, $matches);
         } catch (Exception $e) {
-            error_log($content . PHP_EOL, 3, '/logs/debug.log');
-            error_log($e . PHP_EOL, 3, '/logs/debug.log');
-            return $log;
+            return (new Monolog\Multiline())->parse($file_handler);
         }
 
         try {
@@ -84,4 +104,25 @@ class Monolog extends Parser
 
         return $log;
     }
+
+    public function advance(mixed &$file_handler, int $offset): void
+    {
+        (new Monolog\Multiline())->advance($file_handler, $offset);
+    }
+
+    protected int $lines;
+    protected function getLines(string $filename): int
+    {
+        if (!isset($this->lines)) {
+            $cnt = 0;
+            $fh = fopen($filename, 'r');
+            while(!feof($fh)) {
+                fgets($fh);
+                $cnt ++;
+            }
+            fclose($fh);
+            $this->lines = $cnt;
+        }
+        return $this->lines;
+    }
 }
diff --git a/app/src/Parser/Monolog/Multiline.php b/app/src/Parser/Monolog/Multiline.php
new file mode 100644
index 0000000..2026f36
--- /dev/null
+++ b/app/src/Parser/Monolog/Multiline.php
@@ -0,0 +1,145 @@
+<?php
+namespace ProVM\Logview\Parser\Monolog;
+
+use Exception;
+use Error;
+use Safe\DateTimeImmutable;
+use ProVM\Common\Define\Log;
+use ProVM\Common\Implement\Parser;
+
+use function Safe\{error_log, fclose, fopen, json_encode, preg_match};
+
+class Multiline extends Parser
+{
+    public function isMultiline(string $filename): bool
+    {
+        return true;
+    }
+
+    public function total(string $filename): int
+    {
+        try {
+            $fh = fopen($filename, 'r');
+            $cnt = 0;
+            while(!feof($fh)) {
+                $line = fgets($fh);
+                if (str_starts_with($line, '[')) {
+                    $cnt ++;
+                }
+            }
+            fclose($fh);
+            return parent::total($filename);
+        } catch (Exception $e) {
+            error_log($e . PHP_EOL, 3, '/logs/total.log');
+            return 0;
+        }
+    }
+
+    public function parse(mixed &$file_handler): Log
+    {
+        $log = parent::parse($file_handler);
+        $content = [$log->getOriginal()];
+        $this->parseError($log, $content[0]);
+        while(!feof($file_handler)) {
+            $line_number = ftell($file_handler);
+            $line = fgets($file_handler);
+            if (!$line or trim($line) === '') {
+                continue;
+            }
+            if (str_starts_with($line, '[')) {
+                fseek($file_handler, $line_number);
+                break;
+            }
+            $content []= $line;
+            if (str_starts_with($line, 'Stack trace')) {
+                $log->setStack([$line]);
+                continue;
+            }
+            if (str_starts_with($line, '#')) {
+                $this->parseStack($log, $line);
+                continue;
+            }
+            $this->parseLine($log, $line);
+        }
+        $log->setOriginal(implode(PHP_EOL, $content));
+        return $log;
+    }
+
+    public function advance(mixed &$file_handler, int $offset): void
+    {
+        if ($offset === 0) {
+            return;
+        }
+        $cnt = 0;
+        while(!feof($file_handler)) {
+            if ($cnt >= $offset) {
+                break;
+            }
+            $line = fgets($file_handler);
+            if (str_starts_with($line, '[')) {
+                $cnt ++;
+            }
+        }
+    }
+
+    protected function parseError(Log &$log, string $line): void
+    {
+        $regex = [
+            "\[(?P<date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}(?:\+|-)\d{2}:\d{2})\]",
+            "\s(?<channel>\w*)",
+            "\.(?<severity>\w*)",
+            ":\s(?<message>.*)",
+        ];
+        $regex = implode('', $regex);
+        try {
+            preg_match("/{$regex}/", $line, $matches);
+        } catch (Exception $e) {
+            return;
+        }
+        try {
+            $extra = [];
+            try {
+                $log->setDate(DateTimeImmutable::createFromFormat('Y-m-d\TH:i:s.uP', $matches['date']));
+            } catch (Exception $e) {
+                $log->setDate(new DateTimeImmutable());
+                $extra['date'] = $matches['date'];
+            }
+            $log->setChannel($matches['channel']);
+            $log->setSeverity($matches['severity']);
+            $message = $matches['message'];
+            if (str_contains($message, 'Stack trace')) {
+                list($msg, $data) = explode('Stack trace:', $message);
+                $message = trim($msg);
+                $regex = '/\s#\d+\s/';
+                $lines = preg_split($regex, $data);
+                array_shift($lines);
+                $log->setStack($lines);
+            }
+            $log->setMessage($message);
+            if ($matches['context'] !== '') {
+                $log->setContext("{{$matches['context']}}");
+            }
+            if (isset($matches['extra']) and $matches['extra'] !== '') {
+                $extra['extra'] = "{{$matches['extra']}}";
+            }
+            if (count($extra) > 0) {
+                $log->setExtra(json_encode($extra, JSON_UNESCAPED_SLASHES));
+            }
+        } catch (Error $e) {
+            error_log($e . PHP_EOL, 3, '/logs/debug.log');
+            error_log(var_export($matches, true) . PHP_EOL, 3, '/logs/debug.log');
+        }
+    }
+    protected function parseStack(Log &$log, string $line): void
+    {
+        $stack = $log->getStack();
+        $stack []= $line;
+        $log->setStack($stack);
+    }
+    protected function parseLine(Log &$log, string $line): void
+    {
+        $extra = explode(PHP_EOL, $log->getExtra()) ?? [];
+        $extra []= $line;
+        $log->setExtra(implode(PHP_EOL, $extra));
+    }
+}
diff --git a/app/src/Parser/PHPDefault.php b/app/src/Parser/PHPDefault.php
index bed3bc2..d4d1bf5 100644
--- a/app/src/Parser/PHPDefault.php
+++ b/app/src/Parser/PHPDefault.php
@@ -4,7 +4,7 @@ namespace ProVM\Logview\Parser;
 use Error;
 use Exception;
 use Safe\DateTimeImmutable;
-use function Safe\{fopen, preg_match_all, preg_match, error_log, json_encode};
+use function Safe\{fopen, fclose, preg_match_all, preg_match, error_log, json_encode};
 use ProVM\Common\Define\Log;
 use ProVM\Common\Implement\Parser;
 
@@ -26,9 +26,10 @@ class PHPDefault extends Parser
             return 0;
         }
     }
-    public function parse(string $content): Log
+    public function parse(mixed &$file_handler): Log
     {
-        $log = parent::parse($content);
+        $log = parent::parse($file_handler);
+        $content = $log->getOriginal();
         $regex = "/\[(?<date>\d{2}-\w{3}-\d{4}\s\d{2}:\d{2}:\d{2}\s\w{3})\]\s(?<level>PHP|User)\s(?<severity>\w+):\s(?<message>.*)/";
         try {
             preg_match($regex, $content, $matches);