Mejoras al cerebro

.gitignore

@@ -1,5 +1,12 @@
 # PyCharm
 /.idea/
+__pycache__/
 
 # PipEnv
-Pipfile.lock
+Pipfile.lock
+
+# Logs
+/logs/
+
+# Data
+/data/

Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+keyboard = "*"
+beautifulsoup4 = "*"
+pytz = "*"
+flask = "*"
+nltk = "*"
+spacy = "*"
+
+[requires]
+python_version = "3.8"

TODO.md

@@ -0,0 +1,35 @@
+# TODO
+
+## Main modulo
+1. Workers para
+    1. [x] Revisar Email
+    1. [ ] Revisar WhatsApp
+    1. [ ] Procesar Texto
+
+## Modulo de Revision de Email
+Para revisar si hay mails nuevos, validando que sea de las personas registradas.
+Si no está registrada se avisa a administrador para saber que hacer.
+Limpieza de Inbox y Spam.
+- [x] Revisar lista de emails
+- [x] Revisar si fue revisado
+- [x] Revisar si corresponde a un "Jefe"
+- [x] Confirmar con cerebro si es un comando
+
+## Modulo de WhatsApp
+Respuestas a mensajes.
+
+## AI para procesar textos
+Spacy permite procesar texto
+Hay que poder "educar" el modelo
+Hay que poder agregar datos
+Hay que poder agregar los correos y mensajes que no se conocen a listado "desconocido"
+
+## Modulo de Recordatorio
+Crear recordatorios y mandar correo o WhatsApp en el momento del recordatorio.
+
+## Modulo de Registro de Eventos
+Ir llevando un registro de eventos.
+
+### Otros
++ [ ] Establecer lenguage de interpretacion para los comandos
++ [ ] Crear AI para interpretacion de comandos en lenguage comun

common/helper/logger.py

@@ -0,0 +1,123 @@
+import os
+import datetime
+import locale
+from threading import Thread
+import queue
+from src.functions import dump_queue
+
+
+def get_today(tz):
+    today = datetime.datetime.now(tz=tz)
+    locale.setlocale(locale.LC_TIME, 'es_ES')
+    return today
+
+
+class Logger:
+    """
+    Clase que lleva el diario de actividades de la secretaria
+    """
+    def __init__(self, log_folder, timezone, name):
+        self.folder = log_folder
+        self.tz = timezone
+        self.name = name
+        self.messages = []
+
+    def get_filename(self):
+        files = [f for f in os.listdir(self.folder) if os.path.isfile(os.path.join(self.folder, f)) and 'diary' in f]
+        today = datetime.datetime.now(tz=self.tz)
+        if len(files) == 0:
+            return os.path.join(self.folder, self.name + '-diary-{0}.log'.format(today.strftime('%Y-%m-%d')))
+        last = files[-1]
+        return os.path.join(self.folder, last)
+
+    def start_new(self):
+        today = datetime.datetime.now(tz=self.tz)
+        filename = os.path.join(self.folder, self.name + '-diary-{0}.log'.format(today.strftime('%Y-%m-%d')))
+        with open(filename, 'w') as f:
+            pass
+
+    def start_log(self):
+        today = datetime.datetime.now(tz=self.tz)
+        locale.setlocale(locale.LC_TIME, 'es_ES')
+        msg = 'Inicio una nueva jornada siendo las {0} del {1}'.format(today.strftime('%H:%M:%S'),
+                                                                  today.strftime('%d de %B de %Y'))
+        self.log(msg)
+
+    def stop_log(self):
+        now = datetime.datetime.now(tz=self.tz)
+        locale.setlocale(locale.LC_TIME, 'es_ES')
+        msg = 'Siendo las {0}, termino mis registros por esta jornada'.format(now.strftime('%H:%M:%S'))
+        self.log(msg)
+        msg = '--------'
+        self.log(msg)
+
+    def log_action(self, action):
+        msg = 'he realizado {0}'.format(action)
+        self.log_msg(msg)
+
+    def log_not_action(self, action):
+        msg = 'no he podido realizar {0}'.format(action)
+        self.log_msg(msg)
+
+    def start_turn(self, action):
+        self.log_msg('Inicio de turno de {0}'.format(action))
+
+    def end_turn(self, action):
+        self.log_msg('Termino de turno de {0}'.format(action))
+
+    def log_msg(self, msg):
+        today = get_today(self.tz)
+        line = 'A las {0} del {1}, {2}'.format(today.strftime('%H:%M:%S'), today.strftime('%d de %B de %Y'), msg)
+        self.log(line)
+
+    def log(self, message):
+        line = message
+        if line[-1] != '.' and line != '--------':
+            line = line.rstrip('.') + '.'
+        self.messages.append(line)
+        if len(self.messages) > 1000:
+            self.start_new()
+        with open(self.get_filename(), 'a') as f:
+            f.write(line + "\n")
+
+
+class Worker(Thread):
+    def __init__(self, params, configs):
+        super().__init__()
+        self.event = params['events']['log_stop']
+        self.queue = params['queues']['log']
+        self.wait = configs.get('supervisor.wait')
+        self.logger = Logger(params['folders']['log'], configs.get('timezone'), params['logger']['name'])
+        self.logging = params['logging']
+
+    def parse_message(self, message):
+        if 'is_start' in message and message['is_start']:
+            self.logger.start_log()
+            return
+        if 'action' in message:
+            if 'not' in message and message['not']:
+                self.logger.log_not_action(message['action'])
+                return
+            self.logger.log_action(message['action'])
+            return
+        if 'start_turn' in message:
+            self.logger.start_turn(message['start_turn'])
+            return
+        if 'end_turn' in message:
+            self.logger.end_turn(message['end_turn'])
+            return
+        self.logger.log_msg(message['message'])
+
+    def run(self):
+        self.logging.log('Starting', caller=type(self))
+        self.queue.put({'is_start': True})
+        while not self.event.is_set():
+            try:
+                message = self.queue.get(timeout=self.wait)
+                self.logging.log('Logger received message', caller=type(self))
+                self.parse_message(message)
+            except queue.Empty:
+                continue
+        [self.parse_message(message) for message in dump_queue(self.queue, self.wait)]
+        self.logger.stop_log()
+        self.logging.log('Exiting', caller=type(self))

common/helper/logging.py

@@ -0,0 +1,44 @@
+import os
+import datetime
+
+
+class Logging:
+    """
+    Clase que registra los eventos del flujo del programa en archivos
+    """
+    def __init__(self, timezone, folder=None, name=''):
+        self.tz = timezone
+        self.folder = folder
+        self.name = name
+        self.filename = self.get_filename()
+        self.log('------')
+
+    def get_filename(self):
+        files = [f for f in os.listdir(self.folder) if os.path.isfile(os.path.join(self.folder, f)) and 'logging' in f]
+        if len(files) > 0:
+            filename = files[-1]
+            if os.path.getsize(os.path.join(self.folder, filename)) < 1024 * 1024:
+                return filename
+        today = datetime.datetime.now(tz=self.tz)
+        filename = '-'.join([self.name, 'logging-{0}.log'.format(today.strftime('%Y-%m-%d'))]).strip('-')
+        base_name = filename
+        n = 1
+        while os.path.isfile(os.path.join(self.folder, filename)):
+            filename = '-'.join([
+                base_name,
+                str(n)
+            ])
+            n += 1
+        return filename
+
+    def log(self, message, caller=None):
+        now = datetime.datetime.now(tz=self.tz)
+        if caller is None:
+            msg = '[{0}] {1}.'.format(now.strftime('%Y-%m-%d %H:%M:%S'), message.strip('.'))
+        else:
+            msg = '[{0}] ({2}) {1}.'.format(now.strftime('%Y-%m-%d %H:%M:%S'), message.strip('.'), caller)
+        print(msg)
+        if self.folder is not None:
+            filename = os.path.join(self.folder, self.filename)
+            with open(filename, 'a') as f:
+                f.write(msg + "\n")

config/email.json

@@ -0,0 +1,24 @@
+{
+  "imap": {
+    "server": "imap.yandex.com",
+    "port": 993,
+    "user": {
+      "name": "secretary@incoviba.cl",
+      "password": "vgvjuwzwizktdpka"
+    },
+    "ssl": true
+  },
+  "smtp": {
+    "server": "smtp.yandex.com",
+    "port": 495,
+    "user": {
+      "name": "secretary@incoviba.cl",
+      "password": "vgvjuwzwizktdpka"
+    },
+    "ssl": true
+  },
+  "max": 5,
+  "consultas": "email_consultas.json",
+  "spam": "email_spam.json",
+  "revisados": "email_revisados.json"
+}

config/supervisor.json

@@ -0,0 +1,4 @@
+{
+  "wait": 15,
+  "timezone": "America/Santiago"
+}

entry/api.py

@@ -0,0 +1,48 @@
+from flask import Flask, redirect, url_for
+import os
+from src.instrucciones import Instrucciones
+import json
+from src.brain.build_data import brain_app
+
+
+app = Flask(__name__)
+app.register_blueprint(brain_app, url_prefix='/brain')
+data_folder = os.path.join(os.path.realpath('..'), 'data')
+
+
+@app.route('/', methods=['GET'])
+def index():
+    return {
+        'api': {
+            'entrypoints': {
+                'bosses': [
+                    'add',
+                    '/'
+                ],
+                'instructions': [
+                    'add',
+                    '/'
+                ],
+                'email': []
+            }
+        }
+    }
+
+
+@app.route('/instructions/', methods=['GET'])
+def instructions():
+    instrucciones = Instrucciones(data_folder)
+    data = {'Instrucciones': [{'Name': i.instruccion, 'Aliases': i.aliases} for i in instrucciones.instrucciones]}
+    return json.dumps(data)
+
+
+@app.route('/instructions/add/<string:instruccion>/<string:alias>')
+def add_instruccion(instruccion, alias):
+    ins = Instrucciones(data_folder)
+    ins.add(instruccion, [alias])
+    ins.save()
+    return redirect(url_for('instructions'))
+
+
+if __name__ == '__main__':
+    app.run(port=8081, debug=True)

entry/email/inbox.py

@@ -0,0 +1,24 @@
+import argparse
+from src.email.functions import connect, check_inbox
+
+
+def main(args):
+    print('Connecting')
+    imap_server = connect(imap_url=args.url, port=args.port, username=args.username, password=args.password,
+                          ssl=args.ssl)
+    print('Checking emails')
+    emails = check_inbox(imap_server)
+    print(len(emails))
+    [print(e) for e in emails]
+    imap_server.close()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-u', '--url')
+    parser.add_argument('-p', '--port')
+    parser.add_argument('-un', '--username')
+    parser.add_argument('-up', '--password')
+    parser.add_argument('--ssl')
+    _args = parser.parse_args()
+    main(_args)

entry/main.py

@@ -0,0 +1,51 @@
+import argparse
+import os
+import pytz
+from common.helper.logging import Logging
+from setup.config import load_config
+from src.supervisor import Supervisor
+from src.bosses import Bosses
+
+
+def main(args):
+    configs = load_config(args.config_folder)
+    configs.set('timezone', pytz.timezone('America/Santiago'))
+    params = {
+        'folders': {
+            'config': args.config_folder,
+            'log': args.log_folder,
+            'data': args.data_folder
+        },
+        'bosses': Bosses(args.data_folder),
+        'logging': Logging(configs.get('timezone'), args.log_folder)
+    }
+
+    params['logging'].log('Starting', caller='main')
+
+    supervisor = Supervisor(configs=configs, params=params)
+
+    supervisor.register_worker('common.helper.logger', 'Worker')
+    supervisor.add_queue('log')
+    supervisor.register_worker('src.email', 'RevisorEmailWorker')
+    supervisor.register_worker('src.email', 'EmailInterpreter')
+    supervisor.add_queue('emails')
+    supervisor.add_queue('questions')
+    supervisor.register_worker('src.action.questions', 'QuestionWorker')
+    supervisor.register_worker('src.action.reminder', 'RemindRegisterWorker')
+    supervisor.register_worker('src.action.reminder', 'ReminderWorker')
+    supervisor.add_queue('reminders')
+    supervisor.add_lock('reminders')
+
+    supervisor.start()
+    supervisor.join()
+
+    params['logging'].log('Waiting for Supervisor', caller='main')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config_folder', default=os.path.join(os.path.realpath('../'), 'config'))
+    parser.add_argument('-d', '--data_folder', default=os.path.join(os.path.realpath('..'), 'data'))
+    parser.add_argument('-l', '--log_folder', default=os.path.join(os.path.realpath('..'), 'logs'))
+    _args = parser.parse_args()
+    main(_args)

setup/config.py

@@ -0,0 +1,70 @@
+import os
+import json
+
+
+class ConfigFile:
+    def __init__(self, name):
+        self.name = name
+        self._data = []
+
+    def load(self, folder):
+        filename = os.path.join(folder, self.name + '.json')
+        with open(filename, 'r') as f:
+            self._data = json.load(f)
+
+    def get(self, name):
+        if '.' not in name and name in self._data:
+            return self._data[name]
+        subs = "']['".join(name.split('.'))
+        substr = "self._data['" + subs + "']"
+        print(substr)
+
+
+class Config:
+    def __init__(self, folder):
+        self.folder = folder
+        self._configs = {}
+
+    def load(self):
+        files = [f for f in os.listdir(self.folder) if os.path.isfile(os.path.join(self.folder, f)) and '.json' in f]
+        configs = {}
+        for f in files:
+            name = f.split('.')[0]
+            filename = os.path.join(self.folder, f)
+            with open(filename, 'r') as fh:
+                configs[name] = json.load(fh)
+        self._configs = configs
+
+    def get(self, name):
+        if '.' not in name and name in self._configs:
+            return self._configs[name]
+        subs = "']['".join(name.split('.'))
+        substr = "self._configs['" + subs + "']"
+        try:
+            sub = eval(substr)
+            return sub
+        except:
+            return None
+
+    def get_all(self, name_list):
+        output = {}
+        for name in name_list:
+            output[name] = self.get(name)
+        return output
+
+    def set(self, name, value):
+        if '.' not in name:
+            self._configs[name] = value
+            return
+        subs = "']['".join(name.split('.'))
+        substr = "self._configs['" + subs + "'] = {0}".format(value)
+        try:
+            eval(substr)
+        except:
+            pass
+
+
+def load_config(folder):
+    config = Config(folder)
+    config.load()
+    return config

src/action/reminder.py

@@ -0,0 +1,112 @@
+import datetime
+import queue
+from threading import Thread
+import os
+import json
+import time
+
+
+class Reminder:
+    def __init__(self, timezone):
+        self.datetime = 0
+        self.message = ''
+        self.recursive = False
+        self.last = 0
+        self.repeats = 0
+        self.tz = timezone
+
+    def should_remind(self):
+        now = datetime.datetime.now(tz=self.tz)
+        event = self.get_remind_time()
+        dif = event - now
+        if dif.days * 24 * 60 * 60 + dif.seconds == 0:
+            return True
+        return False
+
+    def get_remind_time(self):
+        if self.recursive:
+            event = self.last + self.datetime
+            return event
+        return self.datetime
+
+    def remind(self):
+        # Do remind
+        if self.recursive:
+            self.last = datetime.datetime.now(tz=self.tz)
+            self.repeats -= 1
+
+
+class RemindRegisterWorker(Thread):
+    def __init__(self, params, configs):
+        super().__init__()
+        self.event = params['events']['stop']
+        self.queue = params['queues']['reminders']
+        self.lock = params['locks']['reminders']
+        self.folder = params['folders']['data']
+        self.logging = params['logging']
+        self.wait = configs.get('supervisor.wait')
+
+    def add_reminder(self, reminder):
+        self.lock.acquire()
+        filename = os.path.join(self.folder, 'reminders.json')
+        with open(filename, 'w+') as f:
+            reminders = json.load(f)
+            reminders.append(reminder)
+            json.dump(reminders, f)
+        self.lock.release()
+
+    def run(self):
+        self.logging.log('Starting', caller=type(self))
+        while not self.event.is_set():
+            self.logging.log('Looping status {0}'.format(not self.event.is_set()),
+                             caller=type(self))
+            try:
+                reminder = self.queue.get(timeout=self.wait)
+                self.add_reminder(reminder)
+            except queue.Empty:
+                pass
+        self.logging.log('Exiting', caller=type(self))
+        return
+
+
+class ReminderWorker(Thread):
+    def __init__(self, params, configs):
+        super().__init__()
+        self.event = params['events']['stop']
+        self.folder = params['folders']['data']
+        self.lock = params['locks']['reminders']
+        self.logging = params['logging']
+        self.wait = configs.get('supervisor.wait')
+        self.tz = configs.get('timezone')
+
+    def check_reminders(self):
+        self.lock.acquire()
+        filename = os.path.join(self.folder, 'reminders.json')
+        if not os.path.isfile(filename):
+            self.lock.release()
+            return
+        with open(filename, 'w+') as f:
+            reminders = json.load(f)
+            new_reminders = []
+            for r in reminders:
+                reminder = Reminder(self.tz)
+                for k, val in r.items():
+                    setattr(reminder, k, val)
+                if reminder.should_remind():
+                    reminder.remind()
+                    if reminder.recursive and reminder.repeats > 0:
+                        new_reminders.append(reminder)
+                else:
+                    new_reminders.append(reminder)
+            json.dump(new_reminders, f)
+        self.lock.release()
+
+    def run(self):
+        self.logging.log('Starting', caller=type(self))
+        while not self.event.is_set():
+            self.logging.log('Looping status {0}'.format(not self.event.is_set()),
+                             caller=type(self))
+            self.check_reminders()
+            time.sleep(self.wait)
+        self.logging.log('Exiting', caller=type(self))
+        return

src/address_book.py

@@ -0,0 +1,40 @@
+import os
+import json
+
+
+class Contact:
+    def __init__(self):
+        self.names = ''
+        self.last_name = ''
+        self.maiden_name = ''
+        self.emails = []
+        self.phones = []
+
+    def full_name(self):
+        return ' '.join([self.names, self.last_name, self.maiden_name])
+
+
+class AddressBook:
+    def __init__(self, data_folder):
+        filename = os.path.join(data_folder, 'address_book.json')
+        with open(filename, 'r') as f:
+            data = json.load(f)
+
+        self.contacts = []
+        for c in data:
+            contact = Contact()
+            for k, val in c.items():
+                setattr(contact, k, val)
+            self.contacts.append(contact)
+
+    def find(self, name):
+        for c in self.contacts:
+            if c.full_name() == name or c.names == name or c.last_name == name:
+                return c
+        return None
+
+    def find_id(self, name):
+        for (k, c) in enumerate(self.contacts):
+            if c.full_name() == name or c.names == name or c.last_name == name:
+                return k
+        return -1

src/bosses.py

@@ -0,0 +1,63 @@
+import os
+import json
+from src.address_book import AddressBook
+
+
+class Boss:
+    def __init__(self):
+        self.full_name = ''
+        self.aliases = []
+        self.contact = None
+
+
+class Bosses:
+    def __init__(self, data_folder):
+        self.filename = os.path.join(data_folder, 'bosses.json')
+        with open(self.filename, 'r') as f:
+            data = json.load(f)
+        self.bosses = []
+        addrs = AddressBook(data_folder)
+        for b in data:
+            boss = Boss()
+            for k, val in b.items():
+                setattr(boss, k, val)
+            boss.contact = addrs.find(boss.full_name)
+            self.bosses.append(boss)
+
+    def is_boss(self, name):
+        for boss in self.bosses:
+            if boss.full_name in name:
+                return True
+            for m in boss.contact.emails:
+                if m in name:
+                    return True
+            for a in boss.aliases:
+                if a in name:
+                    return True
+        return False
+
+    def add_boss(self, name, aliases: list = None):
+        if self.is_boss(name):
+            return
+        b = Boss()
+        b.full_name = name
+        if aliases is not None:
+            b.aliases = aliases
+
+    def get(self, name):
+        if not self.is_boss(name):
+            return None
+        for i, boss in enumerate(self.bosses):
+            if boss.full_name in name:
+                return i
+            for m in boss.contact.emails:
+                if m in name:
+                    return i
+            for a in boss.aliases:
+                if a in name:
+                    return i
+
+    def save(self):
+        data = [{'full_name': boss.full_name, 'aliases': boss.aliases} for boss in self.bosses]
+        with open(self.filename, 'w') as f:
+            json.dump(data, f, indent=4)

src/brain/aiml.py

@@ -0,0 +1,3 @@
+class Aiml:
+    def __init__(self):
+        pass

src/brain/brain.py

@@ -0,0 +1,29 @@
+import os
+import spacy
+from src.instrucciones import Instrucciones
+
+
+class Brain:
+    def __init__(self, data_folder):
+        self.folder = data_folder
+        self.filename = os.path.join(data_folder, 'brain.json')
+        self.nlp = None
+
+        self.load_nlp(data_folder)
+
+    def load_nlp(self, data_folder):
+        folder = os.path.join(data_folder, 'model')
+        self.nlp = spacy.load(folder)
+
+    def save_model(self):
+        folder = os.path.join(self.folder, 'model')
+        self.nlp.to_disk(folder)
+
+    def get_command(self, phrase):
+        doc = self.nlp(phrase)
+        command = max(doc.cats, key=doc.cats.get)
+        return command
+
+    def get_response(self, command, phrase):
+        doc = self.nlp(phrase)
+        return doc

src/brain/build_data.py

@@ -0,0 +1,20 @@
+import argparse
+import os
+from flask import Blueprint, request
+import json
+
+
+brain_app = Blueprint('brain_blueprint', __name__)
+
+
+@brain_app.route('/', methods=['GET'])
+def index():
+    return {
+        'api': '/brain'
+    }
+
+
+@brain_app.route('/training/data/add', methods=['POST'])
+def add_data():
+    input_data = json.loads(request.data)
+    return input_data

src/brain/build_model.py

@@ -0,0 +1,33 @@
+import argparse
+import os
+import spacy
+from src.instrucciones import Instrucciones
+
+
+def load_model(commands):
+    nlp = spacy.load('es_core_news_sm')
+    if 'textcat' not in nlp.pipe_names:
+        textcat = nlp.create_pipe('textcat')
+        nlp.add_pipe(textcat)
+        textcat.add_label('test')
+        for c in commands.instrucciones:
+            textcat.add_label(c.instruccion)
+    return nlp
+
+
+def save_model(data_folder, model):
+    folder = os.path.join(data_folder, 'model')
+    model.to_disk(folder)
+
+
+def main(args):
+    commands = Instrucciones(args.data_folder)
+    model = load_model(commands)
+    save_model(args.data_folder, model)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--data_folder')
+    _args = parser.parse_args()
+    main(_args)

src/brain/train_model.py

@@ -0,0 +1,33 @@
+import argparse
+import os
+import spacy
+import json
+
+
+def load_model(folder):
+    return spacy.load(folder)
+
+
+def get_data(folder):
+    files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
+    data = []
+    for filename in files:
+        with open(filename, 'r') as f:
+            data += json.load(f)
+    return data
+
+
+def train_model(model, data):
+    optimizer = model.begin_training()
+    pass
+
+
+def main(args):
+    pass
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--data_folder')
+    _args = parser.parse_args()
+    main(_args)

src/communication/__init__.py

@@ -0,0 +1 @@
+from .message import Message

src/communication/message.py

@@ -0,0 +1,9 @@
+class Message:
+    def __init__(self, mtype, sender, datetime, text, original, subject):
+        self.type = mtype
+        self.sender = sender
+        self.datetime = datetime
+        self.text = text
+        self.checked = False
+        self.original = original
+        self.subject = subject

src/email/definitions.py

@@ -0,0 +1,19 @@
+from email.parser import BytesParser
+from email.policy import default as DefaultPolicy
+
+
+class Email:
+    def __init__(self, uid):
+        self.uid = uid
+        self.message = ''
+
+    def get(self, imap):
+        status, raw_data = imap.uid('fetch', self.uid, '(RFC822)')
+        if status != 'OK':
+            raise Exception('Could not recover message {0}'.format(self.uid))
+        self.message = BytesParser(policy=DefaultPolicy).parsebytes(text=raw_data[0][1])
+
+    def delete(self, imap):
+        status, result = imap.uid('STORE', self.uid, '+FLAGS', '(\\Deleted)')
+        if status != 'OK':
+            raise Exception('Could not flag message {0}'.format(self.uid))

src/email/functions.py

@@ -0,0 +1,39 @@
+import keyboard
+import imaplib
+from src.email.definitions import Email
+
+
+def exit_thread(stop, logger):
+    logger.log('Starting exit thread', caller='exit_thread')
+    keyboard.wait('Esc')
+    logger.log('Escape pressed', caller='exit_thread')
+    stop.set()
+    logger.log('Exit signal sent', caller='exit_thread')
+
+
+def connect(imap_url, port, username, password, ssl=False):
+    if ssl:
+        server = imaplib.IMAP4_SSL(imap_url, port=port)
+    else:
+        server = imaplib.IMAP4(imap_url, port=port)
+    server.login(username, password)
+    return server
+
+
+def check_inbox(server):
+    status, msg = server.select('INBOX')
+    if status != 'OK':
+        return None
+
+    status, ids = server.uid('search', None, 'All')
+    if status != 'OK':
+        return None
+
+    ids = ids[0].decode().split()
+    emails = []
+    for mid in ids:
+        em = Email(mid)
+        em.get(server)
+        emails.append(em)
+
+    return emails

src/email/main.py

@@ -0,0 +1,72 @@
+import argparse
+import os
+from common.helper.logging import Logging
+from setup.config import load_config
+import pytz
+from src.bosses import Bosses
+from src.instrucciones import Instrucciones
+from src.email.supervisor import Email
+from src.brain.brain import Brain
+
+
+def set_params(args, configs):
+    log_name = 'email'
+    params = {
+        'folders': {
+            'config': args.config_folder,
+            'log': args.log_folder,
+            'data': args.data_folder
+        },
+        'bosses': Bosses(args.data_folder),
+        'instrucciones': Instrucciones(args.data_folder),
+        'brain': Brain(args.data_folder),
+        'logging': Logging(configs.get('timezone'), args.log_folder, log_name),
+        'logger': {
+            'name': log_name
+        },
+        'filenames': {
+            'consultas': os.path.join(args.data_folder, configs.get('email.consultas')),
+            'spam': os.path.join(args.data_folder, configs.get('email.spam')),
+            'revisados': os.path.join(args.data_folder, configs.get('email.revisados'))
+        }
+    }
+    return params
+
+
+def set_setup():
+    return {
+        'workers': [
+            ('common.helper.logger', 'Worker'),
+            ('src.email.workers', 'Obtenedor'),
+            ('src.email.workers', 'Validador'),
+            ('src.email.workers', 'Consultador'),
+            ('src.email.workers', 'Borrador'),
+            ('src.email.workers', 'Procesador')
+        ],
+        'queues': ['log', 'emails', 'valid', 'invalid', 'borrar'],
+        'events': [],
+        'locks': []
+    }
+
+
+def main(args):
+    configs = load_config(args.config_folder)
+    configs.set('timezone', pytz.timezone(configs.get('supervisor.timezone')))
+
+    params = set_params(args, configs)
+
+    setup = set_setup()
+
+    email = Email(configs=configs, params=params, setup=setup)
+
+    email.start()
+    email.join()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config_folder', default=os.path.join(os.path.realpath('../..'), 'config'))
+    parser.add_argument('-d', '--data_folder', default=os.path.join(os.path.realpath('../..'), 'data'))
+    parser.add_argument('-l', '--log_folder', default=os.path.join(os.path.realpath('../..'), 'logs'))
+    _args = parser.parse_args()
+    main(_args)

src/email/supervisor.py

@@ -0,0 +1,107 @@
+from threading import Thread, Event, Lock
+from queue import Queue
+import importlib
+from src.functions import exit_thread
+import time
+
+
+class Email(Thread):
+    """
+    Email module supervisor thread
+    """
+    def __init__(self, configs, params, setup):
+        super(Email, self).__init__()
+
+        self.configs = configs
+        self.params = params
+        self.registry = {}
+        self.workers = []
+        self.working = []
+        self.worker_status = []
+
+        self.add_event('stop')
+        self.add_event('log_stop')
+
+        self.setup(setup)
+
+    def setup(self, data):
+        for (m, n) in data['workers']:
+            self.register_worker(m, n)
+        for q in data['queues']:
+            self.add_queue(q)
+        for e in data['events']:
+            self.add_event(e)
+        for lo in data['locks']:
+            self.add_lock(lo)
+
+    def register_worker(self, module, name):
+        if module not in self.registry:
+            self.registry[module] = []
+        self.registry[module].append(name)
+
+    def add_worker(self, worker):
+        self.workers.append(worker)
+
+    def start_worker(self, module, name):
+        worker = getattr(module, name)
+        worker = worker(configs=self.configs, params=self.params)
+        self.add_worker(worker)
+        self.working.append((module, name))
+        worker.start()
+        self.worker_status.append(True)
+
+    def start_workers(self):
+        for module_name, workers in self.registry.items():
+            module = importlib.import_module(module_name)
+            for name in workers:
+                self.start_worker(module, name)
+
+    def check_workers(self):
+        stopped = 0
+        for (k, w) in enumerate(self.workers):
+            if not self.worker_status[k]:
+                stopped += 1
+                continue
+            if not w.is_alive():
+                self.params['logging'].log('Worker {0} stopped'.format(type(w)))
+                self.params['queues']['log'].put({'not': True, 'action': type(w)})
+                stopped += 1
+                self.worker_status[k] = False
+                (m, n) = self.working[k]
+                # Restart worker
+                self.start_worker(m, n)
+        if stopped == len(self.workers):
+            return False
+        return True
+
+    def join_workers(self):
+        [w.join(self.configs.get('supervisor.wait')) for w in self.workers if w.is_alive()]
+
+    def add_queue(self, name):
+        if 'queues' not in self.params or self.params['queues'] is None:
+            self.params['queues'] = {}
+        self.params['queues'][name] = Queue()
+
+    def add_event(self, name):
+        if 'events' not in self.params or self.params['events'] is None:
+            self.params['events'] = {}
+        self.params['events'][name] = Event()
+
+    def add_lock(self, name):
+        if 'locks' not in self.params or self.params['locks'] is None:
+            self.params['locks'] = {}
+        self.params['locks'][name] = Lock()
+
+    def run(self) -> None:
+        self.start_workers()
+        worker = Thread(target=exit_thread, args=(self.params['events']['stop'], self.params['logging']))
+        worker.start()
+
+        while not self.params['events']['stop'].is_set():
+            if not self.check_workers():
+                break
+            time.sleep(self.configs.get('supervisor.wait'))
+        self.params['logging'].log('Waiting for workers', type(self))
+        self.params['events']['log_stop'].set()
+        self.join_workers()
+        worker.join()

src/email/workers.py

@@ -0,0 +1,302 @@
+import queue
+from src.email.functions import connect, check_inbox
+import time
+from types import SimpleNamespace
+from src.worker import Worker
+from bs4 import BeautifulSoup
+import re
+import email.utils
+from src.communication import Message
+import json
+from src.functions import dump_queue
+from pprint import pprint
+
+
+class Obtenedor(Worker):
+    """
+    Trabajador que obtiene la lista de correos del inbox configurado
+    """
+    def __init__(self, configs, params):
+        super(Obtenedor, self).__init__(configs, params)
+
+        self.name = 'Email:Obtenedor'
+        self.url = configs.get('email.imap.server')
+        self.port = configs.get('email.imap.port')
+        user = {'user': '', 'password': ''}
+        self.user = SimpleNamespace(**user)
+        self.user.name = configs.get('email.imap.user.name')
+        self.user.password = configs.get('email.imap.user.password')
+        self.ssl = configs.get('email.imap.ssl')
+
+        self.filename = params['filenames']['revisados']
+        self.revisados = []
+        self.load_revisados()
+
+        self.queue = params['queues']['emails']
+        self.frec = configs.get('supervisor.wait')
+
+    def is_revisado(self, uid):
+        return uid in self.revisados
+
+    def load_revisados(self):
+        data = []
+        try:
+            with open(self.filename, 'r') as f:
+                data = json.load(f)
+        except FileNotFoundError:
+            pass
+        self.revisados = data
+
+    def save_revisados(self):
+        data = []
+        try:
+            with open(self.filename, 'r') as f:
+                data = json.load(f)
+        except FileNotFoundError:
+            pass
+        for uid in self.revisados:
+            if uid not in data:
+                data.append(uid)
+        with open(self.filename, 'w') as f:
+            json.dump(data, f)
+
+    def add_revisado(self, uid):
+        if self.is_revisado(uid):
+            return
+        self.revisados.append(uid)
+
+    def build_message(self, email_part):
+        output = []
+        if email_part.is_multipart():
+            for part in email_part.get_payload():
+                output += self.build_message(part)
+        else:
+            html = email_part.get_payload(decode=True).decode('utf-8')
+            bs = BeautifulSoup(html, 'html.parser')
+            if bs.body:
+                html = bs.body.get_text()
+            else:
+                html = bs.get_text()
+            html = re.sub(' +', ' ', re.sub("\n+", ' ', html)).strip(' ')
+            output.append(html)
+        return output
+
+    def run(self) -> None:
+        self.start_turn()
+        while not self.stop.is_set():
+            e = 0
+            with connect(self.url, self.port, self.user.name, self.user.password, self.ssl) as imap:
+                self.logger.log('Getting emails', type(self))
+                emails = check_inbox(imap)
+                if emails is None:
+                    self.logger.log('No emails found', type(self))
+                    continue
+                for em in emails:
+                    if self.is_revisado(em.uid):
+                        continue
+                    sender = em.message['from']
+                    # text = ' '.join([em.message['subject'] + '.'] + self.build_message(em.message))
+                    text = self.build_message(em.message)
+                    msg = Message('email', text=text, original=em, sender=sender, subject=str(em.message['subject']),
+                                  datetime=email.utils.parsedate_to_datetime(em.message['Date']))
+                    self.queue.put(msg)
+                    self.add_revisado(em.uid)
+                    e += 1
+            self.logger.log('{0} new emails found'.format(e), type(self))
+            self.diary.put({'message': 'Obtenidos {0} correos nuevos'.format(e)})
+            time.sleep(self.frec)
+        # self.save_revisados()
+        self.end_turn()
+
+
+class Validador(Worker):
+    """
+    Trabajador que valida segun las reglas establecidas
+    Reglas:
+    1. Listado de jefes, con sus correos en la libreta de contactos -> validos
+    2. Instrucciones conocidas -> invalidos, pero para revisar
+    3. Listado de spam -> borrar
+    """
+    def __init__(self, configs, params):
+        super(Validador, self).__init__(configs=configs, params=params)
+
+        self.name = 'Email:Validador'
+        self.emails = params['queues']['emails']
+        self.validos = params['queues']['valid']
+        self.invalidos = params['queues']['invalid']
+        self.borrar = params['queues']['borrar']
+        self.bosses = params['bosses']
+        self.instrucciones = params['instrucciones']
+
+        self.frec = configs.get('supervisor.wait')
+
+    def validar_bosses(self, sender):
+        return self.bosses.is_boss(sender)
+
+    def validar_instrucciones(self, message):
+        return self.instrucciones.is_valid(message.subject)
+
+    def validar(self, message):
+        if self.validar_bosses(message.sender):
+            self.validos.put(message)
+            return
+        if self.validar_instrucciones(message):
+            self.invalidos.put(message)
+            return
+        self.borrar.put(message.original.uid)
+
+    def run(self):
+        self.start_turn()
+        while not self.stop.is_set():
+            try:
+                em = self.emails.get(timeout=self.frec)
+            except queue.Empty:
+                continue
+
+            self.validar(em)
+        # Cleanup
+        [self.validar(em) for em in dump_queue(self.emails, self.frec)]
+        self.end_turn()
+
+
+class Consultador(Worker):
+    """
+    Trabajador que registra los correos que no son de jefes para consulta
+    """
+    def __init__(self, configs, params):
+        super(Consultador, self).__init__(configs=configs, params=params)
+
+        self.name = 'Email:Consultador'
+        self.filename = params['filenames']['consultas']
+        self.invalidos = params['queues']['invalid']
+        self.frec = configs.get('supervisor.wait')
+        self.max = configs.get('email.max')
+        self.mensajes = []
+
+    def is_full(self):
+        return len(self.mensajes) >= self.max
+
+    def save_messages(self):
+        data = []
+        try:
+            with open(self.filename, 'r') as f:
+                data = json.load(f)
+        except FileNotFoundError:
+            pass
+        for m in self.mensajes:
+            if m not in data:
+                data.append(m)
+        with open(self.filename, 'w') as f:
+            json.dump(data, f, indent=2)
+        self.mensajes = []
+
+    def parse_message(self, message):
+        msg = {
+            'sender': message.sender,
+            'uid': message.original.uid,
+            'time': message.datetime.strftime('%Y-%m-%d %H:%M:%S'),
+            'message': message.text
+        }
+        self.mensajes.append(msg)
+
+    def run(self) -> None:
+        self.start_turn()
+        while not self.stop.is_set():
+            try:
+                em = self.invalidos.get(timeout=self.frec)
+            except queue.Empty:
+                continue
+            self.parse_message(em)
+            if self.is_full():
+                self.save_messages()
+        [self.parse_message(message) for message in dump_queue(self.invalidos, self.frec)]
+        self.save_messages()
+        self.end_turn()
+
+
+class Borrador(Worker):
+    """
+    Trabajador que borra los correos marcados para borrar
+    """
+    def __init__(self, configs, params):
+        super(Borrador, self).__init__(configs=configs, params=params)
+
+        self.name = 'Email:Borrador'
+
+        self.queue = params['queues']['borrar']
+        self.frec = configs.get('supervisor.wait')
+        self.max = configs.get('email.max')
+
+        self.url = configs.get('email.imap.server')
+        self.port = configs.get('email.imap.port')
+        user = {'user': '', 'password': ''}
+        self.user = SimpleNamespace(**user)
+        self.user.name = configs.get('email.imap.user.name')
+        self.user.password = configs.get('email.imap.user.password')
+        self.ssl = configs.get('email.imap.ssl')
+
+        self.borrar = []
+
+    def is_full(self):
+        return len(self.borrar) >= self.max
+
+    def do_borrar(self):
+        with connect(self.url, self.port, self.user.name, self.user.password, self.ssl) as imap:
+            status, msg = imap.select('INBOX')
+            if status != 'OK':
+                return
+
+            for uid in self.borrar:
+                print('Borrar ', uid)
+                # status, ids = imap.uid('store', uid, '+FLAGS', b'\\Deleted')
+                # if status != 'OK':
+                #    continue
+
+            # imap.expunge()
+        self.borrar = []
+
+    def run(self) -> None:
+        self.start_turn()
+        while not self.stop.is_set():
+            try:
+                uid = self.queue.get(timeout=self.frec)
+            except queue.Empty:
+                continue
+            self.borrar.append(uid)
+            if self.is_full():
+                self.do_borrar()
+        # Cleanup
+        [self.borrar.append(uid) for uid in dump_queue(self.queue, self.frec)]
+        self.do_borrar()
+        self.end_turn()
+
+
+class Procesador(Worker):
+    """
+    Trabajador que revisa los correos validos y los procesa de acuerdo a las instrucciones
+    """
+    def __init__(self, configs, params):
+        super(Procesador, self).__init__(configs=configs, params=params)
+
+        self.name = 'Email:Procesador'
+        self.queue = params['queues']['valid']
+        self.frec = configs.get('supervisor.wait')
+        self.brain = params['brain']
+
+    def cleanup(self):
+        self.brain.save_model()
+
+    def run(self) -> None:
+        self.start_turn()
+        while not self.stop.is_set():
+            try:
+                em = self.queue.get(timeout=self.frec)
+            except queue.Empty:
+                continue
+            command = self.brain.get_command(em.subject)
+            pprint((em.subject, command))
+            for t in em.text:
+                contenido = self.brain.get_response(command, t)
+
+        self.cleanup()
+        self.end_turn()

src/functions.py

@@ -0,0 +1,21 @@
+import keyboard
+import queue as q
+
+
+def exit_thread(stop, logger):
+    logger.log('Starting exit thread', caller='exit_thread')
+    keyboard.wait('Esc')
+    logger.log('Escape pressed', caller='exit_thread')
+    stop.set()
+    logger.log('Exit signal sent', caller='exit_thread')
+
+
+def dump_queue(queue, wait):
+    data = []
+    while True:
+        try:
+            m = queue.get(timeout=wait)
+        except q.Empty:
+            break
+        data.append(m)
+    return data

src/instrucciones.py

@@ -0,0 +1,96 @@
+import os
+import json
+
+
+class Command:
+    def __init__(self):
+        self.command = ''
+
+
+class Commands:
+    def __init__(self, data_folder):
+        self.filename = os.path.join(data_folder, 'commands.json')
+
+        self.commands = []
+        self.load_commands(self.filename)
+
+    def load_commands(self, filename):
+        data = []
+        if os.path.isfile(filename):
+            with open(filename, 'r') as f:
+                data = json.load(f)
+        for c in data:
+            cmd = Command()
+            cmd.command = c
+            self.commands.append(cmd)
+        
+    def get(self, command):
+        for i, c in enumerate(self.commands):
+            if command == c.command:
+                return i
+        return None
+            
+    def find(self, command):
+        return self.commands[self.get(command=command)]
+
+
+class Instruccion:
+    def __init__(self):
+        self.instruccion = ''
+        self.command = None
+        self.params = {}
+
+
+class Instrucciones:
+    def __init__(self, data_folder):
+        self.filename = os.path.join(data_folder, 'instrucciones.json')
+        self.commands = Commands(data_folder)
+
+        self.instrucciones = []
+        self.load_instrucciones(self.filename)
+        self.idx = 0
+
+    def load_instrucciones(self, filename):
+        data = []
+        if os.path.isfile(filename):
+            with open(filename, 'r') as f:
+                data = json.load(f)
+        for d in data:
+            i = Instruccion()
+            i.instruccion = d['name']
+            if 'params' in d:
+                for param, val in d['params'].items():
+                    i.params[param] = val
+            if 'command' in d:
+                i.command = self.commands.find(d['command'])
+            self.instrucciones.append(i)
+
+    def get(self, instruccion):
+        if not self.is_valid(instruccion):
+            return None
+        for i, ins in enumerate(self.instrucciones):
+            if instruccion == ins.instruccion:
+                return i
+
+    def find(self, instruccion):
+        if not self.is_valid(instruccion):
+            return None
+        return self.instrucciones[self.get(instruccion)]
+
+    def is_valid(self, instruccion):
+        for i in self.instrucciones:
+            if instruccion == i.instruccion:
+                return True
+        return False
+
+    def add(self, instruccion, aliases: list = None):
+        if self.is_valid(instruccion):
+            return
+        ins = Instruccion()
+        ins.instruccion = instruccion
+        self.instrucciones.append(ins)
+
+    def save(self):
+        data = [{'instruccion': i.instruccion, 'params': i.params.items()} for i in self.instrucciones]
+        with open(self.filename, 'w') as f:
+            json.dump(data, f, indent=4)

src/supervisor.py

@@ -0,0 +1,93 @@
+from threading import Thread, Event, Lock
+from queue import Queue
+import keyboard
+import time
+import importlib
+
+
+class Supervisor(Thread):
+    def __init__(self, configs, params):
+        super().__init__()
+        self._configs = configs
+        self.params = params
+        self._workers = []
+        self._status = []
+        self._registry = {}
+        self.add_event('stop')
+
+    def add_worker(self, worker):
+        self._workers.append(worker)
+
+    def start_workers(self):
+        [w.start() for w in self._workers]
+        [self._status.append(True) for w in self._workers]
+
+    def check_workers(self):
+        stopped = 0
+        for (k, w) in enumerate(self._workers):
+            if not self._status[k]:
+                stopped += 1
+                continue
+            if not w.is_alive():
+                self.params['logging'].log('Worker {0} stopped'.format(type(w)))
+                self.params['queues']['log'].put({'not': True, 'action': type(w)})
+                stopped += 1
+                self._status[k] = False
+        if stopped == len(self._workers):
+            return False
+        return True
+
+    def join_workers(self):
+        [w.join(self._configs.get('supervisor.wait')) for w in self._workers]
+
+    def add_queue(self, name):
+        if 'queues' not in self.params or self.params['queues'] is None:
+            self.params['queues'] = {}
+        self.params['queues'][name] = Queue()
+
+    def add_event(self, name):
+        if 'events' not in self.params or self.params['events'] is None:
+            self.params['events'] = {}
+        self.params['events'][name] = Event()
+
+    def add_lock(self, name):
+        if 'locks' not in self.params or self.params['locks'] is None:
+            self.params['locks'] = {}
+        self.params['locks'][name] = Lock()
+
+    def register_worker(self, module, name):
+        if module not in self._registry:
+            self._registry[module] = []
+        self._registry[module].append(name)
+
+    def run(self):
+        for (module_name, ws) in self._registry.items():
+            module = importlib.import_module(module_name)
+            for w in ws:
+                worker = getattr(module, w)
+                self.add_worker(worker(configs=self._configs, params=self.params))
+
+        self.add_worker(Thread(target=exit_thread, args=(self.params['events']['stop'], self.params['logging'])))
+
+        self.params['queues']['log'].put({'is_start': True})
+        self.start_workers()
+        while not self.params['events']['stop'].is_set():
+            self.params['logging'].log('Looping main status {0}'.format(not self.params['events']['stop'].is_set()),
+                                       caller=type(self))
+            if not self.check_workers():
+                break
+            self.params['logging'].log('Waiting {0} secs'.format(self._configs.get('supervisor.wait')),
+                                       caller=type(self))
+            time.sleep(self._configs.get('supervisor.wait'))
+        self.params['logging'].log('Exiting', caller=type(self))
+        self.join_workers()
+        self.params['logging'].log('Waiting for Workers', caller=type(self))
+        return
+
+
+def exit_thread(event, logging):
+    logging.log('Starting exit thread', caller='exit_thread')
+    keyboard.wait('Esc')
+    logging.log('Escape pressed', caller='exit_thread')
+    event.set()
+    logging.log('Exit signal sent', caller='exit_thread')

src/worker.py

@@ -0,0 +1,17 @@
+from threading import Thread
+
+
+class Worker(Thread):
+    def __init__(self, configs, params):
+        super(Worker, self).__init__()
+        self.stop = params['events']['stop']
+        self.diary = params['queues']['log']
+        self.logger = params['logging']
+
+    def start_turn(self):
+        self.diary.put({'start_turn': self.name})
+        self.logger.log('Starting', type(self))
+
+    def end_turn(self):
+        self.diary.put({'end_turn': self.name})
+        self.logger.log('Exiting', type(self))