From 81090e785388ec4bf4afa03b013096476e899210 Mon Sep 17 00:00:00 2001 From: Copetan <14883143+Copetan@users.noreply.github.com> Date: Tue, 4 Feb 2020 20:27:17 -0800 Subject: [PATCH] Adding a test i'm trying with pipes and stuff --- common/nlp.py | 17 ++ connectors/connector_common.py | 282 ++++++++++++++++----------------- 2 files changed, 158 insertions(+), 141 deletions(-) diff --git a/common/nlp.py b/common/nlp.py index e6625b6..ed877cf 100644 --- a/common/nlp.py +++ b/common/nlp.py @@ -30,7 +30,24 @@ def hashtag_pipe(doc): merged_hashtag = False return doc + def discord_emote_pipe(doc): + merged_emote = False + while True: + for token_index, token in enumerate(doc): + if token.text == "<": + if doc[token_index + 1].text == ":": + start_index = token.idx + end_index = start_index + len(doc[token_index + 1].text) + 1 + if doc.merge(start_index, end_index) is not None: + merged_emote = True + break + if not merged_emote: + break + merged_emote = False + return doc + nlp.add_pipe(hashtag_pipe) + nlp.add_pipe(discord_emote_pipe) return nlp diff --git a/connectors/connector_common.py b/connectors/connector_common.py index 9a9437a..85a718b 100644 --- a/connectors/connector_common.py +++ b/connectors/connector_common.py @@ -11,163 +11,163 @@ class ConnectorRecvMessage(object): - def __init__(self, text: str, learn: bool=False, reply=True): - self.text = text - self.learn = learn - self.reply = reply + def __init__(self, text: str, learn: bool = False, reply=True): + self.text = text + self.learn = learn + self.reply = reply class ConnectorReplyGenerator(object): - def __init__(self, markov_model: MarkovTrieDb, - structure_scheduler: StructureModelScheduler): - self._markov_model = markov_model - self._structure_scheduler = structure_scheduler - self._nlp = None - - def give_nlp(self, nlp): - self._nlp = nlp - - def generate(self, message: str, doc: Doc = None, ignore_topics: List[str] = []) -> Optional[str]: - - if doc is None: - filtered_message = MarkovFilters.filter_input(message) - doc = self._nlp(filtered_message) - - subjects = [] - for token in doc: - if token.text in ignore_topics: - continue - markov_word = self._markov_model.select(token.text) - if markov_word is not None: - subjects.append(markov_word) - if len(subjects) == 0: - return "I wasn't trained on that!" - - def structure_generator(): - sentence_stats_manager = InputTextStatManager() - while True: - choices, p_values = sentence_stats_manager.probabilities() - if len(choices) > 0: - num_sentences = np.random.choice(choices, p=p_values) - else: - num_sentences = np.random.randint(1, 5) - yield self._structure_scheduler.predict(num_sentences=num_sentences) - - generator = MarkovGenerator(structure_generator=structure_generator(), subjects=subjects) - - reply_words = [] - sentences = generator.generate(db=self._markov_model) - if sentences is None: - return "Huh?" - for sentence in sentences: - for word_idx, word in enumerate(sentence): - if not word.compound: - text = CapitalizationMode.transform(word.mode, word.text) - else: - text = word.text - reply_words.append(text) - - reply = " ".join(reply_words) - filtered_reply = MarkovFilters.smooth_output(reply) - - return filtered_reply + def __init__(self, markov_model: MarkovTrieDb, + structure_scheduler: StructureModelScheduler): + self._markov_model = markov_model + self._structure_scheduler = structure_scheduler + self._nlp = None + + def give_nlp(self, nlp): + self._nlp = nlp + + def generate(self, message: str, doc: Doc = None, ignore_topics: List[str] = []) -> Optional[str]: + + if doc is None: + filtered_message = MarkovFilters.filter_input(message) + doc = self._nlp(filtered_message) + + subjects = [] + for token in doc: + if token.text in ignore_topics: + continue + markov_word = self._markov_model.select(token.text) + if markov_word is not None: + subjects.append(markov_word) + if len(subjects) == 0: + return "I wasn't trained on that!" + + def structure_generator(): + sentence_stats_manager = InputTextStatManager() + while True: + choices, p_values = sentence_stats_manager.probabilities() + if len(choices) > 0: + num_sentences = np.random.choice(choices, p=p_values) + else: + num_sentences = np.random.randint(1, 5) + yield self._structure_scheduler.predict(num_sentences=num_sentences) + + generator = MarkovGenerator(structure_generator=structure_generator(), subjects=subjects) + + reply_words = [] + sentences = generator.generate(db=self._markov_model) + if sentences is None: + return "Huh?" + for sentence in sentences: + for word_idx, word in enumerate(sentence): + if not word.compound: + text = CapitalizationMode.transform(word.mode, word.text) + else: + text = word.text + reply_words.append(text) + + reply = " ".join(reply_words) + filtered_reply = MarkovFilters.smooth_output(reply) + + return filtered_reply class ConnectorWorker(Process): - def __init__(self, name, read_queue: Queue, write_queue: Queue, shutdown_event: Event): - Process.__init__(self, name=name) - self._read_queue = read_queue - self._write_queue = write_queue - self._shutdown_event = shutdown_event - self._frontend = None + def __init__(self, name, read_queue: Queue, write_queue: Queue, shutdown_event: Event): + Process.__init__(self, name=name) + self._read_queue = read_queue + self._write_queue = write_queue + self._shutdown_event = shutdown_event + self._frontend = None - def send(self, message: ConnectorRecvMessage): - return self._write_queue.put(message) + def send(self, message: ConnectorRecvMessage): + return self._write_queue.put(message) - def recv(self) -> Optional[str]: - return self._read_queue.get() + def recv(self) -> Optional[str]: + return self._read_queue.get() - def run(self): - pass + def run(self): + pass class ConnectorScheduler(object): - def __init__(self, shutdown_event: Event): - self._read_queue = Queue() - self._write_queue = Queue() - self._shutdown_event = shutdown_event - self._worker = None + def __init__(self, shutdown_event: Event): + self._read_queue = Queue() + self._write_queue = Queue() + self._shutdown_event = shutdown_event + self._worker = None - def recv(self, timeout: Optional[float]) -> Optional[ConnectorRecvMessage]: - try: - return self._read_queue.get(timeout=timeout) - except Empty: - return None + def recv(self, timeout: Optional[float]) -> Optional[ConnectorRecvMessage]: + try: + return self._read_queue.get(timeout=timeout) + except Empty: + return None - def send(self, message: str): - self._write_queue.put(message) + def send(self, message: str): + self._write_queue.put(message) - def start(self): - self._worker.start() + def start(self): + self._worker.start() - def shutdown(self): - self._worker.join() + def shutdown(self): + self._worker.join() class Connector(object): - def __init__(self, reply_generator: ConnectorReplyGenerator, connectors_event: Event): - self._reply_generator = reply_generator - self._scheduler = None - self._thread = Thread(target=self.run) - self._write_queue = Queue() - self._read_queue = Queue() - self._frontends_event = connectors_event - self._shutdown_event = Event() - self._muted = True - - def give_nlp(self, nlp): - self._reply_generator.give_nlp(nlp) - - def start(self): - self._scheduler.start() - self._thread.start() - - def run(self): - while not self._shutdown_event.is_set(): - message = self._scheduler.recv(timeout=0.2) - if self._muted: - self._scheduler.send(None) - elif message is not None: - # Receive the message and put it in a queue - self._read_queue.put(message) - # Notify main program to wakeup and check for messages - self._frontends_event.set() - # Send the reply - reply = self._write_queue.get() - self._scheduler.send(reply) - - def send(self, message: str): - self._write_queue.put(message) - - def recv(self) -> Optional[ConnectorRecvMessage]: - if not self._read_queue.empty(): - return self._read_queue.get() - return None - - def shutdown(self): - # Shutdown event signals both our thread and process to shutdown - self._shutdown_event.set() - self._scheduler.shutdown() - self._thread.join() - - def generate(self, message: str, doc: Doc=None) -> str: - return self._reply_generator.generate(message, doc) - - def mute(self): - self._muted = True - - def unmute(self): - self._muted = False - - def empty(self): - return self._read_queue.empty() + def __init__(self, reply_generator: ConnectorReplyGenerator, connectors_event: Event): + self._reply_generator = reply_generator + self._scheduler = None + self._thread = Thread(target=self.run) + self._write_queue = Queue() + self._read_queue = Queue() + self._frontends_event = connectors_event + self._shutdown_event = Event() + self._muted = True + + def give_nlp(self, nlp): + self._reply_generator.give_nlp(nlp) + + def start(self): + self._scheduler.start() + self._thread.start() + + def run(self): + while not self._shutdown_event.is_set(): + message = self._scheduler.recv(timeout=0.2) + if self._muted: + self._scheduler.send(None) + elif message is not None: + # Receive the message and put it in a queue + self._read_queue.put(message) + # Notify main program to wakeup and check for messages + self._frontends_event.set() + # Send the reply + reply = self._write_queue.get() + self._scheduler.send(reply) + + def send(self, message: str): + self._write_queue.put(message) + + def recv(self) -> Optional[ConnectorRecvMessage]: + if not self._read_queue.empty(): + return self._read_queue.get() + return None + + def shutdown(self): + # Shutdown event signals both our thread and process to shutdown + self._shutdown_event.set() + self._scheduler.shutdown() + self._thread.join() + + def generate(self, message: str, doc: Doc = None) -> str: + return self._reply_generator.generate(message, doc) + + def mute(self): + self._muted = True + + def unmute(self): + self._muted = False + + def empty(self): + return self._read_queue.empty()