diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..35ad88d3d --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +# Python +__pycache__/ + +# Distribution / packaging +init +.Python +env/ +venv/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +.pytest_cache/ + +# Notebooks +.ipynb_checkpoints/ +*.ipynb + +# Data +*.csv +*.pickle +*.txt +*.xls +*.xlsx +*.tar.gz + +# Editors +.idea +.vscode + +# Files +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..cb71364b0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,11 @@ +Copyright 2021 Assistance Publique - Hôpitaux de Paris + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 000000000..94d2b96e1 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# NLPTools + +A simple library to group together the different pre-processing pipelines that are used at AP-HP, as Spacy component. diff --git a/nlptools/__init__.py b/nlptools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nlptools/rules/__init__.py b/nlptools/rules/__init__.py new file mode 100644 index 000000000..2a631d0d4 --- /dev/null +++ b/nlptools/rules/__init__.py @@ -0,0 +1,2 @@ +import nlptools.rules.sections +import nlptools.rules.pollution diff --git a/nlptools/rules/base.py b/nlptools/rules/base.py new file mode 100644 index 000000000..ca6fb7c01 --- /dev/null +++ b/nlptools/rules/base.py @@ -0,0 +1,95 @@ +from itertools import chain +from typing import List, Optional, Tuple + +from spacy.tokens import Doc, Span + +if not Doc.has_extension('note_id'): + Doc.set_extension('note_id', default=None) + + +class BaseComponent(object): + """ + Base component that contains the logic for : + - boundaries selections + - match filtering + """ + + split_on_punct = True + + @staticmethod + def _filter_matches(matches: List[Span]) -> List[Span]: + """ + Filter matches to remove duplicates and inclusions. + + Arguments + --------- + matches: List of matches (spans). + + Returns + ------- + filtered_matches: List of filtered matches. + """ + + filtered_matches = [] + + for span in matches: + + if not set(range(span.start, span.end)).intersection( + chain(*[set(range(s.start, s.end)) for s in filtered_matches]) + ): + filtered_matches.append(span) + + else: + s = set(range(span.start, span.end)) + + for match in filtered_matches[:]: + m = set(range(match.start, match.end)) + + if m & s: + tokens = sorted(list(s | m)) + start, end = tokens[0], tokens[-1] + + new_span = Span(span.doc, start, end + 1, label=span.label_) + + filtered_matches.remove(match) + filtered_matches.append(new_span) + break + + return filtered_matches + + def _boundaries(self, doc: Doc, terminations: Optional[List[Span]] = None) -> List[Tuple[int, int]]: + """ + Create sub sentences based sentences and terminations found in text. + + Parameters + ---------- + doc: spaCy Doc object + terminations: List of tuples with (match_id, start, end) + + Returns + ------- + boundaries: List of tuples with (start, end) of spans + """ + + if terminations is None: + terminations = [] + + sent_starts = [sent.start for sent in doc.sents] + termination_starts = [t.start for t in terminations] + + if self.split_on_punct: + punctuations = [t.i for t in doc if t.is_punct and '-' not in t.text] + else: + punctuations = [] + + starts = sent_starts + termination_starts + punctuations + [len(doc)] + + # Remove duplicates + starts = list(set(starts)) + + # Sort starts + starts.sort() + + boundaries = [(start, end) for start, end in zip(starts[:-1], starts[1:])] + + return boundaries diff --git a/nlptools/rules/pollution/__init__.py b/nlptools/rules/pollution/__init__.py new file mode 100644 index 000000000..d28a3ae82 --- /dev/null +++ b/nlptools/rules/pollution/__init__.py @@ -0,0 +1,2 @@ +from .pollution import Pollution +from .terms import pollutions diff --git a/nlptools/rules/pollution/pollution.py b/nlptools/rules/pollution/pollution.py new file mode 100644 index 000000000..598f9283f --- /dev/null +++ b/nlptools/rules/pollution/pollution.py @@ -0,0 +1,227 @@ +from typing import List, Tuple, Dict, Optional + +from spacy.language import Language +from spacy.tokens import Token, Span, Doc +from spaczz.matcher import RegexMatcher + +from nlptools.rules.base import BaseComponent +from nlptools.rules.pollution import terms + +import numpy as np + + +if not Doc.has_extension('note_id'): + Doc.set_extension('note_id', default=None) + + +def clean_getter(doc: Doc) -> List[Token]: + """ + Gets a list of tokens with pollution removed. + + Arguments + --------- + doc: Spacy Doc object. + + Returns + ------- + tokens: List of clean tokens. + """ + + tokens = [] + + for token in doc: + if not token._.pollution: + tokens.append(token) + + return tokens + + +def clean2original(doc: Doc) -> np.ndarray: + """ + Creates an alignment array to convert spans from the cleaned + textual representation to the original text object. + + Arguments + --------- + doc: Spacy Doc object. + + Returns + ------- + alignement: Alignment array. + """ + + lengths = np.array([len(token.text_with_ws) for token in doc]) + pollution = np.array([token._.pollution for token in doc]) + + current = 0 + + clean = [] + + for l, p in zip(lengths, pollution): + if not p: + current += l + clean.append(current) + clean = np.array(clean) + + alignment = np.stack([lengths.cumsum(), clean]) + + return alignment + + +def align(doc: Doc, index: int) -> int: + """ + Aligns a character found in the clean text with + its index in the original text. + + Arguments + --------- + doc: Spacy Doc object. + index: Character index in the clean text. + + Returns + ------- + index: Character index in the original text. + """ + + if index < 0: + index = len(doc._.clean_) - index + + alignment = clean2original(doc) + offset = alignment[0] - alignment[1] + + return index + offset[alignment[1] < index][-1] + +def char_clean2original( + doc: Doc, + start: int, + end: int, + alignment_mode: Optional[str] = 'strict', +) -> Span: + """ + Returns a Spacy Span object from character span computed on the clean text. + + Arguments + --------- + doc: Spacy Doc object + start: Character index of the beginning of the expression in the clean text. + end: Character index of the end of the expression in the clean text. + alignment_mode: Alignment mode. See https://spacy.io/api/doc#char_span. + + Returns + ------- + span: Span in the original text. + """ + + start, end = align(doc, start), align(doc, end) + return doc.char_span(start, end, alignment_mode=alignment_mode) + + +class Pollution(BaseComponent): + """ + Tags pollution tokens. + + Populates a number of Spacy extensions : + - `Token._.pollution` : indicates whether the token is a pollution + - `Doc._.clean` : lists non-pollution tokens + - `Doc._.clean_` : original text with pollutions removed. + - `Doc._.char_clean_span` : method to create a Span using character + indices extracted using the cleaned text. + """ + + def __init__( + self, + nlp: Language, + pollution: Dict[str, str], + ): + + self.nlp = nlp + + self.pollution = pollution + + if not Token.has_extension('pollution'): + Token.set_extension('pollution', default=False) + + if not Doc.has_extension('clean'): + Doc.set_extension('clean', getter=clean_getter) + + if not Doc.has_extension('clean_'): + Doc.set_extension('clean_', getter=lambda doc: ''.join([t.text_with_ws for t in doc._.clean])) + + if not Doc.has_extension('char_clean_span'): + Doc.set_extension('char_clean_span', method=char_clean2original) + + self.build_patterns() + + def build_patterns(self) -> None: + """ + Builds the patterns for phrase matching. + """ + + # efficiently build spaCy matcher patterns + self.matcher = RegexMatcher(self.nlp.vocab) + + for term in self.pollution.values(): + self.matcher.add("pollution", [term]) + + def process_pollutions(self, doc: Doc) -> Tuple[List[Span], List[Span], List[Span]]: + """ + Find pollutions in doc and clean candidate negations to remove pseudo negations + + Parameters + ---------- + doc: spaCy Doc object + + Returns + ------- + pollution: list of pollution spans + """ + + matches = self.matcher(doc) + pollutions = [ + Span(doc, start, end, label='neg_pseudo') + for match_id, start, end, ratio in matches + if match_id == "pollution" + ] + + pollutions = self._filter_matches(pollutions) + + return pollutions + + def __call__(self, doc: Doc) -> Doc: + """ + Tags pollutions. + + Parameters + ---------- + doc: spaCy Doc object + + Returns + ------- + doc: spaCy Doc object, annotated for negation + """ + pollutions = self.process_pollutions(doc) + + for pollution in pollutions: + + for token in pollution: + token._.pollution = True + + # for token in doc: + # if token.is_space: + # token._.pollution = True + + return doc + + +default_config = dict( + pollution=terms.pollution, +) + + +@Language.factory("pollution", default_config=default_config) +def create_pollution_component( + nlp: Language, + name: str, + pollution: Dict[str, str], +): + return Pollution(nlp, pollution=pollution) diff --git a/nlptools/rules/pollution/terms.py b/nlptools/rules/pollution/terms.py new file mode 100644 index 000000000..8f82641e5 --- /dev/null +++ b/nlptools/rules/pollution/terms.py @@ -0,0 +1,17 @@ +# Droits et information des patients +informations = ( + r"(?s)(=====+\s*)?(L\s*e\s*s\sdonnées\s*administratives,\s*sociales\s*|" + r"I?nfo\s*rmation\s*aux?\s*patients?|" + r"L’AP-HP\s*collecte\s*vos\s*données\s*administratives|" + r"L’Assistance\s*Publique\s*-\s*Hôpitaux\s*de\s*Paris\s*\(?AP-HP\)?\s*a\s*créé\s*une\s*base\s*de\s*données)" + r".{,2000}https?:\/\/recherche\.aphp\.fr\/eds\/droit-opposition[\s\.]*" +) + +# Exemple : NBNbWbWbNbWbNBNbNbWbWbNBNbWbNbNbWbNBNbW... +bars = r"(?i)\b([nbw]{5,}|[_\-]{5,})\b" + + +pollutions = dict( + informations=informations, + bars=bars, +) diff --git a/nlptools/rules/sections/__init__.py b/nlptools/rules/sections/__init__.py new file mode 100644 index 000000000..a7050814a --- /dev/null +++ b/nlptools/rules/sections/__init__.py @@ -0,0 +1 @@ +from .sections import Sections diff --git a/nlptools/rules/sections/sections.py b/nlptools/rules/sections/sections.py new file mode 100644 index 000000000..b1eacfca6 --- /dev/null +++ b/nlptools/rules/sections/sections.py @@ -0,0 +1,112 @@ +from itertools import chain +from typing import List, Tuple, Dict + +from spacy.language import Language +from spacy.matcher import PhraseMatcher +from spacy.tokens import Doc, Span + +from nlptools.rules.base import BaseComponent + + +if not Doc.has_extension('note_id'): + Doc.set_extension('note_id', default=None) + + +class Sections(BaseComponent): + """ + Divides the document into sections. + """ + + def __init__( + self, + nlp: Language, + sections: Dict[str, List[str]], + ): + + self.nlp = nlp + + self.sections = sections + + if not Doc.has_extension('sections'): + Doc.set_extension('sections', default=[]) + + self.build_patterns() + + def build_patterns(self) -> None: + # efficiently build spaCy matcher patterns + self.matcher = PhraseMatcher(self.nlp.vocab) + + for section, expressions in self.sections.items(): + patterns = list(self.nlp.tokenizer.pipe(expressions)) + self.matcher.add(section, None, *patterns) + + def process(self, doc: Doc) -> Tuple[List[Span], List[Span], List[Span]]: + """ + Find section references in doc and filter out duplicates and inclusions + + Parameters + ---------- + doc: spaCy Doc object + + Returns + ------- + sections: List of Spans referring to sections. + """ + matches = self.matcher(doc) + + sections = [ + Span(doc, start, end, label=self.nlp.vocab.strings[match_id]) + for match_id, start, end in matches + ] + + sections = self._filter_matches(sections) + + return sections + + def __call__(self, doc: Doc) -> Doc: + """ + Divides the doc into sections + + Parameters + ---------- + doc: spaCy Doc object + + Returns + ------- + doc: spaCy Doc object, annotated for sections + """ + sections = self.process(doc) + + ents = [] + + for s1, s2 in zip(sections[:-1], sections[1:]): + section = Span(doc, s1.start, s2.start, label=s1.label) + ents.append(section) + + if sections: + ents.append(Span(doc, sections[-1].start, len(doc), label=sections[-1].label)) + + doc._.sections = ents + + return doc + + +try: + # We use a list provided by PyMedExt : + # https://github.com/equipe22/pymedext_eds/blob/master/pymedext_eds/constants.py + from pymedext_eds.constants import SECTION_DICT as SECTIONS +except ImportError: + SECTIONS = dict() + +default_config = dict( + sections=SECTIONS, +) + + +@Language.factory("sections", default_config=default_config) +def create_negation_component( + nlp: Language, + name: str, + sections: Dict[str, List[str]], +): + return Sections(nlp, sections=sections) diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 000000000..56884cad5 --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,3 @@ +# Notebooks + +Check out the pipeline notebook to experiment with baseline components written in Spacy. diff --git a/notebooks/pipeline.md b/notebooks/pipeline.md new file mode 100644 index 000000000..82a2e055f --- /dev/null +++ b/notebooks/pipeline.md @@ -0,0 +1,75 @@ +--- +jupyter: + jupytext: + formats: ipynb,md + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.11.2 + kernelspec: + display_name: spacy + language: python + name: spacy +--- + +```python +%reload_ext autoreload +%autoreload 2 +``` + +```python +# Importation du "contexte", ie la bibliothèque sans installation +import context +``` + +```python +import spacy +``` + +```python +import negparhyp.baseline +``` + +# Baselines + +```python +nlp = spacy.blank('fr') +``` + +```python +nlp.add_pipe('sentencizer') +nlp.add_pipe('sections') +nlp.add_pipe('negation') +nlp.add_pipe('hypothesis') +nlp.add_pipe('context') +``` + +```python +text = "Le patient est admis pour des douleurs dans le bras droit, mais n'a pas de problème de locomotion. " \ + "Historique d'AVC dans la famille. pourrait être un cas de rhume." +``` + +```python +doc = nlp(text) +``` + +```python +print(f'{"Token":<16}{"Polarity":<12} {"Hypothesis":<12} {"Context":<5}') +print(f'{"-----":<16}{"--------":<12} {"----------":<12} {"-------":<5}') + +for token in doc: + print(f'{token.text:<16}{token._.polarity_:<12} {token._.hypothesis_:<12} {token._.context_:<5}') +``` + +```python +doc._.family +``` + +```python +doc._.negations +``` + +```python +doc._.hypothesis +``` diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..4b3c8cdd8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import context +import spacy +from pytest import fixture + + +@fixture(scope='session') +def nlp(): + model = spacy.blank('fr') + + model.add_pipe("sentencizer") + model.add_pipe('section') + model.add_pipe('pollution') + + return model diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 000000000..9855d5528 --- /dev/null +++ b/tests/context.py @@ -0,0 +1,5 @@ +import os +import sys + +REPO_PATH = os.path.abspath(os.path.join(os.path.dirname("__file__"), '..')) +sys.path.insert(0, REPO_PATH) diff --git a/tests/examples.py b/tests/examples.py new file mode 100644 index 000000000..def2ff8c6 --- /dev/null +++ b/tests/examples.py @@ -0,0 +1,108 @@ +context_examples = [ + { + 'text': "Le père du patient a eu un cancer du colon.", + 'entities': [ + { + 'start': 27, + 'end': 42, + 'context': 'FAMILY', + 'context_pymedext': 'family' + } + ] + }, + { + 'text': "Antécédents familiaux : diabète.", + 'entities': [ + { + 'start': 24, + 'end': 31, + 'context': 'FAMILY', + 'context_pymedext': 'family' + } + ] + }, + { + 'text': "Un relevé sanguin a été effectué.", + 'entities': [ + { + 'start': 3, + 'end': 17, + 'context': 'PATIENT', + 'context_pymedext': 'patient' + } + ] + } +] + +hypothesis_examples = [ + { + 'text': "Plusieurs diagnostics sont envisagés.", + 'entities': [ + { + 'start': 10, + 'end': 21, + 'hypothesis': 'HYP', + 'hypothesis_pymedext': 'hypothesis' + } + ] + }, + { + 'text': "Suspicion de diabète.", + 'entities': [ + { + 'start': 13, + 'end': 20, + 'hypothesis': 'HYP', + 'hypothesis_pymedext': 'hypothesis' + } + ] + }, + { + 'text': "Le ligament est rompu.", + 'entities': [ + { + 'start': 16, + 'end': 21, + 'hypothesis': 'CERT', + 'hypothesis_pymedext': 'certain' + } + ] + } +] + +negation_examples = [ + { + 'text': "Le patient n'est pas malade.", + 'entities': [ + { + 'start': 21, + 'end': 27, + 'polarity': 'NEG', + 'polarity_pymedext': 'neg' + } + ] + }, + { + 'text': "Aucun traitement.", + 'entities': [ + { + 'start': 6, + 'end': 16, + 'polarity': 'NEG', + 'polarity_pymedext': 'neg' + } + ] + }, + { + 'text': "Le scan révèle une grosseur.", + 'entities': [ + { + 'start': 8, + 'end': 14, + 'polarity': 'AFF', + 'polarity_pymedext': 'aff' + } + ] + } + +] diff --git a/tests/readme.md b/tests/readme.md new file mode 100644 index 000000000..ebb99af7f --- /dev/null +++ b/tests/readme.md @@ -0,0 +1,20 @@ +# Testing the algorithm + +Various tests for the components of the Spacy pipeline. + +We decided to design tests entity-wise, meaning that we only check the validity +of the computed modality on a set of entities. This design choice is motivated by +the fact that : + +1. That's what we actually care about. We want our pipeline to detect negation, + family context, patient history and hypothesis relative to a given entity. + +2. Deciding on the span of an annotation (negated, hypothesis, etc) is tricky. + Consider the example : `"Le patient n'est pas malade."`. Should the negated span + correspond to `["est", "malade"]`, `["malade"]`, `["n'", "est", "pas", "malade", "."]` ? + +3. Depending on the design of the algorithm, the span might be off, even though it + can correctly assign polarity to a given entity (but considered that the punctuation + was negated as well). + By relaxing the need to infer the correct span, we avoid giving an unfair disadvantage + to an otherwise great algorithm.