clean_parallel.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import re
import argparse


# The variables below need to be adjusted for a language pair and dataset.
# To add a new language, define the list of alpha characters in the dict below.

MIN_LENGTH = 1      # minimum number of words in a sentence, should be > 0
MAX_LENGTH = 150    # maximum number of words in a sentence
RATIO_LENGTH = 0.5  # minimum length ratio of source/target and target/source

RATIO_ALPHA_WORDS = 0.4  # minimum fraction of "real" words in a source sentence
RATIO_ALPHA_CHARS = 0.5  # minimum fraction of alpha characters in a source sentence

CHARS = {
    'bg': r'[АаБбВвГгДддЕеЖжЗзИиЙйКкkasЛлМмНнОоПпРрСсТтУуФфХхЦцЧчШшЩщЪъЬьЮюЯя]',
    'cs': r'[a-zÁáČčĎďÉéěÍíŇňÓóŘřŠšŤťÚúůÝýŽž]',
    'ca': r'[a-zÀàÈèÉéÍíÒòÓóÚúÇç]',
    'da': r'[a-zÆæØøÅå]',
    'de': r'[a-zÄäÖöÜüß]',
    'en': r'[a-z]',
    'el': r'[a-zΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩω]',
    'es': r'[a-zÁáÉéÍíÓóÚúñÑ]',
    'et': r'[a-zÕõÄäÖöÜü]',
    'eu': r'[a-zñÑ]',
    'fi': r'[a-zÅåÄäÖö]',
    'fr': r'[a-zÂâÁáÀàâÇçÉéÈèÊêÓóÒòÔôŒœÜüÛûŸÿ]',
    'ga': r'[abcdefghilmnoprstuáéíóúÁÉÍÓÚ]',
    'gl': r'[a-zÁáÉéÍíÓóÚúÑñ]',
    'hr': r'[abcčČćĆdđĐefghijklmnoprsšŠtuvzžŽ]',
    'hu': r'[a-zÁáÉéÍíÓóÖöŐőŰű]',
    'is': r'[abdefghijklmnoprstuvxyÁáðÐÉéÍíÓóÚúÝýÞþÆæÖö]',
    'it': r'[a-zàÀèÈéÉìÌíÍîÎòÒóÓùÙúÚ]',
    'lt': r'[aąbcČčdeĘęĖėfghiĮįyjklmnoprsŠštuŲųŪūvzŽž]',
    'lv': r'[aĀābcČčdeĒēfgĢģhiĪījkĶķlĻļmnŅņoprsŠštuŪūvzŽž]',
    'mt': r'[abĊċdefĠġghĦħiiejklmnopqrstuvwxŻżz]',
    'nb': r'[a-zÂâÁáÀàâÉéÈèÊêÓóÒòÔôÜüÆæØøÅå]',
    'nl': r'[a-zÂâÁáÀàâÉéÈèÊêÓóÒòÔôÚú]',
    'no': r'[a-zÂâÁáÀàâÉéÈèÊêÓóÒòÔôÜüÆæØøÅå]',
    'nn': r'[a-zÂâÁáÀàâÉéÈèÊêÓóÒòÔôÜüÆæØøÅå]',
    'pl': r'[a-zĄąĆćĘꣳŃńÓ󌜏źŻż]',
    'pt': r'[a-zÂâÁáÀàÃãÇçÉéÈèÊêÍíÌìÓóÒòÔôÕõÚúÙù]',
    'ro': r'[a-zĂăÂâÎîȘșȚț]',
    'ru': r'[а-я]',
    'sk': r'[a-záäÁÄčČďĎžéÉíÍĺĹľĽňŇóÓôÔŕŔšŠťŤúÚýÝžŽ]',
    'sl': r'[abcčČdđĐefghijklmnoprsšŠtuvzžŽ]',
    'sv': r'[a-zÅåÄäÖö]',
}


def main():
    args = parse_user_args()

    for i, line in enumerate(sys.stdin):
        fields = line.strip().split('\t')
        if len(fields) < 2:
            continue

        src = fields[-2].strip()
        trg = fields[-1].strip()

        skip = clean_parallel(src, trg, args.src_lang, args.trg_lang)
        if skip:
            if args.debug:
                sys.stderr.write("{}\t{}".format(skip, line))
            continue
        sys.stdout.write(line)


def clean_parallel(src, trg, src_lang, trg_lang):
    if src.lower() == trg.lower():
        return "IDENTICAL"

    src_toks = src.split()
    trg_toks = trg.split()
    src_len = len(src_toks)
    trg_len = len(trg_toks)

    if not src_len or not trg_len:
        return "EMPTY"
    
    # https://stackoverflow.com/questions/23680976/python-removing-non-latin-characters
    #if re.search(u'[^\x00-\x7F\x80-\xFF\u0100-\u017F\u0180-\u024F\u1E00-\u1EFF]', src):
    #    return "SRC_NON_LATIN"
    
    #if re.search(u'[^\x00-\x7F\x80-\xFF\u0100-\u017F\u0180-\u024F\u1E00-\u1EFF]', trg):
    #    return "TRG_NON_LATIN"
    
    ratio_len = src_len / float(trg_len)
    if ratio_len < RATIO_LENGTH or ratio_len > (1. / RATIO_LENGTH):
        return "RATIO_LENGTH"

    if src_len < MIN_LENGTH or trg_len < MIN_LENGTH:
        return "TOO_SHORT"

    if src_len > MAX_LENGTH or trg_len > MAX_LENGTH:
        return "TOO_LONG"

    if src_lang in CHARS:
        num_alpha = sum(
            [1 if re.match(CHARS[src_lang], t, re.IGNORECASE) else 0 for t in src_toks])
        if num_alpha / float(src_len) < RATIO_ALPHA_WORDS:
            return "RATIO_ALPHA_SRC"

        char_alpha = len(re.findall(CHARS[src_lang], src, re.IGNORECASE))
        if char_alpha / float(len(src.replace(' ', ''))) < RATIO_ALPHA_CHARS:
            return "RATIO_CHARS_SRC"

    if trg_lang in CHARS:
        num_alpha = sum(
            [1 if re.match(CHARS[trg_lang], t, re.IGNORECASE) else 0 for t in trg_toks])
        if num_alpha / float(trg_len) < RATIO_ALPHA_WORDS:
            return "RATIO_ALPHA_TRG"

        char_alpha = len(re.findall(CHARS[trg_lang], trg, re.IGNORECASE))
        if char_alpha / float(len(trg.replace(' ', ''))) < RATIO_ALPHA_CHARS:
            return "RATIO_CHARS_TRG"

    return None


def parse_user_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("-l1", "--src-lang", default='es')
    parser.add_argument("-l2", "--trg-lang", default='en')
    parser.add_argument("--debug", action='store_true')
    return parser.parse_args()


if __name__ == "__main__":
    main()