Code 1

[stestagg]

import json
import operator
from collections import defaultdict


BIGRAMS_PER_LANG = defaultdict(lambda: defaultdict(int))

def parse_sample(output, text):
    # words = []
    for part in text.split():
        part = part.lower()
        if len(part) < 2:
            continue
        for first, second in zip(part, part[1:]):
             output[first+second] += 1


def load_dataset(filename):
    f = open(filename, "r")
    try:
        objects = []
        # text = f.readlines()
        for line in f:
            o = json.loads(line)
            yield o["lang"], o["text"]
    finally:
        f.close()

def get_top_bigrams(bigram_freqs):
    return [bi for bi, count in sorted(bigram_freqs.items(), key=operator.itemgetter(1), reverse=True)[0:5]]


def main():
    train = load_dataset("train_200.json")
    for lang, text in train:
        parse_sample(BIGRAMS_PER_LANG[lang], text)
    top_bigrams = {}
    for key, value in BIGRAMS_PER_LANG.items():
        top_bigrams[key] = get_top_bigrams(value)
    with open('output.json', 'w') as fp:
        json.dump(top_bigrams, fp, indent=2)


if __name__ == '__main__':
    main()
    # test_loader()