nlp-lab/Project/Tools/stream_language_detector.py

19 lines
573 B
Python
Raw Normal View History

2018-05-22 20:17:24 +02:00
#!/usr/bin/env python3
from langdetect import detect
import fileinput as fi
import sys
# just a little script to detect languages of lines starting
# with the json keyword 'text' and writing the language as json value to stdout.
# other lines are just passing through so that this script can be used in a shell pipeline
for line in fi.input():
2018-05-28 14:55:43 +02:00
if line.startswith(' "text"'):
2018-05-22 20:17:24 +02:00
try:
2018-05-28 14:55:43 +02:00
sys.stdout.write(' "lang": "' + detect(line[10:]) + '"\n')
2018-05-22 20:17:24 +02:00
except Exception:
sys.stdout.write(' "lang": "NaN"\n')
sys.stdout.write(line)