diff --git a/Project/Tools/stream_language_detector.py b/Project/Tools/stream_language_detector.py new file mode 100755 index 0000000..23666d5 --- /dev/null +++ b/Project/Tools/stream_language_detector.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from langdetect import detect +import fileinput as fi +import sys + +# just a little script to detect languages of lines starting +# with the json keyword 'text' and writing the language as json value to stdout. +# other lines are just passing through so that this script can be used in a shell pipeline + +for line in fi.input(): + s = str.split(line.lstrip(' '), ":") + if s[0] == '"text"': + try: + sys.stdout.write(' "lang": "' + detect(s[1]) + '"\n') + except Exception: + sys.stdout.write(' "lang": "NaN"\n') + sys.stdout.write(line) + + diff --git a/Project/Tools/twitter2messages.sh b/Project/Tools/twitter2messages.sh index 3d22b0e..bf80470 100755 --- a/Project/Tools/twitter2messages.sh +++ b/Project/Tools/twitter2messages.sh @@ -65,7 +65,7 @@ perform_and_exit echo "filter by emoji list:" perform_and_exit echo $elist | tr -d '"' | tr -d ',' #perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT" -perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT" +perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT" # ↑ such obvious, much selfexplaining 💁😈