stream language detector

Project/Tools/stream_language_detector.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from langdetect import detect
+import fileinput as fi
+import sys
+
+# just a little script to detect languages of lines starting 
+# with the json keyword 'text' and writing the language as json value to stdout.
+# other lines are just passing through so that this script can be used in a shell pipeline
+
+for line in fi.input():
+    s = str.split(line.lstrip(' '), ":")
+    if s[0] == '"text"':
+        try:
+            sys.stdout.write('  "lang": "' + detect(s[1]) + '"\n')
+        except Exception:
+            sys.stdout.write('  "lang": "NaN"\n')
+    sys.stdout.write(line)
+
+

Project/Tools/twitter2messages.sh

@@ -65,7 +65,7 @@ perform_and_exit echo "filter by emoji list:"
 perform_and_exit echo $elist | tr -d '"' | tr -d ','
 
 #perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
-perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4  | tee /dev/tty > "$OUTPUT"
+perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT"
 
 # ↑ such obvious, much selfexplaining 💁😈