From 8c4b167057bbdc0d9cf017bd14e98774870055d7 Mon Sep 17 00:00:00 2001
From: Jonas Weinz <s6jowein@uni-bonn.de>
Date: Tue, 22 May 2018 20:17:24 +0200
Subject: [PATCH] stream language detector

---
 Project/Tools/stream_language_detector.py | 19 +++++++++++++++++++
 Project/Tools/twitter2messages.sh         |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100755 Project/Tools/stream_language_detector.py

diff --git a/Project/Tools/stream_language_detector.py b/Project/Tools/stream_language_detector.py
new file mode 100755
index 0000000..23666d5
--- /dev/null
+++ b/Project/Tools/stream_language_detector.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from langdetect import detect
+import fileinput as fi
+import sys
+
+# just a little script to detect languages of lines starting 
+# with the json keyword 'text' and writing the language as json value to stdout.
+# other lines are just passing through so that this script can be used in a shell pipeline
+
+for line in fi.input():
+    s = str.split(line.lstrip(' '), ":")
+    if s[0] == '"text"':
+        try:
+            sys.stdout.write('  "lang": "' + detect(s[1]) + '"\n')
+        except Exception:
+            sys.stdout.write('  "lang": "NaN"\n')
+    sys.stdout.write(line)
+
+
diff --git a/Project/Tools/twitter2messages.sh b/Project/Tools/twitter2messages.sh
index 3d22b0e..bf80470 100755
--- a/Project/Tools/twitter2messages.sh
+++ b/Project/Tools/twitter2messages.sh
@@ -65,7 +65,7 @@ perform_and_exit echo "filter by emoji list:"
 perform_and_exit echo $elist | tr -d '"' | tr -d ','
 
 #perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
-perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4  | tee /dev/tty > "$OUTPUT"
+perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT"
 
 # ↑ such obvious, much selfexplaining 💁😈