jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

faster twitter parsing

Browse Source
This commit is contained in:
Jonas Weinz 2018-05-21 23:42:55 +02:00
parent 1eb68b1bbc
commit c71e761bab
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Project/Tools/twitter2messages.sh
View File

@ -64,7 +64,9 @@ perform_and_exit export elist=\"`head -c -1 "$SCRIPTPATH/emoji-list.txt" | tr '\
perform_and_exit echo "filter by emoji list:"
perform_and_exit echo $elist | tr -d '"' | tr -d ','
perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
#perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT"
# ↑ such obvious, much selfexplaining 💁😈
confirm_action