From c71e761bab5ea6c17c257b9816f64ad127544d43 Mon Sep 17 00:00:00 2001 From: Jonas Weinz Date: Mon, 21 May 2018 23:42:55 +0200 Subject: [PATCH] faster twitter parsing --- Project/Tools/twitter2messages.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Project/Tools/twitter2messages.sh b/Project/Tools/twitter2messages.sh index 9bb20a3..d5538ef 100755 --- a/Project/Tools/twitter2messages.sh +++ b/Project/Tools/twitter2messages.sh @@ -64,7 +64,9 @@ perform_and_exit export elist=\"`head -c -1 "$SCRIPTPATH/emoji-list.txt" | tr '\ perform_and_exit echo "filter by emoji list:" perform_and_exit echo $elist | tr -d '"' | tr -d ',' -perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT" +#perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT" +perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT" + # ↑ such obvious, much selfexplaining 💁😈 confirm_action