stream language detector
This commit is contained in:
		
							
								
								
									
										19
									
								
								Project/Tools/stream_language_detector.py
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										19
									
								
								Project/Tools/stream_language_detector.py
									
									
									
									
									
										Executable file
									
								
							| @ -0,0 +1,19 @@ | ||||
| #!/usr/bin/env python3 | ||||
| from langdetect import detect | ||||
| import fileinput as fi | ||||
| import sys | ||||
|  | ||||
| # just a little script to detect languages of lines starting  | ||||
| # with the json keyword 'text' and writing the language as json value to stdout. | ||||
| # other lines are just passing through so that this script can be used in a shell pipeline | ||||
|  | ||||
| for line in fi.input(): | ||||
|     s = str.split(line.lstrip(' '), ":") | ||||
|     if s[0] == '"text"': | ||||
|         try: | ||||
|             sys.stdout.write('  "lang": "' + detect(s[1]) + '"\n') | ||||
|         except Exception: | ||||
|             sys.stdout.write('  "lang": "NaN"\n') | ||||
|     sys.stdout.write(line) | ||||
|  | ||||
|  | ||||
| @ -65,7 +65,7 @@ perform_and_exit echo "filter by emoji list:" | ||||
| perform_and_exit echo $elist | tr -d '"' | tr -d ',' | ||||
|  | ||||
| #perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT" | ||||
| perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4  | tee /dev/tty > "$OUTPUT" | ||||
| perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 1 -B 4 | tee /dev/tty > "$OUTPUT" | ||||
|  | ||||
| # ↑ such obvious, much selfexplaining 💁😈 | ||||
|  | ||||
|  | ||||
		Reference in New Issue
	
	Block a user