custom json filter
This commit is contained in:
		
							
								
								
									
										140
									
								
								Project/Tools/json_stream_filter.py
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										140
									
								
								Project/Tools/json_stream_filter.py
									
									
									
									
									
										Executable file
									
								
							| @ -0,0 +1,140 @@ | |||||||
|  | #!/usr/bin/env python3 | ||||||
|  |  | ||||||
|  | import fileinput as fi | ||||||
|  | import json | ||||||
|  | import sys | ||||||
|  | import re | ||||||
|  |  | ||||||
|  | JSON_INDENT=2 | ||||||
|  |  | ||||||
|  | # a little helper tool for processing and manipulating json streams. | ||||||
|  | # just because jq is a little bit limited in its abilities to process json values. | ||||||
|  | # especially here to filter out emojis with regexp and put filtered elements into new key-value pairs | ||||||
|  |  | ||||||
|  | class regex_tuple(object): | ||||||
|  |     ''' | ||||||
|  |     a regex tuple object consists of: | ||||||
|  |       * regexp_str: regexp string | ||||||
|  |       * keys_to_process: keys which are processed | ||||||
|  |       * key_to_store_match | ||||||
|  |     ''' | ||||||
|  |     def __init__(self, regexp_str, keys_to_process, replace_str = None, key_to_store_match = None): | ||||||
|  |         self.regexp_str = regexp_str | ||||||
|  |         self.keys_to_process = keys_to_process | ||||||
|  |         self.key_to_store_match = key_to_store_match | ||||||
|  |         self.replace_str = replace_str | ||||||
|  |         self.regexp_obj = re.compile(self.regexp_str) | ||||||
|  |      | ||||||
|  |     @staticmethod | ||||||
|  |     def create_from_string(s): | ||||||
|  |         # delimiter is the first char: | ||||||
|  |         d = s[0] | ||||||
|  |         vals = s[1:].split(d) | ||||||
|  |         if len(vals) != 4: | ||||||
|  |             sys.stderr.write("Error creating regex object from string: " + s + "\n") | ||||||
|  |             return None | ||||||
|  |         reg_str = vals[0] | ||||||
|  |         reg_key = vals[1] | ||||||
|  |         reg_rep_str = vals[2] if vals[2] != "null" else None | ||||||
|  |         reg_store = vals[3] if vals[3] != "null" and len(vals[3]) > 0 else None | ||||||
|  |  | ||||||
|  |         return regex_tuple(reg_str,[reg_key],reg_rep_str, reg_store) | ||||||
|  |  | ||||||
|  | class json_streamer(object): | ||||||
|  |     def __init__(self, output_keys, regex_tuple_list, json_indent=JSON_INDENT): | ||||||
|  |         self.output_keys = output_keys | ||||||
|  |         self.regex_tuple_list = regex_tuple_list | ||||||
|  |         self.json_indent = json_indent | ||||||
|  |         self.regex_dict = None | ||||||
|  |         self.create_regex_dict() | ||||||
|  |      | ||||||
|  |     def create_regex_dict(self): | ||||||
|  |         d = {} | ||||||
|  |         for item in self.regex_tuple_list: | ||||||
|  |             for k in item.keys_to_process: | ||||||
|  |                 if k in d: | ||||||
|  |                     d[k].append(item) | ||||||
|  |                 else: | ||||||
|  |                     d[k] = [item] | ||||||
|  |         self.regex_dict = d | ||||||
|  |      | ||||||
|  |     def process_json_object(self, stream_dict): | ||||||
|  |         # for every registered key, look whether we have one in our stream object | ||||||
|  |         for key, r_list in self.regex_dict.items(): | ||||||
|  |             if key in stream_dict: | ||||||
|  |                 for r in r_list: | ||||||
|  |                     if r.key_to_store_match is not None: | ||||||
|  |                         # looking for all occurences and storing them in an on json key | ||||||
|  |                         matches = r.regexp_obj.findall(stream_dict[key]) | ||||||
|  |                         stream_dict[r.key_to_store_match] = matches # TODO: can not handle multiple rules storing in the same key!!! | ||||||
|  |                     if r.replace_str is not None: | ||||||
|  |                         # replacing all occurences | ||||||
|  |                         stream_dict[key] = r.regexp_obj.sub(r.replace_str, stream_dict[key]) | ||||||
|  |      | ||||||
|  |     def main_stream(self, stream_input = sys.stdin, stream_output = sys.stdout, stream_error = sys.stderr): | ||||||
|  |         processed_buffer= "" | ||||||
|  |         depth = 0               # bracket depth (in case we have to handle nested json objects) | ||||||
|  |         inside_quotes = False   # used for deteting whether we are inside a string in order to ignore brackets inside quotes | ||||||
|  |         line_counter = 0 | ||||||
|  |         current_batch_start_line = 0 | ||||||
|  |         success_batch_counter = 0 | ||||||
|  |         fail_batch_counter = 0 | ||||||
|  |  | ||||||
|  |         prev_c = '' | ||||||
|  |  | ||||||
|  |         for line in stream_input: | ||||||
|  |             line_counter+=1 | ||||||
|  |             for c in line: | ||||||
|  |                 if c == '{' and not inside_quotes: | ||||||
|  |                     if depth == 0: | ||||||
|  |                         current_batch_start_line = line_counter | ||||||
|  |                     depth += 1 | ||||||
|  |                     processed_buffer += c | ||||||
|  |                 elif c == '}' and depth > 0 and not inside_quotes: | ||||||
|  |                     depth -= 1 | ||||||
|  |                     processed_buffer += c | ||||||
|  |                     if depth == 0: | ||||||
|  |                         try: | ||||||
|  |                             d = json.loads(processed_buffer) | ||||||
|  |                             self.process_json_object(d) | ||||||
|  |                             stream_output.write(json.dumps(d, indent=self.json_indent, ensure_ascii=False)) | ||||||
|  |                             processed_buffer = "" | ||||||
|  |                             success_batch_counter += 1 | ||||||
|  |                         except json.decoder.JSONDecodeError: | ||||||
|  |                             stream_error.write("Error processing json object. Ignoring the following lines (starting at line "  | ||||||
|  |                                 + str(current_batch_start_line) + "):\n\n") | ||||||
|  |                             stream_error.write(processed_buffer + "\n\n") | ||||||
|  |                             processed_buffer = "" | ||||||
|  |                             fail_batch_counter += 1 | ||||||
|  |  | ||||||
|  |                 elif depth > 0: | ||||||
|  |                     processed_buffer += c | ||||||
|  |                 else: | ||||||
|  |                     stream_output.write(c) | ||||||
|  |  | ||||||
|  |                 # flipping quotes status (and don't forget to exlude escaped quotes!) | ||||||
|  |                 if c == '"' and prev_c != '\\': | ||||||
|  |                     inside_quotes = not inside_quotes | ||||||
|  |                  | ||||||
|  |                 # setting previous c. only exception if a double backslash occurs ignoring it. | ||||||
|  |                 # Because that breaks our escaped character detection | ||||||
|  |                 prev_c = c if not (c == '\\' and prev_c == '\\') else '' | ||||||
|  |          | ||||||
|  |         stream_error.write("\n\nReached EOF. #Processed objects: " + str(success_batch_counter) + ", #failed objects: " + str(fail_batch_counter) + "\n\n") | ||||||
|  |  | ||||||
|  | if __name__ == "__main__": | ||||||
|  |  | ||||||
|  |     args = sys.argv[1:] | ||||||
|  |  | ||||||
|  |     if len(args) == 0: | ||||||
|  |         print("missing arguments") | ||||||
|  |         sys.exit(-1) | ||||||
|  |      | ||||||
|  |     reg_tuples = [] | ||||||
|  |     while len(args) > 0: | ||||||
|  |         reg_tuples.append(regex_tuple.create_from_string(args[0])) | ||||||
|  |         args = args[1:] | ||||||
|  |      | ||||||
|  |     streamer = json_streamer(None, reg_tuples) | ||||||
|  |     streamer.main_stream() | ||||||
|  |  | ||||||
		Reference in New Issue
	
	Block a user