master-thesis/json_buffered_reader.py

103 lines
2.8 KiB
Python
Raw Permalink Normal View History

2019-05-08 17:39:41 +02:00
#!/usr/bin/env python3
import json
class JSON_buffered_reader(object):
def __init__(self, filename: str, serialization_array_depth: int = 1):
self.filename = filename
self.n = 0
self.serialization_array_depth = serialization_array_depth
self.array_d = 0
self.object_d = 0
self.file_obj = None
self.buffer = r''
self.json_queue = []
self.eof = False
self._open()
def _open(self):
self.n = 0
self.array_d = 0
self.object_d = 0
self.file_obj = open(self.filename, 'r')
def _close(self):
self.file_obj.close()
def _process_next_line(self):
line = self.file_obj.readline()
if len(line) == 0:
self.eof = True
self._close()
return
2019-05-14 21:46:16 +02:00
prev_c = ''
in_quotes = False
2019-05-08 17:39:41 +02:00
for c in line:
2019-05-14 21:46:16 +02:00
if prev_c == '\\':
2019-05-08 17:39:41 +02:00
self.buffer += c
2019-05-14 21:46:16 +02:00
prev_c = '' if c == '\\' else c
2019-05-08 17:39:41 +02:00
continue
elif c == '\n':
continue
2019-05-14 21:46:16 +02:00
if not in_quotes:
if c == '[':
if self.array_d >= self.serialization_array_depth:
self.buffer += c
self.array_d += 1
elif c == ']':
if self.array_d >= self.serialization_array_depth:
self.buffer += c
self.array_d -= 1
elif c == '{':
self.object_d += 1
self.buffer += c
elif c == '}':
self.object_d -= 1
self.buffer += c
elif c == '"':
in_quotes = True
self.buffer += c
elif c == ',':
if not(self.array_d == self.serialization_array_depth and self.object_d == 0):
self.buffer += c
elif c == '\n' or c == ' ' or c == '\t':
continue
else:
self.buffer += c
2019-05-08 17:39:41 +02:00
else:
2019-05-14 21:46:16 +02:00
if c == '"':
in_quotes = False
2019-05-08 17:39:41 +02:00
self.buffer += c
assert self.object_d >= 0
assert self.array_d >= 0
2019-05-14 21:46:16 +02:00
prev_c = c
2019-05-08 17:39:41 +02:00
if self.object_d == 0:
2019-05-14 21:46:16 +02:00
if self.array_d == self.serialization_array_depth and len(self.buffer) > 0:
2019-05-08 17:39:41 +02:00
self.json_queue.append(self.buffer)
self.buffer = r''
def __iter__(self):
return self
def __next__(self):
while len(self.json_queue) == 0:
if self.eof:
2019-05-14 21:46:16 +02:00
raise StopIteration
2019-05-08 17:39:41 +02:00
self._process_next_line()
2019-05-14 21:46:16 +02:00
tmp = self.json_queue.pop(0)
return json.loads(tmp)