jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dump put emoji counts

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-19 16:46:19 +02:00
parent 8300ed4510
commit f4766912e1
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Project/simple_approach/simple_twitter_learning.py
View File

@ -24,6 +24,8 @@ import pickle
import operator import operator
from sklearn.pipeline import Pipeline from sklearn.pipeline import Pipeline
import json import json
import datetime
nltk.download('punkt') nltk.download('punkt')
nltk.download('averaged_perceptron_tagger') nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet') nltk.download('wordnet')
@ -227,6 +229,7 @@ class sample_data_manager(object):
""" """
assert np.min(file_index_range) >= 0 and np.max(file_index_range) < self.n_files assert np.min(file_index_range) >= 0 and np.max(file_index_range) < self.n_files
n = len(file_index_range) n = len(file_index_range)
for i in file_index_range: for i in file_index_range:
print("reading file: " + self.json_files[i] + "...") print("reading file: " + self.json_files[i] + "...")
raw_data_i = pd.read_json(self.json_files[i], encoding="utf-8") raw_data_i = pd.read_json(self.json_files[i], encoding="utf-8")
@ -259,7 +262,7 @@ class sample_data_manager(object):
tmp = [np.nanmean(emoji2sent(e, only_emoticons=only_emoticons), axis=0, dtype=float) for e in emojis_i] tmp = [np.nanmean(emoji2sent(e, only_emoticons=only_emoticons), axis=0, dtype=float) for e in emojis_i]
c = 0 c = 0
for t in tmp: for t in tmp:
# only to find and debug wrong formatted data
if str(type(t)) != "<class 'numpy.ndarray'>": if str(type(t)) != "<class 'numpy.ndarray'>":
print(t, type(t)) print(t, type(t))
print(emojis_i[c]) print(emojis_i[c])
@ -359,6 +362,12 @@ class sample_data_manager(object):
self.emoji_weights['X'] = 0 # dummy values self.emoji_weights['X'] = 0 # dummy values
self.emoji_count['X'] = 0 self.emoji_count['X'] = 0
# dump count data to json:
f = open("count_from_read_progress_" + str(datetime.datetime.now()) + ".json", 'w')
f.write(json.dumps(self.emoji_count, ensure_ascii=False))
f.close()
def get_emoji_count(self): def get_emoji_count(self):
""" """
@return: descending list of tuples in form (<emoji as character>, <emoji count>) @return: descending list of tuples in form (<emoji as character>, <emoji count>)
@ -371,7 +380,7 @@ class sample_data_manager(object):
def filter_by_top_emojis(self,n_top = 20): def filter_by_top_emojis(self,n_top = 20):
""" """
filgter out messages not containing one of the `n_top` emojis filter out messages not containing one of the `n_top` emojis
@param n_top: number of top emojis used for filtering @param n_top: number of top emojis used for filtering
""" """