jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added possibility to filter by emoji list

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-20 12:49:55 +02:00
parent 5a0a701e5c
commit b4eb220973
2 changed files with 31 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Project/simple_approach/Continous_Learner.ipynb
View File

@ -145,7 +145,7 @@
{ {
"data": { "data": {
"application/vnd.jupyter.widget-view+json": { "application/vnd.jupyter.widget-view+json": {
"model_id": "43f80c7d9c024a57b4009b77296f1ab0", "model_id": "31e69854333f4c599b037b6c27f30f20",
"version_major": 2, "version_major": 2,
"version_minor": 0 "version_minor": 0
}, },
@ -175,7 +175,8 @@
" [\n", " [\n",
" (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", " (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
" (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\"),\n", " (widgets.BoundedIntText(value=20,disabled=True,min=-1, max=100), \"n_top_emojis\"),\n",
" (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\")\n", " (widgets.Dropdown(options=[\"latest\", \"mean\"], value=\"latest\"), \"label_criteria\"),\n",
" (widgets.Text(value=\"\"), \"custom_emojis\")\n",
" ],\n", " ],\n",
" [\n", " [\n",
" (widgets.Button(disabled=True),\"load_data\")\n", " (widgets.Button(disabled=True),\"load_data\")\n",
@ -449,6 +450,8 @@
" \n", " \n",
" emoji_mean = shown_widgets[\"label_criteria\"].value == \"mean\"\n", " emoji_mean = shown_widgets[\"label_criteria\"].value == \"mean\"\n",
" \n", " \n",
" custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n",
" \n",
" sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", " sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n",
" n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", " n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n",
" file_range=range(r[0], r[1]),\n", " file_range=range(r[0], r[1]),\n",
@ -456,7 +459,8 @@
" read_progress_callback=p_r.update,\n", " read_progress_callback=p_r.update,\n",
" stem_progress_callback=p_s.update if lemm_and_stemm else None,\n", " stem_progress_callback=p_s.update if lemm_and_stemm else None,\n",
" apply_stemming = lemm_and_stemm,\n", " apply_stemming = lemm_and_stemm,\n",
" emoji_mean=emoji_mean)\n", " emoji_mean=emoji_mean,\n",
" custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None)\n",
" shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", " shown_widgets[\"batch_size\"].max = len(sdm.labels)\n",
" \n", " \n",
" \n", " \n",
@ -679,6 +683,15 @@
"shown_widgets[\"test_input\"].observe(test_input)" "shown_widgets[\"test_input\"].observe(test_input)"
] ]
}, },
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"sdm"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,

17
Project/simple_approach/simple_twitter_learning.py
View File

@ -164,7 +164,7 @@ def batch_lemm(sentences):
class sample_data_manager(object): class sample_data_manager(object):
@staticmethod @staticmethod
def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False): def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None):
""" """
generate, read and process train data in one step. generate, read and process train data in one step.
@ -184,7 +184,10 @@ class sample_data_manager(object):
sdm.generate_emoji_count_and_weights() sdm.generate_emoji_count_and_weights()
if n_top_emojis > 0: if custom_target_emojis is not None:
sdm.filter_by_emoji_list(custom_target_emojis)
elif n_top_emojis > 0:
sdm.filter_by_top_emojis(n_top=n_top_emojis) sdm.filter_by_top_emojis(n_top=n_top_emojis)
if n_kmeans_cluster > 0: if n_kmeans_cluster > 0:
@ -393,6 +396,16 @@ class sample_data_manager(object):
self.emojis = self.emojis[in_top] self.emojis = self.emojis[in_top]
print("remaining samples after top emoji filtering: ", len(self.labels)) print("remaining samples after top emoji filtering: ", len(self.labels))
def filter_by_emoji_list(self, custom_target_emojis):
assert self.labels is not None
in_list = [edist.sentiment_vector_to_emoji(x) in custom_target_emojis for x in self.labels]
self.labels = self.labels[in_list]
self.plain_text = self.plain_text[in_list]
self.emojis = self.emojis[in_list]
print("remaining samples after custom emoji filtering: ", len(self.labels))
def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5): def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5):
""" """
generate binary labels using kmeans. generate binary labels using kmeans.