jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added ability to filter by sentence length

Browse Source
This commit is contained in:
Jonas Weinz 2018-07-21 10:32:34 +02:00
parent 1c8c15b0d5
commit 96e4606880
2 changed files with 46 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
Project/simple_approach/Continous_Learner.ipynb
View File

@ -11,7 +11,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -43,6 +43,19 @@
"[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n", "[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n",
"[nltk_data] Package wordnet is already up-to-date!\n" "[nltk_data] Package wordnet is already up-to-date!\n"
] ]
},
{
"ename": "NameError",
"evalue": "name 'min_words' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-2-ce00b6a80bda>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0msimple_twitter_learning\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_extraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCountVectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTfidfVectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHashingVectorizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Dokumente/gitRepos/NLP-LAB/Project/simple_approach/simple_twitter_learning.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 166\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0msample_data_manager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 167\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 168\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgenerate_and_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0monly_emoticons\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapply_stemming\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_top_emojis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_range\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_kmeans_cluster\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mread_progress_callback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstem_progress_callback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0memoji_mean\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_target_emojis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Dokumente/gitRepos/NLP-LAB/Project/simple_approach/simple_twitter_learning.py\u001b[0m in \u001b[0;36msample_data_manager\u001b[0;34m()\u001b[0m\n\u001b[1;32m 412\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"remaining samples after custom emoji filtering: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 413\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 414\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mfilter_by_sentence_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmin_words\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 415\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplain_text\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mNameError\u001b[0m: name 'min_words' is not defined"
]
} }
], ],
"source": [ "source": [
@ -69,7 +82,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -124,48 +137,9 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/markdown": [
"----"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## User Interface"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3e7d23dfb4b24f888d95bbd416565026",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Tab(children=(VBox(children=(HBox(children=(Text(value='./data_en/', description='root_path'), Button(descript…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"mp(\"----\")\n", "mp(\"----\")\n",
"mp(\"## User Interface\")\n", "mp(\"## User Interface\")\n",
@ -179,7 +153,8 @@
" [\n", " [\n",
" (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n", " (widgets.IntRangeSlider(disabled=True, min=0, max=0), \"file_range\"),\n",
" (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\"),\n", " (widgets.Checkbox(value=True,disabled=True), \"only_emoticons\"),\n",
" (widgets.Checkbox(value=False,disabled=True), \"apply_lemmatization_and_stemming\")\n", " (widgets.Checkbox(value=False,disabled=True), \"apply_lemmatization_and_stemming\"),\n",
" (widgets.BoundedIntText(value=5,min=0, max=10), \"min_words\")\n",
" ],\n", " ],\n",
" [\n", " [\n",
" (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n", " (widgets.BoundedIntText(value=-1,disabled=True,min=-1, max=10), \"k_means_cluster\"),\n",
@ -280,7 +255,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -298,7 +273,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -386,7 +361,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -410,7 +385,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -461,6 +436,8 @@
" \n", " \n",
" custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n", " custom_emojis = list(shown_widgets[\"custom_emojis\"].value)\n",
" \n", " \n",
" min_words = shown_widgets[\"min_words\"].value\n",
" \n",
" sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n", " sdm = stl.sample_data_manager.generate_and_read(path=shown_widgets[\"root_path\"].value,\n",
" n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n", " n_top_emojis=shown_widgets[\"n_top_emojis\"].value,\n",
" file_range=range(r[0], r[1]),\n", " file_range=range(r[0], r[1]),\n",
@ -469,7 +446,8 @@
" stem_progress_callback=p_s.update if lemm_and_stemm else None,\n", " stem_progress_callback=p_s.update if lemm_and_stemm else None,\n",
" apply_stemming = lemm_and_stemm,\n", " apply_stemming = lemm_and_stemm,\n",
" emoji_mean=emoji_mean,\n", " emoji_mean=emoji_mean,\n",
" custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None)\n", " custom_target_emojis=custom_emojis if len(custom_emojis) > 0 else None,\n",
" min_words=min_words)\n",
" shown_widgets[\"batch_size\"].max = len(sdm.labels)\n", " shown_widgets[\"batch_size\"].max = len(sdm.labels)\n",
" \n", " \n",
" \n", " \n",
@ -487,7 +465,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -531,7 +509,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -668,7 +646,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -692,45 +670,6 @@
"shown_widgets[\"test_input\"].observe(test_input)" "shown_widgets[\"test_input\"].observe(test_input)"
] ]
}, },
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"sdm"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'NoneType' object has no attribute 'pipeline'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-13-beaf1df9153b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_steps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'vectorizer'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"I am sad\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'pipeline'"
]
}
],
"source": [
"v = pm.pipeline.named_steps['vectorizer'].transform([\"I am sad\"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pm.pipeline.named_steps['keras_model'].predict([v])"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,

17
Project/simple_approach/simple_twitter_learning.py
View File

@ -165,7 +165,7 @@ def batch_lemm(sentences):
class sample_data_manager(object): class sample_data_manager(object):
@staticmethod @staticmethod
def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None): def generate_and_read(path:str, only_emoticons=True, apply_stemming=True, n_top_emojis=-1, file_range=None, n_kmeans_cluster=-1, read_progress_callback=None, stem_progress_callback=None, emoji_mean=False, custom_target_emojis = None, min_words=0):
""" """
generate, read and process train data in one step. generate, read and process train data in one step.
@ -194,6 +194,10 @@ class sample_data_manager(object):
if n_kmeans_cluster > 0: if n_kmeans_cluster > 0:
sdm.generate_kmeans_binary_label(only_emoticons=only_emoticons, n_clusters=n_kmeans_cluster) sdm.generate_kmeans_binary_label(only_emoticons=only_emoticons, n_clusters=n_kmeans_cluster)
if min_words > 0:
sdm.filter_by_sentence_length(min_words=min_words)
return sdm return sdm
@ -407,6 +411,17 @@ class sample_data_manager(object):
self.emojis = self.emojis[in_list] self.emojis = self.emojis[in_list]
print("remaining samples after custom emoji filtering: ", len(self.labels)) print("remaining samples after custom emoji filtering: ", len(self.labels))
def filter_by_sentence_length(self, min_words):
assert self.plain_text is not None
is_long = [True if len(x.split()) >= min_words else False for x in self.plain_text]
self.labels = self.labels[is_long]
self.plain_text = self.plain_text[is_long]
self.emojis = self.emojis[is_long]
print("remaining samples after sentence length filtering: ", len(self.labels))
def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5): def generate_kmeans_binary_label(self, only_emoticons=True, n_clusters=5):
""" """
generate binary labels using kmeans. generate binary labels using kmeans.