changing de database again

data/nltk_corpus.ipynb

@@ -2,15 +2,15 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 35,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 1161192/1161192 [00:02<00:00, 577463.34it/s]\n",
+      "100%|██████████| 1161192/1161192 [00:01<00:00, 600059.17it/s]\n",
-      "117659it [00:04, 26815.49it/s]\n"
+      "117659it [00:04, 27148.22it/s]\n"
      ]
     }
    ],
@@ -20,6 +20,8 @@
     "import nltk\n",
     "from tqdm.auto import tqdm\n",
     "import json\n",
+    "import unidecode\n",
+    "\n",
     "\n",
     "# Create a frequency distribution of words in the Brown corpus\n",
     "fd = nltk.FreqDist([word.lower() for word in tqdm(brown.words())])\n",
@@ -42,12 +44,13 @@
     "            if ant_name != word and word not in ant_name:\n",
     "                antonyms.add(ant_name)\n",
     "    word_fixed = word.replace(\"-\",\"\").replace(\"_\",\"\").replace(\" \",\"\")\n",
+    "    word_fixed = unidecode.unidecode(word_fixed)\n",
     "    entry = {\n",
     "        \"word\": word_fixed,\n",
     "        \"senses\": senses,\n",
     "        \"synonyms\": list(synonyms),\n",
     "        \"antonyms\": list(antonyms),\n",
-    "        \"frequency\": fd[word.lower()]\n",
+    "        \"frequency\": len(senses)\n",
     "    }\n",
     "    dictionary[word_fixed] = entry\n",
     "\n",
@@ -56,6 +59,26 @@
     "    json.dump(dictionary, outfile, indent=4)\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['care must be exercised when this substance is to be deflagrated']"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "synset.examples()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 21,
@@ -124,6 +147,8 @@
     "        lemma = lemma.replace(\"ö\", \"oe\")\n",
     "        lemma = lemma.replace(\"ä\", \"ae\")\n",
     "\n",
+    "        lemma = unidecode.unidecode(lemma)\n",
+    "\n",
     "        senses = [remove_umlaute(s) for s in senses]\n",
     "\n",
     "        entry = {\n",
@@ -191,13 +216,6 @@
     "s = l.synsets()[0]"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": 113,
@@ -23456,13 +23474,6 @@
    "source": [
     "dictionary"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {