updated german dictionary

data/better_grid_building2.ipynb

@@ -1188,24 +1188,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 30,
    "source": [
-    "get_database(\"de\")['sol']"
+    "get_database(\"de\")['ore']"
    ],
    "outputs": [
     {
      "output_type": "execute_result",
      "data": {
       "text/plain": [
-       "{'word': 'Sol',\n",
+       "{'word': 'Ore',\n",
-       " 'senses': ['Chemie: eine kolloidale Suspension von festen Partikeln oder Makromolekülen in einer flüssigen Phase'],\n",
+       " 'senses': ['Münze, die in Dänemark, Norwegen und Schweden verwendet wird'],\n",
        " 'synonyms': [],\n",
-       " 'antonyms': ['Gel'],\n",
+       " 'antonyms': [],\n",
-       " 'num_translations': 6}"
+       " 'num_translations': 10}"
       ]
      },
      "metadata": {},
-     "execution_count": 9
+     "execution_count": 30
     }
    ],
    "metadata": {}

data/data_tests_de.ipynb

@@ -3,8 +3,6 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "import json\n",
     "import re\n",
@@ -19,13 +17,13 @@
     "import wikitextparser as wtp\n",
     "import numpy as np\n",
     "import unidecode"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "def extract_data(wikitext: str, title: str):\n",
     "\n",
@@ -113,23 +111,13 @@
     "\n",
     "    \n",
     "    return data\n"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": 3,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stderr",
-     "text": [
-      "989351it [04:32, 3636.28it/s]\n"
-     ]
-    }
-   ],
    "source": [
     "bzfile_path = \"dewiktionary-20210201-pages-articles.xml.bz2\"\n",
     "bz = BZ2File(bzfile_path)\n",
@@ -144,6 +132,9 @@
     "    word = word.replace(\"ä\", \"ae\")\n",
     "    word = word.replace(\"ü\", \"ue\")\n",
     "    word = word.replace(\"ö\", \"oe\")\n",
+    "    word = word.replace(\"Ä\", \"Ae\")\n",
+    "    word = word.replace(\"Ü\", \"Ue\")\n",
+    "    word = word.replace(\"Ö\", \"Oe\")\n",
     "    word = word.replace(\"ß\", \"ss\")\n",
     "\n",
     "    \n",
@@ -163,22 +154,30 @@
     "\n",
     "    i += 1\n",
     "\n"
-   ]
+   ],
+   "outputs": [
+    {
+     "output_type": "stream",
+     "name": "stderr",
+     "text": [
+      "989351it [07:14, 2274.95it/s]\n"
+     ]
+    }
+   ],
+   "metadata": {
+    "tags": []
+   }
   },
   {
+   "cell_type": "markdown",
    "source": [
     "## filter database from links and only keep \"relevant\" items"
    ],
-   "cell_type": "markdown",
    "metadata": {}
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
    "source": [
     "def clean_text(s:str):\n",
     "\n",
@@ -234,14 +233,133 @@
     "\n",
     "    return text\n",
     "\n"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {
+    "tags": []
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "source": [
+    "def word_wise_filtering(sentence: str):\n",
+    "    words = sentence.split()\n",
+    "    for i, word in enumerate(words.copy()):\n",
+    "        if \"|\" in word:\n",
+    "            splitted = word.split(\"|\")\n",
+    "            if len(splitted) == 2:\n",
+    "                words[i] = splitted[1]\n",
+    "    return \" \".join(words)\n"
+   ],
+   "outputs": [],
+   "metadata": {}
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "source": [
+    "parse(\"{}[{:d}]{}\", \"[12]\") is None"
+   ],
+   "outputs": [
+    {
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "metadata": {},
+     "execution_count": 23
+    }
+   ],
+   "metadata": {}
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "source": [
+    "word_wise_filtering(\"hallo welten|welt|df\")"
+   ],
+   "outputs": [
+    {
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": [
+       "'hallo welten|welt|df'"
+      ]
+     },
+     "metadata": {},
+     "execution_count": 14
+    }
+   ],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": 7,
-   "metadata": {
+   "source": [
-    "tags": []
+    "filtered_db = {}\n",
-   },
+    "\n",
+    "for key, item in tqdm.tqdm(db.items()):\n",
+    "    senses = item['senses']\n",
+    "    synonyms = item['synonyms']\n",
+    "    antonyms = item['antonyms']\n",
+    "\n",
+    "    def filter_entries(entries: list):\n",
+    "        new_list = []\n",
+    "        for entry in entries:\n",
+    "            try:\n",
+    "                cleaned_entry = clean_text(entry)\n",
+    "                # sort out bad hints:\n",
+    "                if \"{\" in cleaned_entry:\n",
+    "                    continue\n",
+    "                if item['word'].lower() in cleaned_entry.lower():\n",
+    "                    continue\n",
+    "                if \"familienname\" in cleaned_entry.lower():\n",
+    "                    continue\n",
+    "                if \"ortsteil von\" in cleaned_entry.lower():\n",
+    "                    continue\n",
+    "                if \"dorf in\" in cleaned_entry.lower():\n",
+    "                    continue\n",
+    "                if parse(\"{}[{:d}]{}\", cleaned_entry) is not None:\n",
+    "                    continue\n",
+    "                if entry.isupper(): # try to sort out initialisms\n",
+    "                    continue\n",
+    "                \n",
+    "                \n",
+    "                cleaned_entry = word_wise_filtering (cleaned_entry)\n",
+    "\n",
+    "                new_list.append(cleaned_entry)\n",
+    "            except:\n",
+    "                #print(\"cannot process item\", entry)\n",
+    "                # just skipping unprocessable items\n",
+    "                pass\n",
+    "\n",
+    "    \n",
+    "        return new_list\n",
+    "    \n",
+    "    item['senses'] = filter_entries(senses)\n",
+    "    item['synonyms'] = filter_entries(synonyms)\n",
+    "    item['antonyms'] = filter_entries(antonyms)\n",
+    "\n",
+    "    # clean key from special characters:\n",
+    "    unaccented_key = unidecode.unidecode(key)\n",
+    "    unaccented_word = unidecode.unidecode(item['word'])\n",
+    "\n",
+    "    def has_digit(s: str):\n",
+    "        for c in s:\n",
+    "            if c.isdigit():\n",
+    "                return True\n",
+    "        return False\n",
+    "\n",
+    "    n_hints = len(item['senses']) + len(item['synonyms']) + len(item['antonyms'])\n",
+    "\n",
+    "    if (n_hints == 1 and item['num_translations'] >= 6) or n_hints > 1:\n",
+    "        if not has_digit(item['word']):\n",
+    "            item['word'] = unaccented_word\n",
+    "            filtered_db[unaccented_key] = item\n"
+   ],
    "outputs": [
     {
      "output_type": "stream",
@@ -305,70 +423,16 @@
      ]
     }
    ],
-   "source": [
+   "metadata": {
-    "filtered_db = {}\n",
+    "tags": []
-    "\n",
+   }
-    "for key, item in tqdm.tqdm(db.items()):\n",
-    "    senses = item['senses']\n",
-    "    synonyms = item['synonyms']\n",
-    "    antonyms = item['antonyms']\n",
-    "\n",
-    "    def filter_entries(entries: list):\n",
-    "        new_list = []\n",
-    "        for entry in entries:\n",
-    "            try:\n",
-    "                cleaned_entry = clean_text(entry)\n",
-    "                # sort out bad hints:\n",
-    "                if \"{\" in cleaned_entry:\n",
-    "                    continue\n",
-    "                if item['word'].lower() in cleaned_entry.lower():\n",
-    "                    continue\n",
-    "                if \"familienname\" in cleaned_entry.lower():\n",
-    "                    continue\n",
-    "                if \"ortsteil von\" in cleaned_entry.lower():\n",
-    "                    continue\n",
-    "                if \"dorf in\" in cleaned_entry.lower():\n",
-    "                    continue\n",
-    "                if entry.isupper(): # try to sort out initialisms\n",
-    "                    continue\n",
-    "                \n",
-    "                cleaned_entry = cleaned_entry.replace(\"|\",\"/\")\n",
-    "\n",
-    "                new_list.append(cleaned_entry)\n",
-    "            except:\n",
-    "                #print(\"cannot process item\", entry)\n",
-    "                # just skipping unprocessable items\n",
-    "                pass\n",
-    "\n",
-    "    \n",
-    "        return new_list\n",
-    "    \n",
-    "    item['senses'] = filter_entries(senses)\n",
-    "    item['synonyms'] = filter_entries(synonyms)\n",
-    "    item['antonyms'] = filter_entries(antonyms)\n",
-    "\n",
-    "    # clean key from special characters:\n",
-    "    unaccented_key = unidecode.unidecode(key)\n",
-    "    unaccented_word = unidecode.unidecode(item['word'])\n",
-    "\n",
-    "    def has_digit(s: str):\n",
-    "        for c in s:\n",
-    "            if c.isdigit():\n",
-    "                return True\n",
-    "        return False\n",
-    "\n",
-    "    n_hints = len(item['senses']) + len(item['synonyms']) + len(item['antonyms'])\n",
-    "\n",
-    "    if (n_hints == 1 and item['num_translations'] >= 6) or n_hints > 1:\n",
-    "        if not has_digit(item['word']):\n",
-    "            item['word'] = unaccented_word\n",
-    "            filtered_db[unaccented_key] = item\n"
-   ]
   },
   {
    "cell_type": "code",
    "execution_count": 13,
-   "metadata": {},
+   "source": [
+    "db['muenchen']"
+   ],
    "outputs": [
     {
      "output_type": "execute_result",
@@ -391,14 +455,14 @@
      "execution_count": 13
     }
    ],
-   "source": [
+   "metadata": {}
-    "db['muenchen']"
-   ]
   },
   {
    "cell_type": "code",
    "execution_count": 16,
-   "metadata": {},
+   "source": [
+    "len(filtered_db)"
+   ],
    "outputs": [
     {
      "output_type": "execute_result",
@@ -411,24 +475,26 @@
      "execution_count": 16
     }
    ],
-   "source": [
+   "metadata": {}
-    "len(filtered_db)"
-   ]
   },
   {
    "cell_type": "code",
    "execution_count": 18,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "with open('de.json', 'w') as f:\n",
     "    json.dump(filtered_db, f, indent = 4)"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": 25,
-   "metadata": {},
+   "source": [
+    "l = [1,2,3]\n",
+    "l.remove(4)\n",
+    "l"
+   ],
    "outputs": [
     {
      "output_type": "error",
@@ -442,18 +508,14 @@
      ]
     }
    ],
-   "source": [
+   "metadata": {}
-    "l = [1,2,3]\n",
-    "l.remove(4)\n",
-    "l"
-   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "source": [],
    "outputs": [],
-   "source": []
+   "metadata": {}
   }
  ],
  "metadata": {