updated german dictionary
This commit is contained in:
		| @ -1188,24 +1188,24 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "execution_count": 30, | ||||
|    "source": [ | ||||
|     "get_database(\"de\")['sol']" | ||||
|     "get_database(\"de\")['ore']" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "execute_result", | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "{'word': 'Sol',\n", | ||||
|        " 'senses': ['Chemie: eine kolloidale Suspension von festen Partikeln oder Makromolekülen in einer flüssigen Phase'],\n", | ||||
|        "{'word': 'Ore',\n", | ||||
|        " 'senses': ['Münze, die in Dänemark, Norwegen und Schweden verwendet wird'],\n", | ||||
|        " 'synonyms': [],\n", | ||||
|        " 'antonyms': ['Gel'],\n", | ||||
|        " 'num_translations': 6}" | ||||
|        " 'antonyms': [],\n", | ||||
|        " 'num_translations': 10}" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "execution_count": 9 | ||||
|      "execution_count": 30 | ||||
|     } | ||||
|    ], | ||||
|    "metadata": {} | ||||
|  | ||||
| @ -3,8 +3,6 @@ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import json\n", | ||||
|     "import re\n", | ||||
| @ -19,13 +17,13 @@ | ||||
|     "import wikitextparser as wtp\n", | ||||
|     "import numpy as np\n", | ||||
|     "import unidecode" | ||||
|    ] | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def extract_data(wikitext: str, title: str):\n", | ||||
|     "\n", | ||||
| @ -113,23 +111,13 @@ | ||||
|     "\n", | ||||
|     "    \n", | ||||
|     "    return data\n" | ||||
|    ] | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": { | ||||
|     "tags": [] | ||||
|    }, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "stream", | ||||
|      "name": "stderr", | ||||
|      "text": [ | ||||
|       "989351it [04:32, 3636.28it/s]\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "bzfile_path = \"dewiktionary-20210201-pages-articles.xml.bz2\"\n", | ||||
|     "bz = BZ2File(bzfile_path)\n", | ||||
| @ -144,6 +132,9 @@ | ||||
|     "    word = word.replace(\"ä\", \"ae\")\n", | ||||
|     "    word = word.replace(\"ü\", \"ue\")\n", | ||||
|     "    word = word.replace(\"ö\", \"oe\")\n", | ||||
|     "    word = word.replace(\"Ä\", \"Ae\")\n", | ||||
|     "    word = word.replace(\"Ü\", \"Ue\")\n", | ||||
|     "    word = word.replace(\"Ö\", \"Oe\")\n", | ||||
|     "    word = word.replace(\"ß\", \"ss\")\n", | ||||
|     "\n", | ||||
|     "    \n", | ||||
| @ -163,22 +154,30 @@ | ||||
|     "\n", | ||||
|     "    i += 1\n", | ||||
|     "\n" | ||||
|    ] | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "stream", | ||||
|      "name": "stderr", | ||||
|      "text": [ | ||||
|       "989351it [07:14, 2274.95it/s]\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "metadata": { | ||||
|     "tags": [] | ||||
|    } | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "source": [ | ||||
|     "## filter database from links and only keep \"relevant\" items" | ||||
|    ], | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "metadata": { | ||||
|     "tags": [] | ||||
|    }, | ||||
|    "outputs": [], | ||||
|    "execution_count": 4, | ||||
|    "source": [ | ||||
|     "def clean_text(s:str):\n", | ||||
|     "\n", | ||||
| @ -234,14 +233,133 @@ | ||||
|     "\n", | ||||
|     "    return text\n", | ||||
|     "\n" | ||||
|    ] | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "metadata": { | ||||
|     "tags": [] | ||||
|    } | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 12, | ||||
|    "source": [ | ||||
|     "def word_wise_filtering(sentence: str):\n", | ||||
|     "    words = sentence.split()\n", | ||||
|     "    for i, word in enumerate(words.copy()):\n", | ||||
|     "        if \"|\" in word:\n", | ||||
|     "            splitted = word.split(\"|\")\n", | ||||
|     "            if len(splitted) == 2:\n", | ||||
|     "                words[i] = splitted[1]\n", | ||||
|     "    return \" \".join(words)\n" | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 23, | ||||
|    "source": [ | ||||
|     "parse(\"{}[{:d}]{}\", \"[12]\") is None" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "execute_result", | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "True" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "execution_count": 23 | ||||
|     } | ||||
|    ], | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 14, | ||||
|    "source": [ | ||||
|     "word_wise_filtering(\"hallo welten|welt|df\")" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "execute_result", | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "'hallo welten|welt|df'" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "execution_count": 14 | ||||
|     } | ||||
|    ], | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "metadata": { | ||||
|     "tags": [] | ||||
|    }, | ||||
|    "source": [ | ||||
|     "filtered_db = {}\n", | ||||
|     "\n", | ||||
|     "for key, item in tqdm.tqdm(db.items()):\n", | ||||
|     "    senses = item['senses']\n", | ||||
|     "    synonyms = item['synonyms']\n", | ||||
|     "    antonyms = item['antonyms']\n", | ||||
|     "\n", | ||||
|     "    def filter_entries(entries: list):\n", | ||||
|     "        new_list = []\n", | ||||
|     "        for entry in entries:\n", | ||||
|     "            try:\n", | ||||
|     "                cleaned_entry = clean_text(entry)\n", | ||||
|     "                # sort out bad hints:\n", | ||||
|     "                if \"{\" in cleaned_entry:\n", | ||||
|     "                    continue\n", | ||||
|     "                if item['word'].lower() in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if \"familienname\" in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if \"ortsteil von\" in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if \"dorf in\" in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if parse(\"{}[{:d}]{}\", cleaned_entry) is not None:\n", | ||||
|     "                    continue\n", | ||||
|     "                if entry.isupper(): # try to sort out initialisms\n", | ||||
|     "                    continue\n", | ||||
|     "                \n", | ||||
|     "                \n", | ||||
|     "                cleaned_entry = word_wise_filtering (cleaned_entry)\n", | ||||
|     "\n", | ||||
|     "                new_list.append(cleaned_entry)\n", | ||||
|     "            except:\n", | ||||
|     "                #print(\"cannot process item\", entry)\n", | ||||
|     "                # just skipping unprocessable items\n", | ||||
|     "                pass\n", | ||||
|     "\n", | ||||
|     "    \n", | ||||
|     "        return new_list\n", | ||||
|     "    \n", | ||||
|     "    item['senses'] = filter_entries(senses)\n", | ||||
|     "    item['synonyms'] = filter_entries(synonyms)\n", | ||||
|     "    item['antonyms'] = filter_entries(antonyms)\n", | ||||
|     "\n", | ||||
|     "    # clean key from special characters:\n", | ||||
|     "    unaccented_key = unidecode.unidecode(key)\n", | ||||
|     "    unaccented_word = unidecode.unidecode(item['word'])\n", | ||||
|     "\n", | ||||
|     "    def has_digit(s: str):\n", | ||||
|     "        for c in s:\n", | ||||
|     "            if c.isdigit():\n", | ||||
|     "                return True\n", | ||||
|     "        return False\n", | ||||
|     "\n", | ||||
|     "    n_hints = len(item['senses']) + len(item['synonyms']) + len(item['antonyms'])\n", | ||||
|     "\n", | ||||
|     "    if (n_hints == 1 and item['num_translations'] >= 6) or n_hints > 1:\n", | ||||
|     "        if not has_digit(item['word']):\n", | ||||
|     "            item['word'] = unaccented_word\n", | ||||
|     "            filtered_db[unaccented_key] = item\n" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "stream", | ||||
| @ -305,70 +423,16 @@ | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "filtered_db = {}\n", | ||||
|     "\n", | ||||
|     "for key, item in tqdm.tqdm(db.items()):\n", | ||||
|     "    senses = item['senses']\n", | ||||
|     "    synonyms = item['synonyms']\n", | ||||
|     "    antonyms = item['antonyms']\n", | ||||
|     "\n", | ||||
|     "    def filter_entries(entries: list):\n", | ||||
|     "        new_list = []\n", | ||||
|     "        for entry in entries:\n", | ||||
|     "            try:\n", | ||||
|     "                cleaned_entry = clean_text(entry)\n", | ||||
|     "                # sort out bad hints:\n", | ||||
|     "                if \"{\" in cleaned_entry:\n", | ||||
|     "                    continue\n", | ||||
|     "                if item['word'].lower() in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if \"familienname\" in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if \"ortsteil von\" in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if \"dorf in\" in cleaned_entry.lower():\n", | ||||
|     "                    continue\n", | ||||
|     "                if entry.isupper(): # try to sort out initialisms\n", | ||||
|     "                    continue\n", | ||||
|     "                \n", | ||||
|     "                cleaned_entry = cleaned_entry.replace(\"|\",\"/\")\n", | ||||
|     "\n", | ||||
|     "                new_list.append(cleaned_entry)\n", | ||||
|     "            except:\n", | ||||
|     "                #print(\"cannot process item\", entry)\n", | ||||
|     "                # just skipping unprocessable items\n", | ||||
|     "                pass\n", | ||||
|     "\n", | ||||
|     "    \n", | ||||
|     "        return new_list\n", | ||||
|     "    \n", | ||||
|     "    item['senses'] = filter_entries(senses)\n", | ||||
|     "    item['synonyms'] = filter_entries(synonyms)\n", | ||||
|     "    item['antonyms'] = filter_entries(antonyms)\n", | ||||
|     "\n", | ||||
|     "    # clean key from special characters:\n", | ||||
|     "    unaccented_key = unidecode.unidecode(key)\n", | ||||
|     "    unaccented_word = unidecode.unidecode(item['word'])\n", | ||||
|     "\n", | ||||
|     "    def has_digit(s: str):\n", | ||||
|     "        for c in s:\n", | ||||
|     "            if c.isdigit():\n", | ||||
|     "                return True\n", | ||||
|     "        return False\n", | ||||
|     "\n", | ||||
|     "    n_hints = len(item['senses']) + len(item['synonyms']) + len(item['antonyms'])\n", | ||||
|     "\n", | ||||
|     "    if (n_hints == 1 and item['num_translations'] >= 6) or n_hints > 1:\n", | ||||
|     "        if not has_digit(item['word']):\n", | ||||
|     "            item['word'] = unaccented_word\n", | ||||
|     "            filtered_db[unaccented_key] = item\n" | ||||
|    ] | ||||
|    "metadata": { | ||||
|     "tags": [] | ||||
|    } | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 13, | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "db['muenchen']" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "execute_result", | ||||
| @ -391,14 +455,14 @@ | ||||
|      "execution_count": 13 | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "db['muenchen']" | ||||
|    ] | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 16, | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "len(filtered_db)" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "execute_result", | ||||
| @ -411,24 +475,26 @@ | ||||
|      "execution_count": 16 | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "len(filtered_db)" | ||||
|    ] | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 18, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "with open('de.json', 'w') as f:\n", | ||||
|     "    json.dump(filtered_db, f, indent = 4)" | ||||
|    ] | ||||
|    ], | ||||
|    "outputs": [], | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 25, | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "l = [1,2,3]\n", | ||||
|     "l.remove(4)\n", | ||||
|     "l" | ||||
|    ], | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "output_type": "error", | ||||
| @ -442,18 +508,14 @@ | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "l = [1,2,3]\n", | ||||
|     "l.remove(4)\n", | ||||
|     "l" | ||||
|    ] | ||||
|    "metadata": {} | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "source": [], | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|    "metadata": {} | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|  | ||||
							
								
								
									
										
											BIN
										
									
								
								data/de.json
									 (Stored with Git LFS)
									
									
									
									
								
							
							
						
						
									
										
											BIN
										
									
								
								data/de.json
									 (Stored with Git LFS)
									
									
									
									
								
							
										
											Binary file not shown.
										
									
								
							
		Reference in New Issue
	
	Block a user