updated german dictionary

This commit is contained in:
Jonas Weinz 2021-08-31 17:26:19 +02:00
parent ccefd63539
commit ff5cbafe2b
4 changed files with 180 additions and 118 deletions

View File

@ -1188,24 +1188,24 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 30,
"source": [
"get_database(\"de\")['sol']"
"get_database(\"de\")['ore']"
],
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'word': 'Sol',\n",
" 'senses': ['Chemie: eine kolloidale Suspension von festen Partikeln oder Makromolekülen in einer flüssigen Phase'],\n",
"{'word': 'Ore',\n",
" 'senses': ['Münze, die in Dänemark, Norwegen und Schweden verwendet wird'],\n",
" 'synonyms': [],\n",
" 'antonyms': ['Gel'],\n",
" 'num_translations': 6}"
" 'antonyms': [],\n",
" 'num_translations': 10}"
]
},
"metadata": {},
"execution_count": 9
"execution_count": 30
}
],
"metadata": {}

View File

@ -3,8 +3,6 @@
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import re\n",
@ -19,13 +17,13 @@
"import wikitextparser as wtp\n",
"import numpy as np\n",
"import unidecode"
]
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def extract_data(wikitext: str, title: str):\n",
"\n",
@ -113,23 +111,13 @@
"\n",
" \n",
" return data\n"
]
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"tags": []
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"989351it [04:32, 3636.28it/s]\n"
]
}
],
"source": [
"bzfile_path = \"dewiktionary-20210201-pages-articles.xml.bz2\"\n",
"bz = BZ2File(bzfile_path)\n",
@ -144,6 +132,9 @@
" word = word.replace(\"ä\", \"ae\")\n",
" word = word.replace(\"ü\", \"ue\")\n",
" word = word.replace(\"ö\", \"oe\")\n",
" word = word.replace(\"Ä\", \"Ae\")\n",
" word = word.replace(\"Ü\", \"Ue\")\n",
" word = word.replace(\"Ö\", \"Oe\")\n",
" word = word.replace(\"ß\", \"ss\")\n",
"\n",
" \n",
@ -163,22 +154,30 @@
"\n",
" i += 1\n",
"\n"
]
],
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"989351it [07:14, 2274.95it/s]\n"
]
}
],
"metadata": {
"tags": []
}
},
{
"cell_type": "markdown",
"source": [
"## filter database from links and only keep \"relevant\" items"
],
"cell_type": "markdown",
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"tags": []
},
"outputs": [],
"execution_count": 4,
"source": [
"def clean_text(s:str):\n",
"\n",
@ -234,14 +233,133 @@
"\n",
" return text\n",
"\n"
]
],
"outputs": [],
"metadata": {
"tags": []
}
},
{
"cell_type": "code",
"execution_count": 12,
"source": [
"def word_wise_filtering(sentence: str):\n",
" words = sentence.split()\n",
" for i, word in enumerate(words.copy()):\n",
" if \"|\" in word:\n",
" splitted = word.split(\"|\")\n",
" if len(splitted) == 2:\n",
" words[i] = splitted[1]\n",
" return \" \".join(words)\n"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 23,
"source": [
"parse(\"{}[{:d}]{}\", \"[12]\") is None"
],
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"True"
]
},
"metadata": {},
"execution_count": 23
}
],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 14,
"source": [
"word_wise_filtering(\"hallo welten|welt|df\")"
],
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'hallo welten|welt|df'"
]
},
"metadata": {},
"execution_count": 14
}
],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"tags": []
},
"source": [
"filtered_db = {}\n",
"\n",
"for key, item in tqdm.tqdm(db.items()):\n",
" senses = item['senses']\n",
" synonyms = item['synonyms']\n",
" antonyms = item['antonyms']\n",
"\n",
" def filter_entries(entries: list):\n",
" new_list = []\n",
" for entry in entries:\n",
" try:\n",
" cleaned_entry = clean_text(entry)\n",
" # sort out bad hints:\n",
" if \"{\" in cleaned_entry:\n",
" continue\n",
" if item['word'].lower() in cleaned_entry.lower():\n",
" continue\n",
" if \"familienname\" in cleaned_entry.lower():\n",
" continue\n",
" if \"ortsteil von\" in cleaned_entry.lower():\n",
" continue\n",
" if \"dorf in\" in cleaned_entry.lower():\n",
" continue\n",
" if parse(\"{}[{:d}]{}\", cleaned_entry) is not None:\n",
" continue\n",
" if entry.isupper(): # try to sort out initialisms\n",
" continue\n",
" \n",
" \n",
" cleaned_entry = word_wise_filtering (cleaned_entry)\n",
"\n",
" new_list.append(cleaned_entry)\n",
" except:\n",
" #print(\"cannot process item\", entry)\n",
" # just skipping unprocessable items\n",
" pass\n",
"\n",
" \n",
" return new_list\n",
" \n",
" item['senses'] = filter_entries(senses)\n",
" item['synonyms'] = filter_entries(synonyms)\n",
" item['antonyms'] = filter_entries(antonyms)\n",
"\n",
" # clean key from special characters:\n",
" unaccented_key = unidecode.unidecode(key)\n",
" unaccented_word = unidecode.unidecode(item['word'])\n",
"\n",
" def has_digit(s: str):\n",
" for c in s:\n",
" if c.isdigit():\n",
" return True\n",
" return False\n",
"\n",
" n_hints = len(item['senses']) + len(item['synonyms']) + len(item['antonyms'])\n",
"\n",
" if (n_hints == 1 and item['num_translations'] >= 6) or n_hints > 1:\n",
" if not has_digit(item['word']):\n",
" item['word'] = unaccented_word\n",
" filtered_db[unaccented_key] = item\n"
],
"outputs": [
{
"output_type": "stream",
@ -305,70 +423,16 @@
]
}
],
"source": [
"filtered_db = {}\n",
"\n",
"for key, item in tqdm.tqdm(db.items()):\n",
" senses = item['senses']\n",
" synonyms = item['synonyms']\n",
" antonyms = item['antonyms']\n",
"\n",
" def filter_entries(entries: list):\n",
" new_list = []\n",
" for entry in entries:\n",
" try:\n",
" cleaned_entry = clean_text(entry)\n",
" # sort out bad hints:\n",
" if \"{\" in cleaned_entry:\n",
" continue\n",
" if item['word'].lower() in cleaned_entry.lower():\n",
" continue\n",
" if \"familienname\" in cleaned_entry.lower():\n",
" continue\n",
" if \"ortsteil von\" in cleaned_entry.lower():\n",
" continue\n",
" if \"dorf in\" in cleaned_entry.lower():\n",
" continue\n",
" if entry.isupper(): # try to sort out initialisms\n",
" continue\n",
" \n",
" cleaned_entry = cleaned_entry.replace(\"|\",\"/\")\n",
"\n",
" new_list.append(cleaned_entry)\n",
" except:\n",
" #print(\"cannot process item\", entry)\n",
" # just skipping unprocessable items\n",
" pass\n",
"\n",
" \n",
" return new_list\n",
" \n",
" item['senses'] = filter_entries(senses)\n",
" item['synonyms'] = filter_entries(synonyms)\n",
" item['antonyms'] = filter_entries(antonyms)\n",
"\n",
" # clean key from special characters:\n",
" unaccented_key = unidecode.unidecode(key)\n",
" unaccented_word = unidecode.unidecode(item['word'])\n",
"\n",
" def has_digit(s: str):\n",
" for c in s:\n",
" if c.isdigit():\n",
" return True\n",
" return False\n",
"\n",
" n_hints = len(item['senses']) + len(item['synonyms']) + len(item['antonyms'])\n",
"\n",
" if (n_hints == 1 and item['num_translations'] >= 6) or n_hints > 1:\n",
" if not has_digit(item['word']):\n",
" item['word'] = unaccented_word\n",
" filtered_db[unaccented_key] = item\n"
]
"metadata": {
"tags": []
}
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"source": [
"db['muenchen']"
],
"outputs": [
{
"output_type": "execute_result",
@ -391,14 +455,14 @@
"execution_count": 13
}
],
"source": [
"db['muenchen']"
]
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"source": [
"len(filtered_db)"
],
"outputs": [
{
"output_type": "execute_result",
@ -411,24 +475,26 @@
"execution_count": 16
}
],
"source": [
"len(filtered_db)"
]
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"with open('de.json', 'w') as f:\n",
" json.dump(filtered_db, f, indent = 4)"
]
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"source": [
"l = [1,2,3]\n",
"l.remove(4)\n",
"l"
],
"outputs": [
{
"output_type": "error",
@ -442,18 +508,14 @@
]
}
],
"source": [
"l = [1,2,3]\n",
"l.remove(4)\n",
"l"
]
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"source": [],
"outputs": [],
"source": []
"metadata": {}
}
],
"metadata": {

BIN
data/de.json (Stored with Git LFS)

Binary file not shown.

BIN
server/de.json (Stored with Git LFS)

Binary file not shown.