changing de database again

This commit is contained in:
Jonas Weinz 2023-04-02 17:36:30 +02:00
parent 77e104bc19
commit ef8aeb13f9
2 changed files with 31 additions and 20 deletions

View File

@ -2,15 +2,15 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 22, "execution_count": 35,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"100%|██████████| 1161192/1161192 [00:02<00:00, 577463.34it/s]\n", "100%|██████████| 1161192/1161192 [00:01<00:00, 600059.17it/s]\n",
"117659it [00:04, 26815.49it/s]\n" "117659it [00:04, 27148.22it/s]\n"
] ]
} }
], ],
@ -20,6 +20,8 @@
"import nltk\n", "import nltk\n",
"from tqdm.auto import tqdm\n", "from tqdm.auto import tqdm\n",
"import json\n", "import json\n",
"import unidecode\n",
"\n",
"\n", "\n",
"# Create a frequency distribution of words in the Brown corpus\n", "# Create a frequency distribution of words in the Brown corpus\n",
"fd = nltk.FreqDist([word.lower() for word in tqdm(brown.words())])\n", "fd = nltk.FreqDist([word.lower() for word in tqdm(brown.words())])\n",
@ -42,12 +44,13 @@
" if ant_name != word and word not in ant_name:\n", " if ant_name != word and word not in ant_name:\n",
" antonyms.add(ant_name)\n", " antonyms.add(ant_name)\n",
" word_fixed = word.replace(\"-\",\"\").replace(\"_\",\"\").replace(\" \",\"\")\n", " word_fixed = word.replace(\"-\",\"\").replace(\"_\",\"\").replace(\" \",\"\")\n",
" word_fixed = unidecode.unidecode(word_fixed)\n",
" entry = {\n", " entry = {\n",
" \"word\": word_fixed,\n", " \"word\": word_fixed,\n",
" \"senses\": senses,\n", " \"senses\": senses,\n",
" \"synonyms\": list(synonyms),\n", " \"synonyms\": list(synonyms),\n",
" \"antonyms\": list(antonyms),\n", " \"antonyms\": list(antonyms),\n",
" \"frequency\": fd[word.lower()]\n", " \"frequency\": len(senses)\n",
" }\n", " }\n",
" dictionary[word_fixed] = entry\n", " dictionary[word_fixed] = entry\n",
"\n", "\n",
@ -56,6 +59,26 @@
" json.dump(dictionary, outfile, indent=4)\n" " json.dump(dictionary, outfile, indent=4)\n"
] ]
}, },
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['care must be exercised when this substance is to be deflagrated']"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"synset.examples()"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 21, "execution_count": 21,
@ -124,6 +147,8 @@
" lemma = lemma.replace(\"ö\", \"oe\")\n", " lemma = lemma.replace(\"ö\", \"oe\")\n",
" lemma = lemma.replace(\"ä\", \"ae\")\n", " lemma = lemma.replace(\"ä\", \"ae\")\n",
"\n", "\n",
" lemma = unidecode.unidecode(lemma)\n",
"\n",
" senses = [remove_umlaute(s) for s in senses]\n", " senses = [remove_umlaute(s) for s in senses]\n",
"\n", "\n",
" entry = {\n", " entry = {\n",
@ -191,13 +216,6 @@
"s = l.synsets()[0]" "s = l.synsets()[0]"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 113, "execution_count": 113,
@ -23456,13 +23474,6 @@
"source": [ "source": [
"dictionary" "dictionary"
] ]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {

BIN
server/de.json (Stored with Git LFS)

Binary file not shown.