From ef8aeb13f946bfd8b67d8d76dd6cf75131103ef8 Mon Sep 17 00:00:00 2001 From: Jonas Weinz Date: Sun, 2 Apr 2023 17:36:30 +0200 Subject: [PATCH] changing de database again --- data/nltk_corpus.ipynb | 47 ++++++++++++++++++++++++++---------------- server/de.json | 4 ++-- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/data/nltk_corpus.ipynb b/data/nltk_corpus.ipynb index b8a289c..205d325 100644 --- a/data/nltk_corpus.ipynb +++ b/data/nltk_corpus.ipynb @@ -2,15 +2,15 @@ "cells": [ { "cell_type": "code", - "execution_count": 22, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 1161192/1161192 [00:02<00:00, 577463.34it/s]\n", - "117659it [00:04, 26815.49it/s]\n" + "100%|██████████| 1161192/1161192 [00:01<00:00, 600059.17it/s]\n", + "117659it [00:04, 27148.22it/s]\n" ] } ], @@ -20,6 +20,8 @@ "import nltk\n", "from tqdm.auto import tqdm\n", "import json\n", + "import unidecode\n", + "\n", "\n", "# Create a frequency distribution of words in the Brown corpus\n", "fd = nltk.FreqDist([word.lower() for word in tqdm(brown.words())])\n", @@ -42,12 +44,13 @@ " if ant_name != word and word not in ant_name:\n", " antonyms.add(ant_name)\n", " word_fixed = word.replace(\"-\",\"\").replace(\"_\",\"\").replace(\" \",\"\")\n", + " word_fixed = unidecode.unidecode(word_fixed)\n", " entry = {\n", " \"word\": word_fixed,\n", " \"senses\": senses,\n", " \"synonyms\": list(synonyms),\n", " \"antonyms\": list(antonyms),\n", - " \"frequency\": fd[word.lower()]\n", + " \"frequency\": len(senses)\n", " }\n", " dictionary[word_fixed] = entry\n", "\n", @@ -56,6 +59,26 @@ " json.dump(dictionary, outfile, indent=4)\n" ] }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['care must be exercised when this substance is to be deflagrated']" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "synset.examples()" + ] + }, { "cell_type": "code", "execution_count": 21, @@ -124,6 +147,8 @@ " lemma = lemma.replace(\"ö\", \"oe\")\n", " lemma = lemma.replace(\"ä\", \"ae\")\n", "\n", + " lemma = unidecode.unidecode(lemma)\n", + "\n", " senses = [remove_umlaute(s) for s in senses]\n", "\n", " entry = {\n", @@ -191,13 +216,6 @@ "s = l.synsets()[0]" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 113, @@ -23456,13 +23474,6 @@ "source": [ "dictionary" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/server/de.json b/server/de.json index 3776734..b26a8a2 100644 --- a/server/de.json +++ b/server/de.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:276f757a01f743a693d60197b804e9f5c9b56b35ab23b6045218cdc679fa3250 -size 31056136 +oid sha256:3c302c54b7d1332629f8e6321a485535d37db8e2ce234ea9ac1fb6a8aa1c68ca +size 16473481