improvements in en db generation
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@ -160,3 +160,5 @@ cython_debug/
|
|||||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
#.idea/
|
#.idea/
|
||||||
|
|
||||||
|
|
||||||
|
.cache/
|
||||||
|
|||||||
@ -24,5 +24,7 @@ python -m multiplayer_crosswords.server.main
|
|||||||
## start the webui
|
## start the webui
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -m multiplayer_crosswords.server.serve_frontend
|
python -m multiplayer_crosswords.server.serve_frontend --port 8080
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Then open your browser and navigate to `http://localhost:8080`.
|
||||||
|
|||||||
671
data/dictionary_generation.ipynb
Normal file
671
data/dictionary_generation.ipynb
Normal file
@ -0,0 +1,671 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "e048da07",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Create Dictionaies for crossword clues\n",
|
||||||
|
"\n",
|
||||||
|
"this notebook creates dictionaries for crossword clues.\n",
|
||||||
|
"\n",
|
||||||
|
"The final dictionaries will be saved as json file, containing a list of entries in the following format:\n",
|
||||||
|
"\n",
|
||||||
|
"```json\n",
|
||||||
|
"{\n",
|
||||||
|
" \"<unique_word_entry>\": {\n",
|
||||||
|
" \"word\": \"<word>\",\n",
|
||||||
|
" \"senses\": [\n",
|
||||||
|
" \"<definition_1>\",\n",
|
||||||
|
" \"<definition_2>\",\n",
|
||||||
|
" \"...\" \n",
|
||||||
|
" ],\n",
|
||||||
|
" \"synonyms\": [\n",
|
||||||
|
" \"<synonym_1>\",\n",
|
||||||
|
" \"<synonym_2>\",\n",
|
||||||
|
" \"...\" \n",
|
||||||
|
" ],\n",
|
||||||
|
" \"antonyms\": [\n",
|
||||||
|
" \"<antonym_1>\",\n",
|
||||||
|
" \"<antonym_2>\",\n",
|
||||||
|
" \"...\" \n",
|
||||||
|
" ],\n",
|
||||||
|
" \"word_frequency\": <frequency_value (from 0 to 100)>\n",
|
||||||
|
"\n",
|
||||||
|
" },\n",
|
||||||
|
"}\n",
|
||||||
|
"```\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "28040681",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Install some dependencies for that notebook"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"id": "f0aecff7",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Requirement already satisfied: tqdm in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (4.67.1)\n",
|
||||||
|
"Requirement already satisfied: pandas in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (2.3.0)\n",
|
||||||
|
"Requirement already satisfied: requests in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (2.32.5)\n",
|
||||||
|
"Requirement already satisfied: ipywidgets in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (8.1.8)\n",
|
||||||
|
"Requirement already satisfied: pydantic in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (2.12.4)\n",
|
||||||
|
"Requirement already satisfied: nltk in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (3.9.2)\n",
|
||||||
|
"Requirement already satisfied: numpy>=1.26.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2.2.6)\n",
|
||||||
|
"Requirement already satisfied: python-dateutil>=2.8.2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n",
|
||||||
|
"Requirement already satisfied: pytz>=2020.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2025.2)\n",
|
||||||
|
"Requirement already satisfied: tzdata>=2022.7 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2025.2)\n",
|
||||||
|
"Requirement already satisfied: charset_normalizer<4,>=2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (3.4.4)\n",
|
||||||
|
"Requirement already satisfied: idna<4,>=2.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (3.11)\n",
|
||||||
|
"Requirement already satisfied: urllib3<3,>=1.21.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (2.5.0)\n",
|
||||||
|
"Requirement already satisfied: certifi>=2017.4.17 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (2025.11.12)\n",
|
||||||
|
"Requirement already satisfied: comm>=0.1.3 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (0.2.3)\n",
|
||||||
|
"Requirement already satisfied: ipython>=6.1.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (9.7.0)\n",
|
||||||
|
"Requirement already satisfied: traitlets>=4.3.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (5.14.3)\n",
|
||||||
|
"Requirement already satisfied: widgetsnbextension~=4.0.14 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (4.0.15)\n",
|
||||||
|
"Requirement already satisfied: jupyterlab_widgets~=3.0.15 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (3.0.16)\n",
|
||||||
|
"Requirement already satisfied: annotated-types>=0.6.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (0.7.0)\n",
|
||||||
|
"Requirement already satisfied: pydantic-core==2.41.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (2.41.5)\n",
|
||||||
|
"Requirement already satisfied: typing-extensions>=4.14.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (4.15.0)\n",
|
||||||
|
"Requirement already satisfied: typing-inspection>=0.4.2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (0.4.2)\n",
|
||||||
|
"Requirement already satisfied: click in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from nltk) (8.3.1)\n",
|
||||||
|
"Requirement already satisfied: joblib in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from nltk) (1.5.2)\n",
|
||||||
|
"Requirement already satisfied: regex>=2021.8.3 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from nltk) (2025.11.3)\n",
|
||||||
|
"Requirement already satisfied: decorator>=4.3.2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (5.2.1)\n",
|
||||||
|
"Requirement already satisfied: ipython-pygments-lexers>=1.0.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (1.1.1)\n",
|
||||||
|
"Requirement already satisfied: jedi>=0.18.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.2)\n",
|
||||||
|
"Requirement already satisfied: matplotlib-inline>=0.1.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.1)\n",
|
||||||
|
"Requirement already satisfied: pexpect>4.3 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n",
|
||||||
|
"Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.52)\n",
|
||||||
|
"Requirement already satisfied: pygments>=2.11.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (2.19.2)\n",
|
||||||
|
"Requirement already satisfied: stack_data>=0.6.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n",
|
||||||
|
"Requirement already satisfied: six>=1.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
|
||||||
|
"Requirement already satisfied: parso<0.9.0,>=0.8.4 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from jedi>=0.18.1->ipython>=6.1.0->ipywidgets) (0.8.5)\n",
|
||||||
|
"Requirement already satisfied: ptyprocess>=0.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n",
|
||||||
|
"Requirement already satisfied: wcwidth in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.14)\n",
|
||||||
|
"Requirement already satisfied: executing>=1.2.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (2.2.1)\n",
|
||||||
|
"Requirement already satisfied: asttokens>=2.1.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (3.0.1)\n",
|
||||||
|
"Requirement already satisfied: pure-eval in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (0.2.3)\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# install dependencies for this notebooks\n",
|
||||||
|
"\n",
|
||||||
|
"!pip install tqdm pandas requests ipywidgets pydantic nltk"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "a0964dfc",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Import Libraries and define Constants and source urls"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"id": "e7d3d24f",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"/tmp/ipykernel_157490/121043459.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
|
||||||
|
" from tqdm.autonotebook import tqdm\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# import necessary libraries\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import requests\n",
|
||||||
|
"from tqdm.autonotebook import tqdm \n",
|
||||||
|
"from pathlib import Path\n",
|
||||||
|
"\n",
|
||||||
|
"# some constants\n",
|
||||||
|
"\n",
|
||||||
|
"CACHE_DIR = Path(\"./.cache\")\n",
|
||||||
|
"CACHE_DIR.mkdir(exist_ok=True)\n",
|
||||||
|
"\n",
|
||||||
|
"CRYPTICS_CROSSWORDS_DB_URL = \"https://cryptics.georgeho.org/data/clues.csv?_stream=on&_size=max\"\n",
|
||||||
|
"CRYPTICS_CROSSWORDS_DB_CSV = CACHE_DIR / \"cryptics_clues.csv\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "61263a61",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Download External Data\n",
|
||||||
|
"\n",
|
||||||
|
"* Crypticts DB (\"https://cryptics.georgeho.org/\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "806a5c51",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n",
|
||||||
|
"[nltk_data] Package wordnet is already up-to-date!\n",
|
||||||
|
"[nltk_data] Downloading package omw-1.4 to /home/jonas/nltk_data...\n",
|
||||||
|
"[nltk_data] Package omw-1.4 is already up-to-date!\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# download the cryptics crosswords database if not already cached \n",
|
||||||
|
"if not CRYPTICS_CROSSWORDS_DB_CSV.exists():\n",
|
||||||
|
" response = requests.get(CRYPTICS_CROSSWORDS_DB_URL)\n",
|
||||||
|
" with open(CRYPTICS_CROSSWORDS_DB_CSV, \"wb\") as f:\n",
|
||||||
|
" f.write(response.content)\n",
|
||||||
|
"\n",
|
||||||
|
"# download wordnet from nltk\n",
|
||||||
|
"import nltk\n",
|
||||||
|
"nltk.download('wordnet')\n",
|
||||||
|
"nltk.download('omw-1.4') # optional, extra languages / lemmas\n",
|
||||||
|
"nltk.download('omw') # try the older omw package"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "26060068",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Define our Datastructures"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"id": "8c81708c",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from pydantic import BaseModel\n",
|
||||||
|
"import re \n",
|
||||||
|
"\n",
|
||||||
|
"class WordEntry(BaseModel):\n",
|
||||||
|
" word: str\n",
|
||||||
|
" senses: list[str]\n",
|
||||||
|
" synonyms: list[str]\n",
|
||||||
|
" antonyms: list[str]\n",
|
||||||
|
" word_frequency: int # frequency rank of the word (0% - 100%)\n",
|
||||||
|
" source: str # source of the word entry (e.g., \"cryptics\", \"wordnet\", etc.)\n",
|
||||||
|
" categories: list[str] # categories or tags associated with the word entry\n",
|
||||||
|
"\n",
|
||||||
|
"class Dictionary(BaseModel):\n",
|
||||||
|
" entries: dict[str, WordEntry] # mapping from word to WordEntry\n",
|
||||||
|
" def add_entry(self, entry: WordEntry):\n",
|
||||||
|
" if entry.word not in self.entries: \n",
|
||||||
|
" self.entries[entry.word] = entry\n",
|
||||||
|
" else:\n",
|
||||||
|
" if entry.source == self.entries[entry.word].source:\n",
|
||||||
|
" # merge entries if word already exists\n",
|
||||||
|
" existing_entry = self.entries[entry.word]\n",
|
||||||
|
" existing_entry.senses = list(set(existing_entry.senses) | set(entry.senses))\n",
|
||||||
|
" existing_entry.synonyms = list(set(existing_entry.synonyms) | set(entry.synonyms))\n",
|
||||||
|
" existing_entry.antonyms = list(set(existing_entry.antonyms) | set(entry.antonyms))\n",
|
||||||
|
" existing_entry.categories = list(set(existing_entry.categories) | set(entry.categories))\n",
|
||||||
|
" existing_entry.word_frequency = max(existing_entry.word_frequency, entry.word_frequency)\n",
|
||||||
|
" else:\n",
|
||||||
|
" # create a new entry\n",
|
||||||
|
" word = entry.word\n",
|
||||||
|
" i = 1\n",
|
||||||
|
" while f\"{word}_{i}\" in self.entries:\n",
|
||||||
|
" i += 1 \n",
|
||||||
|
" self.entries[f\"{word}_{i}\"] = entry\n",
|
||||||
|
"\n",
|
||||||
|
" "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b1397355",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Parse Data (EN)\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"id": "56988415",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"en_db = Dictionary (entries={})"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "d5b35c1b",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Parse cryptics DB"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"id": "e9e1ee43",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "4c5f6f1c7c594df6946e51686feeee34",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/42 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# csv structure: we will use the column clue as senses and the column answer (lowercase) as word.\n",
|
||||||
|
"# words that have spaces will be skipped\n",
|
||||||
|
"\n",
|
||||||
|
"# read the file in batches to avoid memory issues \n",
|
||||||
|
"batch_size = 1000\n",
|
||||||
|
"\n",
|
||||||
|
"# Calculate total lines safely\n",
|
||||||
|
"try:\n",
|
||||||
|
" total_lines = sum(1 for line in open(CRYPTICS_CROSSWORDS_DB_CSV))\n",
|
||||||
|
"except Exception as e:\n",
|
||||||
|
" print(f\"Error counting lines: {e}\")\n",
|
||||||
|
" total_lines = 0\n",
|
||||||
|
"\n",
|
||||||
|
"cryptics_cols = \"rowid\tclue\tanswer\tdefinition\tclue_number\tpuzzle_date\tpuzzle_name\tsource_url\tsource\".split()\n",
|
||||||
|
"\n",
|
||||||
|
"for start_row in tqdm(range(1, total_lines, batch_size)):\n",
|
||||||
|
" try:\n",
|
||||||
|
" # Use on_bad_lines='skip' to handle rows with too many fields\n",
|
||||||
|
" df = pd.read_csv(CRYPTICS_CROSSWORDS_DB_CSV, skiprows=start_row, nrows=batch_size, names=cryptics_cols, on_bad_lines='skip')\n",
|
||||||
|
" except Exception as e:\n",
|
||||||
|
" print(f\"Error reading batch starting at {start_row}: {e}\")\n",
|
||||||
|
" continue\n",
|
||||||
|
"\n",
|
||||||
|
" for index, row in df.iterrows():\n",
|
||||||
|
" # Check if answer is a string (handles NaN)\n",
|
||||||
|
" if not isinstance(row['answer'], str):\n",
|
||||||
|
" continue\n",
|
||||||
|
"\n",
|
||||||
|
" word = row['answer'].lower()\n",
|
||||||
|
" if ' ' in word:\n",
|
||||||
|
" continue\n",
|
||||||
|
" \n",
|
||||||
|
" # Check if clue is a string\n",
|
||||||
|
" if not isinstance(row['clue'], str):\n",
|
||||||
|
" continue\n",
|
||||||
|
"\n",
|
||||||
|
" clue = row['clue']\n",
|
||||||
|
" # replace - and _ with empty string\n",
|
||||||
|
" word = word.replace(\"-\", \"\").replace(\"_\", \"\")\n",
|
||||||
|
" word = word.lower()\n",
|
||||||
|
"\n",
|
||||||
|
" # remove numbers in parentheses or brackets (e.g. (5), [4], (3,4), [1-9])\n",
|
||||||
|
" clue = re.sub(r'\\s*[(\\[][\\d,\\-\\s]+[)\\]]$', '', clue).strip()\n",
|
||||||
|
"\n",
|
||||||
|
" # if the word is not alphabetic, skip it\n",
|
||||||
|
" if not word.isalpha():\n",
|
||||||
|
" continue \n",
|
||||||
|
" en_db.add_entry(WordEntry(\n",
|
||||||
|
" word=word,\n",
|
||||||
|
" senses=[clue],\n",
|
||||||
|
" synonyms=[],\n",
|
||||||
|
" antonyms=[],\n",
|
||||||
|
" word_frequency=-1, # placeholder frequency\n",
|
||||||
|
" source=\"cryptics\",\n",
|
||||||
|
" categories=[\"cryptic_clue\"]\n",
|
||||||
|
" ))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "62734af4",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Parse Wordnet Data\n",
|
||||||
|
"\n",
|
||||||
|
"* import necessary stuff:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"id": "32280691",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from nltk.corpus import wordnet as wn\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "e13bf2f3",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"* Parse synsets"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"id": "b21721e2",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "c411da19479944298117158962a08ea3",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/117659 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"all_synsets = wn.all_synsets()\n",
|
||||||
|
"\n",
|
||||||
|
"def frequency_metric(lemma, word):\n",
|
||||||
|
" # derive a frequency metric, based on polysemy and lemma count:\n",
|
||||||
|
" # 1. SemCor frequency count for this specific sense\n",
|
||||||
|
" semcor_count = lemma.count()\n",
|
||||||
|
" # 2. Polysemy: number of synsets for this word\n",
|
||||||
|
" polysemy_count = len(wn.synsets(word))\n",
|
||||||
|
" \n",
|
||||||
|
" return semcor_count + polysemy_count\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"for synset in tqdm(list(all_synsets)):\n",
|
||||||
|
" #print(synset.name(), synset.definition() )\n",
|
||||||
|
"\n",
|
||||||
|
" # find the first \"good\" lemma name (only alphabetic characters) \n",
|
||||||
|
" # WE NEED LEMMA OBJECTS NOW, NOT JUST NAMES\n",
|
||||||
|
" good_lemmas = [lemma for lemma in synset.lemmas() if lemma.name().isalpha()] \n",
|
||||||
|
" if not good_lemmas:\n",
|
||||||
|
" continue \n",
|
||||||
|
"\n",
|
||||||
|
" target_lemma = good_lemmas[0]\n",
|
||||||
|
" word = target_lemma.name().lower()\n",
|
||||||
|
" clue = synset.definition()\n",
|
||||||
|
"\n",
|
||||||
|
" # Calculate frequency\n",
|
||||||
|
" raw_metric = frequency_metric(target_lemma, word)\n",
|
||||||
|
" \n",
|
||||||
|
" # Normalize to 0-100 range\n",
|
||||||
|
" # Values can range from 1 to >200 for very common words.\n",
|
||||||
|
" # We apply a factor and clamp.\n",
|
||||||
|
" # Using factor 1.0 means 100 count -> 100 frequency.\n",
|
||||||
|
" word_frequency = min(100, int(raw_metric))\n",
|
||||||
|
"\n",
|
||||||
|
" en_db.add_entry(WordEntry(\n",
|
||||||
|
" word=word,\n",
|
||||||
|
" senses=[clue],\n",
|
||||||
|
" synonyms=[],\n",
|
||||||
|
" antonyms=[],\n",
|
||||||
|
" word_frequency=word_frequency,\n",
|
||||||
|
" source=\"wordnet\",\n",
|
||||||
|
" categories=[\"wordnet\"]\n",
|
||||||
|
" ))\n",
|
||||||
|
" \n",
|
||||||
|
" #break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "1e246fd0",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Create German Database using OpenThesaurus\n",
|
||||||
|
"\n",
|
||||||
|
"* download data first:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"id": "435d0b78",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"OpenThesaurus available at .cache/openthesaurus.txt\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from multiplayer_crosswords.data_utils import download_openthesaurus\n",
|
||||||
|
"\n",
|
||||||
|
"openthesaurus_path = download_openthesaurus()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "0de30aac",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"* parse the data"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "07e4e97d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "be16b393",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Save extracted databases\n",
|
||||||
|
"\n",
|
||||||
|
"Dump the db to disk as json"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"id": "69b67091",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "ef2252adfaf240c981a18567f5d23b45",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/96407 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "ef2252adfaf240c981a18567f5d23b45",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/96407 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Wrote 74357 entries to en.json\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"EN_PATH = Path(\"./en.json\")\n",
|
||||||
|
"\n",
|
||||||
|
"import json\n",
|
||||||
|
"\n",
|
||||||
|
"# entries to include:\n",
|
||||||
|
"INCLUDED_SOURCES = {\n",
|
||||||
|
" #\"cryptics\",\n",
|
||||||
|
" \"wordnet\"\n",
|
||||||
|
"} \n",
|
||||||
|
"\n",
|
||||||
|
"with open(EN_PATH, \"w\") as f:\n",
|
||||||
|
" f.write(\"{\\n\")\n",
|
||||||
|
" i = 0\n",
|
||||||
|
" for key, value in tqdm(en_db.entries.items()):\n",
|
||||||
|
"\n",
|
||||||
|
" if value.source not in INCLUDED_SOURCES:\n",
|
||||||
|
" continue \n",
|
||||||
|
" \n",
|
||||||
|
" # dump json\n",
|
||||||
|
" if i > 0:\n",
|
||||||
|
" f.write(\",\\n\")\n",
|
||||||
|
" d_value = value.model_dump()\n",
|
||||||
|
" as_json = json.dumps(\n",
|
||||||
|
" d_value, indent=4\n",
|
||||||
|
" )\n",
|
||||||
|
" as_json = \"\\n \".join(as_json.split(\"\\n\"))\n",
|
||||||
|
" as_json = \" \\\"\" + key + \"\\\": \" + as_json\n",
|
||||||
|
" f.write(as_json)\n",
|
||||||
|
" i += 1\n",
|
||||||
|
" \n",
|
||||||
|
" f.write(\"\\n}\\n\")\n",
|
||||||
|
" print(f\"Wrote {i} entries to {EN_PATH}\" )"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"id": "c3f8d049",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"test_synset = wn.synsets(\"house\")[0]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 27,
|
||||||
|
"id": "3870ea2c",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 27,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"test_synset.a"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 26,
|
||||||
|
"id": "7e7033cf",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"60"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 26,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"test_word_str = \"table\"\n",
|
||||||
|
"\n",
|
||||||
|
"frequency_metric(wn.synsets(test_word_str)[0].lemmas()[0], test_word_str ) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "f91aee6f",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "multiplayer-crosswords-W02cfZ32-py3.12",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.12.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
34
multiplayer_crosswords/data_utils.py
Normal file
34
multiplayer_crosswords/data_utils.py
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import zipfile
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def download_openthesaurus(cache_dir = None) -> Path | None:
|
||||||
|
|
||||||
|
if cache_dir is None:
|
||||||
|
cache_dir = Path("./.cache")
|
||||||
|
else:
|
||||||
|
cache_dir = Path(cache_dir)
|
||||||
|
|
||||||
|
# Define paths
|
||||||
|
OPENTHESAURUS_URL = "https://www.openthesaurus.de/export/OpenThesaurus-Textversion.zip"
|
||||||
|
OPENTHESAURUS_ZIP = cache_dir / "OpenThesaurus-Textversion.zip"
|
||||||
|
OPENTHESAURUS_TXT = cache_dir / "openthesaurus.txt"
|
||||||
|
|
||||||
|
if not OPENTHESAURUS_TXT.exists():
|
||||||
|
if not OPENTHESAURUS_ZIP.exists():
|
||||||
|
print("Downloading OpenThesaurus...")
|
||||||
|
r = requests.get(OPENTHESAURUS_URL)
|
||||||
|
with open(OPENTHESAURUS_ZIP, "wb") as f:
|
||||||
|
f.write(r.content)
|
||||||
|
|
||||||
|
print("Extracting...")
|
||||||
|
with zipfile.ZipFile(OPENTHESAURUS_ZIP, 'r') as zip_ref:
|
||||||
|
zip_ref.extractall(cache_dir)
|
||||||
|
|
||||||
|
print(f"OpenThesaurus available at {OPENTHESAURUS_TXT}")
|
||||||
|
|
||||||
|
return OPENTHESAURUS_TXT
|
||||||
|
|
||||||
@ -59,9 +59,16 @@ def load_dictionary(p: str | Path) -> Dictionary:
|
|||||||
|
|
||||||
if len(hints) > 0:
|
if len(hints) > 0:
|
||||||
|
|
||||||
|
difficulty = 1.0
|
||||||
|
word_freq = obj.get("word_frequency", -1)
|
||||||
|
if word_freq >= 0:
|
||||||
|
# Map 0-100 frequency to 1.0-0.0 difficulty
|
||||||
|
# Higher frequency -> Lower difficulty
|
||||||
|
difficulty = max(0.0, min(1.0, 1.0 - (word_freq / 100.0)))
|
||||||
|
|
||||||
w = Word(word=word,
|
w = Word(word=word,
|
||||||
hints=hints,
|
hints=hints,
|
||||||
difficulty=1)
|
difficulty=difficulty)
|
||||||
dict_obj.add_word(w)
|
dict_obj.add_word(w)
|
||||||
load_dictionary._cache[cache_key] = dict_obj
|
load_dictionary._cache[cache_key] = dict_obj
|
||||||
return dict_obj
|
return dict_obj
|
||||||
@ -70,4 +77,4 @@ def load_en_dictionary() -> Dictionary:
|
|||||||
return load_dictionary(Path(__file__).parent.parent / "data" / "en.json")
|
return load_dictionary(Path(__file__).parent.parent / "data" / "en.json")
|
||||||
|
|
||||||
def load_de_dictionary() -> Dictionary:
|
def load_de_dictionary() -> Dictionary:
|
||||||
return load_dictionary(Path(__file__).parent.parent / "data" / "de.json")
|
return load_dictionary(Path(__file__).parent.parent / "data" / "de.json")
|
||||||
|
|||||||
Reference in New Issue
Block a user