{ "cells": [ { "cell_type": "markdown", "id": "e048da07", "metadata": {}, "source": [ "# Create Dictionaies for crossword clues\n", "\n", "this notebook creates dictionaries for crossword clues.\n", "\n", "The final dictionaries will be saved as json file, containing a list of entries in the following format:\n", "\n", "```json\n", "{\n", " \"\": {\n", " \"word\": \"\",\n", " \"senses\": [\n", " \"\",\n", " \"\",\n", " \"...\" \n", " ],\n", " \"synonyms\": [\n", " \"\",\n", " \"\",\n", " \"...\" \n", " ],\n", " \"antonyms\": [\n", " \"\",\n", " \"\",\n", " \"...\" \n", " ],\n", " \"word_frequency\": \n", "\n", " },\n", "}\n", "```\n" ] }, { "cell_type": "markdown", "id": "28040681", "metadata": {}, "source": [ "### Install some dependencies for that notebook" ] }, { "cell_type": "code", "execution_count": 1, "id": "f0aecff7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: tqdm in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (4.67.1)\n", "Requirement already satisfied: pandas in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (2.3.0)\n", "Requirement already satisfied: requests in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (2.32.5)\n", "Requirement already satisfied: ipywidgets in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (8.1.8)\n", "Requirement already satisfied: pydantic in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (2.12.4)\n", "Requirement already satisfied: nltk in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (3.9.2)\n", "Requirement already satisfied: numpy>=1.26.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2.2.6)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2025.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pandas) (2025.2)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (3.4.4)\n", "Requirement already satisfied: idna<4,>=2.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (3.11)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (2.5.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from requests) (2025.11.12)\n", "Requirement already satisfied: comm>=0.1.3 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (0.2.3)\n", "Requirement already satisfied: ipython>=6.1.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (9.7.0)\n", "Requirement already satisfied: traitlets>=4.3.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (5.14.3)\n", "Requirement already satisfied: widgetsnbextension~=4.0.14 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (4.0.15)\n", "Requirement already satisfied: jupyterlab_widgets~=3.0.15 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipywidgets) (3.0.16)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.41.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (2.41.5)\n", "Requirement already satisfied: typing-extensions>=4.14.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (4.15.0)\n", "Requirement already satisfied: typing-inspection>=0.4.2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pydantic) (0.4.2)\n", "Requirement already satisfied: click in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from nltk) (8.3.1)\n", "Requirement already satisfied: joblib in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from nltk) (1.5.2)\n", "Requirement already satisfied: regex>=2021.8.3 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from nltk) (2025.11.3)\n", "Requirement already satisfied: decorator>=4.3.2 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (5.2.1)\n", "Requirement already satisfied: ipython-pygments-lexers>=1.0.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (1.1.1)\n", "Requirement already satisfied: jedi>=0.18.1 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.2)\n", "Requirement already satisfied: matplotlib-inline>=0.1.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.1)\n", "Requirement already satisfied: pexpect>4.3 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.52)\n", "Requirement already satisfied: pygments>=2.11.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (2.19.2)\n", "Requirement already satisfied: stack_data>=0.6.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", "Requirement already satisfied: six>=1.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from jedi>=0.18.1->ipython>=6.1.0->ipywidgets) (0.8.5)\n", "Requirement already satisfied: ptyprocess>=0.5 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", "Requirement already satisfied: wcwidth in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.14)\n", "Requirement already satisfied: executing>=1.2.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (2.2.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (3.0.1)\n", "Requirement already satisfied: pure-eval in /home/jonas/.cache/pypoetry/virtualenvs/multiplayer-crosswords-W02cfZ32-py3.12/lib/python3.12/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (0.2.3)\n" ] } ], "source": [ "# install dependencies for this notebooks\n", "\n", "!pip install tqdm pandas requests ipywidgets pydantic nltk" ] }, { "cell_type": "markdown", "id": "a0964dfc", "metadata": {}, "source": [ "### Import Libraries and define Constants and source urls" ] }, { "cell_type": "code", "execution_count": 2, "id": "e7d3d24f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_157490/121043459.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", " from tqdm.autonotebook import tqdm\n" ] } ], "source": [ "# import necessary libraries\n", "import pandas as pd\n", "import requests\n", "from tqdm.autonotebook import tqdm \n", "from pathlib import Path\n", "\n", "# some constants\n", "\n", "CACHE_DIR = Path(\"./.cache\")\n", "CACHE_DIR.mkdir(exist_ok=True)\n", "\n", "CRYPTICS_CROSSWORDS_DB_URL = \"https://cryptics.georgeho.org/data/clues.csv?_stream=on&_size=max\"\n", "CRYPTICS_CROSSWORDS_DB_CSV = CACHE_DIR / \"cryptics_clues.csv\"" ] }, { "cell_type": "markdown", "id": "61263a61", "metadata": {}, "source": [ "## Download External Data\n", "\n", "* Crypticts DB (\"https://cryptics.georgeho.org/\")" ] }, { "cell_type": "code", "execution_count": null, "id": "806a5c51", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package wordnet to /home/jonas/nltk_data...\n", "[nltk_data] Package wordnet is already up-to-date!\n", "[nltk_data] Downloading package omw-1.4 to /home/jonas/nltk_data...\n", "[nltk_data] Package omw-1.4 is already up-to-date!\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# download the cryptics crosswords database if not already cached \n", "if not CRYPTICS_CROSSWORDS_DB_CSV.exists():\n", " response = requests.get(CRYPTICS_CROSSWORDS_DB_URL)\n", " with open(CRYPTICS_CROSSWORDS_DB_CSV, \"wb\") as f:\n", " f.write(response.content)\n", "\n", "# download wordnet from nltk\n", "import nltk\n", "nltk.download('wordnet')\n", "nltk.download('omw-1.4') # optional, extra languages / lemmas\n", "nltk.download('omw') # try the older omw package" ] }, { "cell_type": "markdown", "id": "26060068", "metadata": {}, "source": [ "## Define our Datastructures" ] }, { "cell_type": "code", "execution_count": 4, "id": "8c81708c", "metadata": {}, "outputs": [], "source": [ "from pydantic import BaseModel\n", "import re \n", "\n", "class WordEntry(BaseModel):\n", " word: str\n", " senses: list[str]\n", " synonyms: list[str]\n", " antonyms: list[str]\n", " word_frequency: int # frequency rank of the word (0% - 100%)\n", " source: str # source of the word entry (e.g., \"cryptics\", \"wordnet\", etc.)\n", " categories: list[str] # categories or tags associated with the word entry\n", "\n", "class Dictionary(BaseModel):\n", " entries: dict[str, WordEntry] # mapping from word to WordEntry\n", " def add_entry(self, entry: WordEntry):\n", " if entry.word not in self.entries: \n", " self.entries[entry.word] = entry\n", " else:\n", " if entry.source == self.entries[entry.word].source:\n", " # merge entries if word already exists\n", " existing_entry = self.entries[entry.word]\n", " existing_entry.senses = list(set(existing_entry.senses) | set(entry.senses))\n", " existing_entry.synonyms = list(set(existing_entry.synonyms) | set(entry.synonyms))\n", " existing_entry.antonyms = list(set(existing_entry.antonyms) | set(entry.antonyms))\n", " existing_entry.categories = list(set(existing_entry.categories) | set(entry.categories))\n", " existing_entry.word_frequency = max(existing_entry.word_frequency, entry.word_frequency)\n", " else:\n", " # create a new entry\n", " word = entry.word\n", " i = 1\n", " while f\"{word}_{i}\" in self.entries:\n", " i += 1 \n", " self.entries[f\"{word}_{i}\"] = entry\n", "\n", " " ] }, { "cell_type": "markdown", "id": "b1397355", "metadata": {}, "source": [ "## Parse Data (EN)\n", "\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "56988415", "metadata": {}, "outputs": [], "source": [ "en_db = Dictionary (entries={})" ] }, { "cell_type": "markdown", "id": "d5b35c1b", "metadata": {}, "source": [ "### Parse cryptics DB" ] }, { "cell_type": "code", "execution_count": 6, "id": "e9e1ee43", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4c5f6f1c7c594df6946e51686feeee34", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/42 [00:00200 for very common words.\n", " # We apply a factor and clamp.\n", " # Using factor 1.0 means 100 count -> 100 frequency.\n", " word_frequency = min(100, int(raw_metric))\n", "\n", " en_db.add_entry(WordEntry(\n", " word=word,\n", " senses=[clue],\n", " synonyms=[],\n", " antonyms=[],\n", " word_frequency=word_frequency,\n", " source=\"wordnet\",\n", " categories=[\"wordnet\"]\n", " ))\n", " \n", " #break" ] }, { "cell_type": "markdown", "id": "1e246fd0", "metadata": {}, "source": [ "## Create German Database using OpenThesaurus\n", "\n", "* download data first:" ] }, { "cell_type": "code", "execution_count": 9, "id": "435d0b78", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "OpenThesaurus available at .cache/openthesaurus.txt\n" ] } ], "source": [ "from multiplayer_crosswords.data_utils import download_openthesaurus\n", "\n", "openthesaurus_path = download_openthesaurus()" ] }, { "cell_type": "markdown", "id": "0de30aac", "metadata": {}, "source": [ "* parse the data" ] }, { "cell_type": "code", "execution_count": null, "id": "07e4e97d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "be16b393", "metadata": {}, "source": [ "### Save extracted databases\n", "\n", "Dump the db to disk as json" ] }, { "cell_type": "code", "execution_count": 9, "id": "69b67091", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ef2252adfaf240c981a18567f5d23b45", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/96407 [00:00 0:\n", " f.write(\",\\n\")\n", " d_value = value.model_dump()\n", " as_json = json.dumps(\n", " d_value, indent=4\n", " )\n", " as_json = \"\\n \".join(as_json.split(\"\\n\"))\n", " as_json = \" \\\"\" + key + \"\\\": \" + as_json\n", " f.write(as_json)\n", " i += 1\n", " \n", " f.write(\"\\n}\\n\")\n", " print(f\"Wrote {i} entries to {EN_PATH}\" )" ] }, { "cell_type": "code", "execution_count": 10, "id": "c3f8d049", "metadata": {}, "outputs": [], "source": [ "test_synset = wn.synsets(\"house\")[0]" ] }, { "cell_type": "code", "execution_count": 27, "id": "3870ea2c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_synset.a" ] }, { "cell_type": "code", "execution_count": 26, "id": "7e7033cf", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "60" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_word_str = \"table\"\n", "\n", "frequency_metric(wn.synsets(test_word_str)[0].lemmas()[0], test_word_str ) " ] }, { "cell_type": "code", "execution_count": null, "id": "f91aee6f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "multiplayer-crosswords-W02cfZ32-py3.12", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.3" } }, "nbformat": 4, "nbformat_minor": 5 }