Merge branch 'master' of ssh://the-cake-is-a-lie.net:20022/jonas/NLP-LAB
This commit is contained in:
		
							
								
								
									
										84
									
								
								Jonas_Solutions/Task03_Instructions.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										84
									
								
								Jonas_Solutions/Task03_Instructions.py
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,84 @@ | ||||
| import numpy as np  | ||||
| import pandas as pd  | ||||
| from sklearn.feature_extraction.text import CountVectorizer | ||||
| from keras.preprocessing.text import Tokenizer | ||||
| from keras.preprocessing.sequence import pad_sequences | ||||
| from keras.models import Sequential | ||||
| from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D | ||||
| from sklearn.model_selection import train_test_split | ||||
| from keras.utils.np_utils import to_categorical | ||||
| import re | ||||
|  | ||||
| ''' | ||||
| Task 3: playing with NN framwork/keras and basic sentiment analysis | ||||
| - use the following model as a baseline and improve it! | ||||
| - export your metadata (just basic hyperparameters and outcomes for test data!) | ||||
| - test data = 0.3 (not in this example, change it!) | ||||
| - random_state = 4222 | ||||
| - no need to cross-validation! | ||||
| ''' | ||||
|  | ||||
| # parameters | ||||
| max_fatures = 500 | ||||
| embed_dim = 128 | ||||
| lstm_out = 196 | ||||
| dropout = 0.1 | ||||
| dropout_1d = 0.4 | ||||
| recurrent_dropout = 0.1 | ||||
| random_state = 1324 | ||||
| validation_size = 1000 | ||||
| batch_size = 16 | ||||
| epochs=2 | ||||
| verbose= 2 | ||||
|  | ||||
| df = pd.read_csv('dataset_sentiment.csv') | ||||
| df = df[['text','sentiment']] | ||||
| print(df[0:10]) | ||||
|  | ||||
| df = df[df.sentiment != "Neutral"] | ||||
| df['text'] = df['text'].apply(lambda x: x.lower()) | ||||
| df['text'] = df['text'].apply(lambda x: x.replace('rt',' ')) | ||||
| df['text'] = df['text'].apply((lambda x: re.sub('[^a-zA-z0-9\s]','',x))) | ||||
|      | ||||
| tok = Tokenizer(num_words=max_fatures, split=' ') | ||||
| tok.fit_on_texts(df['text'].values) | ||||
| X = tok.texts_to_sequences(df['text'].values) | ||||
| X = pad_sequences(X) | ||||
|  | ||||
| nn = Sequential() | ||||
| nn.add(Embedding(max_fatures, embed_dim, input_length = X.shape[1])) | ||||
| nn.add(SpatialDropout1D(dropout_1d)) | ||||
| nn.add(LSTM(lstm_out, dropout=dropout, recurrent_dropout=recurrent_dropout)) | ||||
| nn.add(Dense(2, activation='softmax')) | ||||
| nn.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics = ['accuracy']) | ||||
| print(nn.summary()) | ||||
|  | ||||
| Y = pd.get_dummies(df['sentiment']).values | ||||
| X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.30, random_state = random_state) | ||||
| nn.fit(X_train, Y_train, epochs = epochs, batch_size=batch_size, verbose=verbose) | ||||
|  | ||||
| X_validate = X_test[-validation_size:] | ||||
| Y_validate = Y_test[-validation_size:] | ||||
| X_test = X_test[:-validation_size] | ||||
| Y_test = Y_test[:-validation_size] | ||||
| score, accuracy = nn.evaluate(X_test, Y_test, verbose = 2, batch_size = batch_size) | ||||
| print("score: %.2f" % (score)) | ||||
| print("acc: %.2f" % (accuracy)) | ||||
|  | ||||
| pos_cnt, neg_cnt, pos_ok, neg_ok = 0, 0, 0, 0 | ||||
| for x in range(len(X_validate)): | ||||
|     result = nn.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 2)[0] | ||||
|     if np.argmax(result) == np.argmax(Y_validate[x]): | ||||
|         if np.argmax(Y_validate[x]) == 0: neg_ok += 1 | ||||
|         else: pos_ok += 1 | ||||
|     if np.argmax(Y_validate[x]) == 0: neg_cnt += 1 | ||||
|     else: pos_cnt += 1 | ||||
|  | ||||
| print("pos_acc", pos_ok/pos_cnt*100, "%") | ||||
| print("neg_acc", neg_ok/neg_cnt*100, "%") | ||||
|  | ||||
| X2 = ['what are u going to say about that? the truth, wassock?!'] | ||||
| X2 = tok.texts_to_sequences(X2) | ||||
| X2 = pad_sequences(X2, maxlen=26, dtype='int32', value=0) | ||||
| print(X2) | ||||
| print(nn.predict(X2, batch_size=1, verbose = 2)[0]) | ||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							
							
								
								
									
										316
									
								
								Jonas_Solutions/Task_03.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										316
									
								
								Jonas_Solutions/Task_03.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,316 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stderr", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "/home/jonas/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", | ||||
|       "  from ._conv import register_converters as _register_converters\n", | ||||
|       "Using TensorFlow backend.\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "import numpy as np \n", | ||||
|     "import pandas as pd \n", | ||||
|     "from sklearn.feature_extraction.text import CountVectorizer\n", | ||||
|     "from keras.preprocessing.text import Tokenizer\n", | ||||
|     "from keras.preprocessing.sequence import pad_sequences\n", | ||||
|     "from keras.models import Sequential\n", | ||||
|     "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n", | ||||
|     "from sklearn.model_selection import train_test_split\n", | ||||
|     "from keras.utils.np_utils import to_categorical\n", | ||||
|     "import re\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     ">**Task 3**: playing with NN framwork/keras and basic sentiment analysis\n", | ||||
|     ">- use the following model as a baseline and improve it!\n", | ||||
|     ">- export your metadata (just basic hyperparameters and outcomes for test data!)\n", | ||||
|     ">- test data = 0.3 (not in this example, change it!)\n", | ||||
|     ">- random_state = 4222\n", | ||||
|     ">- no need to cross-validation!\n", | ||||
|     "\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "dataset already downloaded\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "\n", | ||||
|     "if [ ! -e 'dataset_sentiment.csv' ]\n", | ||||
|     "then\n", | ||||
|     "    echo \"downloading dataset\"\n", | ||||
|     "    wget https://raw.githubusercontent.com/SmartDataAnalytics/MA-INF-4222-NLP-Lab/master/2018_SoSe/exercises/dataset_sentiment.csv\n", | ||||
|     "else\n", | ||||
|     "    echo \"dataset already downloaded\"\n", | ||||
|     "fi" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "# parameters\n", | ||||
|     "max_fatures = 500\n", | ||||
|     "embed_dim = 128\n", | ||||
|     "lstm_out = 196\n", | ||||
|     "dropout = 0.1\n", | ||||
|     "dropout_1d = 0.4\n", | ||||
|     "recurrent_dropout = 0.1\n", | ||||
|     "random_state = 1324\n", | ||||
|     "validation_size = 1000\n", | ||||
|     "batch_size = 16\n", | ||||
|     "epochs=2\n", | ||||
|     "verbose= 2" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 13, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "                                                text sentiment\n", | ||||
|       "0  RT @NancyLeeGrahn: How did everyone feel about...   Neutral\n", | ||||
|       "1  RT @ScottWalker: Didn't catch the full #GOPdeb...  Positive\n", | ||||
|       "2  RT @TJMShow: No mention of Tamir Rice and the ...   Neutral\n", | ||||
|       "3  RT @RobGeorge: That Carly Fiorina is trending ...  Positive\n", | ||||
|       "4  RT @DanScavino: #GOPDebate w/ @realDonaldTrump...  Positive\n", | ||||
|       "5  RT @GregAbbott_TX: @TedCruz: \"On my first day ...  Positive\n", | ||||
|       "6  RT @warriorwoman91: I liked her and was happy ...  Negative\n", | ||||
|       "7  Going on #MSNBC Live with @ThomasARoberts arou...   Neutral\n", | ||||
|       "8  Deer in the headlights RT @lizzwinstead: Ben C...  Negative\n", | ||||
|       "9  RT @NancyOsborne180: Last night's debate prove...  Negative\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "df = pd.read_csv('dataset_sentiment.csv')\n", | ||||
|     "df = df[['text','sentiment']]\n", | ||||
|     "print(df[0:10])\n", | ||||
|     "\n", | ||||
|     "df = df[df.sentiment != \"Neutral\"]\n", | ||||
|     "df['text'] = df['text'].apply(lambda x: x.lower())\n", | ||||
|     "df['text'] = df['text'].apply(lambda x: x.replace('rt',' '))\n", | ||||
|     "df['text'] = df['text'].apply((lambda x: re.sub('[^a-zA-Z0-9\\s]','',x)))" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 22, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "tok = Tokenizer(num_words=max_fatures, split=' ')\n", | ||||
|     "tok.fit_on_texts(df['text'].values)\n", | ||||
|     "X = tok.texts_to_sequences(df['text'].values)\n", | ||||
|     "X = pad_sequences(X)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 15, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "_________________________________________________________________\n", | ||||
|       "Layer (type)                 Output Shape              Param #   \n", | ||||
|       "=================================================================\n", | ||||
|       "embedding_2 (Embedding)      (None, 26, 128)           64000     \n", | ||||
|       "_________________________________________________________________\n", | ||||
|       "spatial_dropout1d_2 (Spatial (None, 26, 128)           0         \n", | ||||
|       "_________________________________________________________________\n", | ||||
|       "lstm_2 (LSTM)                (None, 196)               254800    \n", | ||||
|       "_________________________________________________________________\n", | ||||
|       "dense_2 (Dense)              (None, 2)                 394       \n", | ||||
|       "=================================================================\n", | ||||
|       "Total params: 319,194\n", | ||||
|       "Trainable params: 319,194\n", | ||||
|       "Non-trainable params: 0\n", | ||||
|       "_________________________________________________________________\n", | ||||
|       "None\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "nn = Sequential()\n", | ||||
|     "nn.add(Embedding(max_fatures, embed_dim, input_length = X.shape[1]))\n", | ||||
|     "nn.add(SpatialDropout1D(dropout_1d))\n", | ||||
|     "nn.add(LSTM(lstm_out, dropout=dropout, recurrent_dropout=recurrent_dropout))\n", | ||||
|     "nn.add(Dense(2, activation='softmax'))\n", | ||||
|     "nn.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])\n", | ||||
|     "print(nn.summary())" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "Epoch 1/2\n", | ||||
|       " - 21s - loss: 0.4322 - acc: 0.8196\n", | ||||
|       "Epoch 2/2\n", | ||||
|       " - 25s - loss: 0.3612 - acc: 0.8509\n" | ||||
|      ] | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "<keras.callbacks.History at 0x7f0646b9e828>" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 7, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "Y = pd.get_dummies(df['sentiment']).values\n", | ||||
|     "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.30, random_state = random_state)\n", | ||||
|     "nn.fit(X_train, Y_train, epochs = epochs, batch_size=batch_size, verbose=verbose)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "score: 0.37\n", | ||||
|       "acc: 0.84\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "X_validate = X_test[-validation_size:]\n", | ||||
|     "Y_validate = Y_test[-validation_size:]\n", | ||||
|     "X_test = X_test[:-validation_size]\n", | ||||
|     "Y_test = Y_test[:-validation_size]\n", | ||||
|     "score, accuracy = nn.evaluate(X_test, Y_test, verbose = 2, batch_size = batch_size)\n", | ||||
|     "print(\"score: %.2f\" % (score))\n", | ||||
|     "print(\"acc: %.2f\" % (accuracy))" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "pos_cnt, neg_cnt, pos_ok, neg_ok = 0, 0, 0, 0\n", | ||||
|     "for x in range(len(X_validate)):\n", | ||||
|     "    result = nn.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 2)[0]\n", | ||||
|     "    if np.argmax(result) == np.argmax(Y_validate[x]):\n", | ||||
|     "        if np.argmax(Y_validate[x]) == 0: neg_ok += 1\n", | ||||
|     "        else: pos_ok += 1\n", | ||||
|     "    if np.argmax(Y_validate[x]) == 0: neg_cnt += 1\n", | ||||
|     "    else: pos_cnt += 1" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "pos_acc 39.58333333333333 %\n", | ||||
|       "neg_acc 95.29702970297029 %\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "print(\"pos_acc\", pos_ok/pos_cnt*100, \"%\")\n", | ||||
|     "print(\"neg_acc\", neg_ok/neg_cnt*100, \"%\")" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "[[  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  48  37\n", | ||||
|       "  311 189   4 144  22  16   1 281]]\n", | ||||
|       "[0.8364928  0.16350722]\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "X2 = ['what are u going to say about that? the truth, wassock?!']\n", | ||||
|     "X2 = tok.texts_to_sequences(X2)\n", | ||||
|     "X2 = pad_sequences(X2, maxlen=26, dtype='int32', value=0)\n", | ||||
|     "print(X2)\n", | ||||
|     "print(nn.predict(X2, batch_size=1, verbose = 2)[0])" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.6.5" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 2 | ||||
| } | ||||
							
								
								
									
										94
									
								
								Jonas_Solutions/output.ttl
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										94
									
								
								Jonas_Solutions/output.ttl
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,94 @@ | ||||
| @prefix dct: <http://purl.org/dc/terms/> . | ||||
| @prefix mexalgo: <http://mex.aksw.org/mex-algo#> . | ||||
| @prefix mexcore: <http://mex.aksw.org/mex-core#> . | ||||
| @prefix mexperf: <http://mex.aksw.org/mex-perf#> . | ||||
| @prefix prov: <http://www.w3.org/ns/prov#> . | ||||
| @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> . | ||||
| @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> . | ||||
| @prefix this: <http://mex.aksw.org/examples/> . | ||||
| @prefix xml: <http://www.w3.org/XML/1998/namespace> . | ||||
| @prefix xsd: <http://www.w3.org/2001/XMLSchema#> . | ||||
|  | ||||
| this:measure1 a mexcore:PerformanceMeasure ; | ||||
|     mexperf:accuracy "0.8478535353535354"^^xsd:float ; | ||||
|     mexperf:precision "0.8225446428571429"^^xsd:float ; | ||||
|     mexperf:recall "0.8998778998778999"^^xsd:float ; | ||||
|     prov:wasGeneratedBy this:conf1 . | ||||
|  | ||||
| this:measure2 a mexcore:PerformanceMeasure ; | ||||
|     mexperf:accuracy "0.5776805251641138"^^xsd:float ; | ||||
|     mexperf:precision "0.5369458128078818"^^xsd:float ; | ||||
|     mexperf:recall "0.5240384615384616"^^xsd:float ; | ||||
|     prov:wasGeneratedBy this:conf2 . | ||||
|  | ||||
| this:measure3a a mexcore:PerformanceMeasure ; | ||||
|     mexperf:accuracy "0.5391414141414141"^^xsd:float ; | ||||
|     mexperf:precision "0.5341130604288499"^^xsd:float ; | ||||
|     mexperf:recall "0.3581699346405229"^^xsd:float ; | ||||
|     prov:wasGeneratedBy this:conf3a . | ||||
|  | ||||
| this:measure3b a mexcore:PerformanceMeasure ; | ||||
|     mexperf:accuracy "0.5142231947483589"^^xsd:float ; | ||||
|     mexperf:precision "0.25"^^xsd:float ; | ||||
|     mexperf:recall "0.03365384615384615"^^xsd:float ; | ||||
|     prov:wasGeneratedBy this:conf3b . | ||||
|  | ||||
| this:measure4 a mexcore:PerformanceMeasure ; | ||||
|     mexperf:accuracy "0.7714856762158561"^^xsd:float ; | ||||
|     mexperf:precision "0.7680865449628127"^^xsd:float ; | ||||
|     mexperf:recall "0.7680865449628127"^^xsd:float ; | ||||
|     prov:wasGeneratedBy this:conf4 . | ||||
|  | ||||
| this:Dataset03 a mexcore:dataset ; | ||||
|     rdfs:label "Dataset03" . | ||||
|  | ||||
| this:conf1 a mexcore:ExperimentConfiguration ; | ||||
|     rdfs:label "conf1" ; | ||||
|     prov:used this:Dataset01, | ||||
|         this:model_a ; | ||||
|     prov:wasStartedBy this:jonas_weinz_task_2 . | ||||
|  | ||||
| this:conf2 a mexcore:ExperimentConfiguration ; | ||||
|     rdfs:label "conf2" ; | ||||
|     prov:used this:Dataset02, | ||||
|         this:model_b ; | ||||
|     prov:wasStartedBy this:jonas_weinz_task_2 . | ||||
|  | ||||
| this:conf3a a mexcore:ExperimentConfiguration ; | ||||
|     rdfs:label "conf3a" ; | ||||
|     prov:used this:Dataset01, | ||||
|         this:model_b ; | ||||
|     prov:wasStartedBy this:jonas_weinz_task_2 . | ||||
|  | ||||
| this:conf3b a mexcore:ExperimentConfiguration ; | ||||
|     rdfs:label "conf3b" ; | ||||
|     prov:used this:Dataset02, | ||||
|         this:model_a ; | ||||
|     prov:wasStartedBy this:jonas_weinz_task_2 . | ||||
|  | ||||
| this:conf4 a mexcore:ExperimentConfiguration ; | ||||
|     rdfs:label "conf4" ; | ||||
|     prov:used this:Dataset03, | ||||
|         this:model_c ; | ||||
|     prov:wasStartedBy this:jonas_weinz_task_2 . | ||||
|  | ||||
| this:model_c a mexalgo:Algorithm ; | ||||
|     rdfs:label "model_c" ; | ||||
|     dct:identifier "MLPClassifier" . | ||||
|  | ||||
| this:Dataset01 a mexcore:dataset ; | ||||
|     rdfs:label "Dataset01" . | ||||
|  | ||||
| this:Dataset02 a mexcore:dataset ; | ||||
|     rdfs:label "Dataset02" . | ||||
|  | ||||
| this:model_a a mexalgo:Algorithm ; | ||||
|     rdfs:label "model_a" ; | ||||
|     dct:identifier "RandomForestClassifier" . | ||||
|  | ||||
| this:model_b a mexalgo:Algorithm ; | ||||
|     rdfs:label "model_b" ; | ||||
|     dct:identifier "MLPClassifier" . | ||||
|  | ||||
| this:jonas_weinz_task_2 a mexcore:Experiment . | ||||
|  | ||||
							
								
								
									
										423
									
								
								Project/Tools/emoji tester old.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										423
									
								
								Project/Tools/emoji tester old.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,423 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import pandas as pd\n", | ||||
|     "from IPython.display import clear_output, Markdown, Math\n", | ||||
|     "import ipywidgets as widgets\n", | ||||
|     "import os" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## file input stuff:\n", | ||||
|     "\n", | ||||
|     "* replace `test.txt` with yout whatsapp log file" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "./whatsapp2csv.sh test.txt" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* read table `test.csv` exported by `whatsapp2csv.sh`" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "messages = pd.read_csv('test.txt.csv', delimiter='\\t')\n", | ||||
|     "messages.head()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "if [ ! -e emoji-data.txt ]\n", | ||||
|     "then\n", | ||||
|     "    echo \"downloading emoji specification\"\n", | ||||
|     "    wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n", | ||||
|     "else\n", | ||||
|     "    echo \"found existing emoji specification\"\n", | ||||
|     "fi\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "emoji_blacklist = set([\n", | ||||
|     "    0x1F3FB,\n", | ||||
|     "    0x1F3FC,\n", | ||||
|     "    0x1F3FD,\n", | ||||
|     "    0x1F3FE,\n", | ||||
|     "    0x1F3FF\n", | ||||
|     "])" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n", | ||||
|     "emoji_data['type'] = emoji_data['type'].str.strip()\n", | ||||
|     "emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n", | ||||
|     "emoji_data" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* now build a set out of the unicode types" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "ord(\"😀\") == int('0x1f600',16)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "emoji_codes = emoji_data['unicode']\n", | ||||
|     "emoji_codes.head()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* we have to iterate over the whole list and extract all given ranges:" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "emoji_code_list = []\n", | ||||
|     "for entry in emoji_codes:\n", | ||||
|     "    # testing whether we have an entry or a range:\n", | ||||
|     "    if '.' in entry:\n", | ||||
|     "        # range\n", | ||||
|     "        a,b = entry.split(\"..\")\n", | ||||
|     "        for i in range(int(a,16),int(b,16) +1):\n", | ||||
|     "            if i not in emoji_blacklist:\n", | ||||
|     "                emoji_code_list.append(i)\n", | ||||
|     "    else:\n", | ||||
|     "        # single entry\n", | ||||
|     "        if i not in emoji_blacklist:\n", | ||||
|     "            emoji_code_list.append(int(entry,16))\n", | ||||
|     "emoji_code_set = set(emoji_code_list)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "# simple test:\n", | ||||
|     "print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* expanding column and fill new emojis" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "messages[\"emojis\"] = None" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "for i in messages.index:\n", | ||||
|     "    emoji_list = []\n", | ||||
|     "    to_remove = []\n", | ||||
|     "    m = messages.iloc[i]['message']\n", | ||||
|     "    for c in str(m):\n", | ||||
|     "        if ord(c) in emoji_code_set:\n", | ||||
|     "            emoji_list.append(c)\n", | ||||
|     "        elif ord(c) in emoji_blacklist:\n", | ||||
|     "            to_remove.append(c)\n", | ||||
|     "            \n", | ||||
|     "    messages.loc[i,'emojis'] = emoji_list\n", | ||||
|     "    #remove emiójis from message\n", | ||||
|     "    for e in (emoji_list + to_remove):\n", | ||||
|     "        m = m.replace(e,\"\")\n", | ||||
|     "    messages.loc[i,'message'] = m\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "messages[:20]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* get a list only containing messaged with emojis" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n", | ||||
|     "emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "display(emoji_messages)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## learning part" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import numpy as np\n", | ||||
|     "import itertools\n", | ||||
|     "import sklearn.utils as sku\n", | ||||
|     "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", | ||||
|     "from sklearn.model_selection import train_test_split" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "labels=[e[-1] for e in emoji_messages['emojis']]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "labels[:10]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "vectorizer = TfidfVectorizer(stop_words='english')\n", | ||||
|     "vec_train = vectorizer.fit_transform(X1)\n", | ||||
|     "vec_test = vectorizer.transform(Xt1)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "from sklearn.ensemble import RandomForestClassifier as RFC\n", | ||||
|     "from sklearn.neural_network import MLPClassifier as MLP\n", | ||||
|     "from sklearn.naive_bayes import MultinomialNB as MNB\n", | ||||
|     "#clf_a = RFC(criterion='entropy', random_state=4222)\n", | ||||
|     "clf_a = MLP()\n", | ||||
|     "clf_a.fit(vec_train, y1)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "pred = clf_a.predict(vectorizer.transform(Xt1))\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n", | ||||
|     "testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n", | ||||
|     "testlist.to_csv('export.csv')\n", | ||||
|     "testlist" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "print(chr(0x1F3F))" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "vec_train[0]" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "\n", | ||||
|     "out = widgets.Output()\n", | ||||
|     "\n", | ||||
|     "t = widgets.Text()\n", | ||||
|     "b = widgets.Button(\n", | ||||
|     "    description='get smiley',\n", | ||||
|     "    disabled=False,\n", | ||||
|     "    button_style='', # 'success', 'info', 'warning', 'danger' or ''\n", | ||||
|     "    tooltip='Click me',\n", | ||||
|     "    icon='check'\n", | ||||
|     ")\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "\n", | ||||
|     "def handle_submit(sender):\n", | ||||
|     "    with out:\n", | ||||
|     "        clear_output()\n", | ||||
|     "    with out:\n", | ||||
|     "        display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n", | ||||
|     "\n", | ||||
|     "b.on_click(handle_submit)\n", | ||||
|     "    \n", | ||||
|     "display(t)\n", | ||||
|     "display(widgets.VBox([b, out]))  " | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.6.5" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 2 | ||||
| } | ||||
							
								
								
									
										676
									
								
								Project/Tools/emoji tester.ipynb
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										676
									
								
								Project/Tools/emoji tester.ipynb
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,676 @@ | ||||
| { | ||||
|  "cells": [ | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 1, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import pandas as pd\n", | ||||
|     "from IPython.display import clear_output, Markdown, Math\n", | ||||
|     "import ipywidgets as widgets\n", | ||||
|     "import os" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def create_widgets(t_text, b_text, out, additional_widgets=[]):\n", | ||||
|     "    texts = []\n", | ||||
|     "    for t in t_text:\n", | ||||
|     "        texts.append(widgets.Text(t))\n", | ||||
|     "    \n", | ||||
|     "    button = widgets.Button(\n", | ||||
|     "        description=b_text,\n", | ||||
|     "        disabled=False,\n", | ||||
|     "        button_style='', # 'success', 'info', 'warning', 'danger' or ''\n", | ||||
|     "        tooltip=b_text,\n", | ||||
|     "        icon='check'\n", | ||||
|     "    )\n", | ||||
|     "    display(widgets.VBox([widgets.HBox(texts + additional_widgets + [button]), out]))\n", | ||||
|     "    return texts + [button]\n", | ||||
|     "\n", | ||||
|     "out_convert = widgets.Output()\n", | ||||
|     "out_build = widgets.Output()\n", | ||||
|     "out_train = widgets.Output()\n", | ||||
|     "out_save = widgets.Output()\n", | ||||
|     "out_read = widgets.Output()\n", | ||||
|     "out_test = widgets.Output()\n", | ||||
|     "\n", | ||||
|     "def mp(msg):\n", | ||||
|     "    display(Markdown(msg))" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "# Emoji Tester\n", | ||||
|     "\n", | ||||
|     "just run all cells at first. Then select on of the actions below." | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## converting plain whatsapp export to csv" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "8c94a8d3b3724ad08359817b2086cf85", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(HBox(children=(Text(value='test.txt'), Button(description='convert whatsapp file to csv', icon='check', style=ButtonStyle(), tooltip='convert whatsapp file to csv'))), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## read csv and build database" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "ee22e300097e49f1ac24b11662f7dc69", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(HBox(children=(Text(value='test.txt.csv'), Checkbox(value=False, description='using only last emoji'), Button(description='read', icon='check', style=ButtonStyle(), tooltip='read'))), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## Train" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "de06136ee346492d80ef39d304cbc31c", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(HBox(children=(Dropdown(description='Learning Method', index=1, options=('DecisionTree', 'MLP', 'RandomForest'), value='MLP'), Checkbox(value=False, description='Using one vs all (very slow, only with multi-label!)'), Button(description='train', icon='check', style=ButtonStyle(), tooltip='train'))), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## save trained classifier" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "b046f3ada7ec4ba7a17965d718552d21", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='save classifier', icon='check', style=ButtonStyle(), tooltip='save classifier'))), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## import trained classifier" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "ac4054e75d8a4214ad040b1b21d6c925", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='import classifier', icon='check', style=ButtonStyle(), tooltip='import classifier'))), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "text/markdown": [ | ||||
|        "## predict emoji on custom text" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "<IPython.core.display.Markdown object>" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     }, | ||||
|     { | ||||
|      "data": { | ||||
|       "application/vnd.jupyter.widget-view+json": { | ||||
|        "model_id": "db631acbe9c94cac907efaf501a69c6a", | ||||
|        "version_major": 2, | ||||
|        "version_minor": 0 | ||||
|       }, | ||||
|       "text/html": [ | ||||
|        "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", | ||||
|        "  that the widgets JavaScript is still loading. If this message persists, it\n", | ||||
|        "  likely means that the widgets JavaScript library is either not installed or\n", | ||||
|        "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n", | ||||
|        "  Widgets Documentation</a> for setup instructions.\n", | ||||
|        "</p>\n", | ||||
|        "<p>\n", | ||||
|        "  If you're reading this message in another frontend (for example, a static\n", | ||||
|        "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n", | ||||
|        "  it may mean that your frontend doesn't currently support widgets.\n", | ||||
|        "</p>\n" | ||||
|       ], | ||||
|       "text/plain": [ | ||||
|        "VBox(children=(HBox(children=(Text(value=''), Checkbox(value=False, description='Show probabilities (only on trees)'), Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='get emoji'))), Output()))" | ||||
|       ] | ||||
|      }, | ||||
|      "metadata": {}, | ||||
|      "output_type": "display_data" | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "mp(\"## converting plain whatsapp export to csv\")\n", | ||||
|     "t_convert, b_convert = create_widgets([\"test.txt\"], \"convert whatsapp file to csv\", out_convert)\n", | ||||
|     "mp(\"## read csv and build database\")\n", | ||||
|     "single_label = widgets.Checkbox(value=False, description='using only last emoji', disable=False)\n", | ||||
|     "t_build, b_build = create_widgets([\"test.txt.csv\"], \"read\", out_build, [single_label])\n", | ||||
|     "mp(\"## Train\")\n", | ||||
|     "d = widgets.Dropdown(options=['DecisionTree', 'MLP', 'RandomForest'], value='MLP', description='Learning Method', disabled=False)\n", | ||||
|     "ova = widgets.Checkbox(value=False, description='Using one vs all (very slow, only with multi-label!)', disabled=False)\n", | ||||
|     "b_train = button = widgets.Button(description=\"train\", disabled=False, button_style='', tooltip=\"train\",icon='check')\n", | ||||
|     "display(widgets.VBox([widgets.HBox([d,ova,b_train]), out_train]))\n", | ||||
|     "mp(\"## save trained classifier\")\n", | ||||
|     "t_save_c, t_save_m, t_save_v, b_save = create_widgets([\"clf.pkl\", \"mlb.pkl\", \"vectorizer.pkl\"], \"save classifier\", out_save)\n", | ||||
|     "mp(\"## import trained classifier\")\n", | ||||
|     "t_read_c, t_read_m, t_read_v, b_read = create_widgets([\"clf.pkl\", \"mlb.pkl\", \"vectorizer.pkl\"], \"import classifier\", out_read)\n", | ||||
|     "mp(\"## predict emoji on custom text\")\n", | ||||
|     "b_prop = widgets.Checkbox(value=False, description='Show probabilities (only on trees)', disabled=False)\n", | ||||
|     "t_test, b_test = create_widgets([\"\"], \"get emoji\", out_test,[b_prop])" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "----\n", | ||||
|     "## Code Section:" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def convert(b):\n", | ||||
|     "    with out_convert:\n", | ||||
|     "        clear_output()\n", | ||||
|     "    with out_convert:\n", | ||||
|     "        mp(\"**converting \" + t_convert.value + \"…**\")\n", | ||||
|     "        import subprocess\n", | ||||
|     "        print(str(subprocess.check_output([\"./whatsapp2csv.sh\", t_convert.value])).strip())\n", | ||||
|     "        mp(\"**done**\")\n", | ||||
|     "\n", | ||||
|     "b_convert.on_click(convert)\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* download emoji specification if not already existing" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "name": "stdout", | ||||
|      "output_type": "stream", | ||||
|      "text": [ | ||||
|       "found existing emoji specification\n" | ||||
|      ] | ||||
|     } | ||||
|    ], | ||||
|    "source": [ | ||||
|     "%%bash\n", | ||||
|     "if [ ! -e emoji-data.txt ]\n", | ||||
|     "then\n", | ||||
|     "    echo \"downloading emoji specification\"\n", | ||||
|     "    wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n", | ||||
|     "else\n", | ||||
|     "    echo \"found existing emoji specification\"\n", | ||||
|     "fi\n" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* stuff for creating emoji database" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "emoji_blacklist = set([\n", | ||||
|     "    0x1F3FB,\n", | ||||
|     "    0x1F3FC,\n", | ||||
|     "    0x1F3FD,\n", | ||||
|     "    0x1F3FE,\n", | ||||
|     "    0x1F3FF,\n", | ||||
|     "    0x2642,\n", | ||||
|     "    0x2640\n", | ||||
|     "])\n", | ||||
|     "\n", | ||||
|     "emoji_code_set = None\n", | ||||
|     "\n", | ||||
|     "def create_emoji_set():\n", | ||||
|     "    global emoji_code_set\n", | ||||
|     "    \n", | ||||
|     "    emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n", | ||||
|     "    emoji_data['type'] = emoji_data['type'].str.strip()\n", | ||||
|     "    emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n", | ||||
|     "    \n", | ||||
|     "    emoji_codes = emoji_data['unicode']\n", | ||||
|     "    emoji_codes.head()\n", | ||||
|     "    \n", | ||||
|     "    emoji_code_list = []\n", | ||||
|     "    for entry in emoji_codes:\n", | ||||
|     "        # testing whether we have an entry or a range:\n", | ||||
|     "        if '.' in entry:\n", | ||||
|     "            # range\n", | ||||
|     "            a,b = entry.split(\"..\")\n", | ||||
|     "            for i in range(int(a,16),int(b,16) +1):\n", | ||||
|     "                if i not in emoji_blacklist:\n", | ||||
|     "                    emoji_code_list.append(i)\n", | ||||
|     "        else:\n", | ||||
|     "            # single entry\n", | ||||
|     "            if i not in emoji_blacklist:\n", | ||||
|     "                emoji_code_list.append(int(entry,16))\n", | ||||
|     "    emoji_code_set = set(emoji_code_list)\n", | ||||
|     "    display(Markdown(\"**imported Emojis** (without modifier):\\n>\" + \"\".join([chr(x) for x in emoji_code_set])))\n", | ||||
|     "    display(Markdown(\"**blacklisted Emojis:**\\n>\" + \"\".join([chr(x) for x in emoji_blacklist])))\n", | ||||
|     "    f = open('emoji-list.txt', 'w')\n", | ||||
|     "    for e in emoji_code_set:\n", | ||||
|     "        f.write(chr(e) + \"\\n\")\n", | ||||
|     "    f.close()" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "markdown", | ||||
|    "metadata": {}, | ||||
|    "source": [ | ||||
|     "* stuff for reading whatsapp messages" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "messages = None\n", | ||||
|     "vectorizer = None\n", | ||||
|     "clf_a = None\n", | ||||
|     "mlb = None\n", | ||||
|     "\n", | ||||
|     "emoji_messages=None" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def read_message_and_build_db(filename):\n", | ||||
|     "    global messages\n", | ||||
|     "    global emoji_messages\n", | ||||
|     "    global vectorizer\n", | ||||
|     "    global clf_a\n", | ||||
|     "    global mlb\n", | ||||
|     "    \n", | ||||
|     "    messages = pd.read_csv(filename, delimiter='\\t')\n", | ||||
|     "    mp(\"**filter messages and creating labels. This can take a while...**\")\n", | ||||
|     "    messages[\"emojis\"] = None\n", | ||||
|     "    \n", | ||||
|     "    msg_batchsize = 1000\n", | ||||
|     "    msg_counter = 0\n", | ||||
|     "    \n", | ||||
|     "    for i in messages.index:\n", | ||||
|     "        \n", | ||||
|     "        msg_counter+=1\n", | ||||
|     "        if msg_counter >= msg_batchsize:\n", | ||||
|     "            print(str(100 * i / messages.shape[0]) + \"%\")\n", | ||||
|     "            msg_counter=0\n", | ||||
|     "        \n", | ||||
|     "        emoji_list = []\n", | ||||
|     "        m = messages.iloc[i]['message']\n", | ||||
|     "        m_new = \"\"\n", | ||||
|     "        for c in str(m):\n", | ||||
|     "            if ord(c) in emoji_code_set:\n", | ||||
|     "                emoji_list.append(c)\n", | ||||
|     "            elif ord(c) not in emoji_blacklist:\n", | ||||
|     "                m_new += c\n", | ||||
|     "        # if single label: only use last found emoji\n", | ||||
|     "        messages.loc[i,'emojis'] = set(emoji_list) if (not single_label.value) or len(emoji_list)==0 else set(emoji_list[-1])\n", | ||||
|     "        #remove emiójis from message\n", | ||||
|     "        messages.loc[i,'message'] = m_new\n", | ||||
|     "    \n", | ||||
|     "    emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n", | ||||
|     "    emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]\n", | ||||
|     "    \n", | ||||
|     "    mp(\"**Done**\")\n", | ||||
|     "    \n", | ||||
|     "    display(emoji_messages)\n", | ||||
|     "\n", | ||||
|     "def train(b):\n", | ||||
|     "    global messages\n", | ||||
|     "    global emoji_messages\n", | ||||
|     "    global vectorizer\n", | ||||
|     "    global clf_a\n", | ||||
|     "    global mlb\n", | ||||
|     "    with out_train:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        # train part:\n", | ||||
|     "        import numpy as np\n", | ||||
|     "        import itertools\n", | ||||
|     "        import sklearn.utils as sku\n", | ||||
|     "        from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n", | ||||
|     "        from sklearn.model_selection import train_test_split\n", | ||||
|     "        from sklearn.preprocessing import MultiLabelBinarizer\n", | ||||
|     "\n", | ||||
|     "        mlb = MultiLabelBinarizer() if not single_label.value else None\n", | ||||
|     "        \n", | ||||
|     "        if not mlb:\n", | ||||
|     "            l = [list(e)[-1] for e in emoji_messages['emojis']]\n", | ||||
|     "        \n", | ||||
|     "        labels=mlb.fit_transform(emoji_messages['emojis']) if mlb else l\n", | ||||
|     "        \n", | ||||
|     "        if mlb:\n", | ||||
|     "            display(Markdown(\"**emojis contained in Dataset:**\\n >\" + \"\".join(mlb.classes_ )))\n", | ||||
|     "        else:\n", | ||||
|     "            display(Markdown(\"**emojis contained in Dataset:**\\n >\" + \"\".join(set(l))))\n", | ||||
|     "\n", | ||||
|     "        X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)\n", | ||||
|     "\n", | ||||
|     "        vectorizer = TfidfVectorizer(stop_words='english')\n", | ||||
|     "        vec_train = vectorizer.fit_transform(X1)\n", | ||||
|     "        vec_test = vectorizer.transform(Xt1)\n", | ||||
|     "\n", | ||||
|     "        mp(\"**train classifier. This can take a very long time… Grab a coffe! 😀**\")\n", | ||||
|     "\n", | ||||
|     "        from sklearn.ensemble import RandomForestClassifier as RFC\n", | ||||
|     "        from sklearn.neural_network import MLPClassifier as MLP\n", | ||||
|     "        #from sklearn.naive_bayes import MultinomialNB as MNB\n", | ||||
|     "        from sklearn.tree import DecisionTreeClassifier as DTC\n", | ||||
|     "        from sklearn.multiclass import OneVsRestClassifier as OVRC\n", | ||||
|     "        clf_a = None\n", | ||||
|     "        if (d.value == \"DecisionTree\"):\n", | ||||
|     "            clf_a = DTC()\n", | ||||
|     "        elif d.value == \"MLP\":\n", | ||||
|     "            clf_a = MLP(hidden_layer_sizes=(64,))\n", | ||||
|     "        elif d.value == \"RandomForest\":\n", | ||||
|     "            RFC(criterion='entropy', random_state=4222)\n", | ||||
|     "\n", | ||||
|     "        if ova.value:\n", | ||||
|     "            clf_a=OVRC(clf_a)\n", | ||||
|     "\n", | ||||
|     "        display(clf_a)\n", | ||||
|     "        clf_a.fit(vec_train, y1)\n", | ||||
|     "\n", | ||||
|     "        mp(\"**training done**\")\n", | ||||
|     "\n", | ||||
|     "        pred = clf_a.predict(vectorizer.transform(Xt1))\n", | ||||
|     "\n", | ||||
|     "        testlist = pd.DataFrame({'message': Xt1, 'pred': mlb.inverse_transform(pred) if mlb else pred, 'teacher': mlb.inverse_transform(yt1) if mlb else yt1})\n", | ||||
|     "        testlist.to_csv('export.csv')\n", | ||||
|     "        display(testlist)\n", | ||||
|     "    \n", | ||||
|     "def build_db(b):\n", | ||||
|     "    with out_build:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        create_emoji_set()\n", | ||||
|     "        read_message_and_build_db(t_build.value)\n", | ||||
|     "b_build.on_click(build_db)\n", | ||||
|     "b_train.on_click(train)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "from sklearn.externals import joblib\n", | ||||
|     "def write_to_file(b):\n", | ||||
|     "    global vectorizer\n", | ||||
|     "    global clf_a\n", | ||||
|     "    global mlb\n", | ||||
|     "    \n", | ||||
|     "    with out_save:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"**write to file...**\")\n", | ||||
|     "        joblib.dump(clf_a, t_save_c.value)\n", | ||||
|     "        if mlb:\n", | ||||
|     "            joblib.dump(mlb, t_save_m.value) \n", | ||||
|     "        joblib.dump(vectorizer, t_save_v.value)\n", | ||||
|     "        mp(\"**done**\")\n", | ||||
|     "b_save.on_click(write_to_file)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def read_from_file(b):\n", | ||||
|     "    global vectorizer\n", | ||||
|     "    global clf_a\n", | ||||
|     "    global mlb\n", | ||||
|     "    \n", | ||||
|     "    with out_read:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        mp(\"**read from file…**\")\n", | ||||
|     "        clf_a = joblib.load(t_read_c.value)\n", | ||||
|     "        if t_read_m.value != \"\":\n", | ||||
|     "            mlb = joblib.load(t_read_m.value)\n", | ||||
|     "        else:\n", | ||||
|     "            mlb = None\n", | ||||
|     "        vectorizer = joblib.load(t_read_v.value)\n", | ||||
|     "        mp(\"**done**\")\n", | ||||
|     "b_read.on_click(read_from_file)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "def predict(b):\n", | ||||
|     "    with out_test:\n", | ||||
|     "        clear_output()\n", | ||||
|     "        v = mlb.inverse_transform(clf_a.predict(vectorizer.transform([t_test.value])))[0] if mlb else clf_a.predict(vectorizer.transform([t_test.value]))[0]\n", | ||||
|     "        mp(\"**prediction:**\\n# \" + (\"\".join(v) if len(v)>0 else \" \"))\n", | ||||
|     "        if b_prop.value:\n", | ||||
|     "            pred = clf_a.predict_proba(vectorizer.transform([t_test.value]))\n", | ||||
|     "            print(mlb.inverse_transform(pred))\n", | ||||
|     "\n", | ||||
|     "b_test.on_click(predict)" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": null, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [] | ||||
|   } | ||||
|  ], | ||||
|  "metadata": { | ||||
|   "kernelspec": { | ||||
|    "display_name": "Python 3", | ||||
|    "language": "python", | ||||
|    "name": "python3" | ||||
|   }, | ||||
|   "language_info": { | ||||
|    "codemirror_mode": { | ||||
|     "name": "ipython", | ||||
|     "version": 3 | ||||
|    }, | ||||
|    "file_extension": ".py", | ||||
|    "mimetype": "text/x-python", | ||||
|    "name": "python", | ||||
|    "nbconvert_exporter": "python", | ||||
|    "pygments_lexer": "ipython3", | ||||
|    "version": "3.6.5" | ||||
|   } | ||||
|  }, | ||||
|  "nbformat": 4, | ||||
|  "nbformat_minor": 2 | ||||
| } | ||||
							
								
								
									
										714
									
								
								Project/Tools/emoji-data.txt
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										714
									
								
								Project/Tools/emoji-data.txt
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,714 @@ | ||||
| # emoji-data.txt | ||||
| # Date: 2018-02-07, 07:55:18 GMT | ||||
| # © 2018 Unicode®, Inc. | ||||
| # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. | ||||
| # For terms of use, see http://www.unicode.org/terms_of_use.html | ||||
| # | ||||
| # Emoji Data for UTS #51 | ||||
| # Version: 11.0 | ||||
| # | ||||
| # For documentation and usage, see http://www.unicode.org/reports/tr51 | ||||
| # | ||||
| # Format:  | ||||
| # <codepoint(s)> ; <property> # <comments>  | ||||
| # Note: there is no guarantee as to the structure of whitespace or comments | ||||
| # | ||||
| # Characters and sequences are listed in code point order. Users should be shown a more natural order. | ||||
| # See the CLDR collation order for Emoji. | ||||
|  | ||||
|  | ||||
| # ================================================ | ||||
|  | ||||
| # All omitted code points have Emoji=No  | ||||
| # @missing: 0000..10FFFF  ; Emoji ; No | ||||
|  | ||||
| 0023          ; Emoji                #  1.1  [1] (#️)       number sign | ||||
| 002A          ; Emoji                #  1.1  [1] (*️)       asterisk | ||||
| 0030..0039    ; Emoji                #  1.1 [10] (0️..9️)    digit zero..digit nine | ||||
| 00A9          ; Emoji                #  1.1  [1] (©️)       copyright | ||||
| 00AE          ; Emoji                #  1.1  [1] (®️)       registered | ||||
| 203C          ; Emoji                #  1.1  [1] (‼️)       double exclamation mark | ||||
| 2049          ; Emoji                #  3.0  [1] (⁉️)       exclamation question mark | ||||
| 2122          ; Emoji                #  1.1  [1] (™️)       trade mark | ||||
| 2139          ; Emoji                #  3.0  [1] (ℹ️)       information | ||||
| 2194..2199    ; Emoji                #  1.1  [6] (↔️..↙️)    left-right arrow..down-left arrow | ||||
| 21A9..21AA    ; Emoji                #  1.1  [2] (↩️..↪️)    right arrow curving left..left arrow curving right | ||||
| 231A..231B    ; Emoji                #  1.1  [2] (⌚..⌛)    watch..hourglass done | ||||
| 2328          ; Emoji                #  1.1  [1] (⌨️)       keyboard | ||||
| 23CF          ; Emoji                #  4.0  [1] (⏏️)       eject button | ||||
| 23E9..23F3    ; Emoji                #  6.0 [11] (⏩..⏳)    fast-forward button..hourglass not done | ||||
| 23F8..23FA    ; Emoji                #  7.0  [3] (⏸️..⏺️)    pause button..record button | ||||
| 24C2          ; Emoji                #  1.1  [1] (Ⓜ️)       circled M | ||||
| 25AA..25AB    ; Emoji                #  1.1  [2] (▪️..▫️)    black small square..white small square | ||||
| 25B6          ; Emoji                #  1.1  [1] (▶️)       play button | ||||
| 25C0          ; Emoji                #  1.1  [1] (◀️)       reverse button | ||||
| 25FB..25FE    ; Emoji                #  3.2  [4] (◻️..◾)    white medium square..black medium-small square | ||||
| 2600..2604    ; Emoji                #  1.1  [5] (☀️..☄️)    sun..comet | ||||
| 260E          ; Emoji                #  1.1  [1] (☎️)       telephone | ||||
| 2611          ; Emoji                #  1.1  [1] (☑️)       ballot box with check | ||||
| 2614..2615    ; Emoji                #  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage | ||||
| 2618          ; Emoji                #  4.1  [1] (☘️)       shamrock | ||||
| 261D          ; Emoji                #  1.1  [1] (☝️)       index pointing up | ||||
| 2620          ; Emoji                #  1.1  [1] (☠️)       skull and crossbones | ||||
| 2622..2623    ; Emoji                #  1.1  [2] (☢️..☣️)    radioactive..biohazard | ||||
| 2626          ; Emoji                #  1.1  [1] (☦️)       orthodox cross | ||||
| 262A          ; Emoji                #  1.1  [1] (☪️)       star and crescent | ||||
| 262E..262F    ; Emoji                #  1.1  [2] (☮️..☯️)    peace symbol..yin yang | ||||
| 2638..263A    ; Emoji                #  1.1  [3] (☸️..☺️)    wheel of dharma..smiling face | ||||
| 2640          ; Emoji                #  1.1  [1] (♀️)       female sign | ||||
| 2642          ; Emoji                #  1.1  [1] (♂️)       male sign | ||||
| 2648..2653    ; Emoji                #  1.1 [12] (♈..♓)    Aries..Pisces | ||||
| 265F..2660    ; Emoji                #  1.1  [2] (♟️..♠️)    chess pawn..spade suit | ||||
| 2663          ; Emoji                #  1.1  [1] (♣️)       club suit | ||||
| 2665..2666    ; Emoji                #  1.1  [2] (♥️..♦️)    heart suit..diamond suit | ||||
| 2668          ; Emoji                #  1.1  [1] (♨️)       hot springs | ||||
| 267B          ; Emoji                #  3.2  [1] (♻️)       recycling symbol | ||||
| 267E..267F    ; Emoji                #  4.1  [2] (♾️..♿)    infinity..wheelchair symbol | ||||
| 2692..2697    ; Emoji                #  4.1  [6] (⚒️..⚗️)    hammer and pick..alembic | ||||
| 2699          ; Emoji                #  4.1  [1] (⚙️)       gear | ||||
| 269B..269C    ; Emoji                #  4.1  [2] (⚛️..⚜️)    atom symbol..fleur-de-lis | ||||
| 26A0..26A1    ; Emoji                #  4.0  [2] (⚠️..⚡)    warning..high voltage | ||||
| 26AA..26AB    ; Emoji                #  4.1  [2] (⚪..⚫)    white circle..black circle | ||||
| 26B0..26B1    ; Emoji                #  4.1  [2] (⚰️..⚱️)    coffin..funeral urn | ||||
| 26BD..26BE    ; Emoji                #  5.2  [2] (⚽..⚾)    soccer ball..baseball | ||||
| 26C4..26C5    ; Emoji                #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind cloud | ||||
| 26C8          ; Emoji                #  5.2  [1] (⛈️)       cloud with lightning and rain | ||||
| 26CE          ; Emoji                #  6.0  [1] (⛎)       Ophiuchus | ||||
| 26CF          ; Emoji                #  5.2  [1] (⛏️)       pick | ||||
| 26D1          ; Emoji                #  5.2  [1] (⛑️)       rescue worker’s helmet | ||||
| 26D3..26D4    ; Emoji                #  5.2  [2] (⛓️..⛔)    chains..no entry | ||||
| 26E9..26EA    ; Emoji                #  5.2  [2] (⛩️..⛪)    shinto shrine..church | ||||
| 26F0..26F5    ; Emoji                #  5.2  [6] (⛰️..⛵)    mountain..sailboat | ||||
| 26F7..26FA    ; Emoji                #  5.2  [4] (⛷️..⛺)    skier..tent | ||||
| 26FD          ; Emoji                #  5.2  [1] (⛽)       fuel pump | ||||
| 2702          ; Emoji                #  1.1  [1] (✂️)       scissors | ||||
| 2705          ; Emoji                #  6.0  [1] (✅)       white heavy check mark | ||||
| 2708..2709    ; Emoji                #  1.1  [2] (✈️..✉️)    airplane..envelope | ||||
| 270A..270B    ; Emoji                #  6.0  [2] (✊..✋)    raised fist..raised hand | ||||
| 270C..270D    ; Emoji                #  1.1  [2] (✌️..✍️)    victory hand..writing hand | ||||
| 270F          ; Emoji                #  1.1  [1] (✏️)       pencil | ||||
| 2712          ; Emoji                #  1.1  [1] (✒️)       black nib | ||||
| 2714          ; Emoji                #  1.1  [1] (✔️)       heavy check mark | ||||
| 2716          ; Emoji                #  1.1  [1] (✖️)       heavy multiplication x | ||||
| 271D          ; Emoji                #  1.1  [1] (✝️)       latin cross | ||||
| 2721          ; Emoji                #  1.1  [1] (✡️)       star of David | ||||
| 2728          ; Emoji                #  6.0  [1] (✨)       sparkles | ||||
| 2733..2734    ; Emoji                #  1.1  [2] (✳️..✴️)    eight-spoked asterisk..eight-pointed star | ||||
| 2744          ; Emoji                #  1.1  [1] (❄️)       snowflake | ||||
| 2747          ; Emoji                #  1.1  [1] (❇️)       sparkle | ||||
| 274C          ; Emoji                #  6.0  [1] (❌)       cross mark | ||||
| 274E          ; Emoji                #  6.0  [1] (❎)       cross mark button | ||||
| 2753..2755    ; Emoji                #  6.0  [3] (❓..❕)    question mark..white exclamation mark | ||||
| 2757          ; Emoji                #  5.2  [1] (❗)       exclamation mark | ||||
| 2763..2764    ; Emoji                #  1.1  [2] (❣️..❤️)    heavy heart exclamation..red heart | ||||
| 2795..2797    ; Emoji                #  6.0  [3] (➕..➗)    heavy plus sign..heavy division sign | ||||
| 27A1          ; Emoji                #  1.1  [1] (➡️)       right arrow | ||||
| 27B0          ; Emoji                #  6.0  [1] (➰)       curly loop | ||||
| 27BF          ; Emoji                #  6.0  [1] (➿)       double curly loop | ||||
| 2934..2935    ; Emoji                #  3.2  [2] (⤴️..⤵️)    right arrow curving up..right arrow curving down | ||||
| 2B05..2B07    ; Emoji                #  4.0  [3] (⬅️..⬇️)    left arrow..down arrow | ||||
| 2B1B..2B1C    ; Emoji                #  5.1  [2] (⬛..⬜)    black large square..white large square | ||||
| 2B50          ; Emoji                #  5.1  [1] (⭐)       star | ||||
| 2B55          ; Emoji                #  5.2  [1] (⭕)       heavy large circle | ||||
| 3030          ; Emoji                #  1.1  [1] (〰️)       wavy dash | ||||
| 303D          ; Emoji                #  3.2  [1] (〽️)       part alternation mark | ||||
| 3297          ; Emoji                #  1.1  [1] (㊗️)       Japanese “congratulations” button | ||||
| 3299          ; Emoji                #  1.1  [1] (㊙️)       Japanese “secret” button | ||||
| 1F004         ; Emoji                #  5.1  [1] (🀄)       mahjong red dragon | ||||
| 1F0CF         ; Emoji                #  6.0  [1] (🃏)       joker | ||||
| 1F170..1F171  ; Emoji                #  6.0  [2] (🅰️..🅱️)    A button (blood type)..B button (blood type) | ||||
| 1F17E         ; Emoji                #  6.0  [1] (🅾️)       O button (blood type) | ||||
| 1F17F         ; Emoji                #  5.2  [1] (🅿️)       P button | ||||
| 1F18E         ; Emoji                #  6.0  [1] (🆎)       AB button (blood type) | ||||
| 1F191..1F19A  ; Emoji                #  6.0 [10] (🆑..🆚)    CL button..VS button | ||||
| 1F1E6..1F1FF  ; Emoji                #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z | ||||
| 1F201..1F202  ; Emoji                #  6.0  [2] (🈁..🈂️)    Japanese “here” button..Japanese “service charge” button | ||||
| 1F21A         ; Emoji                #  5.2  [1] (🈚)       Japanese “free of charge” button | ||||
| 1F22F         ; Emoji                #  5.2  [1] (🈯)       Japanese “reserved” button | ||||
| 1F232..1F23A  ; Emoji                #  6.0  [9] (🈲..🈺)    Japanese “prohibited” button..Japanese “open for business” button | ||||
| 1F250..1F251  ; Emoji                #  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button | ||||
| 1F300..1F320  ; Emoji                #  6.0 [33] (🌀..🌠)    cyclone..shooting star | ||||
| 1F321         ; Emoji                #  7.0  [1] (🌡️)       thermometer | ||||
| 1F324..1F32C  ; Emoji                #  7.0  [9] (🌤️..🌬️)    sun behind small cloud..wind face | ||||
| 1F32D..1F32F  ; Emoji                #  8.0  [3] (🌭..🌯)    hot dog..burrito | ||||
| 1F330..1F335  ; Emoji                #  6.0  [6] (🌰..🌵)    chestnut..cactus | ||||
| 1F336         ; Emoji                #  7.0  [1] (🌶️)       hot pepper | ||||
| 1F337..1F37C  ; Emoji                #  6.0 [70] (🌷..🍼)    tulip..baby bottle | ||||
| 1F37D         ; Emoji                #  7.0  [1] (🍽️)       fork and knife with plate | ||||
| 1F37E..1F37F  ; Emoji                #  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn | ||||
| 1F380..1F393  ; Emoji                #  6.0 [20] (🎀..🎓)    ribbon..graduation cap | ||||
| 1F396..1F397  ; Emoji                #  7.0  [2] (🎖️..🎗️)    military medal..reminder ribbon | ||||
| 1F399..1F39B  ; Emoji                #  7.0  [3] (🎙️..🎛️)    studio microphone..control knobs | ||||
| 1F39E..1F39F  ; Emoji                #  7.0  [2] (🎞️..🎟️)    film frames..admission tickets | ||||
| 1F3A0..1F3C4  ; Emoji                #  6.0 [37] (🎠..🏄)    carousel horse..person surfing | ||||
| 1F3C5         ; Emoji                #  7.0  [1] (🏅)       sports medal | ||||
| 1F3C6..1F3CA  ; Emoji                #  6.0  [5] (🏆..🏊)    trophy..person swimming | ||||
| 1F3CB..1F3CE  ; Emoji                #  7.0  [4] (🏋️..🏎️)    person lifting weights..racing car | ||||
| 1F3CF..1F3D3  ; Emoji                #  8.0  [5] (🏏..🏓)    cricket game..ping pong | ||||
| 1F3D4..1F3DF  ; Emoji                #  7.0 [12] (🏔️..🏟️)    snow-capped mountain..stadium | ||||
| 1F3E0..1F3F0  ; Emoji                #  6.0 [17] (🏠..🏰)    house..castle | ||||
| 1F3F3..1F3F5  ; Emoji                #  7.0  [3] (🏳️..🏵️)    white flag..rosette | ||||
| 1F3F7         ; Emoji                #  7.0  [1] (🏷️)       label | ||||
| 1F3F8..1F3FF  ; Emoji                #  8.0  [8] (🏸..🏿)    badminton..dark skin tone | ||||
| 1F400..1F43E  ; Emoji                #  6.0 [63] (🐀..🐾)    rat..paw prints | ||||
| 1F43F         ; Emoji                #  7.0  [1] (🐿️)       chipmunk | ||||
| 1F440         ; Emoji                #  6.0  [1] (👀)       eyes | ||||
| 1F441         ; Emoji                #  7.0  [1] (👁️)       eye | ||||
| 1F442..1F4F7  ; Emoji                #  6.0[182] (👂..📷)    ear..camera | ||||
| 1F4F8         ; Emoji                #  7.0  [1] (📸)       camera with flash | ||||
| 1F4F9..1F4FC  ; Emoji                #  6.0  [4] (📹..📼)    video camera..videocassette | ||||
| 1F4FD         ; Emoji                #  7.0  [1] (📽️)       film projector | ||||
| 1F4FF         ; Emoji                #  8.0  [1] (📿)       prayer beads | ||||
| 1F500..1F53D  ; Emoji                #  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button | ||||
| 1F549..1F54A  ; Emoji                #  7.0  [2] (🕉️..🕊️)    om..dove | ||||
| 1F54B..1F54E  ; Emoji                #  8.0  [4] (🕋..🕎)    kaaba..menorah | ||||
| 1F550..1F567  ; Emoji                #  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty | ||||
| 1F56F..1F570  ; Emoji                #  7.0  [2] (🕯️..🕰️)    candle..mantelpiece clock | ||||
| 1F573..1F579  ; Emoji                #  7.0  [7] (🕳️..🕹️)    hole..joystick | ||||
| 1F57A         ; Emoji                #  9.0  [1] (🕺)       man dancing | ||||
| 1F587         ; Emoji                #  7.0  [1] (🖇️)       linked paperclips | ||||
| 1F58A..1F58D  ; Emoji                #  7.0  [4] (🖊️..🖍️)    pen..crayon | ||||
| 1F590         ; Emoji                #  7.0  [1] (🖐️)       hand with fingers splayed | ||||
| 1F595..1F596  ; Emoji                #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute | ||||
| 1F5A4         ; Emoji                #  9.0  [1] (🖤)       black heart | ||||
| 1F5A5         ; Emoji                #  7.0  [1] (🖥️)       desktop computer | ||||
| 1F5A8         ; Emoji                #  7.0  [1] (🖨️)       printer | ||||
| 1F5B1..1F5B2  ; Emoji                #  7.0  [2] (🖱️..🖲️)    computer mouse..trackball | ||||
| 1F5BC         ; Emoji                #  7.0  [1] (🖼️)       framed picture | ||||
| 1F5C2..1F5C4  ; Emoji                #  7.0  [3] (🗂️..🗄️)    card index dividers..file cabinet | ||||
| 1F5D1..1F5D3  ; Emoji                #  7.0  [3] (🗑️..🗓️)    wastebasket..spiral calendar | ||||
| 1F5DC..1F5DE  ; Emoji                #  7.0  [3] (🗜️..🗞️)    clamp..rolled-up newspaper | ||||
| 1F5E1         ; Emoji                #  7.0  [1] (🗡️)       dagger | ||||
| 1F5E3         ; Emoji                #  7.0  [1] (🗣️)       speaking head | ||||
| 1F5E8         ; Emoji                #  7.0  [1] (🗨️)       left speech bubble | ||||
| 1F5EF         ; Emoji                #  7.0  [1] (🗯️)       right anger bubble | ||||
| 1F5F3         ; Emoji                #  7.0  [1] (🗳️)       ballot box with ballot | ||||
| 1F5FA         ; Emoji                #  7.0  [1] (🗺️)       world map | ||||
| 1F5FB..1F5FF  ; Emoji                #  6.0  [5] (🗻..🗿)    mount fuji..moai | ||||
| 1F600         ; Emoji                #  6.1  [1] (😀)       grinning face | ||||
| 1F601..1F610  ; Emoji                #  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face | ||||
| 1F611         ; Emoji                #  6.1  [1] (😑)       expressionless face | ||||
| 1F612..1F614  ; Emoji                #  6.0  [3] (😒..😔)    unamused face..pensive face | ||||
| 1F615         ; Emoji                #  6.1  [1] (😕)       confused face | ||||
| 1F616         ; Emoji                #  6.0  [1] (😖)       confounded face | ||||
| 1F617         ; Emoji                #  6.1  [1] (😗)       kissing face | ||||
| 1F618         ; Emoji                #  6.0  [1] (😘)       face blowing a kiss | ||||
| 1F619         ; Emoji                #  6.1  [1] (😙)       kissing face with smiling eyes | ||||
| 1F61A         ; Emoji                #  6.0  [1] (😚)       kissing face with closed eyes | ||||
| 1F61B         ; Emoji                #  6.1  [1] (😛)       face with tongue | ||||
| 1F61C..1F61E  ; Emoji                #  6.0  [3] (😜..😞)    winking face with tongue..disappointed face | ||||
| 1F61F         ; Emoji                #  6.1  [1] (😟)       worried face | ||||
| 1F620..1F625  ; Emoji                #  6.0  [6] (😠..😥)    angry face..sad but relieved face | ||||
| 1F626..1F627  ; Emoji                #  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face | ||||
| 1F628..1F62B  ; Emoji                #  6.0  [4] (😨..😫)    fearful face..tired face | ||||
| 1F62C         ; Emoji                #  6.1  [1] (😬)       grimacing face | ||||
| 1F62D         ; Emoji                #  6.0  [1] (😭)       loudly crying face | ||||
| 1F62E..1F62F  ; Emoji                #  6.1  [2] (😮..😯)    face with open mouth..hushed face | ||||
| 1F630..1F633  ; Emoji                #  6.0  [4] (😰..😳)    anxious face with sweat..flushed face | ||||
| 1F634         ; Emoji                #  6.1  [1] (😴)       sleeping face | ||||
| 1F635..1F640  ; Emoji                #  6.0 [12] (😵..🙀)    dizzy face..weary cat face | ||||
| 1F641..1F642  ; Emoji                #  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face | ||||
| 1F643..1F644  ; Emoji                #  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes | ||||
| 1F645..1F64F  ; Emoji                #  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands | ||||
| 1F680..1F6C5  ; Emoji                #  6.0 [70] (🚀..🛅)    rocket..left luggage | ||||
| 1F6CB..1F6CF  ; Emoji                #  7.0  [5] (🛋️..🛏️)    couch and lamp..bed | ||||
| 1F6D0         ; Emoji                #  8.0  [1] (🛐)       place of worship | ||||
| 1F6D1..1F6D2  ; Emoji                #  9.0  [2] (🛑..🛒)    stop sign..shopping cart | ||||
| 1F6E0..1F6E5  ; Emoji                #  7.0  [6] (🛠️..🛥️)    hammer and wrench..motor boat | ||||
| 1F6E9         ; Emoji                #  7.0  [1] (🛩️)       small airplane | ||||
| 1F6EB..1F6EC  ; Emoji                #  7.0  [2] (🛫..🛬)    airplane departure..airplane arrival | ||||
| 1F6F0         ; Emoji                #  7.0  [1] (🛰️)       satellite | ||||
| 1F6F3         ; Emoji                #  7.0  [1] (🛳️)       passenger ship | ||||
| 1F6F4..1F6F6  ; Emoji                #  9.0  [3] (🛴..🛶)    kick scooter..canoe | ||||
| 1F6F7..1F6F8  ; Emoji                # 10.0  [2] (🛷..🛸)    sled..flying saucer | ||||
| 1F6F9         ; Emoji                # 11.0  [1] (🛹)       skateboard | ||||
| 1F910..1F918  ; Emoji                #  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns | ||||
| 1F919..1F91E  ; Emoji                #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers | ||||
| 1F91F         ; Emoji                # 10.0  [1] (🤟)       love-you gesture | ||||
| 1F920..1F927  ; Emoji                #  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face | ||||
| 1F928..1F92F  ; Emoji                # 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head | ||||
| 1F930         ; Emoji                #  9.0  [1] (🤰)       pregnant woman | ||||
| 1F931..1F932  ; Emoji                # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together | ||||
| 1F933..1F93A  ; Emoji                #  9.0  [8] (🤳..🤺)    selfie..person fencing | ||||
| 1F93C..1F93E  ; Emoji                #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball | ||||
| 1F940..1F945  ; Emoji                #  9.0  [6] (🥀..🥅)    wilted flower..goal net | ||||
| 1F947..1F94B  ; Emoji                #  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform | ||||
| 1F94C         ; Emoji                # 10.0  [1] (🥌)       curling stone | ||||
| 1F94D..1F94F  ; Emoji                # 11.0  [3] (🥍..🥏)    lacrosse..flying disc | ||||
| 1F950..1F95E  ; Emoji                #  9.0 [15] (🥐..🥞)    croissant..pancakes | ||||
| 1F95F..1F96B  ; Emoji                # 10.0 [13] (🥟..🥫)    dumpling..canned food | ||||
| 1F96C..1F970  ; Emoji                # 11.0  [5] (🥬..🥰)    leafy green..smiling face with 3 hearts | ||||
| 1F973..1F976  ; Emoji                # 11.0  [4] (🥳..🥶)    partying face..cold face | ||||
| 1F97A         ; Emoji                # 11.0  [1] (🥺)       pleading face | ||||
| 1F97C..1F97F  ; Emoji                # 11.0  [4] (🥼..🥿)    lab coat..woman’s flat shoe | ||||
| 1F980..1F984  ; Emoji                #  8.0  [5] (🦀..🦄)    crab..unicorn face | ||||
| 1F985..1F991  ; Emoji                #  9.0 [13] (🦅..🦑)    eagle..squid | ||||
| 1F992..1F997  ; Emoji                # 10.0  [6] (🦒..🦗)    giraffe..cricket | ||||
| 1F998..1F9A2  ; Emoji                # 11.0 [11] (🦘..🦢)    kangaroo..swan | ||||
| 1F9B0..1F9B9  ; Emoji                # 11.0 [10] (🦰..🦹)    red-haired..supervillain | ||||
| 1F9C0         ; Emoji                #  8.0  [1] (🧀)       cheese wedge | ||||
| 1F9C1..1F9C2  ; Emoji                # 11.0  [2] (🧁..🧂)    cupcake..salt | ||||
| 1F9D0..1F9E6  ; Emoji                # 10.0 [23] (🧐..🧦)    face with monocle..socks | ||||
| 1F9E7..1F9FF  ; Emoji                # 11.0 [25] (🧧..🧿)    red envelope..nazar amulet | ||||
|  | ||||
| # Total elements: 1250 | ||||
|  | ||||
| # ================================================ | ||||
|  | ||||
| # All omitted code points have Emoji_Presentation=No  | ||||
| # @missing: 0000..10FFFF  ; Emoji_Presentation ; No | ||||
|  | ||||
| 231A..231B    ; Emoji_Presentation   #  1.1  [2] (⌚..⌛)    watch..hourglass done | ||||
| 23E9..23EC    ; Emoji_Presentation   #  6.0  [4] (⏩..⏬)    fast-forward button..fast down button | ||||
| 23F0          ; Emoji_Presentation   #  6.0  [1] (⏰)       alarm clock | ||||
| 23F3          ; Emoji_Presentation   #  6.0  [1] (⏳)       hourglass not done | ||||
| 25FD..25FE    ; Emoji_Presentation   #  3.2  [2] (◽..◾)    white medium-small square..black medium-small square | ||||
| 2614..2615    ; Emoji_Presentation   #  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage | ||||
| 2648..2653    ; Emoji_Presentation   #  1.1 [12] (♈..♓)    Aries..Pisces | ||||
| 267F          ; Emoji_Presentation   #  4.1  [1] (♿)       wheelchair symbol | ||||
| 2693          ; Emoji_Presentation   #  4.1  [1] (⚓)       anchor | ||||
| 26A1          ; Emoji_Presentation   #  4.0  [1] (⚡)       high voltage | ||||
| 26AA..26AB    ; Emoji_Presentation   #  4.1  [2] (⚪..⚫)    white circle..black circle | ||||
| 26BD..26BE    ; Emoji_Presentation   #  5.2  [2] (⚽..⚾)    soccer ball..baseball | ||||
| 26C4..26C5    ; Emoji_Presentation   #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind cloud | ||||
| 26CE          ; Emoji_Presentation   #  6.0  [1] (⛎)       Ophiuchus | ||||
| 26D4          ; Emoji_Presentation   #  5.2  [1] (⛔)       no entry | ||||
| 26EA          ; Emoji_Presentation   #  5.2  [1] (⛪)       church | ||||
| 26F2..26F3    ; Emoji_Presentation   #  5.2  [2] (⛲..⛳)    fountain..flag in hole | ||||
| 26F5          ; Emoji_Presentation   #  5.2  [1] (⛵)       sailboat | ||||
| 26FA          ; Emoji_Presentation   #  5.2  [1] (⛺)       tent | ||||
| 26FD          ; Emoji_Presentation   #  5.2  [1] (⛽)       fuel pump | ||||
| 2705          ; Emoji_Presentation   #  6.0  [1] (✅)       white heavy check mark | ||||
| 270A..270B    ; Emoji_Presentation   #  6.0  [2] (✊..✋)    raised fist..raised hand | ||||
| 2728          ; Emoji_Presentation   #  6.0  [1] (✨)       sparkles | ||||
| 274C          ; Emoji_Presentation   #  6.0  [1] (❌)       cross mark | ||||
| 274E          ; Emoji_Presentation   #  6.0  [1] (❎)       cross mark button | ||||
| 2753..2755    ; Emoji_Presentation   #  6.0  [3] (❓..❕)    question mark..white exclamation mark | ||||
| 2757          ; Emoji_Presentation   #  5.2  [1] (❗)       exclamation mark | ||||
| 2795..2797    ; Emoji_Presentation   #  6.0  [3] (➕..➗)    heavy plus sign..heavy division sign | ||||
| 27B0          ; Emoji_Presentation   #  6.0  [1] (➰)       curly loop | ||||
| 27BF          ; Emoji_Presentation   #  6.0  [1] (➿)       double curly loop | ||||
| 2B1B..2B1C    ; Emoji_Presentation   #  5.1  [2] (⬛..⬜)    black large square..white large square | ||||
| 2B50          ; Emoji_Presentation   #  5.1  [1] (⭐)       star | ||||
| 2B55          ; Emoji_Presentation   #  5.2  [1] (⭕)       heavy large circle | ||||
| 1F004         ; Emoji_Presentation   #  5.1  [1] (🀄)       mahjong red dragon | ||||
| 1F0CF         ; Emoji_Presentation   #  6.0  [1] (🃏)       joker | ||||
| 1F18E         ; Emoji_Presentation   #  6.0  [1] (🆎)       AB button (blood type) | ||||
| 1F191..1F19A  ; Emoji_Presentation   #  6.0 [10] (🆑..🆚)    CL button..VS button | ||||
| 1F1E6..1F1FF  ; Emoji_Presentation   #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z | ||||
| 1F201         ; Emoji_Presentation   #  6.0  [1] (🈁)       Japanese “here” button | ||||
| 1F21A         ; Emoji_Presentation   #  5.2  [1] (🈚)       Japanese “free of charge” button | ||||
| 1F22F         ; Emoji_Presentation   #  5.2  [1] (🈯)       Japanese “reserved” button | ||||
| 1F232..1F236  ; Emoji_Presentation   #  6.0  [5] (🈲..🈶)    Japanese “prohibited” button..Japanese “not free of charge” button | ||||
| 1F238..1F23A  ; Emoji_Presentation   #  6.0  [3] (🈸..🈺)    Japanese “application” button..Japanese “open for business” button | ||||
| 1F250..1F251  ; Emoji_Presentation   #  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button | ||||
| 1F300..1F320  ; Emoji_Presentation   #  6.0 [33] (🌀..🌠)    cyclone..shooting star | ||||
| 1F32D..1F32F  ; Emoji_Presentation   #  8.0  [3] (🌭..🌯)    hot dog..burrito | ||||
| 1F330..1F335  ; Emoji_Presentation   #  6.0  [6] (🌰..🌵)    chestnut..cactus | ||||
| 1F337..1F37C  ; Emoji_Presentation   #  6.0 [70] (🌷..🍼)    tulip..baby bottle | ||||
| 1F37E..1F37F  ; Emoji_Presentation   #  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn | ||||
| 1F380..1F393  ; Emoji_Presentation   #  6.0 [20] (🎀..🎓)    ribbon..graduation cap | ||||
| 1F3A0..1F3C4  ; Emoji_Presentation   #  6.0 [37] (🎠..🏄)    carousel horse..person surfing | ||||
| 1F3C5         ; Emoji_Presentation   #  7.0  [1] (🏅)       sports medal | ||||
| 1F3C6..1F3CA  ; Emoji_Presentation   #  6.0  [5] (🏆..🏊)    trophy..person swimming | ||||
| 1F3CF..1F3D3  ; Emoji_Presentation   #  8.0  [5] (🏏..🏓)    cricket game..ping pong | ||||
| 1F3E0..1F3F0  ; Emoji_Presentation   #  6.0 [17] (🏠..🏰)    house..castle | ||||
| 1F3F4         ; Emoji_Presentation   #  7.0  [1] (🏴)       black flag | ||||
| 1F3F8..1F3FF  ; Emoji_Presentation   #  8.0  [8] (🏸..🏿)    badminton..dark skin tone | ||||
| 1F400..1F43E  ; Emoji_Presentation   #  6.0 [63] (🐀..🐾)    rat..paw prints | ||||
| 1F440         ; Emoji_Presentation   #  6.0  [1] (👀)       eyes | ||||
| 1F442..1F4F7  ; Emoji_Presentation   #  6.0[182] (👂..📷)    ear..camera | ||||
| 1F4F8         ; Emoji_Presentation   #  7.0  [1] (📸)       camera with flash | ||||
| 1F4F9..1F4FC  ; Emoji_Presentation   #  6.0  [4] (📹..📼)    video camera..videocassette | ||||
| 1F4FF         ; Emoji_Presentation   #  8.0  [1] (📿)       prayer beads | ||||
| 1F500..1F53D  ; Emoji_Presentation   #  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button | ||||
| 1F54B..1F54E  ; Emoji_Presentation   #  8.0  [4] (🕋..🕎)    kaaba..menorah | ||||
| 1F550..1F567  ; Emoji_Presentation   #  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty | ||||
| 1F57A         ; Emoji_Presentation   #  9.0  [1] (🕺)       man dancing | ||||
| 1F595..1F596  ; Emoji_Presentation   #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute | ||||
| 1F5A4         ; Emoji_Presentation   #  9.0  [1] (🖤)       black heart | ||||
| 1F5FB..1F5FF  ; Emoji_Presentation   #  6.0  [5] (🗻..🗿)    mount fuji..moai | ||||
| 1F600         ; Emoji_Presentation   #  6.1  [1] (😀)       grinning face | ||||
| 1F601..1F610  ; Emoji_Presentation   #  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face | ||||
| 1F611         ; Emoji_Presentation   #  6.1  [1] (😑)       expressionless face | ||||
| 1F612..1F614  ; Emoji_Presentation   #  6.0  [3] (😒..😔)    unamused face..pensive face | ||||
| 1F615         ; Emoji_Presentation   #  6.1  [1] (😕)       confused face | ||||
| 1F616         ; Emoji_Presentation   #  6.0  [1] (😖)       confounded face | ||||
| 1F617         ; Emoji_Presentation   #  6.1  [1] (😗)       kissing face | ||||
| 1F618         ; Emoji_Presentation   #  6.0  [1] (😘)       face blowing a kiss | ||||
| 1F619         ; Emoji_Presentation   #  6.1  [1] (😙)       kissing face with smiling eyes | ||||
| 1F61A         ; Emoji_Presentation   #  6.0  [1] (😚)       kissing face with closed eyes | ||||
| 1F61B         ; Emoji_Presentation   #  6.1  [1] (😛)       face with tongue | ||||
| 1F61C..1F61E  ; Emoji_Presentation   #  6.0  [3] (😜..😞)    winking face with tongue..disappointed face | ||||
| 1F61F         ; Emoji_Presentation   #  6.1  [1] (😟)       worried face | ||||
| 1F620..1F625  ; Emoji_Presentation   #  6.0  [6] (😠..😥)    angry face..sad but relieved face | ||||
| 1F626..1F627  ; Emoji_Presentation   #  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face | ||||
| 1F628..1F62B  ; Emoji_Presentation   #  6.0  [4] (😨..😫)    fearful face..tired face | ||||
| 1F62C         ; Emoji_Presentation   #  6.1  [1] (😬)       grimacing face | ||||
| 1F62D         ; Emoji_Presentation   #  6.0  [1] (😭)       loudly crying face | ||||
| 1F62E..1F62F  ; Emoji_Presentation   #  6.1  [2] (😮..😯)    face with open mouth..hushed face | ||||
| 1F630..1F633  ; Emoji_Presentation   #  6.0  [4] (😰..😳)    anxious face with sweat..flushed face | ||||
| 1F634         ; Emoji_Presentation   #  6.1  [1] (😴)       sleeping face | ||||
| 1F635..1F640  ; Emoji_Presentation   #  6.0 [12] (😵..🙀)    dizzy face..weary cat face | ||||
| 1F641..1F642  ; Emoji_Presentation   #  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face | ||||
| 1F643..1F644  ; Emoji_Presentation   #  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes | ||||
| 1F645..1F64F  ; Emoji_Presentation   #  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands | ||||
| 1F680..1F6C5  ; Emoji_Presentation   #  6.0 [70] (🚀..🛅)    rocket..left luggage | ||||
| 1F6CC         ; Emoji_Presentation   #  7.0  [1] (🛌)       person in bed | ||||
| 1F6D0         ; Emoji_Presentation   #  8.0  [1] (🛐)       place of worship | ||||
| 1F6D1..1F6D2  ; Emoji_Presentation   #  9.0  [2] (🛑..🛒)    stop sign..shopping cart | ||||
| 1F6EB..1F6EC  ; Emoji_Presentation   #  7.0  [2] (🛫..🛬)    airplane departure..airplane arrival | ||||
| 1F6F4..1F6F6  ; Emoji_Presentation   #  9.0  [3] (🛴..🛶)    kick scooter..canoe | ||||
| 1F6F7..1F6F8  ; Emoji_Presentation   # 10.0  [2] (🛷..🛸)    sled..flying saucer | ||||
| 1F6F9         ; Emoji_Presentation   # 11.0  [1] (🛹)       skateboard | ||||
| 1F910..1F918  ; Emoji_Presentation   #  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns | ||||
| 1F919..1F91E  ; Emoji_Presentation   #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers | ||||
| 1F91F         ; Emoji_Presentation   # 10.0  [1] (🤟)       love-you gesture | ||||
| 1F920..1F927  ; Emoji_Presentation   #  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face | ||||
| 1F928..1F92F  ; Emoji_Presentation   # 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head | ||||
| 1F930         ; Emoji_Presentation   #  9.0  [1] (🤰)       pregnant woman | ||||
| 1F931..1F932  ; Emoji_Presentation   # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together | ||||
| 1F933..1F93A  ; Emoji_Presentation   #  9.0  [8] (🤳..🤺)    selfie..person fencing | ||||
| 1F93C..1F93E  ; Emoji_Presentation   #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball | ||||
| 1F940..1F945  ; Emoji_Presentation   #  9.0  [6] (🥀..🥅)    wilted flower..goal net | ||||
| 1F947..1F94B  ; Emoji_Presentation   #  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform | ||||
| 1F94C         ; Emoji_Presentation   # 10.0  [1] (🥌)       curling stone | ||||
| 1F94D..1F94F  ; Emoji_Presentation   # 11.0  [3] (🥍..🥏)    lacrosse..flying disc | ||||
| 1F950..1F95E  ; Emoji_Presentation   #  9.0 [15] (🥐..🥞)    croissant..pancakes | ||||
| 1F95F..1F96B  ; Emoji_Presentation   # 10.0 [13] (🥟..🥫)    dumpling..canned food | ||||
| 1F96C..1F970  ; Emoji_Presentation   # 11.0  [5] (🥬..🥰)    leafy green..smiling face with 3 hearts | ||||
| 1F973..1F976  ; Emoji_Presentation   # 11.0  [4] (🥳..🥶)    partying face..cold face | ||||
| 1F97A         ; Emoji_Presentation   # 11.0  [1] (🥺)       pleading face | ||||
| 1F97C..1F97F  ; Emoji_Presentation   # 11.0  [4] (🥼..🥿)    lab coat..woman’s flat shoe | ||||
| 1F980..1F984  ; Emoji_Presentation   #  8.0  [5] (🦀..🦄)    crab..unicorn face | ||||
| 1F985..1F991  ; Emoji_Presentation   #  9.0 [13] (🦅..🦑)    eagle..squid | ||||
| 1F992..1F997  ; Emoji_Presentation   # 10.0  [6] (🦒..🦗)    giraffe..cricket | ||||
| 1F998..1F9A2  ; Emoji_Presentation   # 11.0 [11] (🦘..🦢)    kangaroo..swan | ||||
| 1F9B0..1F9B9  ; Emoji_Presentation   # 11.0 [10] (🦰..🦹)    red-haired..supervillain | ||||
| 1F9C0         ; Emoji_Presentation   #  8.0  [1] (🧀)       cheese wedge | ||||
| 1F9C1..1F9C2  ; Emoji_Presentation   # 11.0  [2] (🧁..🧂)    cupcake..salt | ||||
| 1F9D0..1F9E6  ; Emoji_Presentation   # 10.0 [23] (🧐..🧦)    face with monocle..socks | ||||
| 1F9E7..1F9FF  ; Emoji_Presentation   # 11.0 [25] (🧧..🧿)    red envelope..nazar amulet | ||||
|  | ||||
| # Total elements: 1032 | ||||
|  | ||||
| # ================================================ | ||||
|  | ||||
| # All omitted code points have Emoji_Modifier=No  | ||||
| # @missing: 0000..10FFFF  ; Emoji_Modifier ; No | ||||
|  | ||||
| 1F3FB..1F3FF  ; Emoji_Modifier       #  8.0  [5] (🏻..🏿)    light skin tone..dark skin tone | ||||
|  | ||||
| # Total elements: 5 | ||||
|  | ||||
| # ================================================ | ||||
|  | ||||
| # All omitted code points have Emoji_Modifier_Base=No  | ||||
| # @missing: 0000..10FFFF  ; Emoji_Modifier_Base ; No | ||||
|  | ||||
| 261D          ; Emoji_Modifier_Base  #  1.1  [1] (☝️)       index pointing up | ||||
| 26F9          ; Emoji_Modifier_Base  #  5.2  [1] (⛹️)       person bouncing ball | ||||
| 270A..270B    ; Emoji_Modifier_Base  #  6.0  [2] (✊..✋)    raised fist..raised hand | ||||
| 270C..270D    ; Emoji_Modifier_Base  #  1.1  [2] (✌️..✍️)    victory hand..writing hand | ||||
| 1F385         ; Emoji_Modifier_Base  #  6.0  [1] (🎅)       Santa Claus | ||||
| 1F3C2..1F3C4  ; Emoji_Modifier_Base  #  6.0  [3] (🏂..🏄)    snowboarder..person surfing | ||||
| 1F3C7         ; Emoji_Modifier_Base  #  6.0  [1] (🏇)       horse racing | ||||
| 1F3CA         ; Emoji_Modifier_Base  #  6.0  [1] (🏊)       person swimming | ||||
| 1F3CB..1F3CC  ; Emoji_Modifier_Base  #  7.0  [2] (🏋️..🏌️)    person lifting weights..person golfing | ||||
| 1F442..1F443  ; Emoji_Modifier_Base  #  6.0  [2] (👂..👃)    ear..nose | ||||
| 1F446..1F450  ; Emoji_Modifier_Base  #  6.0 [11] (👆..👐)    backhand index pointing up..open hands | ||||
| 1F466..1F469  ; Emoji_Modifier_Base  #  6.0  [4] (👦..👩)    boy..woman | ||||
| 1F46E         ; Emoji_Modifier_Base  #  6.0  [1] (👮)       police officer | ||||
| 1F470..1F478  ; Emoji_Modifier_Base  #  6.0  [9] (👰..👸)    bride with veil..princess | ||||
| 1F47C         ; Emoji_Modifier_Base  #  6.0  [1] (👼)       baby angel | ||||
| 1F481..1F483  ; Emoji_Modifier_Base  #  6.0  [3] (💁..💃)    person tipping hand..woman dancing | ||||
| 1F485..1F487  ; Emoji_Modifier_Base  #  6.0  [3] (💅..💇)    nail polish..person getting haircut | ||||
| 1F4AA         ; Emoji_Modifier_Base  #  6.0  [1] (💪)       flexed biceps | ||||
| 1F574..1F575  ; Emoji_Modifier_Base  #  7.0  [2] (🕴️..🕵️)    man in suit levitating..detective | ||||
| 1F57A         ; Emoji_Modifier_Base  #  9.0  [1] (🕺)       man dancing | ||||
| 1F590         ; Emoji_Modifier_Base  #  7.0  [1] (🖐️)       hand with fingers splayed | ||||
| 1F595..1F596  ; Emoji_Modifier_Base  #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute | ||||
| 1F645..1F647  ; Emoji_Modifier_Base  #  6.0  [3] (🙅..🙇)    person gesturing NO..person bowing | ||||
| 1F64B..1F64F  ; Emoji_Modifier_Base  #  6.0  [5] (🙋..🙏)    person raising hand..folded hands | ||||
| 1F6A3         ; Emoji_Modifier_Base  #  6.0  [1] (🚣)       person rowing boat | ||||
| 1F6B4..1F6B6  ; Emoji_Modifier_Base  #  6.0  [3] (🚴..🚶)    person biking..person walking | ||||
| 1F6C0         ; Emoji_Modifier_Base  #  6.0  [1] (🛀)       person taking bath | ||||
| 1F6CC         ; Emoji_Modifier_Base  #  7.0  [1] (🛌)       person in bed | ||||
| 1F918         ; Emoji_Modifier_Base  #  8.0  [1] (🤘)       sign of the horns | ||||
| 1F919..1F91C  ; Emoji_Modifier_Base  #  9.0  [4] (🤙..🤜)    call me hand..right-facing fist | ||||
| 1F91E         ; Emoji_Modifier_Base  #  9.0  [1] (🤞)       crossed fingers | ||||
| 1F91F         ; Emoji_Modifier_Base  # 10.0  [1] (🤟)       love-you gesture | ||||
| 1F926         ; Emoji_Modifier_Base  #  9.0  [1] (🤦)       person facepalming | ||||
| 1F930         ; Emoji_Modifier_Base  #  9.0  [1] (🤰)       pregnant woman | ||||
| 1F931..1F932  ; Emoji_Modifier_Base  # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together | ||||
| 1F933..1F939  ; Emoji_Modifier_Base  #  9.0  [7] (🤳..🤹)    selfie..person juggling | ||||
| 1F93D..1F93E  ; Emoji_Modifier_Base  #  9.0  [2] (🤽..🤾)    person playing water polo..person playing handball | ||||
| 1F9B5..1F9B6  ; Emoji_Modifier_Base  # 11.0  [2] (🦵..🦶)    leg..foot | ||||
| 1F9B8..1F9B9  ; Emoji_Modifier_Base  # 11.0  [2] (🦸..🦹)    superhero..supervillain | ||||
| 1F9D1..1F9DD  ; Emoji_Modifier_Base  # 10.0 [13] (🧑..🧝)    adult..elf | ||||
|  | ||||
| # Total elements: 106 | ||||
|  | ||||
| # ================================================ | ||||
|  | ||||
| # All omitted code points have Emoji_Component=No  | ||||
| # @missing: 0000..10FFFF  ; Emoji_Component ; No | ||||
|  | ||||
| 0023          ; Emoji_Component      #  1.1  [1] (#️)       number sign | ||||
| 002A          ; Emoji_Component      #  1.1  [1] (*️)       asterisk | ||||
| 0030..0039    ; Emoji_Component      #  1.1 [10] (0️..9️)    digit zero..digit nine | ||||
| 200D          ; Emoji_Component      #  1.1  [1] ()        zero width joiner | ||||
| 20E3          ; Emoji_Component      #  3.0  [1] (⃣)       combining enclosing keycap | ||||
| FE0F          ; Emoji_Component      #  3.2  [1] ()        VARIATION SELECTOR-16 | ||||
| 1F1E6..1F1FF  ; Emoji_Component      #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z | ||||
| 1F3FB..1F3FF  ; Emoji_Component      #  8.0  [5] (🏻..🏿)    light skin tone..dark skin tone | ||||
| 1F9B0..1F9B3  ; Emoji_Component      # 11.0  [4] (🦰..🦳)    red-haired..white-haired | ||||
| E0020..E007F  ; Emoji_Component      #  3.1 [96] (..)      tag space..cancel tag | ||||
|  | ||||
| # Total elements: 146 | ||||
|  | ||||
| # ================================================ | ||||
|  | ||||
| # All omitted code points have Extended_Pictographic=No  | ||||
| # @missing: 0000..10FFFF  ; Extended_Pictographic ; No | ||||
|  | ||||
| 00A9          ; Extended_Pictographic#  1.1  [1] (©️)       copyright | ||||
| 00AE          ; Extended_Pictographic#  1.1  [1] (®️)       registered | ||||
| 203C          ; Extended_Pictographic#  1.1  [1] (‼️)       double exclamation mark | ||||
| 2049          ; Extended_Pictographic#  3.0  [1] (⁉️)       exclamation question mark | ||||
| 2122          ; Extended_Pictographic#  1.1  [1] (™️)       trade mark | ||||
| 2139          ; Extended_Pictographic#  3.0  [1] (ℹ️)       information | ||||
| 2194..2199    ; Extended_Pictographic#  1.1  [6] (↔️..↙️)    left-right arrow..down-left arrow | ||||
| 21A9..21AA    ; Extended_Pictographic#  1.1  [2] (↩️..↪️)    right arrow curving left..left arrow curving right | ||||
| 231A..231B    ; Extended_Pictographic#  1.1  [2] (⌚..⌛)    watch..hourglass done | ||||
| 2328          ; Extended_Pictographic#  1.1  [1] (⌨️)       keyboard | ||||
| 2388          ; Extended_Pictographic#  3.0  [1] (⎈️)       HELM SYMBOL | ||||
| 23CF          ; Extended_Pictographic#  4.0  [1] (⏏️)       eject button | ||||
| 23E9..23F3    ; Extended_Pictographic#  6.0 [11] (⏩..⏳)    fast-forward button..hourglass not done | ||||
| 23F8..23FA    ; Extended_Pictographic#  7.0  [3] (⏸️..⏺️)    pause button..record button | ||||
| 24C2          ; Extended_Pictographic#  1.1  [1] (Ⓜ️)       circled M | ||||
| 25AA..25AB    ; Extended_Pictographic#  1.1  [2] (▪️..▫️)    black small square..white small square | ||||
| 25B6          ; Extended_Pictographic#  1.1  [1] (▶️)       play button | ||||
| 25C0          ; Extended_Pictographic#  1.1  [1] (◀️)       reverse button | ||||
| 25FB..25FE    ; Extended_Pictographic#  3.2  [4] (◻️..◾)    white medium square..black medium-small square | ||||
| 2600..2605    ; Extended_Pictographic#  1.1  [6] (☀️..★️)    sun..BLACK STAR | ||||
| 2607..2612    ; Extended_Pictographic#  1.1 [12] (☇️..☒️)    LIGHTNING..BALLOT BOX WITH X | ||||
| 2614..2615    ; Extended_Pictographic#  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage | ||||
| 2616..2617    ; Extended_Pictographic#  3.2  [2] (☖️..☗️)    WHITE SHOGI PIECE..BLACK SHOGI PIECE | ||||
| 2618          ; Extended_Pictographic#  4.1  [1] (☘️)       shamrock | ||||
| 2619          ; Extended_Pictographic#  3.0  [1] (☙️)       REVERSED ROTATED FLORAL HEART BULLET | ||||
| 261A..266F    ; Extended_Pictographic#  1.1 [86] (☚️..♯️)    BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN | ||||
| 2670..2671    ; Extended_Pictographic#  3.0  [2] (♰️..♱️)    WEST SYRIAC CROSS..EAST SYRIAC CROSS | ||||
| 2672..267D    ; Extended_Pictographic#  3.2 [12] (♲️..♽️)    UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL | ||||
| 267E..267F    ; Extended_Pictographic#  4.1  [2] (♾️..♿)    infinity..wheelchair symbol | ||||
| 2680..2685    ; Extended_Pictographic#  3.2  [6] (⚀️..⚅️)    DIE FACE-1..DIE FACE-6 | ||||
| 2690..2691    ; Extended_Pictographic#  4.0  [2] (⚐️..⚑️)    WHITE FLAG..BLACK FLAG | ||||
| 2692..269C    ; Extended_Pictographic#  4.1 [11] (⚒️..⚜️)    hammer and pick..fleur-de-lis | ||||
| 269D          ; Extended_Pictographic#  5.1  [1] (⚝️)       OUTLINED WHITE STAR | ||||
| 269E..269F    ; Extended_Pictographic#  5.2  [2] (⚞️..⚟️)    THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT | ||||
| 26A0..26A1    ; Extended_Pictographic#  4.0  [2] (⚠️..⚡)    warning..high voltage | ||||
| 26A2..26B1    ; Extended_Pictographic#  4.1 [16] (⚢️..⚱️)    DOUBLED FEMALE SIGN..funeral urn | ||||
| 26B2          ; Extended_Pictographic#  5.0  [1] (⚲️)       NEUTER | ||||
| 26B3..26BC    ; Extended_Pictographic#  5.1 [10] (⚳️..⚼️)    CERES..SESQUIQUADRATE | ||||
| 26BD..26BF    ; Extended_Pictographic#  5.2  [3] (⚽..⚿️)    soccer ball..SQUARED KEY | ||||
| 26C0..26C3    ; Extended_Pictographic#  5.1  [4] (⛀️..⛃️)    WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING | ||||
| 26C4..26CD    ; Extended_Pictographic#  5.2 [10] (⛄..⛍️)    snowman without snow..DISABLED CAR | ||||
| 26CE          ; Extended_Pictographic#  6.0  [1] (⛎)       Ophiuchus | ||||
| 26CF..26E1    ; Extended_Pictographic#  5.2 [19] (⛏️..⛡️)    pick..RESTRICTED LEFT ENTRY-2 | ||||
| 26E2          ; Extended_Pictographic#  6.0  [1] (⛢️)       ASTRONOMICAL SYMBOL FOR URANUS | ||||
| 26E3          ; Extended_Pictographic#  5.2  [1] (⛣️)       HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE | ||||
| 26E4..26E7    ; Extended_Pictographic#  6.0  [4] (⛤️..⛧️)    PENTAGRAM..INVERTED PENTAGRAM | ||||
| 26E8..26FF    ; Extended_Pictographic#  5.2 [24] (⛨️..⛿️)    BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE | ||||
| 2700          ; Extended_Pictographic#  7.0  [1] (✀️)       BLACK SAFETY SCISSORS | ||||
| 2701..2704    ; Extended_Pictographic#  1.1  [4] (✁️..✄️)    UPPER BLADE SCISSORS..WHITE SCISSORS | ||||
| 2705          ; Extended_Pictographic#  6.0  [1] (✅)       white heavy check mark | ||||
| 2708..2709    ; Extended_Pictographic#  1.1  [2] (✈️..✉️)    airplane..envelope | ||||
| 270A..270B    ; Extended_Pictographic#  6.0  [2] (✊..✋)    raised fist..raised hand | ||||
| 270C..2712    ; Extended_Pictographic#  1.1  [7] (✌️..✒️)    victory hand..black nib | ||||
| 2714          ; Extended_Pictographic#  1.1  [1] (✔️)       heavy check mark | ||||
| 2716          ; Extended_Pictographic#  1.1  [1] (✖️)       heavy multiplication x | ||||
| 271D          ; Extended_Pictographic#  1.1  [1] (✝️)       latin cross | ||||
| 2721          ; Extended_Pictographic#  1.1  [1] (✡️)       star of David | ||||
| 2728          ; Extended_Pictographic#  6.0  [1] (✨)       sparkles | ||||
| 2733..2734    ; Extended_Pictographic#  1.1  [2] (✳️..✴️)    eight-spoked asterisk..eight-pointed star | ||||
| 2744          ; Extended_Pictographic#  1.1  [1] (❄️)       snowflake | ||||
| 2747          ; Extended_Pictographic#  1.1  [1] (❇️)       sparkle | ||||
| 274C          ; Extended_Pictographic#  6.0  [1] (❌)       cross mark | ||||
| 274E          ; Extended_Pictographic#  6.0  [1] (❎)       cross mark button | ||||
| 2753..2755    ; Extended_Pictographic#  6.0  [3] (❓..❕)    question mark..white exclamation mark | ||||
| 2757          ; Extended_Pictographic#  5.2  [1] (❗)       exclamation mark | ||||
| 2763..2767    ; Extended_Pictographic#  1.1  [5] (❣️..❧️)    heavy heart exclamation..ROTATED FLORAL HEART BULLET | ||||
| 2795..2797    ; Extended_Pictographic#  6.0  [3] (➕..➗)    heavy plus sign..heavy division sign | ||||
| 27A1          ; Extended_Pictographic#  1.1  [1] (➡️)       right arrow | ||||
| 27B0          ; Extended_Pictographic#  6.0  [1] (➰)       curly loop | ||||
| 27BF          ; Extended_Pictographic#  6.0  [1] (➿)       double curly loop | ||||
| 2934..2935    ; Extended_Pictographic#  3.2  [2] (⤴️..⤵️)    right arrow curving up..right arrow curving down | ||||
| 2B05..2B07    ; Extended_Pictographic#  4.0  [3] (⬅️..⬇️)    left arrow..down arrow | ||||
| 2B1B..2B1C    ; Extended_Pictographic#  5.1  [2] (⬛..⬜)    black large square..white large square | ||||
| 2B50          ; Extended_Pictographic#  5.1  [1] (⭐)       star | ||||
| 2B55          ; Extended_Pictographic#  5.2  [1] (⭕)       heavy large circle | ||||
| 3030          ; Extended_Pictographic#  1.1  [1] (〰️)       wavy dash | ||||
| 303D          ; Extended_Pictographic#  3.2  [1] (〽️)       part alternation mark | ||||
| 3297          ; Extended_Pictographic#  1.1  [1] (㊗️)       Japanese “congratulations” button | ||||
| 3299          ; Extended_Pictographic#  1.1  [1] (㊙️)       Japanese “secret” button | ||||
| 1F000..1F02B  ; Extended_Pictographic#  5.1 [44] (🀀️..🀫️)    MAHJONG TILE EAST WIND..MAHJONG TILE BACK | ||||
| 1F02C..1F02F  ; Extended_Pictographic#   NA  [4] (️..️)    <reserved-1F02C>..<reserved-1F02F> | ||||
| 1F030..1F093  ; Extended_Pictographic#  5.1[100] (🀰️..🂓️)    DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 | ||||
| 1F094..1F09F  ; Extended_Pictographic#   NA [12] (️..️)    <reserved-1F094>..<reserved-1F09F> | ||||
| 1F0A0..1F0AE  ; Extended_Pictographic#  6.0 [15] (🂠️..🂮️)    PLAYING CARD BACK..PLAYING CARD KING OF SPADES | ||||
| 1F0AF..1F0B0  ; Extended_Pictographic#   NA  [2] (️..️)    <reserved-1F0AF>..<reserved-1F0B0> | ||||
| 1F0B1..1F0BE  ; Extended_Pictographic#  6.0 [14] (🂱️..🂾️)    PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS | ||||
| 1F0BF         ; Extended_Pictographic#  7.0  [1] (🂿️)       PLAYING CARD RED JOKER | ||||
| 1F0C0         ; Extended_Pictographic#   NA  [1] (️)       <reserved-1F0C0> | ||||
| 1F0C1..1F0CF  ; Extended_Pictographic#  6.0 [15] (🃁️..🃏)    PLAYING CARD ACE OF DIAMONDS..joker | ||||
| 1F0D0         ; Extended_Pictographic#   NA  [1] (️)       <reserved-1F0D0> | ||||
| 1F0D1..1F0DF  ; Extended_Pictographic#  6.0 [15] (🃑️..🃟️)    PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER | ||||
| 1F0E0..1F0F5  ; Extended_Pictographic#  7.0 [22] (🃠️..🃵️)    PLAYING CARD FOOL..PLAYING CARD TRUMP-21 | ||||
| 1F0F6..1F0FF  ; Extended_Pictographic#   NA [10] (️..️)    <reserved-1F0F6>..<reserved-1F0FF> | ||||
| 1F10D..1F10F  ; Extended_Pictographic#   NA  [3] (🄍️..🄏️)    <reserved-1F10D>..<reserved-1F10F> | ||||
| 1F12F         ; Extended_Pictographic# 11.0  [1] (🄯️)       COPYLEFT SYMBOL | ||||
| 1F16C..1F16F  ; Extended_Pictographic#   NA  [4] (🅬️..🅯️)    <reserved-1F16C>..<reserved-1F16F> | ||||
| 1F170..1F171  ; Extended_Pictographic#  6.0  [2] (🅰️..🅱️)    A button (blood type)..B button (blood type) | ||||
| 1F17E         ; Extended_Pictographic#  6.0  [1] (🅾️)       O button (blood type) | ||||
| 1F17F         ; Extended_Pictographic#  5.2  [1] (🅿️)       P button | ||||
| 1F18E         ; Extended_Pictographic#  6.0  [1] (🆎)       AB button (blood type) | ||||
| 1F191..1F19A  ; Extended_Pictographic#  6.0 [10] (🆑..🆚)    CL button..VS button | ||||
| 1F1AD..1F1E5  ; Extended_Pictographic#   NA [57] (🆭️..️)    <reserved-1F1AD>..<reserved-1F1E5> | ||||
| 1F201..1F202  ; Extended_Pictographic#  6.0  [2] (🈁..🈂️)    Japanese “here” button..Japanese “service charge” button | ||||
| 1F203..1F20F  ; Extended_Pictographic#   NA [13] (️..️)    <reserved-1F203>..<reserved-1F20F> | ||||
| 1F21A         ; Extended_Pictographic#  5.2  [1] (🈚)       Japanese “free of charge” button | ||||
| 1F22F         ; Extended_Pictographic#  5.2  [1] (🈯)       Japanese “reserved” button | ||||
| 1F232..1F23A  ; Extended_Pictographic#  6.0  [9] (🈲..🈺)    Japanese “prohibited” button..Japanese “open for business” button | ||||
| 1F23C..1F23F  ; Extended_Pictographic#   NA  [4] (️..️)    <reserved-1F23C>..<reserved-1F23F> | ||||
| 1F249..1F24F  ; Extended_Pictographic#   NA  [7] (️..️)    <reserved-1F249>..<reserved-1F24F> | ||||
| 1F250..1F251  ; Extended_Pictographic#  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button | ||||
| 1F252..1F25F  ; Extended_Pictographic#   NA [14] (️..️)    <reserved-1F252>..<reserved-1F25F> | ||||
| 1F260..1F265  ; Extended_Pictographic# 10.0  [6] (🉠️..🉥️)    ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI | ||||
| 1F266..1F2FF  ; Extended_Pictographic#   NA[154] (️..️)    <reserved-1F266>..<reserved-1F2FF> | ||||
| 1F300..1F320  ; Extended_Pictographic#  6.0 [33] (🌀..🌠)    cyclone..shooting star | ||||
| 1F321..1F32C  ; Extended_Pictographic#  7.0 [12] (🌡️..🌬️)    thermometer..wind face | ||||
| 1F32D..1F32F  ; Extended_Pictographic#  8.0  [3] (🌭..🌯)    hot dog..burrito | ||||
| 1F330..1F335  ; Extended_Pictographic#  6.0  [6] (🌰..🌵)    chestnut..cactus | ||||
| 1F336         ; Extended_Pictographic#  7.0  [1] (🌶️)       hot pepper | ||||
| 1F337..1F37C  ; Extended_Pictographic#  6.0 [70] (🌷..🍼)    tulip..baby bottle | ||||
| 1F37D         ; Extended_Pictographic#  7.0  [1] (🍽️)       fork and knife with plate | ||||
| 1F37E..1F37F  ; Extended_Pictographic#  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn | ||||
| 1F380..1F393  ; Extended_Pictographic#  6.0 [20] (🎀..🎓)    ribbon..graduation cap | ||||
| 1F394..1F39F  ; Extended_Pictographic#  7.0 [12] (🎔️..🎟️)    HEART WITH TIP ON THE LEFT..admission tickets | ||||
| 1F3A0..1F3C4  ; Extended_Pictographic#  6.0 [37] (🎠..🏄)    carousel horse..person surfing | ||||
| 1F3C5         ; Extended_Pictographic#  7.0  [1] (🏅)       sports medal | ||||
| 1F3C6..1F3CA  ; Extended_Pictographic#  6.0  [5] (🏆..🏊)    trophy..person swimming | ||||
| 1F3CB..1F3CE  ; Extended_Pictographic#  7.0  [4] (🏋️..🏎️)    person lifting weights..racing car | ||||
| 1F3CF..1F3D3  ; Extended_Pictographic#  8.0  [5] (🏏..🏓)    cricket game..ping pong | ||||
| 1F3D4..1F3DF  ; Extended_Pictographic#  7.0 [12] (🏔️..🏟️)    snow-capped mountain..stadium | ||||
| 1F3E0..1F3F0  ; Extended_Pictographic#  6.0 [17] (🏠..🏰)    house..castle | ||||
| 1F3F1..1F3F7  ; Extended_Pictographic#  7.0  [7] (🏱️..🏷️)    WHITE PENNANT..label | ||||
| 1F3F8..1F3FA  ; Extended_Pictographic#  8.0  [3] (🏸..🏺)    badminton..amphora | ||||
| 1F400..1F43E  ; Extended_Pictographic#  6.0 [63] (🐀..🐾)    rat..paw prints | ||||
| 1F43F         ; Extended_Pictographic#  7.0  [1] (🐿️)       chipmunk | ||||
| 1F440         ; Extended_Pictographic#  6.0  [1] (👀)       eyes | ||||
| 1F441         ; Extended_Pictographic#  7.0  [1] (👁️)       eye | ||||
| 1F442..1F4F7  ; Extended_Pictographic#  6.0[182] (👂..📷)    ear..camera | ||||
| 1F4F8         ; Extended_Pictographic#  7.0  [1] (📸)       camera with flash | ||||
| 1F4F9..1F4FC  ; Extended_Pictographic#  6.0  [4] (📹..📼)    video camera..videocassette | ||||
| 1F4FD..1F4FE  ; Extended_Pictographic#  7.0  [2] (📽️..📾️)    film projector..PORTABLE STEREO | ||||
| 1F4FF         ; Extended_Pictographic#  8.0  [1] (📿)       prayer beads | ||||
| 1F500..1F53D  ; Extended_Pictographic#  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button | ||||
| 1F546..1F54A  ; Extended_Pictographic#  7.0  [5] (🕆️..🕊️)    WHITE LATIN CROSS..dove | ||||
| 1F54B..1F54F  ; Extended_Pictographic#  8.0  [5] (🕋..🕏️)    kaaba..BOWL OF HYGIEIA | ||||
| 1F550..1F567  ; Extended_Pictographic#  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty | ||||
| 1F568..1F579  ; Extended_Pictographic#  7.0 [18] (🕨️..🕹️)    RIGHT SPEAKER..joystick | ||||
| 1F57A         ; Extended_Pictographic#  9.0  [1] (🕺)       man dancing | ||||
| 1F57B..1F5A3  ; Extended_Pictographic#  7.0 [41] (🕻️..🖣️)    LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX | ||||
| 1F5A4         ; Extended_Pictographic#  9.0  [1] (🖤)       black heart | ||||
| 1F5A5..1F5FA  ; Extended_Pictographic#  7.0 [86] (🖥️..🗺️)    desktop computer..world map | ||||
| 1F5FB..1F5FF  ; Extended_Pictographic#  6.0  [5] (🗻..🗿)    mount fuji..moai | ||||
| 1F600         ; Extended_Pictographic#  6.1  [1] (😀)       grinning face | ||||
| 1F601..1F610  ; Extended_Pictographic#  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face | ||||
| 1F611         ; Extended_Pictographic#  6.1  [1] (😑)       expressionless face | ||||
| 1F612..1F614  ; Extended_Pictographic#  6.0  [3] (😒..😔)    unamused face..pensive face | ||||
| 1F615         ; Extended_Pictographic#  6.1  [1] (😕)       confused face | ||||
| 1F616         ; Extended_Pictographic#  6.0  [1] (😖)       confounded face | ||||
| 1F617         ; Extended_Pictographic#  6.1  [1] (😗)       kissing face | ||||
| 1F618         ; Extended_Pictographic#  6.0  [1] (😘)       face blowing a kiss | ||||
| 1F619         ; Extended_Pictographic#  6.1  [1] (😙)       kissing face with smiling eyes | ||||
| 1F61A         ; Extended_Pictographic#  6.0  [1] (😚)       kissing face with closed eyes | ||||
| 1F61B         ; Extended_Pictographic#  6.1  [1] (😛)       face with tongue | ||||
| 1F61C..1F61E  ; Extended_Pictographic#  6.0  [3] (😜..😞)    winking face with tongue..disappointed face | ||||
| 1F61F         ; Extended_Pictographic#  6.1  [1] (😟)       worried face | ||||
| 1F620..1F625  ; Extended_Pictographic#  6.0  [6] (😠..😥)    angry face..sad but relieved face | ||||
| 1F626..1F627  ; Extended_Pictographic#  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face | ||||
| 1F628..1F62B  ; Extended_Pictographic#  6.0  [4] (😨..😫)    fearful face..tired face | ||||
| 1F62C         ; Extended_Pictographic#  6.1  [1] (😬)       grimacing face | ||||
| 1F62D         ; Extended_Pictographic#  6.0  [1] (😭)       loudly crying face | ||||
| 1F62E..1F62F  ; Extended_Pictographic#  6.1  [2] (😮..😯)    face with open mouth..hushed face | ||||
| 1F630..1F633  ; Extended_Pictographic#  6.0  [4] (😰..😳)    anxious face with sweat..flushed face | ||||
| 1F634         ; Extended_Pictographic#  6.1  [1] (😴)       sleeping face | ||||
| 1F635..1F640  ; Extended_Pictographic#  6.0 [12] (😵..🙀)    dizzy face..weary cat face | ||||
| 1F641..1F642  ; Extended_Pictographic#  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face | ||||
| 1F643..1F644  ; Extended_Pictographic#  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes | ||||
| 1F645..1F64F  ; Extended_Pictographic#  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands | ||||
| 1F680..1F6C5  ; Extended_Pictographic#  6.0 [70] (🚀..🛅)    rocket..left luggage | ||||
| 1F6C6..1F6CF  ; Extended_Pictographic#  7.0 [10] (🛆️..🛏️)    TRIANGLE WITH ROUNDED CORNERS..bed | ||||
| 1F6D0         ; Extended_Pictographic#  8.0  [1] (🛐)       place of worship | ||||
| 1F6D1..1F6D2  ; Extended_Pictographic#  9.0  [2] (🛑..🛒)    stop sign..shopping cart | ||||
| 1F6D3..1F6D4  ; Extended_Pictographic# 10.0  [2] (🛓️..🛔️)    STUPA..PAGODA | ||||
| 1F6D5..1F6DF  ; Extended_Pictographic#   NA [11] (🛕️..🛟️)    <reserved-1F6D5>..<reserved-1F6DF> | ||||
| 1F6E0..1F6EC  ; Extended_Pictographic#  7.0 [13] (🛠️..🛬)    hammer and wrench..airplane arrival | ||||
| 1F6ED..1F6EF  ; Extended_Pictographic#   NA  [3] (️..️)    <reserved-1F6ED>..<reserved-1F6EF> | ||||
| 1F6F0..1F6F3  ; Extended_Pictographic#  7.0  [4] (🛰️..🛳️)    satellite..passenger ship | ||||
| 1F6F4..1F6F6  ; Extended_Pictographic#  9.0  [3] (🛴..🛶)    kick scooter..canoe | ||||
| 1F6F7..1F6F8  ; Extended_Pictographic# 10.0  [2] (🛷..🛸)    sled..flying saucer | ||||
| 1F6F9         ; Extended_Pictographic# 11.0  [1] (🛹)       skateboard | ||||
| 1F6FA..1F6FF  ; Extended_Pictographic#   NA  [6] (🛺️..️)    <reserved-1F6FA>..<reserved-1F6FF> | ||||
| 1F774..1F77F  ; Extended_Pictographic#   NA [12] (🝴️..🝿️)    <reserved-1F774>..<reserved-1F77F> | ||||
| 1F7D5..1F7D8  ; Extended_Pictographic# 11.0  [4] (🟕️..🟘️)    CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE | ||||
| 1F7D9..1F7FF  ; Extended_Pictographic#   NA [39] (🟙️..️)    <reserved-1F7D9>..<reserved-1F7FF> | ||||
| 1F80C..1F80F  ; Extended_Pictographic#   NA  [4] (️..️)    <reserved-1F80C>..<reserved-1F80F> | ||||
| 1F848..1F84F  ; Extended_Pictographic#   NA  [8] (️..️)    <reserved-1F848>..<reserved-1F84F> | ||||
| 1F85A..1F85F  ; Extended_Pictographic#   NA  [6] (️..️)    <reserved-1F85A>..<reserved-1F85F> | ||||
| 1F888..1F88F  ; Extended_Pictographic#   NA  [8] (️..️)    <reserved-1F888>..<reserved-1F88F> | ||||
| 1F8AE..1F8FF  ; Extended_Pictographic#   NA [82] (️..️)    <reserved-1F8AE>..<reserved-1F8FF> | ||||
| 1F90C..1F90F  ; Extended_Pictographic#   NA  [4] (🤌️..🤏️)    <reserved-1F90C>..<reserved-1F90F> | ||||
| 1F910..1F918  ; Extended_Pictographic#  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns | ||||
| 1F919..1F91E  ; Extended_Pictographic#  9.0  [6] (🤙..🤞)    call me hand..crossed fingers | ||||
| 1F91F         ; Extended_Pictographic# 10.0  [1] (🤟)       love-you gesture | ||||
| 1F920..1F927  ; Extended_Pictographic#  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face | ||||
| 1F928..1F92F  ; Extended_Pictographic# 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head | ||||
| 1F930         ; Extended_Pictographic#  9.0  [1] (🤰)       pregnant woman | ||||
| 1F931..1F932  ; Extended_Pictographic# 10.0  [2] (🤱..🤲)    breast-feeding..palms up together | ||||
| 1F933..1F93A  ; Extended_Pictographic#  9.0  [8] (🤳..🤺)    selfie..person fencing | ||||
| 1F93C..1F93E  ; Extended_Pictographic#  9.0  [3] (🤼..🤾)    people wrestling..person playing handball | ||||
| 1F93F         ; Extended_Pictographic#   NA  [1] (🤿️)       <reserved-1F93F> | ||||
| 1F940..1F945  ; Extended_Pictographic#  9.0  [6] (🥀..🥅)    wilted flower..goal net | ||||
| 1F947..1F94B  ; Extended_Pictographic#  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform | ||||
| 1F94C         ; Extended_Pictographic# 10.0  [1] (🥌)       curling stone | ||||
| 1F94D..1F94F  ; Extended_Pictographic# 11.0  [3] (🥍..🥏)    lacrosse..flying disc | ||||
| 1F950..1F95E  ; Extended_Pictographic#  9.0 [15] (🥐..🥞)    croissant..pancakes | ||||
| 1F95F..1F96B  ; Extended_Pictographic# 10.0 [13] (🥟..🥫)    dumpling..canned food | ||||
| 1F96C..1F970  ; Extended_Pictographic# 11.0  [5] (🥬..🥰)    leafy green..smiling face with 3 hearts | ||||
| 1F971..1F972  ; Extended_Pictographic#   NA  [2] (🥱️..🥲️)    <reserved-1F971>..<reserved-1F972> | ||||
| 1F973..1F976  ; Extended_Pictographic# 11.0  [4] (🥳..🥶)    partying face..cold face | ||||
| 1F977..1F979  ; Extended_Pictographic#   NA  [3] (🥷️..🥹️)    <reserved-1F977>..<reserved-1F979> | ||||
| 1F97A         ; Extended_Pictographic# 11.0  [1] (🥺)       pleading face | ||||
| 1F97B         ; Extended_Pictographic#   NA  [1] (🥻️)       <reserved-1F97B> | ||||
| 1F97C..1F97F  ; Extended_Pictographic# 11.0  [4] (🥼..🥿)    lab coat..woman’s flat shoe | ||||
| 1F980..1F984  ; Extended_Pictographic#  8.0  [5] (🦀..🦄)    crab..unicorn face | ||||
| 1F985..1F991  ; Extended_Pictographic#  9.0 [13] (🦅..🦑)    eagle..squid | ||||
| 1F992..1F997  ; Extended_Pictographic# 10.0  [6] (🦒..🦗)    giraffe..cricket | ||||
| 1F998..1F9A2  ; Extended_Pictographic# 11.0 [11] (🦘..🦢)    kangaroo..swan | ||||
| 1F9A3..1F9AF  ; Extended_Pictographic#   NA [13] (🦣️..🦯️)    <reserved-1F9A3>..<reserved-1F9AF> | ||||
| 1F9B0..1F9B9  ; Extended_Pictographic# 11.0 [10] (🦰..🦹)    red-haired..supervillain | ||||
| 1F9BA..1F9BF  ; Extended_Pictographic#   NA  [6] (🦺️..🦿️)    <reserved-1F9BA>..<reserved-1F9BF> | ||||
| 1F9C0         ; Extended_Pictographic#  8.0  [1] (🧀)       cheese wedge | ||||
| 1F9C1..1F9C2  ; Extended_Pictographic# 11.0  [2] (🧁..🧂)    cupcake..salt | ||||
| 1F9C3..1F9CF  ; Extended_Pictographic#   NA [13] (🧃️..🧏️)    <reserved-1F9C3>..<reserved-1F9CF> | ||||
| 1F9D0..1F9E6  ; Extended_Pictographic# 10.0 [23] (🧐..🧦)    face with monocle..socks | ||||
| 1F9E7..1F9FF  ; Extended_Pictographic# 11.0 [25] (🧧..🧿)    red envelope..nazar amulet | ||||
| 1FA00..1FA5F  ; Extended_Pictographic#   NA [96] (🨀️..️)    <reserved-1FA00>..<reserved-1FA5F> | ||||
| 1FA60..1FA6D  ; Extended_Pictographic# 11.0 [14] (🩠️..🩭️)    XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER | ||||
| 1FA6E..1FFFD  ; Extended_Pictographic#   NA[1424] (️..️)   <reserved-1FA6E>..<reserved-1FFFD> | ||||
|  | ||||
| # Total elements: 3793 | ||||
|  | ||||
| #EOF | ||||
							
								
								
									
										1027
									
								
								Project/Tools/emoji-list.txt
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1027
									
								
								Project/Tools/emoji-list.txt
									
									
									
									
									
										Normal file
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							
							
								
								
									
										18
									
								
								Project/Tools/stream_language_detector.py
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										18
									
								
								Project/Tools/stream_language_detector.py
									
									
									
									
									
										Executable file
									
								
							| @ -0,0 +1,18 @@ | ||||
| #!/usr/bin/env python3 | ||||
| from langdetect import detect | ||||
| import fileinput as fi | ||||
| import sys | ||||
|  | ||||
| # just a little script to detect languages of lines starting  | ||||
| # with the json keyword 'text' and writing the language as json value to stdout. | ||||
| # other lines are just passing through so that this script can be used in a shell pipeline | ||||
|  | ||||
| for line in fi.input(): | ||||
|     if line.startswith('  "text"'): | ||||
|         try: | ||||
|             sys.stdout.write('  "lang": "' + detect(line[10:]) + '"\n') | ||||
|         except Exception: | ||||
|             sys.stdout.write('  "lang": "NaN"\n') | ||||
|     sys.stdout.write(line) | ||||
|  | ||||
|  | ||||
							
								
								
									
										72
									
								
								Project/Tools/twitter2messages.sh
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										72
									
								
								Project/Tools/twitter2messages.sh
									
									
									
									
									
										Executable file
									
								
							| @ -0,0 +1,72 @@ | ||||
| #!/usr/bin/env bash | ||||
|  | ||||
| SCRIPT=`realpath $0` | ||||
| SCRIPTPATH=`dirname $SCRIPT` | ||||
|  | ||||
| # toolset:--------------------------------------------------------------------- | ||||
|  | ||||
| command 2> >(while read line; do echo -e "\e[01;31m$line\e[0m" >&2; done) | ||||
|  | ||||
| function lineprint { | ||||
|     printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' = | ||||
| } | ||||
|  | ||||
| function message { | ||||
|     lineprint | ||||
|     printf "$1\n" | ||||
|     lineprint | ||||
| } | ||||
|  | ||||
| function error_message { | ||||
|     lineprint | ||||
|     printf "$1\n" >&2 | ||||
|     lineprint | ||||
| } | ||||
|  | ||||
| current_action="IDLE" | ||||
|  | ||||
| function confirm_action { | ||||
|     message "successfully finished action: $current_action" | ||||
| } | ||||
|  | ||||
| function set_action { | ||||
|     current_action="$1" | ||||
|     message "$1" | ||||
| } | ||||
|  | ||||
| function perform { | ||||
|     "$@" | ||||
|     local status=$? | ||||
|     if [ $status -ne 0 ] | ||||
|     then | ||||
|         error_message "$current_action failed!" | ||||
|     fi | ||||
|     return $status | ||||
| } | ||||
|  | ||||
| function perform_and_exit { | ||||
|     perform "$@" || exit 1 | ||||
| } | ||||
|  | ||||
| # ----------------------------------------------------------------------------- | ||||
|  | ||||
| INPUT=$1 | ||||
| OUTPUT=$2 | ||||
| if [ $# -ne 2 ] | ||||
| then | ||||
|     error_message "Error: no input file given. Usage: $0 <Filepath> <outputfile>" | ||||
|     exit 1 | ||||
| fi | ||||
|  | ||||
| set_action "processing all files in $INPUT and write to $OUTPUT" | ||||
|  | ||||
| perform_and_exit export elist=\"`head -c -1 "$SCRIPTPATH/emoji-list.txt" | tr '\n' ',' | sed 's/,/\",\"/g'`\" | ||||
| perform_and_exit echo "filter by emoji list:" | ||||
| perform_and_exit echo $elist | tr -d '"' | tr -d ',' | ||||
|  | ||||
| #perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT" | ||||
| perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text, lang: .lang, reply_to: .in_reply_to_status_id} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 3 -B 4 | tee /dev/tty > "$OUTPUT" | ||||
|  | ||||
| # ↑ such obvious, much selfexplaining 💁😈 | ||||
|  | ||||
| confirm_action | ||||
							
								
								
									
										85
									
								
								Project/Tools/whatsapp2csv.sh
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										85
									
								
								Project/Tools/whatsapp2csv.sh
									
									
									
									
									
										Executable file
									
								
							| @ -0,0 +1,85 @@ | ||||
| #!/usr/bin/env bash | ||||
|  | ||||
| SCRIPT=`realpath $0` | ||||
| SCRIPTPATH=`dirname $SCRIPT` | ||||
|  | ||||
| # toolset:--------------------------------------------------------------------- | ||||
|  | ||||
| command 2> >(while read line; do echo -e "\e[01;31m$line\e[0m" >&2; done) | ||||
|  | ||||
| function lineprint { | ||||
|     printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' = | ||||
| } | ||||
|  | ||||
| function message { | ||||
|     lineprint | ||||
|     printf "$1\n" | ||||
|     lineprint | ||||
| } | ||||
|  | ||||
| function error_message { | ||||
|     lineprint | ||||
|     printf "$1\n" >&2 | ||||
|     lineprint | ||||
| } | ||||
|  | ||||
| current_action="IDLE" | ||||
|  | ||||
| function confirm_action { | ||||
|     message "successfully finished action: $current_action" | ||||
| } | ||||
|  | ||||
| function set_action { | ||||
|     current_action="$1" | ||||
|     message "$1" | ||||
| } | ||||
|  | ||||
| function perform { | ||||
|     "$@" | ||||
|     local status=$? | ||||
|     if [ $status -ne 0 ] | ||||
|     then | ||||
|         error_message "$current_action failed!" | ||||
|     fi | ||||
|     return $status | ||||
| } | ||||
|  | ||||
| function perform_and_exit { | ||||
|     perform "$@" || exit 1 | ||||
| } | ||||
|  | ||||
| # ----------------------------------------------------------------------------- | ||||
| # reading input | ||||
|  | ||||
| INPUT=$1 | ||||
| OUTPUT="$INPUT.csv" | ||||
|  | ||||
| if [ $# -eq 0 ] | ||||
| then | ||||
|     error_message "Error: no input file given. Usage: $0 <Whatsapp textbackup file>" | ||||
|     exit 1 | ||||
| fi | ||||
|  | ||||
| set_action "processing File: ${INPUT}" | ||||
|  | ||||
| if [ ! -e "$INPUT" ] | ||||
| then | ||||
|     error_message "Error: file '$INPUT' not found" | ||||
|     exit 1 | ||||
| fi | ||||
|  | ||||
| perform_and_exit echo -e "date\ttime\tperson\tmessage" > "$OUTPUT" | ||||
|  | ||||
| # doing the following things in the pipeline below: | ||||
| # | ||||
| # 1. merging multiline messages by replacing newlines with spaces if they are not starting with a number followed by '/' (not stating with a date) | ||||
| # 2. delete all lines containing the keywords: "added, creates, end-to-end, Media ommited" (which are most probably system messages) | ||||
| # 3. replace separators of whatsapp's txt-format with '\t' as separator for csv | ||||
| # 4. delete all double-quotes. Because people are too stupid to type them in pairs and that breaks some csv-interpreters 🤦♂ | ||||
|  | ||||
| perform_and_exit sed ':a;N;/\n[0-9]\{1,\}\+\//!s/\n/ /;ta;P;D' "$INPUT" | grep -v -E 'left|added|created|end-to-end|Media omitted|import java|<VirtualHost' | sed 's/, /\t/1; s/- /\t/1; s/: /\t/1;' | tr -d '"' >> "$OUTPUT" | ||||
|  | ||||
| confirm_action | ||||
|  | ||||
| message "Wrote output to $OUTPUT" | ||||
|  | ||||
		Reference in New Issue
	
	Block a user