From 2e10be4469e1c65f0e817ddab3f827f33098c189 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 14 Jan 2022 19:53:59 -0700 Subject: [PATCH] included option for variable range of images per listing --- Shoe Classifier_InceptionResNetV2.ipynb | 124 ++++++++--------- Shoe Classifier_VGG16.ipynb | 165 +++++++++------------- Shoe Classifier_VGG19.ipynb | 51 +++++-- Shoe Classifier_Xception.ipynb | 174 +++++++++--------------- ebay_api.py | 36 +++-- 5 files changed, 245 insertions(+), 305 deletions(-) diff --git a/Shoe Classifier_InceptionResNetV2.ipynb b/Shoe Classifier_InceptionResNetV2.ipynb index fc9ce7f..3ea3e8c 100644 --- a/Shoe Classifier_InceptionResNetV2.ipynb +++ b/Shoe Classifier_InceptionResNetV2.ipynb @@ -39,30 +39,24 @@ "metadata": {}, "outputs": [], "source": [ - "def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n", - "\n", - " if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n", - " print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n", - " return model\n", - "\n", - " for layer in model.layers:\n", - " for attr in ['kernel_regularizer']:\n", - " if hasattr(layer, attr):\n", - " setattr(layer, attr, regularizer)\n", - "\n", - " # When we change the layers attributes, the change only happens in the model config file\n", - " model_json = model.to_json()\n", - "\n", - " # Save the weights before reloading the model.\n", - " tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n", - " model.save_weights(tmp_weights_path)\n", - "\n", - " # load the model from the config\n", - " model = tf.keras.models.model_from_json(model_json)\n", - " \n", - " # Reload the model weights\n", - " model.load_weights(tmp_weights_path, by_name=True)\n", - " return model" + "def dict_pics_jup():\n", + " '''\n", + " {source:target} dict used to replace source urls with image location as input\n", + " '''\n", + " target_dir = os.getcwd() + os.sep + \"training_images\"\n", + " with open('temp_pics_source_list.txt') as f:\n", + " temp_pics_source_list = json.load(f)\n", + " \n", + " dict_pics = {}\n", + " for k in temp_pics_source_list:\n", + " patt_1 = re.search(r'[^/]+(?=/\\$_|.(\\.jpg|\\.jpeg|\\.png))', k, re.IGNORECASE)\n", + " patt_2 = re.search(r'(\\.jpg|\\.jpeg|\\.png)', k, re.IGNORECASE)\n", + " if patt_1 and patt_2 is not None:\n", + " tag = patt_1.group() + patt_2.group().lower()\n", + " file_name = target_dir + os.sep + tag\n", + " dict_pics.update({k:file_name})\n", + " print(\"{source:target} dictionary created @ \" + target_dir)\n", + " return dict_pics" ] }, { @@ -93,21 +87,33 @@ } ], "source": [ - "def dict_pics():\n", - " target_dir = os.getcwd() + os.sep + \"training_images\"\n", - " with open('temp_pics_source_list.txt') as f:\n", - " temp_pics_source_list = json.load(f)\n", - " dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n", - " print(\"{source:target} dictionary created @ \" + target_dir)\n", - " return dict_pics\n", + "dict_pics = dict_pics_jup()\n", + "\n", + "with open('women_cat_list.txt') as f:\n", + " women_cats = json.load(f)\n", + "with open('men_cat_list.txt') as f:\n", + " men_cats = json.load(f)\n", + " \n", + "with open('temp_pics_source_list.txt') as f:\n", + " tempics = json.load(f)\n", + "# list of image urls that did not get named properly which will be removed from the dataframe\n", + "drop_row_vals = []\n", + "for pic in tempics:\n", + " try:\n", + " dict_pics[pic]\n", + " except KeyError:\n", + " drop_row_vals.append(pic)\n", + "\n", + "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", + "ddf = df[df.PictureURL.isin(drop_row_vals)==False] # remove improperly named image files\n", + "df = ddf[ddf.PrimaryCategoryID.isin(men_cats)==False] # removes rows of womens categories\n", "\n", - "dict_pics = dict_pics()\n", "blah = pd.Series(df.PictureURL)\n", "df = df.drop(labels=['PictureURL'], axis=1)\n", + "\n", "blah = blah.apply(lambda x: dict_pics[x])\n", "df = pd.concat([blah, df],axis=1)\n", - "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n", - "# removes non-existent image paths" + "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers" ] }, { @@ -117,8 +123,6 @@ "metadata": {}, "outputs": [], "source": [ - "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", - "\n", "df=df.sample(frac=1)" ] }, @@ -155,16 +159,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found 4542 validated image filenames belonging to 13 classes.\n", - "Found 1135 validated image filenames belonging to 13 classes.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n", - " warnings.warn(\n" + "Found 4444 validated image filenames belonging to 7 classes.\n", + "Found 1111 validated image filenames belonging to 7 classes.\n" ] } ], @@ -2656,12 +2652,12 @@ " global_average_pooling2d (Glob (None, 1536) 0 ['conv_7b_ac[0][0]'] \n", " alAveragePooling2D) \n", " \n", - " dense (Dense) (None, 13) 19981 ['global_average_pooling2d[0][0]'\n", + " dense (Dense) (None, 7) 10759 ['global_average_pooling2d[0][0]'\n", " ] \n", " \n", "==================================================================================================\n", - "Total params: 54,356,717\n", - "Trainable params: 54,296,173\n", + "Total params: 54,347,495\n", + "Trainable params: 54,286,951\n", "Non-trainable params: 60,544\n", "__________________________________________________________________________________________________\n" ] @@ -2671,7 +2667,7 @@ "for layer in base_model.layers:\n", " layer.trainable = True\n", " \n", - "output = Dense(13, activation='softmax')(base_model.output)\n", + "output = Dense(7, activation='softmax')(base_model.output)\n", "model = tf.keras.Model(base_model.input, output)\n", "#model = add_regularization(model)\n", "model.summary()\n" @@ -2715,33 +2711,21 @@ "output_type": "stream", "text": [ "Epoch 1/30\n", - "142/142 [==============================] - 71s 426ms/step - loss: 1.5727 - accuracy: 0.4613 - val_loss: 2.5192 - val_accuracy: 0.1233\n", + "139/139 [==============================] - 65s 390ms/step - loss: 1.0030 - accuracy: 0.6427 - val_loss: 2.1331 - val_accuracy: 0.1620\n", "Epoch 2/30\n", - "142/142 [==============================] - 56s 395ms/step - loss: 0.6991 - accuracy: 0.7644 - val_loss: 1.9969 - val_accuracy: 0.3568\n", + "139/139 [==============================] - 51s 365ms/step - loss: 0.3751 - accuracy: 0.8756 - val_loss: 1.7818 - val_accuracy: 0.4707\n", "Epoch 3/30\n", - "142/142 [==============================] - 56s 395ms/step - loss: 0.2355 - accuracy: 0.9302 - val_loss: 1.9622 - val_accuracy: 0.4626\n", + "139/139 [==============================] - 50s 359ms/step - loss: 0.1193 - accuracy: 0.9622 - val_loss: 1.5616 - val_accuracy: 0.6166\n", "Epoch 4/30\n", - "142/142 [==============================] - 56s 393ms/step - loss: 0.1015 - accuracy: 0.9707 - val_loss: 1.8987 - val_accuracy: 0.5207\n", + "139/139 [==============================] - 51s 363ms/step - loss: 0.0830 - accuracy: 0.9750 - val_loss: 1.3374 - val_accuracy: 0.6922\n", "Epoch 5/30\n", - "142/142 [==============================] - 57s 398ms/step - loss: 0.0899 - accuracy: 0.9736 - val_loss: 1.8865 - val_accuracy: 0.5727\n", + "139/139 [==============================] - 50s 360ms/step - loss: 0.0609 - accuracy: 0.9815 - val_loss: 1.3717 - val_accuracy: 0.6976\n", "Epoch 6/30\n", - "142/142 [==============================] - 57s 402ms/step - loss: 0.0895 - accuracy: 0.9742 - val_loss: 1.9440 - val_accuracy: 0.5419\n", + "139/139 [==============================] - 51s 366ms/step - loss: 0.0593 - accuracy: 0.9818 - val_loss: 1.2600 - val_accuracy: 0.7192\n", "Epoch 7/30\n", - "142/142 [==============================] - 58s 406ms/step - loss: 0.0784 - accuracy: 0.9762 - val_loss: 2.2314 - val_accuracy: 0.5145\n", + "139/139 [==============================] - 51s 363ms/step - loss: 0.0468 - accuracy: 0.9860 - val_loss: 1.1707 - val_accuracy: 0.7264\n", "Epoch 8/30\n", - "142/142 [==============================] - 58s 406ms/step - loss: 0.0536 - accuracy: 0.9852 - val_loss: 2.0704 - val_accuracy: 0.5480\n", - "Epoch 9/30\n", - "142/142 [==============================] - 57s 399ms/step - loss: 0.0672 - accuracy: 0.9802 - val_loss: 2.3144 - val_accuracy: 0.5189\n", - "Epoch 10/30\n", - "142/142 [==============================] - 56s 395ms/step - loss: 0.0423 - accuracy: 0.9875 - val_loss: 2.2718 - val_accuracy: 0.5330\n", - "Epoch 11/30\n", - "142/142 [==============================] - 57s 400ms/step - loss: 0.0325 - accuracy: 0.9903 - val_loss: 2.2889 - val_accuracy: 0.5498\n", - "Epoch 12/30\n", - "142/142 [==============================] - 56s 396ms/step - loss: 0.0591 - accuracy: 0.9822 - val_loss: 2.1700 - val_accuracy: 0.5577\n", - "Epoch 13/30\n", - "142/142 [==============================] - 56s 395ms/step - loss: 0.0387 - accuracy: 0.9899 - val_loss: 2.3132 - val_accuracy: 0.5410\n", - "Epoch 14/30\n", - " 95/142 [===================>..........] - ETA: 14s - loss: 0.0344 - accuracy: 0.9888" + " 89/139 [==================>...........] - ETA: 14s - loss: 0.0309 - accuracy: 0.9905" ] } ], diff --git a/Shoe Classifier_VGG16.ipynb b/Shoe Classifier_VGG16.ipynb index 19bfd86..d6d1c99 100644 --- a/Shoe Classifier_VGG16.ipynb +++ b/Shoe Classifier_VGG16.ipynb @@ -83,6 +83,33 @@ "metadata": { "scrolled": true }, + "outputs": [], + "source": [ + "def dict_pics_jup():\n", + " '''\n", + " {source:target} dict used to replace source urls with image location as input\n", + " '''\n", + " target_dir = os.getcwd() + os.sep + \"training_images\"\n", + " with open('temp_pics_source_list.txt') as f:\n", + " temp_pics_source_list = json.load(f)\n", + " \n", + " dict_pics = {}\n", + " for k in temp_pics_source_list:\n", + " patt_1 = re.search(r'[^/]+(?=/\\$_|.(\\.jpg|\\.jpeg|\\.png))', k, re.IGNORECASE)\n", + " patt_2 = re.search(r'(\\.jpg|\\.jpeg|\\.png)', k, re.IGNORECASE)\n", + " if patt_1 and patt_2 is not None:\n", + " tag = patt_1.group() + patt_2.group().lower()\n", + " file_name = target_dir + os.sep + tag\n", + " dict_pics.update({k:file_name})\n", + " print(\"{source:target} dictionary created @ \" + target_dir)\n", + " return dict_pics" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7a6146e6", + "metadata": {}, "outputs": [ { "name": "stdout", @@ -93,31 +120,33 @@ } ], "source": [ - "def dict_pics():\n", - " target_dir = os.getcwd() + os.sep + \"training_images\"\n", - " with open('temp_pics_source_list.txt') as f:\n", - " temp_pics_source_list = json.load(f)\n", - " dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n", - " print(\"{source:target} dictionary created @ \" + target_dir)\n", - " return dict_pics\n", + "dict_pics = dict_pics_jup()\n", + "\n", + "with open('women_cat_list.txt') as f:\n", + " women_cats = json.load(f)\n", + "with open('men_cat_list.txt') as f:\n", + " men_cats = json.load(f)\n", + " \n", + "with open('temp_pics_source_list.txt') as f:\n", + " tempics = json.load(f)\n", + "# list of image urls that did not get named properly which will be removed from the dataframe\n", + "drop_row_vals = []\n", + "for pic in tempics:\n", + " try:\n", + " dict_pics[pic]\n", + " except KeyError:\n", + " drop_row_vals.append(pic)\n", + "\n", + "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", + "ddf = df[df.PictureURL.isin(drop_row_vals)==False] # remove improperly named image files\n", + "df = ddf[ddf.PrimaryCategoryID.isin(men_cats)==False] # removes rows of womens categories\n", "\n", - "dict_pics = dict_pics()\n", "blah = pd.Series(df.PictureURL)\n", "df = df.drop(labels=['PictureURL'], axis=1)\n", + "\n", "blah = blah.apply(lambda x: dict_pics[x])\n", "df = pd.concat([blah, df],axis=1)\n", "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n", - "# removes non-existent image paths" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "7a6146e6", - "metadata": {}, - "outputs": [], - "source": [ - "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", "\n", "df=df.sample(frac=1)" ] @@ -131,7 +160,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Counter({'11498': 546, '11504': 546, '11505': 546, '11632': 546, '15709': 546, '24087': 546, '45333': 546, '53120': 546, '53548': 546, '53557': 546, '55793': 546, '62107': 546, '95672': 546})\n" + "Counter({'11632': 6505, '45333': 6505, '53548': 6505, '53557': 6505, '55793': 6505, '62107': 6505, '95672': 6505})\n" ] } ], @@ -163,16 +192,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found 4542 validated image filenames belonging to 13 classes.\n", - "Found 1135 validated image filenames belonging to 13 classes.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n", - " warnings.warn(\n" + "Found 29143 validated image filenames belonging to 7 classes.\n", + "Found 7285 validated image filenames belonging to 7 classes.\n" ] } ], @@ -302,14 +323,14 @@ "for layer in base_model.layers[:-1]:\n", " updated_model.add(layer)\n", " if layer.name in ['fc1', 'fc2']:\n", - " updated_model.add(Dropout(.75))\n", + " updated_model.add(Dropout(.50))\n", "\n", "model = updated_model\n", "\n", "for layer in model.layers:\n", " layer.trainable = True\n", "\n", - "model.add(Dense(units=13, activation='softmax'))" + "model.add(Dense(units=7, activation='softmax'))" ] }, { @@ -374,11 +395,11 @@ " \n", " dropout_1 (Dropout) (None, 4096) 0 \n", " \n", - " dense (Dense) (None, 13) 53261 \n", + " dense (Dense) (None, 7) 28679 \n", " \n", "=================================================================\n", - "Total params: 134,313,805\n", - "Trainable params: 134,313,805\n", + "Total params: 134,289,223\n", + "Trainable params: 134,289,223\n", "Non-trainable params: 0\n", "_________________________________________________________________\n" ] @@ -402,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "9cd2ba27", "metadata": { "scrolled": false @@ -413,77 +434,19 @@ "output_type": "stream", "text": [ "Epoch 1/30\n", - "142/142 [==============================] - 62s 404ms/step - loss: 2.7986 - accuracy: 0.0771 - val_loss: 2.5716 - val_accuracy: 0.0670\n", + "911/911 [==============================] - 329s 356ms/step - loss: 1.8477 - accuracy: 0.2577 - val_loss: 1.6306 - val_accuracy: 0.3669\n", "Epoch 2/30\n", - "142/142 [==============================] - 53s 370ms/step - loss: 2.6351 - accuracy: 0.0790 - val_loss: 2.5728 - val_accuracy: 0.0802\n", + "911/911 [==============================] - 322s 353ms/step - loss: 1.4882 - accuracy: 0.4353 - val_loss: 1.4317 - val_accuracy: 0.4784\n", "Epoch 3/30\n", - "142/142 [==============================] - 53s 370ms/step - loss: 2.6157 - accuracy: 0.0852 - val_loss: 2.5786 - val_accuracy: 0.0705\n", + "911/911 [==============================] - 323s 354ms/step - loss: 1.3046 - accuracy: 0.5158 - val_loss: 1.2747 - val_accuracy: 0.5235\n", "Epoch 4/30\n", - "142/142 [==============================] - 53s 371ms/step - loss: 2.6087 - accuracy: 0.0821 - val_loss: 2.5501 - val_accuracy: 0.1013\n", + "911/911 [==============================] - 319s 350ms/step - loss: 1.1691 - accuracy: 0.5681 - val_loss: 1.2090 - val_accuracy: 0.5529\n", "Epoch 5/30\n", - "142/142 [==============================] - 53s 371ms/step - loss: 2.5420 - accuracy: 0.1182 - val_loss: 2.4237 - val_accuracy: 0.1401\n", + "911/911 [==============================] - 317s 348ms/step - loss: 1.0389 - accuracy: 0.6185 - val_loss: 1.1774 - val_accuracy: 0.5682\n", "Epoch 6/30\n", - "142/142 [==============================] - 52s 368ms/step - loss: 2.4275 - accuracy: 0.1640 - val_loss: 2.3050 - val_accuracy: 0.1982\n", + "911/911 [==============================] - 317s 348ms/step - loss: 0.9125 - accuracy: 0.6656 - val_loss: 1.2237 - val_accuracy: 0.5639\n", "Epoch 7/30\n", - "142/142 [==============================] - 52s 368ms/step - loss: 2.3270 - accuracy: 0.2008 - val_loss: 2.2103 - val_accuracy: 0.2273\n", - "Epoch 8/30\n", - "142/142 [==============================] - 54s 378ms/step - loss: 2.2011 - accuracy: 0.2294 - val_loss: 2.0607 - val_accuracy: 0.2872\n", - "Epoch 9/30\n", - "142/142 [==============================] - 54s 378ms/step - loss: 2.0790 - accuracy: 0.2781 - val_loss: 1.9376 - val_accuracy: 0.3419\n", - "Epoch 10/30\n", - "142/142 [==============================] - 52s 368ms/step - loss: 1.8708 - accuracy: 0.3417 - val_loss: 2.0407 - val_accuracy: 0.3198\n", - "Epoch 11/30\n", - "142/142 [==============================] - 52s 368ms/step - loss: 1.7588 - accuracy: 0.3743 - val_loss: 1.8968 - val_accuracy: 0.3401\n", - "Epoch 12/30\n", - "142/142 [==============================] - 52s 369ms/step - loss: 1.5927 - accuracy: 0.4251 - val_loss: 1.8735 - val_accuracy: 0.3542\n", - "Epoch 13/30\n", - "142/142 [==============================] - 53s 369ms/step - loss: 1.4704 - accuracy: 0.4626 - val_loss: 1.8489 - val_accuracy: 0.3753\n", - "Epoch 14/30\n", - "142/142 [==============================] - 52s 367ms/step - loss: 1.2452 - accuracy: 0.5484 - val_loss: 1.9595 - val_accuracy: 0.3656\n", - "Epoch 15/30\n", - "142/142 [==============================] - 52s 370ms/step - loss: 1.1410 - accuracy: 0.5885 - val_loss: 1.9159 - val_accuracy: 0.3930\n", - "Epoch 16/30\n", - "142/142 [==============================] - 52s 367ms/step - loss: 0.9705 - accuracy: 0.6510 - val_loss: 2.1161 - val_accuracy: 0.3921\n", - "Epoch 17/30\n", - "142/142 [==============================] - 52s 368ms/step - loss: 0.7992 - accuracy: 0.7041 - val_loss: 2.1271 - val_accuracy: 0.4097\n", - "Epoch 18/30\n", - "142/142 [==============================] - 52s 369ms/step - loss: 0.7544 - accuracy: 0.7305 - val_loss: 2.0093 - val_accuracy: 0.4361\n", - "Epoch 19/30\n", - "142/142 [==============================] - 52s 367ms/step - loss: 0.5651 - accuracy: 0.7961 - val_loss: 2.4199 - val_accuracy: 0.3850\n", - "Epoch 20/30\n", - "142/142 [==============================] - 52s 366ms/step - loss: 0.4566 - accuracy: 0.8342 - val_loss: 2.4058 - val_accuracy: 0.4352\n", - "Epoch 21/30\n", - "142/142 [==============================] - 52s 364ms/step - loss: 0.3841 - accuracy: 0.8622 - val_loss: 2.5407 - val_accuracy: 0.4141\n", - "Epoch 22/30\n", - "142/142 [==============================] - 52s 366ms/step - loss: 0.2496 - accuracy: 0.9086 - val_loss: 3.0696 - val_accuracy: 0.4088\n", - "Epoch 23/30\n", - "142/142 [==============================] - 52s 368ms/step - loss: 0.2847 - accuracy: 0.8985 - val_loss: 2.7929 - val_accuracy: 0.4273\n", - "Epoch 24/30\n", - "142/142 [==============================] - 52s 369ms/step - loss: 0.2035 - accuracy: 0.9293 - val_loss: 3.0300 - val_accuracy: 0.4132\n", - "Epoch 25/30\n", - "142/142 [==============================] - ETA: 0s - loss: 0.1872 - accuracy: 0.9377" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1250\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1251\u001b[0m steps_per_execution=self._steps_per_execution)\n\u001b[0;32m-> 1252\u001b[0;31m val_logs = self.evaluate(\n\u001b[0m\u001b[1;32m 1253\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_x\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1254\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[1;32m 1535\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_r\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1537\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1538\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1539\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 151\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 908\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 909\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 912\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 947\u001b[0m \u001b[0;31m# In this case we have not created variables on the first call. So we can\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 948\u001b[0m \u001b[0;31m# run the first trace but we should fail if variables are created.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 949\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 950\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_variables\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mALLOW_DYNAMIC_VARIABLE_CREATION\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 951\u001b[0m raise ValueError(\"Creating variables on a non-first call to a function\"\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 3128\u001b[0m (graph_function,\n\u001b[1;32m 3129\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 3130\u001b[0;31m return graph_function._call_flat(\n\u001b[0m\u001b[1;32m 3131\u001b[0m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 3132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1957\u001b[0m and executing_eagerly):\n\u001b[1;32m 1958\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1959\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1960\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1961\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 597\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 599\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 59\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 60\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "147/911 [===>..........................] - ETA: 3:39 - loss: 0.7312 - accuracy: 0.7256" ] } ], diff --git a/Shoe Classifier_VGG19.ipynb b/Shoe Classifier_VGG19.ipynb index eaee681..1d20bb4 100644 --- a/Shoe Classifier_VGG19.ipynb +++ b/Shoe Classifier_VGG19.ipynb @@ -94,20 +94,23 @@ ], "source": [ "def dict_pics_jup():\n", + " '''\n", + " {source:target} dict used to replace source urls with image location as input\n", + " '''\n", " target_dir = os.getcwd() + os.sep + \"training_images\"\n", " with open('temp_pics_source_list.txt') as f:\n", " temp_pics_source_list = json.load(f)\n", - " dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n", + " \n", + " dict_pics = {}\n", + " for k in temp_pics_source_list:\n", + " patt_1 = re.search(r'[^/]+(?=/\\$_|.(\\.jpg|\\.jpeg|\\.png))', k, re.IGNORECASE)\n", + " patt_2 = re.search(r'(\\.jpg|\\.jpeg|\\.png)', k, re.IGNORECASE)\n", + " if patt_1 and patt_2 is not None:\n", + " tag = patt_1.group() + patt_2.group().lower()\n", + " file_name = target_dir + os.sep + tag\n", + " dict_pics.update({k:file_name})\n", " print(\"{source:target} dictionary created @ \" + target_dir)\n", - " return dict_pics\n", - "\n", - "dict_pics = dict_pics_jup()\n", - "blah = pd.Series(df.PictureURL)\n", - "df = df.drop(labels=['PictureURL'], axis=1)\n", - "blah = blah.apply(lambda x: dict_pics[x])\n", - "df = pd.concat([blah, df],axis=1)\n", - "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n", - "# removes non-existent image paths" + " return dict_pics" ] }, { @@ -117,7 +120,33 @@ "metadata": {}, "outputs": [], "source": [ + "dict_pics = dict_pics_jup()\n", + "\n", + "with open('women_cat_list.txt') as f:\n", + " women_cats = json.load(f)\n", + "with open('men_cat_list.txt') as f:\n", + " men_cats = json.load(f)\n", + " \n", + "with open('temp_pics_source_list.txt') as f:\n", + " tempics = json.load(f)\n", + "# list of image urls that did not get named properly which will be removed from the dataframe\n", + "drop_row_vals = []\n", + "for pic in tempics:\n", + " try:\n", + " dict_pics[pic]\n", + " except KeyError:\n", + " drop_row_vals.append(pic)\n", + "\n", "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", + "ddf = df[df.PictureURL.isin(drop_row_vals)==False] # remove improperly named image files\n", + "df = ddf[ddf.PrimaryCategoryID.isin(men_cats)==False] # removes rows of womens categories\n", + "\n", + "blah = pd.Series(df.PictureURL)\n", + "df = df.drop(labels=['PictureURL'], axis=1)\n", + "\n", + "blah = blah.apply(lambda x: dict_pics[x])\n", + "df = pd.concat([blah, df],axis=1)\n", + "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n", "\n", "df=df.sample(frac=1)" ] @@ -304,7 +333,7 @@ "#model.add(Dense(64, activation='softmax'))\n", "# model.add(Dropout(.25))\n", "\n", - "model.add(Dense(units=13, activation='softmax'))" + "model.add(Dense(units=7, activation='softmax'))" ] }, { diff --git a/Shoe Classifier_Xception.ipynb b/Shoe Classifier_Xception.ipynb index d9d36a5..303f505 100644 --- a/Shoe Classifier_Xception.ipynb +++ b/Shoe Classifier_Xception.ipynb @@ -36,50 +36,17 @@ { "cell_type": "code", "execution_count": 2, - "id": "8d94196d", + "id": "a5c72863", "metadata": {}, "outputs": [], "source": [ - "def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n", - "\n", - " if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n", - " print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n", - " return model\n", - "\n", - " for layer in model.layers:\n", - " for attr in ['kernel_regularizer']:\n", - " if hasattr(layer, attr):\n", - " setattr(layer, attr, regularizer)\n", - "\n", - " # When we change the layers attributes, the change only happens in the model config file\n", - " model_json = model.to_json()\n", - "\n", - " # Save the weights before reloading the model.\n", - " tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n", - " model.save_weights(tmp_weights_path)\n", - "\n", - " # load the model from the config\n", - " model = tf.keras.models.model_from_json(model_json)\n", - " \n", - " # Reload the model weights\n", - " model.load_weights(tmp_weights_path, by_name=True)\n", - " return model" + "#image_faults.faulty_images() # removes faulty images\n", + "df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)\n" ] }, { "cell_type": "code", "execution_count": 3, - "id": "a5c72863", - "metadata": {}, - "outputs": [], - "source": [ - "image_faults.faulty_images() # removes faulty images\n", - "df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -93,19 +60,23 @@ " \n", " dict_pics = {}\n", " for k in temp_pics_source_list:\n", - " patt_1 = re.search(r'[^/]+(?=/\\$_|.(\\.jpg|\\.jpeg|\\.png))', k, re.IGNORECASE)\n", - " patt_2 = re.search(r'(\\.jpg|\\.jpeg|\\.png)', k, re.IGNORECASE)\n", - " if patt_1 and patt_2 is not None:\n", - " tag = patt_1.group() + patt_2.group().lower()\n", - " file_name = target_dir + os.sep + tag\n", - " dict_pics.update({k:file_name})\n", + " try:\n", + " \n", + " patt_1 = re.search(r'[^/]+(?=/\\$_|.(\\.jpg|\\.jpeg|\\.png))', k, re.IGNORECASE)\n", + " patt_2 = re.search(r'(\\.jpg|\\.jpeg|\\.png)', k, re.IGNORECASE)\n", + " if patt_1 and patt_2 is not None:\n", + " tag = patt_1.group() + patt_2.group().lower()\n", + " file_name = target_dir + os.sep + tag\n", + " dict_pics.update({k:file_name})\n", + " except TypeError:\n", + " print(k)\n", " print(\"{source:target} dictionary created @ \" + target_dir)\n", " return dict_pics\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "1057a442", "metadata": { "scrolled": true @@ -115,12 +86,19 @@ "name": "stdout", "output_type": "stream", "text": [ + "nan\n", "{source:target} dictionary created @ /tf/training_images\n" ] } ], "source": [ "dict_pics = dict_pics_jup()\n", + "\n", + "with open('women_cat_list.txt') as f:\n", + " women_cats = json.load(f)\n", + "with open('men_cat_list.txt') as f:\n", + " men_cats = json.load(f)\n", + " \n", "with open('temp_pics_source_list.txt') as f:\n", " tempics = json.load(f)\n", "# list of image urls that did not get named properly which will be removed from the dataframe\n", @@ -130,9 +108,11 @@ " dict_pics[pic]\n", " except KeyError:\n", " drop_row_vals.append(pic)\n", - " \n", - "df = df[df.PictureURL.isin(drop_row_vals)==False]\n", - "# TODO drop men's or women's categories here\n", + "\n", + "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", + "ddf = df[df.PictureURL.isin(drop_row_vals)==False] # remove improperly named image files\n", + "df = ddf[ddf.PrimaryCategoryID.isin(men_cats)==False] # removes rows of womens categories\n", + "\n", "blah = pd.Series(df.PictureURL)\n", "df = df.drop(labels=['PictureURL'], axis=1)\n", "\n", @@ -143,31 +123,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "7a6146e6", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "17" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", - "\n", - "df=df.sample(frac=1)\n", - "len(drop_row_vals)" + "df=df.sample(frac=1)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "114cc3c0", "metadata": {}, "outputs": [], @@ -179,36 +145,28 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "506aa5cf", "metadata": {}, "outputs": [], "source": [ - "train, test = train_test_split(train, test_size=0.1, random_state=42)\n", + "train, test = train_test_split(train, test_size=0.2, random_state=42)\n", "# stratify=train['PrimaryCategoryID']\n", "# train['PrimaryCategoryID'].value_counts()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "4d72eb90", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n", - " warnings.warn(\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "Found 53005 validated image filenames belonging to 13 classes.\n", - "Found 13251 validated image filenames belonging to 13 classes.\n" + "Found 12276 validated image filenames belonging to 7 classes.\n", + "Found 3068 validated image filenames belonging to 7 classes.\n" ] } ], @@ -222,6 +180,7 @@ " #height_shift_range= 0.2,\n", " #rotation_range= 90,\n", " preprocessing_function=tf.keras.applications.xception.preprocess_input)\n", + "\n", "train_generator=datagen.flow_from_dataframe(\n", " dataframe=train[:len(train)],\n", " directory='./training_images',\n", @@ -248,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "7b70f37f", "metadata": {}, "outputs": [], @@ -258,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "id": "1ed54bf5", "metadata": {}, "outputs": [], @@ -275,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "id": "85934565", "metadata": {}, "outputs": [], @@ -286,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "id": "6322bcad", "metadata": {}, "outputs": [ @@ -306,7 +265,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "07fd25c6", "metadata": {}, "outputs": [], @@ -319,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "b31af79e", "metadata": {}, "outputs": [], @@ -330,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "fe06f2bf", "metadata": {}, "outputs": [ @@ -741,13 +700,13 @@ " global_average_pooling2d (Glob (None, 2048) 0 ['block14_sepconv2_act[0][0]'] \n", " alAveragePooling2D) \n", " \n", - " dense (Dense) (None, 13) 26637 ['global_average_pooling2d[0][0]'\n", + " dense (Dense) (None, 7) 14343 ['global_average_pooling2d[0][0]'\n", " ] \n", " \n", "==================================================================================================\n", - "Total params: 20,888,117\n", - "Trainable params: 20,833,589\n", - "Non-trainable params: 54,528\n", + "Total params: 20,875,823\n", + "Trainable params: 14,343\n", + "Non-trainable params: 20,861,480\n", "__________________________________________________________________________________________________\n" ] } @@ -756,7 +715,7 @@ "for layer in base_model.layers:\n", " layer.trainable = True\n", " \n", - "output = Dense(13, activation='softmax')(base_model.output)\n", + "output = Dense(7, activation='softmax')(base_model.output)\n", "model = tf.keras.Model(base_model.input, output)\n", "#model = add_regularization(model)\n", "model.summary()\n" @@ -764,7 +723,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "id": "ea620129", "metadata": {}, "outputs": [], @@ -777,7 +736,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "id": "fd5d1246", "metadata": {}, "outputs": [], @@ -789,7 +748,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "id": "9cd2ba27", "metadata": { "scrolled": false @@ -800,13 +759,7 @@ "output_type": "stream", "text": [ "Epoch 1/30\n", - "829/829 [==============================] - 786s 942ms/step - loss: 1.5037 - accuracy: 0.4896 - val_loss: 1.2946 - val_accuracy: 0.5520\n", - "Epoch 2/30\n", - "829/829 [==============================] - 726s 875ms/step - loss: 0.8550 - accuracy: 0.7117 - val_loss: 1.3593 - val_accuracy: 0.5593\n", - "Epoch 3/30\n", - "829/829 [==============================] - 750s 905ms/step - loss: 0.3322 - accuracy: 0.8993 - val_loss: 1.5304 - val_accuracy: 0.5542\n", - "Epoch 4/30\n", - "172/829 [=====>........................] - ETA: 7:57 - loss: 0.1030 - accuracy: 0.9787" + "134/192 [===================>..........] - ETA: 31s - loss: 8.9588 - accuracy: 0.1273" ] }, { @@ -816,21 +769,16 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m \u001b[0;31m# No error, now safe to assign to logs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1220\u001b[0m \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1221\u001b[0;31m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1222\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1223\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m 434\u001b[0m \"\"\"\n\u001b[1;32m 435\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 436\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 437\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 294\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 296\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 297\u001b[0m raise ValueError(\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m 314\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 318\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 353\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 354\u001b[0;31m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 355\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m 1030\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1031\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1032\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1033\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1034\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m 1102\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1103\u001b[0m \u001b[0;31m# Only block async when verbose = 1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1104\u001b[0;31m \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_to_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1105\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36msync_to_numpy_or_python_type\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 554\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 555\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 556\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m return pack_sequence_as(\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m expand_composites=expand_composites)\n\u001b[1;32m 871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m return pack_sequence_as(\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m expand_composites=expand_composites)\n\u001b[1;32m 871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36m_to_single_numpy_or_python_type\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m 548\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 551\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1147\u001b[0m \"\"\"\n\u001b[1;32m 1148\u001b[0m \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1149\u001b[0;31m \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1150\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1113\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1114\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1115\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1116\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1117\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1214\u001b[0m _r=1):\n\u001b[1;32m 1215\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1216\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1217\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1218\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 151\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 908\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 909\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 912\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 940\u001b[0m \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 941\u001b[0m \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 942\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 943\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 944\u001b[0m \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 3128\u001b[0m (graph_function,\n\u001b[1;32m 3129\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 3130\u001b[0;31m return graph_function._call_flat(\n\u001b[0m\u001b[1;32m 3131\u001b[0m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 3132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1957\u001b[0m and executing_eagerly):\n\u001b[1;32m 1958\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1959\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1960\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1961\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 597\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 599\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 59\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 60\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } diff --git a/ebay_api.py b/ebay_api.py index 9a77e43..0368612 100644 --- a/ebay_api.py +++ b/ebay_api.py @@ -183,8 +183,6 @@ class ShoppingApi: parent_cats = ['3034', '93427'] # Women's and Men's shoe departments cat_list = [] - # TODO make sep lists for women's and men's shoe cats. Needed to train - # mens and women's cats separately. This might improve val. acc. during training with open('temp_oauth_token.txt') as f: access_token = json.load(f) @@ -408,17 +406,35 @@ class CurateData: ''' expand = input("expand image list or use primary listing image? (y or n): ") if ('y' or 'Y') in expand: - expanded_class = class_training.explode('PictureURL').reset_index(drop=True) - expanded_class = expanded_class.dropna(subset=['PictureURL']) - expanded_class = expanded_class.drop_duplicates(subset=['PictureURL']).reset_index(drop=True) + count = input('how many images? All [A] or the first images?') + if 'A' in count: + expanded_class = class_training.explode('PictureURL').reset_index(drop=True) + expanded_class = expanded_class.dropna(subset=['PictureURL']) + expanded_class = expanded_class.drop_duplicates(subset=['PictureURL']).reset_index(drop=True) - expanded_dropd = dropd.explode('PictureURL').reset_index(drop=True) - expanded_dropd = expanded_dropd.dropna(subset=['PictureURL']) - expanded_dropd = expanded_dropd.drop_duplicates(subset=['PictureURL']).reset_index(drop=True) + expanded_dropd = dropd.explode('PictureURL').reset_index(drop=True) + expanded_dropd = expanded_dropd.dropna(subset=['PictureURL']) + expanded_dropd = expanded_dropd.drop_duplicates(subset=['PictureURL']).reset_index(drop=True) - expanded_dropd = self.extract_df(expanded_dropd) # convert lists to values + expanded_dropd = self.extract_df(expanded_dropd) # convert lists to values - temp_pics_source_list = list(set(expanded_class.PictureURL.to_list())) + temp_pics_source_list = list(set(expanded_class.PictureURL.to_list())) + else: + count = int(count) + class_training['PictureURL'] = class_training['PictureURL'].apply(lambda x: x[0:count] if len(x)>0 else np.nan) + expanded_class = class_training.explode('PictureURL').reset_index(drop=True) + expanded_class = expanded_class.dropna(subset=['PictureURL']) + expanded_class = expanded_class.drop_duplicates(subset=['PictureURL']).reset_index(drop=True) + + dropd = dropd.dropna(subset=['PictureURL']) + dropd['PictureURL'] = dropd['PictureURL'].apply(lambda x: x[0:count] if len(x)>0 else np.nan) + expanded_dropd = dropd.explode('PictureURL').reset_index(drop=True) + expanded_dropd = expanded_dropd.dropna(subset=['PictureURL']) + + expanded_dropd = self.extract_df(expanded_dropd) # convert lists to values + + # retrieves picture URLs from master raw_data.txt and rewrites temp_pics_source_list.txt + temp_pics_source_list = list(set(expanded_class.PictureURL.to_list())) # TODO else: class_training['PictureURL'] = class_training['PictureURL'].apply(lambda x: x[0] if len(x)>0 else np.nan)