scott/ebay-ml-lister
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

commit before checkout 428e0379ef for ref

Browse Source
This commit is contained in:
scott 2022-01-03 13:32:42 -07:00
parent 525f46cc34
commit 7d8be32ba8
6 changed files with 4418 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

471
Shoe Classifier-Copy1.ipynb
View File

@ -1,471 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "572dc7fb",
"metadata": {},
"outputs": [],
"source": [
"from matplotlib import pyplot as plt\n",
"from matplotlib.image import imread\n",
"import pandas as pd\n",
"from collections import Counter\n",
"import json\n",
"import os\n",
"import re\n",
"import tempfile\n",
"import numpy as np\n",
"from os.path import exists\n",
"from imblearn.under_sampling import RandomUnderSampler\n",
"from PIL import ImageFile\n",
"import sklearn as sk\n",
"from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n",
"import tensorflow as tf\n",
"import tensorflow.keras\n",
"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
"from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\n",
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.optimizers import Adam\n",
"# custom modules\n",
"import image_faults\n",
"\n",
"ImageFile.LOAD_TRUNCATED_IMAGES = True"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n",
"\n",
" if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n",
" print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n",
" return model\n",
"\n",
" for layer in model.layers:\n",
" for attr in ['kernel_regularizer']:\n",
" if hasattr(layer, attr):\n",
" setattr(layer, attr, regularizer)\n",
"\n",
" # When we change the layers attributes, the change only happens in the model config file\n",
" model_json = model.to_json()\n",
"\n",
" # Save the weights before reloading the model.\n",
" tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n",
" model.save_weights(tmp_weights_path)\n",
"\n",
" # load the model from the config\n",
" model = tf.keras.models.model_from_json(model_json)\n",
" \n",
" # Reload the model weights\n",
" model.load_weights(tmp_weights_path, by_name=True)\n",
" return model"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a5c72863",
"metadata": {},
"outputs": [],
"source": [
"# image_faults.faulty_images() # removes faulty images\n",
"df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1057a442",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{source:target} dictionary created @ /tf/training_images\n"
]
}
],
"source": [
"def dict_pics():\n",
" target_dir = os.getcwd() + os.sep + \"training_images\"\n",
" with open('temp_pics_source_list.txt') as f:\n",
" temp_pics_source_list = json.load(f)\n",
" dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n",
" print(\"{source:target} dictionary created @ \" + target_dir)\n",
" return dict_pics\n",
"\n",
"dict_pics = dict_pics()\n",
"blah = pd.Series(df.PictureURL)\n",
"df = df.drop(labels=['PictureURL'], axis=1)\n",
"blah = blah.apply(lambda x: dict_pics[x])\n",
"df = pd.concat([blah, df],axis=1)\n",
"df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n",
"# removes non-existent image paths"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "7a6146e6",
"metadata": {},
"outputs": [],
"source": [
"df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n",
"\n",
"df=df.sample(frac=1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"undersample = RandomUnderSampler(sampling_strategy='auto')\n",
"train, y_under = undersample.fit_resample(df, df['PrimaryCategoryID'])\n",
"# print(Counter(train['PrimaryCategoryID']))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "506aa5cf",
"metadata": {},
"outputs": [],
"source": [
"train, test = train_test_split(train, test_size=0.1, random_state=42)\n",
"# stratify=train['PrimaryCategoryID']\n",
"# train['PrimaryCategoryID'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "4d72eb90",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found 5110 validated image filenames belonging to 13 classes.\n",
"Found 1277 validated image filenames belonging to 13 classes.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n",
" warnings.warn(\n"
]
}
],
"source": [
"datagen = ImageDataGenerator(rescale=1./255., \n",
" validation_split=.2,\n",
" #samplewise_std_normalization=True,\n",
" #horizontal_flip= True,\n",
" #vertical_flip= True,\n",
" #width_shift_range= 0.2,\n",
" #height_shift_range= 0.2,\n",
" #rotation_range= 90,\n",
" preprocessing_function=tf.keras.applications.vgg16.preprocess_input)\n",
"train_generator=datagen.flow_from_dataframe(\n",
" dataframe=train[:len(train)],\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=32,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(224,224),\n",
" subset='training'\n",
" )\n",
"validation_generator=datagen.flow_from_dataframe(\n",
" dataframe=train[:len(train)], # is using train right?\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=32,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(224,224),\n",
" subset='validation'\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7b70f37f",
"metadata": {},
"outputs": [],
"source": [
"imgs, labels = next(train_generator)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "1ed54bf5",
"metadata": {},
"outputs": [],
"source": [
"def plotImages(images_arr):\n",
" fig, axes = plt.subplots(1, 10, figsize=(20,20))\n",
" axes = axes.flatten()\n",
" for img, ax in zip( images_arr, axes):\n",
" ax.imshow(img)\n",
" ax.axis('off')\n",
" plt.tight_layout()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "85934565",
"metadata": {},
"outputs": [],
"source": [
"#plotImages(imgs)\n",
"#print(labels)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "6322bcad",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1\n"
]
}
],
"source": [
"physical_devices = tf.config.list_physical_devices('GPU')\n",
"print(len(physical_devices))\n",
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "07fd25c6",
"metadata": {},
"outputs": [],
"source": [
"# see https://www.kaggle.com/dmitrypukhov/cnn-with-imagedatagenerator-flow-from-dataframe for train/test/val split \n",
"# example\n",
"\n",
"# may need to either create a test dataset from the original dataset or just download a new one"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "b31af79e",
"metadata": {},
"outputs": [],
"source": [
"vgg19_model = tf.keras.applications.vgg16.VGG16(weights='imagenet')\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "fe06f2bf",
"metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
"for layer in vgg19_model.layers[:-1]:\n",
" model.add(layer)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "7d3cc82c",
"metadata": {},
"outputs": [],
"source": [
"for layer in model.layers:\n",
" layer.trainable = True"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "ea620129",
"metadata": {},
"outputs": [],
"source": [
"#model.add(Dropout(.5))\n",
"#model.add(Dense(64, activation='softmax'))\n",
"# model.add(Dropout(.25))\n",
"model.add(Dense(units=13, activation='softmax'))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "c774d787",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"model = add_regularization(model)\n",
"#model.summary()\n"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "fd5d1246",
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer=Adam(learning_rate=1e-5), loss='categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"# sparse_categorical_crossentropy"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "9cd2ba27",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/30\n",
"160/160 [==============================] - 59s 360ms/step - loss: 2.7627 - accuracy: 0.1125 - val_loss: 2.7406 - val_accuracy: 0.1237\n",
"Epoch 2/30\n",
"160/160 [==============================] - 56s 351ms/step - loss: 2.7151 - accuracy: 0.1399 - val_loss: 2.7219 - val_accuracy: 0.1402\n",
"Epoch 3/30\n",
"160/160 [==============================] - 56s 351ms/step - loss: 2.6875 - accuracy: 0.1566 - val_loss: 2.6897 - val_accuracy: 0.1629\n",
"Epoch 4/30\n",
"160/160 [==============================] - 56s 353ms/step - loss: 2.6820 - accuracy: 0.1726 - val_loss: 2.6867 - val_accuracy: 0.1684\n",
"Epoch 5/30\n",
"160/160 [==============================] - 57s 355ms/step - loss: 2.6579 - accuracy: 0.1771 - val_loss: 2.6919 - val_accuracy: 0.1558\n",
"Epoch 6/30\n",
"160/160 [==============================] - 56s 353ms/step - loss: 2.6361 - accuracy: 0.1994 - val_loss: 2.6813 - val_accuracy: 0.1832\n",
"Epoch 7/30\n",
"160/160 [==============================] - 56s 352ms/step - loss: 2.6196 - accuracy: 0.2084 - val_loss: 2.6592 - val_accuracy: 0.1950\n",
"Epoch 8/30\n",
"160/160 [==============================] - 57s 353ms/step - loss: 2.6031 - accuracy: 0.2172 - val_loss: 2.6693 - val_accuracy: 0.1770\n",
"Epoch 9/30\n",
"160/160 [==============================] - 57s 355ms/step - loss: 2.5878 - accuracy: 0.2274 - val_loss: 2.6543 - val_accuracy: 0.2091\n",
"Epoch 10/30\n",
"160/160 [==============================] - 56s 350ms/step - loss: 2.5687 - accuracy: 0.2450 - val_loss: 2.6551 - val_accuracy: 0.1942\n",
"Epoch 11/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.5543 - accuracy: 0.2568 - val_loss: 2.6591 - val_accuracy: 0.2020\n",
"Epoch 12/30\n",
"160/160 [==============================] - 56s 352ms/step - loss: 2.5403 - accuracy: 0.2685 - val_loss: 2.6513 - val_accuracy: 0.1973\n",
"Epoch 13/30\n",
"160/160 [==============================] - 56s 352ms/step - loss: 2.5311 - accuracy: 0.2695 - val_loss: 2.6445 - val_accuracy: 0.2060\n",
"Epoch 14/30\n",
"160/160 [==============================] - 56s 351ms/step - loss: 2.5217 - accuracy: 0.2775 - val_loss: 2.6476 - val_accuracy: 0.2044\n",
"Epoch 15/30\n",
"160/160 [==============================] - 56s 351ms/step - loss: 2.5147 - accuracy: 0.2830 - val_loss: 2.6419 - val_accuracy: 0.2036\n",
"Epoch 16/30\n",
"160/160 [==============================] - 56s 351ms/step - loss: 2.5084 - accuracy: 0.2851 - val_loss: 2.6396 - val_accuracy: 0.2200\n",
"Epoch 17/30\n",
"160/160 [==============================] - 56s 348ms/step - loss: 2.5025 - accuracy: 0.2879 - val_loss: 2.6463 - val_accuracy: 0.2302\n",
"Epoch 18/30\n",
"160/160 [==============================] - 56s 350ms/step - loss: 2.4971 - accuracy: 0.2918 - val_loss: 2.6346 - val_accuracy: 0.2208\n",
"Epoch 19/30\n",
"160/160 [==============================] - 56s 353ms/step - loss: 2.4924 - accuracy: 0.2967 - val_loss: 2.6366 - val_accuracy: 0.2208\n",
"Epoch 20/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.4882 - accuracy: 0.2988 - val_loss: 2.6317 - val_accuracy: 0.2271\n",
"Epoch 21/30\n",
"160/160 [==============================] - 56s 349ms/step - loss: 2.4854 - accuracy: 0.3004 - val_loss: 2.6431 - val_accuracy: 0.2240\n",
"Epoch 22/30\n",
"160/160 [==============================] - 56s 352ms/step - loss: 2.4784 - accuracy: 0.3068 - val_loss: 2.6345 - val_accuracy: 0.2114\n",
"Epoch 23/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.4722 - accuracy: 0.3106 - val_loss: 2.6276 - val_accuracy: 0.2294\n",
"Epoch 24/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.4687 - accuracy: 0.3100 - val_loss: 2.6383 - val_accuracy: 0.2177\n",
"Epoch 25/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.4649 - accuracy: 0.3108 - val_loss: 2.6322 - val_accuracy: 0.2122\n",
"Epoch 26/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.4644 - accuracy: 0.3141 - val_loss: 2.6243 - val_accuracy: 0.2247\n",
"Epoch 27/30\n",
"160/160 [==============================] - 56s 352ms/step - loss: 2.4599 - accuracy: 0.3188 - val_loss: 2.6332 - val_accuracy: 0.2138\n",
"Epoch 28/30\n",
"160/160 [==============================] - 57s 353ms/step - loss: 2.4550 - accuracy: 0.3229 - val_loss: 2.6287 - val_accuracy: 0.2232\n",
"Epoch 29/30\n",
"160/160 [==============================] - 57s 354ms/step - loss: 2.4502 - accuracy: 0.3217 - val_loss: 2.6216 - val_accuracy: 0.2287\n",
"Epoch 30/30\n",
"160/160 [==============================] - 56s 351ms/step - loss: 2.4506 - accuracy: 0.3190 - val_loss: 2.6329 - val_accuracy: 0.1793\n"
]
},
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x7f7803569ac0>"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit(x=train_generator,\n",
" steps_per_epoch=len(train_generator),\n",
" validation_data=validation_generator,\n",
" validation_steps=len(validation_generator),\n",
" epochs=30,\n",
" verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "63f791af",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

2787
Shoe Classifier_InceptionResNetV2.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

529
Shoe Classifier_VGG16.ipynb Normal file
View File

@ -0,0 +1,529 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "572dc7fb",
"metadata": {},
"outputs": [],
"source": [
"from matplotlib import pyplot as plt\n",
"from matplotlib.image import imread\n",
"import pandas as pd\n",
"from collections import Counter\n",
"import json\n",
"import os\n",
"import re\n",
"import tempfile\n",
"import numpy as np\n",
"from os.path import exists\n",
"from imblearn.under_sampling import RandomUnderSampler\n",
"from PIL import ImageFile\n",
"import sklearn as sk\n",
"from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n",
"import tensorflow as tf\n",
"import tensorflow.keras\n",
"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
"from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\n",
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.optimizers import Adam\n",
"# custom modules\n",
"import image_faults\n",
"\n",
"ImageFile.LOAD_TRUNCATED_IMAGES = True"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n",
"\n",
" if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n",
" print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n",
" return model\n",
"\n",
" for layer in model.layers:\n",
" for attr in ['kernel_regularizer']:\n",
" if hasattr(layer, attr):\n",
" setattr(layer, attr, regularizer)\n",
"\n",
" # When we change the layers attributes, the change only happens in the model config file\n",
" model_json = model.to_json()\n",
"\n",
" # Save the weights before reloading the model.\n",
" tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n",
" model.save_weights(tmp_weights_path)\n",
"\n",
" # load the model from the config\n",
" model = tf.keras.models.model_from_json(model_json)\n",
" \n",
" # Reload the model weights\n",
" model.load_weights(tmp_weights_path, by_name=True)\n",
" return model"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a5c72863",
"metadata": {},
"outputs": [],
"source": [
"# image_faults.faulty_images() # removes faulty images\n",
"df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1057a442",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{source:target} dictionary created @ /tf/training_images\n"
]
}
],
"source": [
"def dict_pics():\n",
" target_dir = os.getcwd() + os.sep + \"training_images\"\n",
" with open('temp_pics_source_list.txt') as f:\n",
" temp_pics_source_list = json.load(f)\n",
" dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n",
" print(\"{source:target} dictionary created @ \" + target_dir)\n",
" return dict_pics\n",
"\n",
"dict_pics = dict_pics()\n",
"blah = pd.Series(df.PictureURL)\n",
"df = df.drop(labels=['PictureURL'], axis=1)\n",
"blah = blah.apply(lambda x: dict_pics[x])\n",
"df = pd.concat([blah, df],axis=1)\n",
"df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n",
"# removes non-existent image paths"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "7a6146e6",
"metadata": {},
"outputs": [],
"source": [
"df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n",
"\n",
"df=df.sample(frac=1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Counter({'11498': 546, '11504': 546, '11505': 546, '11632': 546, '15709': 546, '24087': 546, '45333': 546, '53120': 546, '53548': 546, '53557': 546, '55793': 546, '62107': 546, '95672': 546})\n"
]
}
],
"source": [
"undersample = RandomUnderSampler(sampling_strategy='auto')\n",
"train, y_under = undersample.fit_resample(df, df['PrimaryCategoryID'])\n",
"print(Counter(train['PrimaryCategoryID']))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "506aa5cf",
"metadata": {},
"outputs": [],
"source": [
"train, test = train_test_split(train, test_size=0.2, random_state=42)\n",
"# stratify=train['PrimaryCategoryID']\n",
"# train['PrimaryCategoryID'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "4d72eb90",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found 4542 validated image filenames belonging to 13 classes.\n",
"Found 1135 validated image filenames belonging to 13 classes.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n",
" warnings.warn(\n"
]
}
],
"source": [
"datagen = ImageDataGenerator(rescale=1./255., \n",
" validation_split=.2,\n",
" #samplewise_std_normalization=True,\n",
" #horizontal_flip= True,\n",
" #vertical_flip= True,\n",
" #width_shift_range= 0.2,\n",
" #height_shift_range= 0.2,\n",
" #rotation_range= 90,\n",
" preprocessing_function=tf.keras.applications.vgg16.preprocess_input)\n",
"train_generator=datagen.flow_from_dataframe(\n",
" dataframe=train[:len(train)],\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=32,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(244,244),\n",
" subset='training'\n",
" )\n",
"validation_generator=datagen.flow_from_dataframe(\n",
" dataframe=train[:len(train)], # is using train right?\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=32,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(244,244),\n",
" subset='validation'\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7b70f37f",
"metadata": {},
"outputs": [],
"source": [
"imgs, labels = next(train_generator)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "1ed54bf5",
"metadata": {},
"outputs": [],
"source": [
"def plotImages(images_arr):\n",
" fig, axes = plt.subplots(1, 10, figsize=(20,20))\n",
" axes = axes.flatten()\n",
" for img, ax in zip( images_arr, axes):\n",
" ax.imshow(img)\n",
" ax.axis('off')\n",
" plt.tight_layout()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "85934565",
"metadata": {},
"outputs": [],
"source": [
"#plotImages(imgs)\n",
"#print(labels)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "6322bcad",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1\n"
]
}
],
"source": [
"physical_devices = tf.config.list_physical_devices('GPU')\n",
"print(len(physical_devices))\n",
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "b31af79e",
"metadata": {},
"outputs": [],
"source": [
"base_model = tf.keras.applications.vgg16.VGG16(weights='imagenet')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "fe06f2bf",
"metadata": {},
"outputs": [],
"source": [
"#model = Sequential()\n",
"#for layer in base_model.layers[:-1]:\n",
"# model.add(layer)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "7d3cc82c",
"metadata": {},
"outputs": [],
"source": [
"# loop through layers, add Dropout after layers 'fc1' and 'fc2'\n",
"updated_model = Sequential()\n",
"for layer in base_model.layers[:-1]:\n",
" updated_model.add(layer)\n",
" if layer.name in ['fc1', 'fc2']:\n",
" updated_model.add(Dropout(.75))\n",
"\n",
"model = updated_model\n",
"\n",
"for layer in model.layers:\n",
" layer.trainable = True\n",
"\n",
"model.add(Dense(units=13, activation='softmax'))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "c774d787",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential\"\n",
"_________________________________________________________________\n",
" Layer (type) Output Shape Param # \n",
"=================================================================\n",
" block1_conv1 (Conv2D) (None, 224, 224, 64) 1792 \n",
" \n",
" block1_conv2 (Conv2D) (None, 224, 224, 64) 36928 \n",
" \n",
" block1_pool (MaxPooling2D) (None, 112, 112, 64) 0 \n",
" \n",
" block2_conv1 (Conv2D) (None, 112, 112, 128) 73856 \n",
" \n",
" block2_conv2 (Conv2D) (None, 112, 112, 128) 147584 \n",
" \n",
" block2_pool (MaxPooling2D) (None, 56, 56, 128) 0 \n",
" \n",
" block3_conv1 (Conv2D) (None, 56, 56, 256) 295168 \n",
" \n",
" block3_conv2 (Conv2D) (None, 56, 56, 256) 590080 \n",
" \n",
" block3_conv3 (Conv2D) (None, 56, 56, 256) 590080 \n",
" \n",
" block3_pool (MaxPooling2D) (None, 28, 28, 256) 0 \n",
" \n",
" block4_conv1 (Conv2D) (None, 28, 28, 512) 1180160 \n",
" \n",
" block4_conv2 (Conv2D) (None, 28, 28, 512) 2359808 \n",
" \n",
" block4_conv3 (Conv2D) (None, 28, 28, 512) 2359808 \n",
" \n",
" block4_pool (MaxPooling2D) (None, 14, 14, 512) 0 \n",
" \n",
" block5_conv1 (Conv2D) (None, 14, 14, 512) 2359808 \n",
" \n",
" block5_conv2 (Conv2D) (None, 14, 14, 512) 2359808 \n",
" \n",
" block5_conv3 (Conv2D) (None, 14, 14, 512) 2359808 \n",
" \n",
" block5_pool (MaxPooling2D) (None, 7, 7, 512) 0 \n",
" \n",
" flatten (Flatten) (None, 25088) 0 \n",
" \n",
" fc1 (Dense) (None, 4096) 102764544 \n",
" \n",
" dropout (Dropout) (None, 4096) 0 \n",
" \n",
" fc2 (Dense) (None, 4096) 16781312 \n",
" \n",
" dropout_1 (Dropout) (None, 4096) 0 \n",
" \n",
" dense (Dense) (None, 13) 53261 \n",
" \n",
"=================================================================\n",
"Total params: 134,313,805\n",
"Trainable params: 134,313,805\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"#model = add_regularization(model)\n",
"model.summary()\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "fd5d1246",
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer=Adam(learning_rate=.0001), loss='categorical_crossentropy',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "9cd2ba27",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/30\n",
"142/142 [==============================] - 62s 404ms/step - loss: 2.7986 - accuracy: 0.0771 - val_loss: 2.5716 - val_accuracy: 0.0670\n",
"Epoch 2/30\n",
"142/142 [==============================] - 53s 370ms/step - loss: 2.6351 - accuracy: 0.0790 - val_loss: 2.5728 - val_accuracy: 0.0802\n",
"Epoch 3/30\n",
"142/142 [==============================] - 53s 370ms/step - loss: 2.6157 - accuracy: 0.0852 - val_loss: 2.5786 - val_accuracy: 0.0705\n",
"Epoch 4/30\n",
"142/142 [==============================] - 53s 371ms/step - loss: 2.6087 - accuracy: 0.0821 - val_loss: 2.5501 - val_accuracy: 0.1013\n",
"Epoch 5/30\n",
"142/142 [==============================] - 53s 371ms/step - loss: 2.5420 - accuracy: 0.1182 - val_loss: 2.4237 - val_accuracy: 0.1401\n",
"Epoch 6/30\n",
"142/142 [==============================] - 52s 368ms/step - loss: 2.4275 - accuracy: 0.1640 - val_loss: 2.3050 - val_accuracy: 0.1982\n",
"Epoch 7/30\n",
"142/142 [==============================] - 52s 368ms/step - loss: 2.3270 - accuracy: 0.2008 - val_loss: 2.2103 - val_accuracy: 0.2273\n",
"Epoch 8/30\n",
"142/142 [==============================] - 54s 378ms/step - loss: 2.2011 - accuracy: 0.2294 - val_loss: 2.0607 - val_accuracy: 0.2872\n",
"Epoch 9/30\n",
"142/142 [==============================] - 54s 378ms/step - loss: 2.0790 - accuracy: 0.2781 - val_loss: 1.9376 - val_accuracy: 0.3419\n",
"Epoch 10/30\n",
"142/142 [==============================] - 52s 368ms/step - loss: 1.8708 - accuracy: 0.3417 - val_loss: 2.0407 - val_accuracy: 0.3198\n",
"Epoch 11/30\n",
"142/142 [==============================] - 52s 368ms/step - loss: 1.7588 - accuracy: 0.3743 - val_loss: 1.8968 - val_accuracy: 0.3401\n",
"Epoch 12/30\n",
"142/142 [==============================] - 52s 369ms/step - loss: 1.5927 - accuracy: 0.4251 - val_loss: 1.8735 - val_accuracy: 0.3542\n",
"Epoch 13/30\n",
"142/142 [==============================] - 53s 369ms/step - loss: 1.4704 - accuracy: 0.4626 - val_loss: 1.8489 - val_accuracy: 0.3753\n",
"Epoch 14/30\n",
"142/142 [==============================] - 52s 367ms/step - loss: 1.2452 - accuracy: 0.5484 - val_loss: 1.9595 - val_accuracy: 0.3656\n",
"Epoch 15/30\n",
"142/142 [==============================] - 52s 370ms/step - loss: 1.1410 - accuracy: 0.5885 - val_loss: 1.9159 - val_accuracy: 0.3930\n",
"Epoch 16/30\n",
"142/142 [==============================] - 52s 367ms/step - loss: 0.9705 - accuracy: 0.6510 - val_loss: 2.1161 - val_accuracy: 0.3921\n",
"Epoch 17/30\n",
"142/142 [==============================] - 52s 368ms/step - loss: 0.7992 - accuracy: 0.7041 - val_loss: 2.1271 - val_accuracy: 0.4097\n",
"Epoch 18/30\n",
"142/142 [==============================] - 52s 369ms/step - loss: 0.7544 - accuracy: 0.7305 - val_loss: 2.0093 - val_accuracy: 0.4361\n",
"Epoch 19/30\n",
"142/142 [==============================] - 52s 367ms/step - loss: 0.5651 - accuracy: 0.7961 - val_loss: 2.4199 - val_accuracy: 0.3850\n",
"Epoch 20/30\n",
"142/142 [==============================] - 52s 366ms/step - loss: 0.4566 - accuracy: 0.8342 - val_loss: 2.4058 - val_accuracy: 0.4352\n",
"Epoch 21/30\n",
"142/142 [==============================] - 52s 364ms/step - loss: 0.3841 - accuracy: 0.8622 - val_loss: 2.5407 - val_accuracy: 0.4141\n",
"Epoch 22/30\n",
"142/142 [==============================] - 52s 366ms/step - loss: 0.2496 - accuracy: 0.9086 - val_loss: 3.0696 - val_accuracy: 0.4088\n",
"Epoch 23/30\n",
"142/142 [==============================] - 52s 368ms/step - loss: 0.2847 - accuracy: 0.8985 - val_loss: 2.7929 - val_accuracy: 0.4273\n",
"Epoch 24/30\n",
"142/142 [==============================] - 52s 369ms/step - loss: 0.2035 - accuracy: 0.9293 - val_loss: 3.0300 - val_accuracy: 0.4132\n",
"Epoch 25/30\n",
"142/142 [==============================] - ETA: 0s - loss: 0.1872 - accuracy: 0.9377"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-18-4cd4443bbf2a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1250\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1251\u001b[0m steps_per_execution=self._steps_per_execution)\n\u001b[0;32m-> 1252\u001b[0;31m val_logs = self.evaluate(\n\u001b[0m\u001b[1;32m 1253\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_x\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1254\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[1;32m 1535\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_r\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1537\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1538\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1539\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 151\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 908\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 909\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 912\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 947\u001b[0m \u001b[0;31m# In this case we have not created variables on the first call. So we can\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 948\u001b[0m \u001b[0;31m# run the first trace but we should fail if variables are created.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 949\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 950\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_variables\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mALLOW_DYNAMIC_VARIABLE_CREATION\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 951\u001b[0m raise ValueError(\"Creating variables on a non-first call to a function\"\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 3128\u001b[0m (graph_function,\n\u001b[1;32m 3129\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 3130\u001b[0;31m return graph_function._call_flat(\n\u001b[0m\u001b[1;32m 3131\u001b[0m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 3132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1957\u001b[0m and executing_eagerly):\n\u001b[1;32m 1958\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1959\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1960\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1961\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 597\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 599\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 59\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 60\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"model.fit(x=train_generator,\n",
" steps_per_epoch=len(train_generator),\n",
" validation_data=validation_generator,\n",
" validation_steps=len(validation_generator),\n",
" epochs=30,\n",
" verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "63f791af",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

386
Shoe Classifier.ipynb → Shoe Classifier_VGG19.ipynb
View File

@ -10,13 +10,17 @@
"from matplotlib import pyplot as plt\n",
"from matplotlib.image import imread\n",
"import pandas as pd\n",
"from collections import Counter\n",
"import json\n",
"import os\n",
"import re\n",
"import tempfile\n",
"import numpy as np\n",
"from os.path import exists\n",
"from imblearn.under_sampling import RandomUnderSampler\n",
"from PIL import ImageFile\n",
"#import sklearn as sk\n",
"import sklearn as sk\n",
"from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n",
"import tensorflow as tf\n",
"import tensorflow.keras\n",
"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
@ -32,6 +36,38 @@
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n",
"\n",
" if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n",
" print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n",
" return model\n",
"\n",
" for layer in model.layers:\n",
" for attr in ['kernel_regularizer']:\n",
" if hasattr(layer, attr):\n",
" setattr(layer, attr, regularizer)\n",
"\n",
" # When we change the layers attributes, the change only happens in the model config file\n",
" model_json = model.to_json()\n",
"\n",
" # Save the weights before reloading the model.\n",
" tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n",
" model.save_weights(tmp_weights_path)\n",
"\n",
" # load the model from the config\n",
" model = tf.keras.models.model_from_json(model_json)\n",
" \n",
" # Reload the model weights\n",
" model.load_weights(tmp_weights_path, by_name=True)\n",
" return model"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a5c72863",
"metadata": {},
"outputs": [],
@ -42,7 +78,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "1057a442",
"metadata": {
"scrolled": true
@ -76,7 +112,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"id": "7a6146e6",
"metadata": {},
"outputs": [],
@ -88,7 +124,38 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Counter({'11498': 3913, '11504': 3913, '11505': 3913, '11632': 3913, '15709': 3913, '24087': 3913, '45333': 3913, '53120': 3913, '53548': 3913, '53557': 3913, '55793': 3913, '62107': 3913, '95672': 3913})\n"
]
}
],
"source": [
"undersample = RandomUnderSampler(sampling_strategy='auto')\n",
"train, y_under = undersample.fit_resample(df, df['PrimaryCategoryID'])\n",
"#print(Counter(train['PrimaryCategoryID']))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "506aa5cf",
"metadata": {},
"outputs": [],
"source": [
"train, test = train_test_split(train, test_size=0.2, random_state=42)\n",
"# stratify=train['PrimaryCategoryID']\n",
"# train['PrimaryCategoryID'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "4d72eb90",
"metadata": {},
"outputs": [
@ -96,56 +163,56 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Found 6217 validated image filenames belonging to 13 classes.\n",
"Found 2664 validated image filenames belonging to 13 classes.\n"
"Found 32547 validated image filenames belonging to 13 classes.\n",
"Found 8136 validated image filenames belonging to 13 classes.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n",
"/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 12 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n",
" warnings.warn(\n"
]
}
],
"source": [
"datagen = ImageDataGenerator(rescale=1./255., \n",
" validation_split=.3,\n",
" #featurewise_std_normalization=True,\n",
" validation_split=.2,\n",
" #samplewise_std_normalization=True,\n",
" #horizontal_flip= True,\n",
" #vertical_flip= True,\n",
" #width_shift_range= 0.2,\n",
" #height_shift_range= 0.2,\n",
" #rotation_range= 180,\n",
" #rotation_range= 90,\n",
" preprocessing_function=tf.keras.applications.vgg16.preprocess_input)\n",
"train_generator=datagen.flow_from_dataframe(\n",
" dataframe=df[:len(df)],\n",
" dataframe=train[:len(train)],\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=32,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(224,224),\n",
" target_size=(244,244),\n",
" subset='training'\n",
" )\n",
"validation_generator=datagen.flow_from_dataframe(\n",
" dataframe=df[:len(df)],\n",
" dataframe=train[:len(train)], # is using train right?\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=32,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(224,224),\n",
" target_size=(244,244),\n",
" subset='validation'\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 9,
"id": "7b70f37f",
"metadata": {},
"outputs": [],
@ -155,7 +222,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 10,
"id": "1ed54bf5",
"metadata": {},
"outputs": [],
@ -172,7 +239,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 11,
"id": "85934565",
"metadata": {},
"outputs": [],
@ -183,7 +250,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 12,
"id": "6322bcad",
"metadata": {},
"outputs": [
@ -203,88 +270,47 @@
},
{
"cell_type": "code",
"execution_count": 10,
"id": "07fd25c6",
"metadata": {},
"outputs": [],
"source": [
"# see https://www.kaggle.com/dmitrypukhov/cnn-with-imagedatagenerator-flow-from-dataframe for train/test/val split \n",
"# example\n",
"\n",
"# may need to either create a test dataset from the original dataset or just download a new one"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 14,
"id": "b31af79e",
"metadata": {},
"outputs": [],
"source": [
"vgg16_model = tf.keras.applications.vgg16.VGG16(weights='imagenet')\n",
"#weights='imagenet'"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "fe06f2bf",
"metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
"for layer in vgg16_model.layers[:-1]:\n",
" model.add(layer)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "7d3cc82c",
"metadata": {},
"outputs": [],
"source": [
"for layer in model.layers:\n",
" layer.trainable = True"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "ea620129",
"metadata": {},
"outputs": [],
"source": [
"model.add(Dense(units=13, activation='softmax'))"
"base_model = tf.keras.applications.vgg19.VGG19(weights='imagenet')"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "c774d787",
"metadata": {
"scrolled": true
},
"id": "fe06f2bf",
"metadata": {},
"outputs": [],
"source": [
"#model.summary()"
"model = Sequential()\n",
"for layer in base_model.layers[:-1]:\n",
" model.add(layer)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "fd5d1246",
"id": "7d3cc82c",
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer=Adam(learning_rate=0.0001), loss='categorical_crossentropy',\n",
" metrics=['accuracy'])"
"for layer in model.layers:\n",
" layer.trainable = True\n",
" \n",
"#model.add(Dropout(.5))\n",
"#model.add(Dense(64, activation='softmax'))\n",
"# model.add(Dropout(.25))\n",
"\n",
"model.add(Dense(units=13, activation='softmax'))"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "9cd2ba27",
"execution_count": 18,
"id": "c774d787",
"metadata": {
"scrolled": true
},
@ -293,97 +319,115 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/100\n",
"195/195 - 83s - loss: 2.5928 - accuracy: 0.1139 - val_loss: 2.3657 - val_accuracy: 0.1674 - 83s/epoch - 426ms/step\n",
"Epoch 2/100\n",
"195/195 - 77s - loss: 2.1473 - accuracy: 0.2582 - val_loss: 1.9276 - val_accuracy: 0.3281 - 77s/epoch - 394ms/step\n",
"Epoch 3/100\n",
"195/195 - 78s - loss: 1.7234 - accuracy: 0.3973 - val_loss: 1.6724 - val_accuracy: 0.4050 - 78s/epoch - 400ms/step\n",
"Epoch 4/100\n",
"195/195 - 78s - loss: 1.4692 - accuracy: 0.4843 - val_loss: 1.5583 - val_accuracy: 0.4662 - 78s/epoch - 402ms/step\n",
"Epoch 5/100\n",
"195/195 - 79s - loss: 1.2598 - accuracy: 0.5477 - val_loss: 1.5135 - val_accuracy: 0.4944 - 79s/epoch - 403ms/step\n",
"Epoch 6/100\n",
"195/195 - 79s - loss: 1.0220 - accuracy: 0.6376 - val_loss: 1.5566 - val_accuracy: 0.4962 - 79s/epoch - 404ms/step\n",
"Epoch 7/100\n",
"195/195 - 78s - loss: 0.8021 - accuracy: 0.7084 - val_loss: 1.7647 - val_accuracy: 0.4711 - 78s/epoch - 398ms/step\n",
"Epoch 8/100\n",
"195/195 - 78s - loss: 0.5998 - accuracy: 0.7804 - val_loss: 1.8439 - val_accuracy: 0.4869 - 78s/epoch - 400ms/step\n",
"Epoch 9/100\n",
"195/195 - 77s - loss: 0.3910 - accuracy: 0.8631 - val_loss: 2.1197 - val_accuracy: 0.4872 - 77s/epoch - 397ms/step\n",
"Epoch 10/100\n",
"195/195 - 78s - loss: 0.2600 - accuracy: 0.9094 - val_loss: 2.3586 - val_accuracy: 0.4703 - 78s/epoch - 402ms/step\n",
"Epoch 11/100\n",
"195/195 - 77s - loss: 0.2066 - accuracy: 0.9279 - val_loss: 2.5012 - val_accuracy: 0.4632 - 77s/epoch - 397ms/step\n",
"Epoch 12/100\n",
"195/195 - 77s - loss: 0.1266 - accuracy: 0.9571 - val_loss: 2.5961 - val_accuracy: 0.4854 - 77s/epoch - 395ms/step\n",
"Epoch 13/100\n",
"195/195 - 77s - loss: 0.1098 - accuracy: 0.9648 - val_loss: 2.9123 - val_accuracy: 0.4602 - 77s/epoch - 395ms/step\n",
"Epoch 14/100\n",
"195/195 - 77s - loss: 0.0794 - accuracy: 0.9744 - val_loss: 2.9060 - val_accuracy: 0.4752 - 77s/epoch - 395ms/step\n",
"Epoch 15/100\n",
"195/195 - 77s - loss: 0.1061 - accuracy: 0.9656 - val_loss: 2.7269 - val_accuracy: 0.4658 - 77s/epoch - 396ms/step\n",
"Epoch 16/100\n",
"195/195 - 77s - loss: 0.0908 - accuracy: 0.9712 - val_loss: 2.9450 - val_accuracy: 0.4621 - 77s/epoch - 396ms/step\n",
"Epoch 17/100\n",
"195/195 - 77s - loss: 0.0820 - accuracy: 0.9736 - val_loss: 2.8900 - val_accuracy: 0.4703 - 77s/epoch - 396ms/step\n",
"Epoch 18/100\n",
"195/195 - 77s - loss: 0.0827 - accuracy: 0.9722 - val_loss: 2.9304 - val_accuracy: 0.4745 - 77s/epoch - 395ms/step\n",
"Epoch 19/100\n",
"195/195 - 77s - loss: 0.0726 - accuracy: 0.9757 - val_loss: 3.2673 - val_accuracy: 0.4580 - 77s/epoch - 393ms/step\n",
"Epoch 20/100\n",
"195/195 - 77s - loss: 0.0560 - accuracy: 0.9821 - val_loss: 3.0101 - val_accuracy: 0.4670 - 77s/epoch - 394ms/step\n",
"Epoch 21/100\n",
"195/195 - 77s - loss: 0.0355 - accuracy: 0.9907 - val_loss: 3.2575 - val_accuracy: 0.4568 - 77s/epoch - 395ms/step\n",
"Epoch 22/100\n",
"195/195 - 78s - loss: 0.0950 - accuracy: 0.9693 - val_loss: 3.0716 - val_accuracy: 0.4568 - 78s/epoch - 399ms/step\n",
"Epoch 23/100\n",
"195/195 - 78s - loss: 0.0307 - accuracy: 0.9910 - val_loss: 3.6348 - val_accuracy: 0.4741 - 78s/epoch - 401ms/step\n",
"Epoch 24/100\n",
"195/195 - 77s - loss: 0.0315 - accuracy: 0.9903 - val_loss: 3.2422 - val_accuracy: 0.4711 - 77s/epoch - 396ms/step\n",
"Epoch 25/100\n",
"195/195 - 78s - loss: 0.0640 - accuracy: 0.9789 - val_loss: 3.5402 - val_accuracy: 0.4298 - 78s/epoch - 402ms/step\n",
"Epoch 26/100\n",
"195/195 - 78s - loss: 0.0916 - accuracy: 0.9715 - val_loss: 3.1916 - val_accuracy: 0.4520 - 78s/epoch - 399ms/step\n",
"Epoch 27/100\n",
"195/195 - 77s - loss: 0.0634 - accuracy: 0.9799 - val_loss: 3.2234 - val_accuracy: 0.4602 - 77s/epoch - 394ms/step\n",
"Epoch 28/100\n",
"195/195 - 78s - loss: 0.0673 - accuracy: 0.9788 - val_loss: 3.4435 - val_accuracy: 0.4647 - 78s/epoch - 400ms/step\n",
"Epoch 29/100\n",
"195/195 - 78s - loss: 0.0440 - accuracy: 0.9870 - val_loss: 3.3068 - val_accuracy: 0.4598 - 78s/epoch - 400ms/step\n",
"Epoch 30/100\n",
"195/195 - 78s - loss: 0.0165 - accuracy: 0.9950 - val_loss: 3.6233 - val_accuracy: 0.4681 - 78s/epoch - 401ms/step\n",
"Epoch 31/100\n",
"195/195 - 79s - loss: 0.0303 - accuracy: 0.9910 - val_loss: 3.7398 - val_accuracy: 0.4572 - 79s/epoch - 403ms/step\n",
"Epoch 32/100\n",
"195/195 - 78s - loss: 0.0554 - accuracy: 0.9815 - val_loss: 3.5560 - val_accuracy: 0.4369 - 78s/epoch - 399ms/step\n",
"Epoch 33/100\n",
"195/195 - 78s - loss: 0.0556 - accuracy: 0.9829 - val_loss: 3.6555 - val_accuracy: 0.4610 - 78s/epoch - 401ms/step\n",
"Epoch 34/100\n",
"195/195 - 78s - loss: 0.0624 - accuracy: 0.9809 - val_loss: 3.3500 - val_accuracy: 0.4617 - 78s/epoch - 401ms/step\n",
"Epoch 35/100\n",
"195/195 - 77s - loss: 0.0367 - accuracy: 0.9886 - val_loss: 3.5968 - val_accuracy: 0.4625 - 77s/epoch - 394ms/step\n",
"Epoch 36/100\n",
"195/195 - 78s - loss: 0.0301 - accuracy: 0.9912 - val_loss: 3.5188 - val_accuracy: 0.4643 - 78s/epoch - 399ms/step\n",
"Epoch 37/100\n",
"195/195 - 79s - loss: 0.0491 - accuracy: 0.9850 - val_loss: 3.3079 - val_accuracy: 0.4471 - 79s/epoch - 403ms/step\n",
"Epoch 38/100\n",
"195/195 - 78s - loss: 0.0577 - accuracy: 0.9821 - val_loss: 3.4042 - val_accuracy: 0.4381 - 78s/epoch - 400ms/step\n",
"Epoch 39/100\n",
"195/195 - 78s - loss: 0.0431 - accuracy: 0.9878 - val_loss: 3.4389 - val_accuracy: 0.4550 - 78s/epoch - 399ms/step\n",
"Epoch 40/100\n",
"195/195 - 77s - loss: 0.0218 - accuracy: 0.9940 - val_loss: 3.1989 - val_accuracy: 0.4854 - 77s/epoch - 397ms/step\n",
"Epoch 41/100\n",
"195/195 - 77s - loss: 0.0296 - accuracy: 0.9921 - val_loss: 3.2759 - val_accuracy: 0.4651 - 77s/epoch - 394ms/step\n",
"Epoch 42/100\n",
"195/195 - 78s - loss: 0.0176 - accuracy: 0.9947 - val_loss: 3.2391 - val_accuracy: 0.4745 - 78s/epoch - 398ms/step\n",
"Epoch 43/100\n",
"195/195 - 79s - loss: 0.0099 - accuracy: 0.9965 - val_loss: 3.5696 - val_accuracy: 0.4553 - 79s/epoch - 405ms/step\n",
"Epoch 44/100\n",
"195/195 - 78s - loss: 0.0516 - accuracy: 0.9852 - val_loss: 3.3857 - val_accuracy: 0.4482 - 78s/epoch - 402ms/step\n",
"Epoch 45/100\n",
"195/195 - 79s - loss: 0.0412 - accuracy: 0.9860 - val_loss: 3.3717 - val_accuracy: 0.4580 - 79s/epoch - 404ms/step\n",
"Epoch 46/100\n"
"Model: \"sequential\"\n",
"_________________________________________________________________\n",
" Layer (type) Output Shape Param # \n",
"=================================================================\n",
" block1_conv1 (Conv2D) (None, 224, 224, 64) 1792 \n",
" \n",
" block1_conv2 (Conv2D) (None, 224, 224, 64) 36928 \n",
" \n",
" block1_pool (MaxPooling2D) (None, 112, 112, 64) 0 \n",
" \n",
" block2_conv1 (Conv2D) (None, 112, 112, 128) 73856 \n",
" \n",
" block2_conv2 (Conv2D) (None, 112, 112, 128) 147584 \n",
" \n",
" block2_pool (MaxPooling2D) (None, 56, 56, 128) 0 \n",
" \n",
" block3_conv1 (Conv2D) (None, 56, 56, 256) 295168 \n",
" \n",
" block3_conv2 (Conv2D) (None, 56, 56, 256) 590080 \n",
" \n",
" block3_conv3 (Conv2D) (None, 56, 56, 256) 590080 \n",
" \n",
" block3_pool (MaxPooling2D) (None, 28, 28, 256) 0 \n",
" \n",
" block4_conv1 (Conv2D) (None, 28, 28, 512) 1180160 \n",
" \n",
" block4_conv2 (Conv2D) (None, 28, 28, 512) 2359808 \n",
" \n",
" block4_conv3 (Conv2D) (None, 28, 28, 512) 2359808 \n",
" \n",
" block4_pool (MaxPooling2D) (None, 14, 14, 512) 0 \n",
" \n",
" block5_conv1 (Conv2D) (None, 14, 14, 512) 2359808 \n",
" \n",
" block5_conv2 (Conv2D) (None, 14, 14, 512) 2359808 \n",
" \n",
" block5_conv3 (Conv2D) (None, 14, 14, 512) 2359808 \n",
" \n",
" block5_pool (MaxPooling2D) (None, 7, 7, 512) 0 \n",
" \n",
" flatten (Flatten) (None, 25088) 0 \n",
" \n",
" fc1 (Dense) (None, 4096) 102764544 \n",
" \n",
" fc2 (Dense) (None, 4096) 16781312 \n",
" \n",
" dense (Dense) (None, 13) 53261 \n",
" \n",
"=================================================================\n",
"Total params: 134,313,805\n",
"Trainable params: 134,313,805\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"#model = add_regularization(model)\n",
"model.summary()\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "fd5d1246",
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer=Adam(learning_rate=.0001), loss='categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"# sparse_categorical_crossentropy"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "9cd2ba27",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/30\n",
"1018/1018 [==============================] - 391s 379ms/step - loss: 2.4329 - accuracy: 0.1571 - val_loss: 2.1902 - val_accuracy: 0.2525\n",
"Epoch 2/30\n",
"1018/1018 [==============================] - 379s 373ms/step - loss: 2.0578 - accuracy: 0.2960 - val_loss: 1.9160 - val_accuracy: 0.3330\n",
"Epoch 3/30\n",
"1018/1018 [==============================] - 381s 374ms/step - loss: 1.8221 - accuracy: 0.3681 - val_loss: 1.8588 - val_accuracy: 0.3535\n",
"Epoch 4/30\n",
"1018/1018 [==============================] - 383s 376ms/step - loss: 1.6406 - accuracy: 0.4272 - val_loss: 1.7819 - val_accuracy: 0.4028\n",
"Epoch 5/30\n",
"1018/1018 [==============================] - 383s 376ms/step - loss: 1.4577 - accuracy: 0.4920 - val_loss: 1.7216 - val_accuracy: 0.4158\n",
"Epoch 6/30\n",
"1018/1018 [==============================] - 379s 372ms/step - loss: 1.2528 - accuracy: 0.5607 - val_loss: 1.7924 - val_accuracy: 0.4140\n",
"Epoch 7/30\n",
"1018/1018 [==============================] - 378s 371ms/step - loss: 1.0030 - accuracy: 0.6469 - val_loss: 1.8017 - val_accuracy: 0.4303\n",
"Epoch 8/30\n",
"1018/1018 [==============================] - 379s 372ms/step - loss: 0.7405 - accuracy: 0.7420 - val_loss: 1.9863 - val_accuracy: 0.4453\n",
"Epoch 9/30\n",
"1018/1018 [==============================] - 379s 372ms/step - loss: 0.4704 - accuracy: 0.8354 - val_loss: 2.3988 - val_accuracy: 0.4263\n",
"Epoch 10/30\n",
"1018/1018 [==============================] - 379s 372ms/step - loss: 0.3059 - accuracy: 0.8944 - val_loss: 2.7526 - val_accuracy: 0.4303\n",
"Epoch 11/30\n",
"1018/1018 [==============================] - 377s 371ms/step - loss: 0.2160 - accuracy: 0.9278 - val_loss: 3.0618 - val_accuracy: 0.4250\n",
"Epoch 12/30\n",
" 437/1018 [===========>..................] - ETA: 2:53 - loss: 0.1370 - accuracy: 0.9536"
]
},
{
@ -393,7 +437,7 @@
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-17-1a4715fb06ea>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-20-4cd4443bbf2a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1214\u001b[0m _r=1):\n\u001b[1;32m 1215\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1216\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1217\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1218\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 151\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
@ -412,8 +456,8 @@
" steps_per_epoch=len(train_generator),\n",
" validation_data=validation_generator,\n",
" validation_steps=len(validation_generator),\n",
" epochs=100,\n",
" verbose=2)"
" epochs=30,\n",
" verbose=1)"
]
},
{

857
Shoe Classifier_Xception.ipynb Normal file
View File

@ -0,0 +1,857 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "572dc7fb",
"metadata": {},
"outputs": [],
"source": [
"from matplotlib import pyplot as plt\n",
"from matplotlib.image import imread\n",
"import pandas as pd\n",
"from collections import Counter\n",
"import json\n",
"import os\n",
"import re\n",
"import tempfile\n",
"import numpy as np\n",
"from os.path import exists\n",
"from imblearn.under_sampling import RandomUnderSampler\n",
"from PIL import ImageFile\n",
"import sklearn as sk\n",
"from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n",
"import tensorflow as tf\n",
"import tensorflow.keras\n",
"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
"from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\n",
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.optimizers import Adam\n",
"# custom modules\n",
"import image_faults\n",
"\n",
"ImageFile.LOAD_TRUNCATED_IMAGES = True"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "8d94196d",
"metadata": {},
"outputs": [],
"source": [
"def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n",
"\n",
" if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n",
" print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n",
" return model\n",
"\n",
" for layer in model.layers:\n",
" for attr in ['kernel_regularizer']:\n",
" if hasattr(layer, attr):\n",
" setattr(layer, attr, regularizer)\n",
"\n",
" # When we change the layers attributes, the change only happens in the model config file\n",
" model_json = model.to_json()\n",
"\n",
" # Save the weights before reloading the model.\n",
" tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n",
" model.save_weights(tmp_weights_path)\n",
"\n",
" # load the model from the config\n",
" model = tf.keras.models.model_from_json(model_json)\n",
" \n",
" # Reload the model weights\n",
" model.load_weights(tmp_weights_path, by_name=True)\n",
" return model"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a5c72863",
"metadata": {},
"outputs": [],
"source": [
"# image_faults.faulty_images() # removes faulty images\n",
"df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1057a442",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{source:target} dictionary created @ /tf/training_images\n"
]
}
],
"source": [
"def dict_pics():\n",
" target_dir = os.getcwd() + os.sep + \"training_images\"\n",
" with open('temp_pics_source_list.txt') as f:\n",
" temp_pics_source_list = json.load(f)\n",
" dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n",
" print(\"{source:target} dictionary created @ \" + target_dir)\n",
" return dict_pics\n",
"\n",
"dict_pics = dict_pics()\n",
"blah = pd.Series(df.PictureURL)\n",
"df = df.drop(labels=['PictureURL'], axis=1)\n",
"blah = blah.apply(lambda x: dict_pics[x])\n",
"df = pd.concat([blah, df],axis=1)\n",
"df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n",
"# removes non-existent image paths"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "7a6146e6",
"metadata": {},
"outputs": [],
"source": [
"df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n",
"\n",
"df=df.sample(frac=1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "114cc3c0",
"metadata": {},
"outputs": [],
"source": [
"undersample = RandomUnderSampler(sampling_strategy='auto')\n",
"train, y_under = undersample.fit_resample(df, df['PrimaryCategoryID'])\n",
"#print(Counter(train['PrimaryCategoryID']))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "506aa5cf",
"metadata": {},
"outputs": [],
"source": [
"train, test = train_test_split(train, test_size=0.1, random_state=42)\n",
"# stratify=train['PrimaryCategoryID']\n",
"# train['PrimaryCategoryID'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "4d72eb90",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found 5110 validated image filenames belonging to 13 classes.\n",
"Found 1277 validated image filenames belonging to 13 classes.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n",
" warnings.warn(\n"
]
}
],
"source": [
"datagen = ImageDataGenerator(rescale=1./255., \n",
" validation_split=.2,\n",
" #samplewise_std_normalization=True,\n",
" #horizontal_flip= True,\n",
" #vertical_flip= True,\n",
" #width_shift_range= 0.2,\n",
" #height_shift_range= 0.2,\n",
" #rotation_range= 90,\n",
" preprocessing_function=tf.keras.applications.xception.preprocess_input)\n",
"train_generator=datagen.flow_from_dataframe(\n",
" dataframe=train[:len(train)],\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=64,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(299,299),\n",
" subset='training'\n",
" )\n",
"validation_generator=datagen.flow_from_dataframe(\n",
" dataframe=train[:len(train)], # is using train right?\n",
" directory='./training_images',\n",
" x_col='PictureURL',\n",
" y_col='PrimaryCategoryID',\n",
" batch_size=64,\n",
" seed=42,\n",
" shuffle=True,\n",
" target_size=(299,299),\n",
" subset='validation'\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7b70f37f",
"metadata": {},
"outputs": [],
"source": [
"imgs, labels = next(train_generator)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "1ed54bf5",
"metadata": {},
"outputs": [],
"source": [
"def plotImages(images_arr):\n",
" fig, axes = plt.subplots(1, 10, figsize=(20,20))\n",
" axes = axes.flatten()\n",
" for img, ax in zip( images_arr, axes):\n",
" ax.imshow(img)\n",
" ax.axis('off')\n",
" plt.tight_layout()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "85934565",
"metadata": {},
"outputs": [],
"source": [
"#plotImages(imgs)\n",
"#print(labels)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "6322bcad",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1\n"
]
}
],
"source": [
"physical_devices = tf.config.list_physical_devices('GPU')\n",
"print(len(physical_devices))\n",
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "07fd25c6",
"metadata": {},
"outputs": [],
"source": [
"# see https://www.kaggle.com/dmitrypukhov/cnn-with-imagedatagenerator-flow-from-dataframe for train/test/val split \n",
"# example\n",
"\n",
"# may need to either create a test dataset from the original dataset or just download a new one"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "b31af79e",
"metadata": {},
"outputs": [],
"source": [
"base_model = tf.keras.applications.xception.Xception(include_top=False, weights='imagenet', pooling='avg')\n",
"#base_model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "fe06f2bf",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"model\"\n",
"__________________________________________________________________________________________________\n",
" Layer (type) Output Shape Param # Connected to \n",
"==================================================================================================\n",
" input_1 (InputLayer) [(None, None, None, 0 [] \n",
" 3)] \n",
" \n",
" block1_conv1 (Conv2D) (None, None, None, 864 ['input_1[0][0]'] \n",
" 32) \n",
" \n",
" block1_conv1_bn (BatchNormaliz (None, None, None, 128 ['block1_conv1[0][0]'] \n",
" ation) 32) \n",
" \n",
" block1_conv1_act (Activation) (None, None, None, 0 ['block1_conv1_bn[0][0]'] \n",
" 32) \n",
" \n",
" block1_conv2 (Conv2D) (None, None, None, 18432 ['block1_conv1_act[0][0]'] \n",
" 64) \n",
" \n",
" block1_conv2_bn (BatchNormaliz (None, None, None, 256 ['block1_conv2[0][0]'] \n",
" ation) 64) \n",
" \n",
" block1_conv2_act (Activation) (None, None, None, 0 ['block1_conv2_bn[0][0]'] \n",
" 64) \n",
" \n",
" block2_sepconv1 (SeparableConv (None, None, None, 8768 ['block1_conv2_act[0][0]'] \n",
" 2D) 128) \n",
" \n",
" block2_sepconv1_bn (BatchNorma (None, None, None, 512 ['block2_sepconv1[0][0]'] \n",
" lization) 128) \n",
" \n",
" block2_sepconv2_act (Activatio (None, None, None, 0 ['block2_sepconv1_bn[0][0]'] \n",
" n) 128) \n",
" \n",
" block2_sepconv2 (SeparableConv (None, None, None, 17536 ['block2_sepconv2_act[0][0]'] \n",
" 2D) 128) \n",
" \n",
" block2_sepconv2_bn (BatchNorma (None, None, None, 512 ['block2_sepconv2[0][0]'] \n",
" lization) 128) \n",
" \n",
" conv2d (Conv2D) (None, None, None, 8192 ['block1_conv2_act[0][0]'] \n",
" 128) \n",
" \n",
" block2_pool (MaxPooling2D) (None, None, None, 0 ['block2_sepconv2_bn[0][0]'] \n",
" 128) \n",
" \n",
" batch_normalization (BatchNorm (None, None, None, 512 ['conv2d[0][0]'] \n",
" alization) 128) \n",
" \n",
" add (Add) (None, None, None, 0 ['block2_pool[0][0]', \n",
" 128) 'batch_normalization[0][0]'] \n",
" \n",
" block3_sepconv1_act (Activatio (None, None, None, 0 ['add[0][0]'] \n",
" n) 128) \n",
" \n",
" block3_sepconv1 (SeparableConv (None, None, None, 33920 ['block3_sepconv1_act[0][0]'] \n",
" 2D) 256) \n",
" \n",
" block3_sepconv1_bn (BatchNorma (None, None, None, 1024 ['block3_sepconv1[0][0]'] \n",
" lization) 256) \n",
" \n",
" block3_sepconv2_act (Activatio (None, None, None, 0 ['block3_sepconv1_bn[0][0]'] \n",
" n) 256) \n",
" \n",
" block3_sepconv2 (SeparableConv (None, None, None, 67840 ['block3_sepconv2_act[0][0]'] \n",
" 2D) 256) \n",
" \n",
" block3_sepconv2_bn (BatchNorma (None, None, None, 1024 ['block3_sepconv2[0][0]'] \n",
" lization) 256) \n",
" \n",
" conv2d_1 (Conv2D) (None, None, None, 32768 ['add[0][0]'] \n",
" 256) \n",
" \n",
" block3_pool (MaxPooling2D) (None, None, None, 0 ['block3_sepconv2_bn[0][0]'] \n",
" 256) \n",
" \n",
" batch_normalization_1 (BatchNo (None, None, None, 1024 ['conv2d_1[0][0]'] \n",
" rmalization) 256) \n",
" \n",
" add_1 (Add) (None, None, None, 0 ['block3_pool[0][0]', \n",
" 256) 'batch_normalization_1[0][0]'] \n",
" \n",
" block4_sepconv1_act (Activatio (None, None, None, 0 ['add_1[0][0]'] \n",
" n) 256) \n",
" \n",
" block4_sepconv1 (SeparableConv (None, None, None, 188672 ['block4_sepconv1_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block4_sepconv1_bn (BatchNorma (None, None, None, 2912 ['block4_sepconv1[0][0]'] \n",
" lization) 728) \n",
" \n",
" block4_sepconv2_act (Activatio (None, None, None, 0 ['block4_sepconv1_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block4_sepconv2 (SeparableConv (None, None, None, 536536 ['block4_sepconv2_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block4_sepconv2_bn (BatchNorma (None, None, None, 2912 ['block4_sepconv2[0][0]'] \n",
" lization) 728) \n",
" \n",
" conv2d_2 (Conv2D) (None, None, None, 186368 ['add_1[0][0]'] \n",
" 728) \n",
" \n",
" block4_pool (MaxPooling2D) (None, None, None, 0 ['block4_sepconv2_bn[0][0]'] \n",
" 728) \n",
" \n",
" batch_normalization_2 (BatchNo (None, None, None, 2912 ['conv2d_2[0][0]'] \n",
" rmalization) 728) \n",
" \n",
" add_2 (Add) (None, None, None, 0 ['block4_pool[0][0]', \n",
" 728) 'batch_normalization_2[0][0]'] \n",
" \n",
" block5_sepconv1_act (Activatio (None, None, None, 0 ['add_2[0][0]'] \n",
" n) 728) \n",
" \n",
" block5_sepconv1 (SeparableConv (None, None, None, 536536 ['block5_sepconv1_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block5_sepconv1_bn (BatchNorma (None, None, None, 2912 ['block5_sepconv1[0][0]'] \n",
" lization) 728) \n",
" \n",
" block5_sepconv2_act (Activatio (None, None, None, 0 ['block5_sepconv1_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block5_sepconv2 (SeparableConv (None, None, None, 536536 ['block5_sepconv2_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block5_sepconv2_bn (BatchNorma (None, None, None, 2912 ['block5_sepconv2[0][0]'] \n",
" lization) 728) \n",
" \n",
" block5_sepconv3_act (Activatio (None, None, None, 0 ['block5_sepconv2_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block5_sepconv3 (SeparableConv (None, None, None, 536536 ['block5_sepconv3_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block5_sepconv3_bn (BatchNorma (None, None, None, 2912 ['block5_sepconv3[0][0]'] \n",
" lization) 728) \n",
" \n",
" add_3 (Add) (None, None, None, 0 ['block5_sepconv3_bn[0][0]', \n",
" 728) 'add_2[0][0]'] \n",
" \n",
" block6_sepconv1_act (Activatio (None, None, None, 0 ['add_3[0][0]'] \n",
" n) 728) \n",
" \n",
" block6_sepconv1 (SeparableConv (None, None, None, 536536 ['block6_sepconv1_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block6_sepconv1_bn (BatchNorma (None, None, None, 2912 ['block6_sepconv1[0][0]'] \n",
" lization) 728) \n",
" \n",
" block6_sepconv2_act (Activatio (None, None, None, 0 ['block6_sepconv1_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block6_sepconv2 (SeparableConv (None, None, None, 536536 ['block6_sepconv2_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block6_sepconv2_bn (BatchNorma (None, None, None, 2912 ['block6_sepconv2[0][0]'] \n",
" lization) 728) \n",
" \n",
" block6_sepconv3_act (Activatio (None, None, None, 0 ['block6_sepconv2_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block6_sepconv3 (SeparableConv (None, None, None, 536536 ['block6_sepconv3_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block6_sepconv3_bn (BatchNorma (None, None, None, 2912 ['block6_sepconv3[0][0]'] \n",
" lization) 728) \n",
" \n",
" add_4 (Add) (None, None, None, 0 ['block6_sepconv3_bn[0][0]', \n",
" 728) 'add_3[0][0]'] \n",
" \n",
" block7_sepconv1_act (Activatio (None, None, None, 0 ['add_4[0][0]'] \n",
" n) 728) \n",
" \n",
" block7_sepconv1 (SeparableConv (None, None, None, 536536 ['block7_sepconv1_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block7_sepconv1_bn (BatchNorma (None, None, None, 2912 ['block7_sepconv1[0][0]'] \n",
" lization) 728) \n",
" \n",
" block7_sepconv2_act (Activatio (None, None, None, 0 ['block7_sepconv1_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block7_sepconv2 (SeparableConv (None, None, None, 536536 ['block7_sepconv2_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block7_sepconv2_bn (BatchNorma (None, None, None, 2912 ['block7_sepconv2[0][0]'] \n",
" lization) 728) \n",
" \n",
" block7_sepconv3_act (Activatio (None, None, None, 0 ['block7_sepconv2_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block7_sepconv3 (SeparableConv (None, None, None, 536536 ['block7_sepconv3_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block7_sepconv3_bn (BatchNorma (None, None, None, 2912 ['block7_sepconv3[0][0]'] \n",
" lization) 728) \n",
" \n",
" add_5 (Add) (None, None, None, 0 ['block7_sepconv3_bn[0][0]', \n",
" 728) 'add_4[0][0]'] \n",
" \n",
" block8_sepconv1_act (Activatio (None, None, None, 0 ['add_5[0][0]'] \n",
" n) 728) \n",
" \n",
" block8_sepconv1 (SeparableConv (None, None, None, 536536 ['block8_sepconv1_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block8_sepconv1_bn (BatchNorma (None, None, None, 2912 ['block8_sepconv1[0][0]'] \n",
" lization) 728) \n",
" \n",
" block8_sepconv2_act (Activatio (None, None, None, 0 ['block8_sepconv1_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block8_sepconv2 (SeparableConv (None, None, None, 536536 ['block8_sepconv2_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block8_sepconv2_bn (BatchNorma (None, None, None, 2912 ['block8_sepconv2[0][0]'] \n",
" lization) 728) \n",
" \n",
" block8_sepconv3_act (Activatio (None, None, None, 0 ['block8_sepconv2_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block8_sepconv3 (SeparableConv (None, None, None, 536536 ['block8_sepconv3_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block8_sepconv3_bn (BatchNorma (None, None, None, 2912 ['block8_sepconv3[0][0]'] \n",
" lization) 728) \n",
" \n",
" add_6 (Add) (None, None, None, 0 ['block8_sepconv3_bn[0][0]', \n",
" 728) 'add_5[0][0]'] \n",
" \n",
" block9_sepconv1_act (Activatio (None, None, None, 0 ['add_6[0][0]'] \n",
" n) 728) \n",
" \n",
" block9_sepconv1 (SeparableConv (None, None, None, 536536 ['block9_sepconv1_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block9_sepconv1_bn (BatchNorma (None, None, None, 2912 ['block9_sepconv1[0][0]'] \n",
" lization) 728) \n",
" \n",
" block9_sepconv2_act (Activatio (None, None, None, 0 ['block9_sepconv1_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block9_sepconv2 (SeparableConv (None, None, None, 536536 ['block9_sepconv2_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block9_sepconv2_bn (BatchNorma (None, None, None, 2912 ['block9_sepconv2[0][0]'] \n",
" lization) 728) \n",
" \n",
" block9_sepconv3_act (Activatio (None, None, None, 0 ['block9_sepconv2_bn[0][0]'] \n",
" n) 728) \n",
" \n",
" block9_sepconv3 (SeparableConv (None, None, None, 536536 ['block9_sepconv3_act[0][0]'] \n",
" 2D) 728) \n",
" \n",
" block9_sepconv3_bn (BatchNorma (None, None, None, 2912 ['block9_sepconv3[0][0]'] \n",
" lization) 728) \n",
" \n",
" add_7 (Add) (None, None, None, 0 ['block9_sepconv3_bn[0][0]', \n",
" 728) 'add_6[0][0]'] \n",
" \n",
" block10_sepconv1_act (Activati (None, None, None, 0 ['add_7[0][0]'] \n",
" on) 728) \n",
" \n",
" block10_sepconv1 (SeparableCon (None, None, None, 536536 ['block10_sepconv1_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block10_sepconv1_bn (BatchNorm (None, None, None, 2912 ['block10_sepconv1[0][0]'] \n",
" alization) 728) \n",
" \n",
" block10_sepconv2_act (Activati (None, None, None, 0 ['block10_sepconv1_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block10_sepconv2 (SeparableCon (None, None, None, 536536 ['block10_sepconv2_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block10_sepconv2_bn (BatchNorm (None, None, None, 2912 ['block10_sepconv2[0][0]'] \n",
" alization) 728) \n",
" \n",
" block10_sepconv3_act (Activati (None, None, None, 0 ['block10_sepconv2_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block10_sepconv3 (SeparableCon (None, None, None, 536536 ['block10_sepconv3_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block10_sepconv3_bn (BatchNorm (None, None, None, 2912 ['block10_sepconv3[0][0]'] \n",
" alization) 728) \n",
" \n",
" add_8 (Add) (None, None, None, 0 ['block10_sepconv3_bn[0][0]', \n",
" 728) 'add_7[0][0]'] \n",
" \n",
" block11_sepconv1_act (Activati (None, None, None, 0 ['add_8[0][0]'] \n",
" on) 728) \n",
" \n",
" block11_sepconv1 (SeparableCon (None, None, None, 536536 ['block11_sepconv1_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block11_sepconv1_bn (BatchNorm (None, None, None, 2912 ['block11_sepconv1[0][0]'] \n",
" alization) 728) \n",
" \n",
" block11_sepconv2_act (Activati (None, None, None, 0 ['block11_sepconv1_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block11_sepconv2 (SeparableCon (None, None, None, 536536 ['block11_sepconv2_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block11_sepconv2_bn (BatchNorm (None, None, None, 2912 ['block11_sepconv2[0][0]'] \n",
" alization) 728) \n",
" \n",
" block11_sepconv3_act (Activati (None, None, None, 0 ['block11_sepconv2_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block11_sepconv3 (SeparableCon (None, None, None, 536536 ['block11_sepconv3_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block11_sepconv3_bn (BatchNorm (None, None, None, 2912 ['block11_sepconv3[0][0]'] \n",
" alization) 728) \n",
" \n",
" add_9 (Add) (None, None, None, 0 ['block11_sepconv3_bn[0][0]', \n",
" 728) 'add_8[0][0]'] \n",
" \n",
" block12_sepconv1_act (Activati (None, None, None, 0 ['add_9[0][0]'] \n",
" on) 728) \n",
" \n",
" block12_sepconv1 (SeparableCon (None, None, None, 536536 ['block12_sepconv1_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block12_sepconv1_bn (BatchNorm (None, None, None, 2912 ['block12_sepconv1[0][0]'] \n",
" alization) 728) \n",
" \n",
" block12_sepconv2_act (Activati (None, None, None, 0 ['block12_sepconv1_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block12_sepconv2 (SeparableCon (None, None, None, 536536 ['block12_sepconv2_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block12_sepconv2_bn (BatchNorm (None, None, None, 2912 ['block12_sepconv2[0][0]'] \n",
" alization) 728) \n",
" \n",
" block12_sepconv3_act (Activati (None, None, None, 0 ['block12_sepconv2_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block12_sepconv3 (SeparableCon (None, None, None, 536536 ['block12_sepconv3_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block12_sepconv3_bn (BatchNorm (None, None, None, 2912 ['block12_sepconv3[0][0]'] \n",
" alization) 728) \n",
" \n",
" add_10 (Add) (None, None, None, 0 ['block12_sepconv3_bn[0][0]', \n",
" 728) 'add_9[0][0]'] \n",
" \n",
" block13_sepconv1_act (Activati (None, None, None, 0 ['add_10[0][0]'] \n",
" on) 728) \n",
" \n",
" block13_sepconv1 (SeparableCon (None, None, None, 536536 ['block13_sepconv1_act[0][0]'] \n",
" v2D) 728) \n",
" \n",
" block13_sepconv1_bn (BatchNorm (None, None, None, 2912 ['block13_sepconv1[0][0]'] \n",
" alization) 728) \n",
" \n",
" block13_sepconv2_act (Activati (None, None, None, 0 ['block13_sepconv1_bn[0][0]'] \n",
" on) 728) \n",
" \n",
" block13_sepconv2 (SeparableCon (None, None, None, 752024 ['block13_sepconv2_act[0][0]'] \n",
" v2D) 1024) \n",
" \n",
" block13_sepconv2_bn (BatchNorm (None, None, None, 4096 ['block13_sepconv2[0][0]'] \n",
" alization) 1024) \n",
" \n",
" conv2d_3 (Conv2D) (None, None, None, 745472 ['add_10[0][0]'] \n",
" 1024) \n",
" \n",
" block13_pool (MaxPooling2D) (None, None, None, 0 ['block13_sepconv2_bn[0][0]'] \n",
" 1024) \n",
" \n",
" batch_normalization_3 (BatchNo (None, None, None, 4096 ['conv2d_3[0][0]'] \n",
" rmalization) 1024) \n",
" \n",
" add_11 (Add) (None, None, None, 0 ['block13_pool[0][0]', \n",
" 1024) 'batch_normalization_3[0][0]'] \n",
" \n",
" block14_sepconv1 (SeparableCon (None, None, None, 1582080 ['add_11[0][0]'] \n",
" v2D) 1536) \n",
" \n",
" block14_sepconv1_bn (BatchNorm (None, None, None, 6144 ['block14_sepconv1[0][0]'] \n",
" alization) 1536) \n",
" \n",
" block14_sepconv1_act (Activati (None, None, None, 0 ['block14_sepconv1_bn[0][0]'] \n",
" on) 1536) \n",
" \n",
" block14_sepconv2 (SeparableCon (None, None, None, 3159552 ['block14_sepconv1_act[0][0]'] \n",
" v2D) 2048) \n",
" \n",
" block14_sepconv2_bn (BatchNorm (None, None, None, 8192 ['block14_sepconv2[0][0]'] \n",
" alization) 2048) \n",
" \n",
" block14_sepconv2_act (Activati (None, None, None, 0 ['block14_sepconv2_bn[0][0]'] \n",
" on) 2048) \n",
" \n",
" global_average_pooling2d (Glob (None, 2048) 0 ['block14_sepconv2_act[0][0]'] \n",
" alAveragePooling2D) \n",
" \n",
" dense (Dense) (None, 13) 26637 ['global_average_pooling2d[0][0]'\n",
" ] \n",
" \n",
"==================================================================================================\n",
"Total params: 20,888,117\n",
"Trainable params: 20,833,589\n",
"Non-trainable params: 54,528\n",
"__________________________________________________________________________________________________\n"
]
}
],
"source": [
"for layer in base_model.layers:\n",
" layer.trainable = True\n",
" \n",
"output = Dense(13, activation='softmax')(base_model.output)\n",
"model = tf.keras.Model(base_model.input, output)\n",
"#model = add_regularization(model)\n",
"model.summary()\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "ea620129",
"metadata": {},
"outputs": [],
"source": [
"#model.add(Dropout(.5))\n",
"#model.add(Dense(64, activation='softmax'))\n",
"# model.add(Dropout(.25))\n",
"#model = add_regularization(model)\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "fd5d1246",
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer=Adam(learning_rate=.0001), loss='categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"# sparse_categorical_crossentropy"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "9cd2ba27",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/30\n",
"80/80 [==============================] - 78s 913ms/step - loss: 1.8419 - accuracy: 0.4153 - val_loss: 2.6144 - val_accuracy: 0.0720\n",
"Epoch 2/30\n",
"80/80 [==============================] - 69s 862ms/step - loss: 0.9255 - accuracy: 0.7121 - val_loss: 2.5804 - val_accuracy: 0.1026\n",
"Epoch 3/30\n",
"80/80 [==============================] - 72s 900ms/step - loss: 0.4003 - accuracy: 0.9092 - val_loss: 2.4443 - val_accuracy: 0.2310\n",
"Epoch 4/30\n",
"80/80 [==============================] - 71s 888ms/step - loss: 0.1224 - accuracy: 0.9847 - val_loss: 2.1299 - val_accuracy: 0.3782\n",
"Epoch 5/30\n",
"80/80 [==============================] - 75s 935ms/step - loss: 0.0371 - accuracy: 0.9975 - val_loss: 1.7368 - val_accuracy: 0.4973\n",
"Epoch 6/30\n",
"80/80 [==============================] - 73s 900ms/step - loss: 0.0167 - accuracy: 0.9992 - val_loss: 1.6747 - val_accuracy: 0.5341\n",
"Epoch 7/30\n",
"80/80 [==============================] - 72s 897ms/step - loss: 0.0097 - accuracy: 0.9998 - val_loss: 1.6494 - val_accuracy: 0.5442\n",
"Epoch 8/30\n",
"80/80 [==============================] - 74s 915ms/step - loss: 0.0062 - accuracy: 0.9998 - val_loss: 1.6659 - val_accuracy: 0.5568\n",
"Epoch 9/30\n",
"80/80 [==============================] - 74s 917ms/step - loss: 0.0044 - accuracy: 1.0000 - val_loss: 1.7088 - val_accuracy: 0.5615\n",
"Epoch 10/30\n",
"80/80 [==============================] - 70s 868ms/step - loss: 0.0035 - accuracy: 1.0000 - val_loss: 1.7540 - val_accuracy: 0.5583\n",
"Epoch 11/30\n",
"80/80 [==============================] - 70s 864ms/step - loss: 0.0027 - accuracy: 0.9998 - val_loss: 1.7894 - val_accuracy: 0.5552\n",
"Epoch 12/30\n",
"80/80 [==============================] - 69s 858ms/step - loss: 0.0020 - accuracy: 1.0000 - val_loss: 1.8126 - val_accuracy: 0.5536\n",
"Epoch 13/30\n",
"80/80 [==============================] - 69s 857ms/step - loss: 0.0019 - accuracy: 1.0000 - val_loss: 1.8496 - val_accuracy: 0.5544\n",
"Epoch 14/30\n",
"80/80 [==============================] - 69s 859ms/step - loss: 0.0015 - accuracy: 1.0000 - val_loss: 1.8646 - val_accuracy: 0.5544\n",
"Epoch 15/30\n",
"30/80 [==========>...................] - ETA: 36s - loss: 0.0011 - accuracy: 1.0000"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-18-4cd4443bbf2a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m \u001b[0;31m# No error, now safe to assign to logs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1220\u001b[0m \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1221\u001b[0;31m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1222\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1223\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m 434\u001b[0m \"\"\"\n\u001b[1;32m 435\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 436\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 437\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 294\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 296\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 297\u001b[0m raise ValueError(\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m 314\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 318\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 353\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 354\u001b[0;31m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 355\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m 1030\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1031\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1032\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1033\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1034\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m 1102\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1103\u001b[0m \u001b[0;31m# Only block async when verbose = 1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1104\u001b[0;31m \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_to_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1105\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36msync_to_numpy_or_python_type\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 554\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 555\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 556\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m return pack_sequence_as(\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m expand_composites=expand_composites)\n\u001b[1;32m 871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m return pack_sequence_as(\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m expand_composites=expand_composites)\n\u001b[1;32m 871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36m_to_single_numpy_or_python_type\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m 548\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 551\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1147\u001b[0m \"\"\"\n\u001b[1;32m 1148\u001b[0m \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1149\u001b[0;31m \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1150\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1113\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1114\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1115\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1116\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1117\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"model.fit(x=train_generator,\n",
" steps_per_epoch=len(train_generator),\n",
" validation_data=validation_generator,\n",
" validation_steps=len(validation_generator),\n",
" epochs=30,\n",
" verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "63f791af",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

91
ebay_api.py
View File

@ -27,20 +27,20 @@ class FindingApi:
Methods for accessing eBay's FindingApi services
'''
def __init__(self, service, idspc):
def __init__(self, service):
self.service = [
'findItemsAdvanced', 'findCompletedItems',
'findItemsByKeywords', 'findItemsIneBayStores', 'findItemsByCategory',
'findItemsByProduct'
][service] # Currently using only index 4, i.e., service = 4
self.idspc = idspc # examples of additional params you may want to add:
# examples of additional params you may want to add:
# 'itemFilter(0).value':'Used' consider using this with findCompletedItems call
# 'itemFilter(1).name':'ListingType'
# 'itemFilter(1).value':'AuctionWithBIN'
# 'StartTimeNewest'
# HideDuplicateItems
def get_data(self, category_id, idspc):
def get_data(self, category_id):
'''
Gets raw JSON data fom FindingApi service call. Currently being used to
@ -51,8 +51,6 @@ class FindingApi:
# days_on_site = (now - startTime).days # as int
ids = []
modTimeFrom = datetime.now() - timedelta(seconds=5) # initialize modTimeFrom value
i = 1
params = {
"OPERATION-NAME":self.service,
"SECURITY-APPNAME":cfg.sec['SECURITY-APPNAME'],
@ -60,62 +58,37 @@ class FindingApi:
"RESPONSE-DATA-FORMAT":"JSON",
"categoryId":category_id,
"paginationInput.entriesPerPage":"100",
"paginationInput.PageNumber":i,
"paginationInput.PageNumber":"1",
"itemFilter(0).name":"Condition",
"itemFilter(0).value":"Used",
"itemFilter.name":"HideDuplicateItems",
"itemFilter.value":"true",
"sortOrder":"StartTimeNewest",
"itemFilter(1).name":"TopRatedSellerOnly",
"itemFilter(1).value":"true",
}
# "itemFilter.name(2)":"modTimeFrom",
# "itemFilter.value(2)":modTimeFrom,
try:
response = requests.get("https://svcs.ebay.com/services/search/FindingService/v1",
params=params, timeout=24)
response.raise_for_status()
while len(ids) < idspc:
except requests.exceptions.RequestException: # appears this works need to be able to continue where you left off or use better timeout?
print('connection error')
return ids
try:
data = response.json()
for item in data['findItemsByCategoryResponse'][0]['searchResult'][0]['item']:
ids.append(item['itemId'][0])
try:
response = requests.get("https://svcs.ebay.com/services/search/FindingService/v1",
params=params, timeout=24)
response.raise_for_status()
ids = list(set(ids))
except requests.exceptions.RequestException: # appears this works need to be able to continue where you left off or use better timeout?
print('connection error')
return ids
try:
data = response.json()
itemSearchURL = data['findItemsByCategoryResponse'][0]['itemSearchURL'][0]
modTimeFrom = data['findItemsByCategoryResponse'][0]['searchResult'][0]['item'][-1]['listingInfo'][0]['startTime'][0]
modTimeFrom = dateutil.parser.isoparse( modTimeFrom )
modTimeFrom = modTimeFrom - timedelta(seconds=5) # TODO NEED BACK TO GMT FORMAT
for item in data['findItemsByCategoryResponse'][0]['searchResult'][0]['item']:
# if item not in ids:
ids.append(item['itemId'][0])
except (AttributeError, KeyError):
print('AttributeError or KeyError. Exiting')
print(response.json())
return ids
#ids = list(set(ids))
except (AttributeError, KeyError):
print('AttributeError or KeyError. Exiting')
print(response.json())
return ids
input('press enter to continue')
i+=1
params = {
"OPERATION-NAME":self.service,
"SECURITY-APPNAME":cfg.sec['SECURITY-APPNAME'],
"SERVICE-VERSION":"1.13.0",
"RESPONSE-DATA-FORMAT":"JSON",
"categoryId":category_id,
"paginationInput.entriesPerPage":"20",
"paginationInput.PageNumber":i,
"itemFilter(0).name":"Condition",
"itemFilter(0).value":"Used",
"itemFilter.name":"HideDuplicateItems",
"itemFilter.value":"true",
"sortOrder":"StartTimeNewest",
}
return ids, data, modTimeFrom, itemSearchURL
return ids
# TODO add some other options to finding call api such as for possibly filtering for used items only. This might give you a better dataset for training. Or maybe a mixture of new and used. Maybe
# try and come up with a way to mathematically determine your odds of maximizing the number of pictures in your training set while reducing the number of useless images. Say for example, if you took a
@ -125,36 +98,32 @@ class FindingApi:
# You may even have more consistency with used shoes since they are "one-off" items without confusing multiple variations and colors. What else you can do is run small training sets on both new and used
# to see which one is more accurate or if a combo of both is more accurate.
def get_ids_from_cats(self): #TODO need to resolve duplicates here to maximize unique ids/data and ShopppingApi call
def get_ids_from_cats(self):
'''
Creates a 20-itemId list to use for the ShoppingApi
call
'''
# target_idspc = self.target_idspc
idspc = self.idspc
itemid_results_list = []
with open('cat_list.txt') as jf:
cat_list = json.load(jf)
for cat in cat_list:
args = [(cat, idspc) for cat in cat_list]
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(lambda p: self.get_data(*p), args):
itemid_results_list.extend(future)
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(self.get_data, cat_list):
itemid_results_list.extend(future)
print(len(itemid_results_list))
a = list(set(itemid_results_list))
print(len(a))
input('press enter to continue')
with open('raw_ids.txt', 'w') as f:
json.dump(itemid_results_list, f)
# 20-ItemID list created to maximize dataset/decrease calls given call constraints
item_id_results = [','.join(itemid_results_list[n:n+20]) for n in list(range(0,
len(itemid_results_list), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
len(itemid_results_list), 20))]
return item_id_results, itemid_results_list
class ShoppingApi: