diff --git a/$tutor$ b/$tutor$ new file mode 100644 index 0000000..426fb57 --- /dev/null +++ b/$tutor$ @@ -0,0 +1,970 @@ +=============================================================================== += W e l c o m e t o t h e V I M T u t o r - Version 1.7 = +=============================================================================== + + Vim is a very powerful editor that has many commands, too many to + explain in a tutor such as this. This tutor is designed to describe + enough of the commands that you will be able to easily use Vim as + an all-purpose editor. + + The approximate time required to complete the tutor is 25-30 minutes, + depending upon how much time is spent with experimentation. + + ATTENTION: + The commands in the lessons will modify the text. Make a copy of this + file to practice on (if you started "vimtutor" this is already a copy). + + It is important to remember that this tutor is set up to teach by + use. That means that you need to execute the commands to learn them + properly. If you only read the text, you will forget the commands! + + Now, make sure that your Caps-Lock key is NOT depressed and press + the j key enough times to move the cursor so that lesson 1.1 + completely fills the screen. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1.1: MOVING THE CURSOR + + + ** To move the cursor, press the h,j,k,l keys as indicated. ** + ^ + k Hint: The h key is at the left and moves left. + < h l > The l key is at the right and moves right. + j The j key looks like a down arrow. + v + 1. Move the cursor around the screen until you are comfortable. + + 2. Hold down the down key (j) until it repeats. + Now you know how to move to the next lesson. + + 3. Using the down key, move to lesson 1.2. + +NOTE: If you are ever unsure about something you typed, press to place + you in Normal mode. Then retype the command you wanted. + +NOTE: The cursor keys should also work. But using hjkl you will be able to + move around much faster, once you get used to it. Really! + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1.2: EXITING VIM + + + !! NOTE: Before executing any of the steps below, read this entire lesson!! + + 1. Press the key (to make sure you are in Normal mode). + + 2. Type: :q! . + This exits the editor, DISCARDING any changes you have made. + + 3. Get back here by executing the command that got you into this tutor. That + might be: vimtutor + + 4. If you have these steps memorized and are confident, execute steps + 1 through 3 to exit and re-enter the editor. + +NOTE: :q! discards any changes you made. In a few lessons you + will learn how to save the changes to a file. + + 5. Move the cursor down to lesson 1.3. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1.3: TEXT EDITING - DELETION + + + ** Press x to delete the character under the cursor. ** + + 1. Move the cursor to the line below marked --->. + + 2. To fix the errors, move the cursor until it is on top of the + character to be deleted. + + 3. Press the x key to delete the unwanted character. + + 4. Repeat steps 2 through 4 until the sentence is correct. + +---> The ccow jumpedd ovverr thhe mooon. + + 5. Now that the line is correct, go on to lesson 1.4. + +NOTE: As you go through this tutor, do not try to memorize, learn by usage. + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1.4: TEXT EDITING - INSERTION + + + ** Press i to insert text. ** + + 1. Move the cursor to the first line below marked --->. + + 2. To make the first line the same as the second, move the cursor on top + of the character BEFORE which the text is to be inserted. + + 3. Press i and type in the necessary additions. + + 4. As each error is fixed press to return to Normal mode. + Repeat steps 2 through 4 to correct the sentence. + +---> There is text misng this . +---> There is some text missing from this line. + + 5. When you are comfortable inserting text move to lesson 1.5. + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1.5: TEXT EDITING - APPENDING + + + ** Press A to append text. ** + + 1. Move the cursor to the first line below marked --->. + It does not matter on what character the cursor is in that line. + + 2. Press A and type in the necessary additions. + + 3. As the text has been appended press to return to Normal mode. + + 4. Move the cursor to the second line marked ---> and repeat + steps 2 and 3 to correct this sentence. + +---> There is some text missing from th + There is some text missing from this line. +---> There is also some text miss + There is also some text missing here. + + 5. When you are comfortable appending text move to lesson 1.6. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1.6: EDITING A FILE + + ** Use :wq to save a file and exit. ** + + !! NOTE: Before executing any of the steps below, read this entire lesson!! + + 1. Exit this tutor as you did in lesson 1.2: :q! + Or, if you have access to another terminal, do the following there. + + 2. At the shell prompt type this command: vim tutor + 'vim' is the command to start the Vim editor, 'tutor' is the name of the + file you wish to edit. Use a file that may be changed. + + 3. Insert and delete text as you learned in the previous lessons. + + 4. Save the file with changes and exit Vim with: :wq + + 5. If you have quit vimtutor in step 1 restart the vimtutor and move down to + the following summary. + + 6. After reading the above steps and understanding them: do it. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 1 SUMMARY + + + 1. The cursor is moved using either the arrow keys or the hjkl keys. + h (left) j (down) k (up) l (right) + + 2. To start Vim from the shell prompt type: vim FILENAME + + 3. To exit Vim type: :q! to trash all changes. + OR type: :wq to save the changes. + + 4. To delete the character at the cursor type: x + + 5. To insert or append text type: + i type inserted text insert before the cursor + A type appended text append after the line + +NOTE: Pressing will place you in Normal mode or will cancel + an unwanted and partially completed command. + +Now continue with lesson 2. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.1: DELETION COMMANDS + + + ** Type dw to delete a word. ** + + 1. Press to make sure you are in Normal mode. + + 2. Move the cursor to the line below marked --->. + + 3. Move the cursor to the beginning of a word that needs to be deleted. + + 4. Type dw to make the word disappear. + + NOTE: The letter d will appear on the last line of the screen as you type + it. Vim is waiting for you to type w . If you see another character + than d you typed something wrong; press and start over. + +---> There are a some words fun that don't belong paper in this sentence. + + 5. Repeat steps 3 and 4 until the sentence is correct and go to lesson 2.2. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.2: MORE DELETION COMMANDS + + + ** Type d$ to delete to the end of the line. ** + + 1. Press to make sure you are in Normal mode. + + 2. Move the cursor to the line below marked --->. + + 3. Move the cursor to the end of the correct line (AFTER the first . ). + + 4. Type d$ to delete to the end of the line. + +---> Somebody typed the end of this line twice. end of this line twice. + + + 5. Move on to lesson 2.3 to understand what is happening. + + + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.3: ON OPERATORS AND MOTIONS + + + Many commands that change text are made from an operator and a motion. + The format for a delete command with the d delete operator is as follows: + + d motion + + Where: + d - is the delete operator. + motion - is what the operator will operate on (listed below). + + A short list of motions: + w - until the start of the next word, EXCLUDING its first character. + e - to the end of the current word, INCLUDING the last character. + $ - to the end of the line, INCLUDING the last character. + + Thus typing de will delete from the cursor to the end of the word. + +NOTE: Pressing just the motion while in Normal mode without an operator will + move the cursor as specified. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.4: USING A COUNT FOR A MOTION + + + ** Typing a number before a motion repeats it that many times. ** + + 1. Move the cursor to the start of the line below marked --->. + + 2. Type 2w to move the cursor two words forward. + + 3. Type 3e to move the cursor to the end of the third word forward. + + 4. Type 0 (zero) to move to the start of the line. + + 5. Repeat steps 2 and 3 with different numbers. + +---> This is just a line with words you can move around in. + + 6. Move on to lesson 2.5. + + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.5: USING A COUNT TO DELETE MORE + + + ** Typing a number with an operator repeats it that many times. ** + + In the combination of the delete operator and a motion mentioned above you + insert a count before the motion to delete more: + d number motion + + 1. Move the cursor to the first UPPER CASE word in the line marked --->. + + 2. Type d2w to delete the two UPPER CASE words. + + 3. Repeat steps 1 and 2 with a different count to delete the consecutive + UPPER CASE words with one command. + +---> this ABC DE line FGHI JK LMN OP of words is Q RS TUV cleaned up. + + + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.6: OPERATING ON LINES + + + ** Type dd to delete a whole line. ** + + Due to the frequency of whole line deletion, the designers of Vi decided + it would be easier to simply type two d's to delete a line. + + 1. Move the cursor to the second line in the phrase below. + 2. Type dd to delete the line. + 3. Now move to the fourth line. + 4. Type 2dd to delete two lines. + +---> 1) Roses are red, +---> 2) Mud is fun, +---> 3) Violets are blue, +---> 4) I have a car, +---> 5) Clocks tell time, +---> 6) Sugar is sweet +---> 7) And so are you. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2.7: THE UNDO COMMAND + + + ** Press u to undo the last commands, U to fix a whole line. ** + + 1. Move the cursor to the line below marked ---> and place it on the + first error. + 2. Type x to delete the first unwanted character. + 3. Now type u to undo the last command executed. + 4. This time fix all the errors on the line using the x command. + 5. Now type a capital U to return the line to its original state. + 6. Now type u a few times to undo the U and preceding commands. + 7. Now type CTRL-R (keeping CTRL key pressed while hitting R) a few times + to redo the commands (undo the undo's). + +---> Fiix the errors oon thhis line and reeplace them witth undo. + + 8. These are very useful commands. Now move on to the lesson 2 Summary. + + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 2 SUMMARY + + + 1. To delete from the cursor up to the next word type: dw + 2. To delete from the cursor to the end of a line type: d$ + 3. To delete a whole line type: dd + + 4. To repeat a motion prepend it with a number: 2w + 5. The format for a change command is: + operator [number] motion + where: + operator - is what to do, such as d for delete + [number] - is an optional count to repeat the motion + motion - moves over the text to operate on, such as w (word), + $ (to the end of line), etc. + + 6. To move to the start of the line use a zero: 0 + + 7. To undo previous actions, type: u (lowercase u) + To undo all the changes on a line, type: U (capital U) + To undo the undo's, type: CTRL-R + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 3.1: THE PUT COMMAND + + + ** Type p to put previously deleted text after the cursor. ** + + 1. Move the cursor to the first line below marked --->. + + 2. Type dd to delete the line and store it in a Vim register. + + 3. Move the cursor to the c) line, ABOVE where the deleted line should go. + + 4. Type p to put the line below the cursor. + + 5. Repeat steps 2 through 4 to put all the lines in correct order. + +---> d) Can you learn too? +---> b) Violets are blue, +---> c) Intelligence is learned, +---> a) Roses are red, + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 3.2: THE REPLACE COMMAND + + + ** Type rx to replace the character at the cursor with x . ** + + 1. Move the cursor to the first line below marked --->. + + 2. Move the cursor so that it is on top of the first error. + + 3. Type r and then the character which should be there. + + 4. Repeat steps 2 and 3 until the first line is equal to the second one. + +---> Whan this lime was tuoed in, someone presswd some wrojg keys! +---> When this line was typed in, someone pressed some wrong keys! + + 5. Now move on to lesson 3.3. + +NOTE: Remember that you should be learning by doing, not memorization. + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 3.3: THE CHANGE OPERATOR + + + ** To change until the end of a word, type ce . ** + + 1. Move the cursor to the first line below marked --->. + + 2. Place the cursor on the u in lubw. + + 3. Type ce and the correct word (in this case, type ine ). + + 4. Press and move to the next character that needs to be changed. + + 5. Repeat steps 3 and 4 until the first sentence is the same as the second. + +---> This lubw has a few wptfd that mrrf changing usf the change operator. +---> This line has a few words that need changing using the change operator. + +Notice that ce deletes the word and places you in Insert mode. + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 3.4: MORE CHANGES USING c + + + ** The change operator is used with the same motions as delete. ** + + 1. The change operator works in the same way as delete. The format is: + + c [number] motion + + 2. The motions are the same, such as w (word) and $ (end of line). + + 3. Move the cursor to the first line below marked --->. + + 4. Move the cursor to the first error. + + 5. Type c$ and type the rest of the line like the second and press . + +---> The end of this line needs some help to make it like the second. +---> The end of this line needs to be corrected using the c$ command. + +NOTE: You can use the Backspace key to correct mistakes while typing. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 3 SUMMARY + + + 1. To put back text that has just been deleted, type p . This puts the + deleted text AFTER the cursor (if a line was deleted it will go on the + line below the cursor). + + 2. To replace the character under the cursor, type r and then the + character you want to have there. + + 3. The change operator allows you to change from the cursor to where the + motion takes you. eg. Type ce to change from the cursor to the end of + the word, c$ to change to the end of a line. + + 4. The format for change is: + + c [number] motion + +Now go on to the next lesson. + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 4.1: CURSOR LOCATION AND FILE STATUS + + ** Type CTRL-G to show your location in the file and the file status. + Type G to move to a line in the file. ** + + NOTE: Read this entire lesson before executing any of the steps!! + + 1. Hold down the Ctrl key and press g . We call this CTRL-G. + A message will appear at the bottom of the page with the filename and the + position in the file. Remember the line number for Step 3. + +NOTE: You may see the cursor position in the lower right corner of the screen + This happens when the 'ruler' option is set (see :help 'ruler' ) + + 2. Press G to move you to the bottom of the file. + Type gg to move you to the start of the file. + + 3. Type the number of the line you were on and then G . This will + return you to the line you were on when you first pressed CTRL-G. + + 4. If you feel confident to do this, execute steps 1 through 3. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 4.2: THE SEARCH COMMAND + + + ** Type / followed by a phrase to search for the phrase. ** + + 1. In Normal mode type the / character. Notice that it and the cursor + appear at the bottom of the screen as with the : command. + + 2. Now type 'errroor' . This is the word you want to search for. + + 3. To search for the same phrase again, simply type n . + To search for the same phrase in the opposite direction, type N . + + 4. To search for a phrase in the backward direction, use ? instead of / . + + 5. To go back to where you came from press CTRL-O (Keep Ctrl down while + pressing the letter o). Repeat to go back further. CTRL-I goes forward. + +---> "errroor" is not the way to spell error; errroor is an error. +NOTE: When the search reaches the end of the file it will continue at the + start, unless the 'wrapscan' option has been reset. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 4.3: MATCHING PARENTHESES SEARCH + + + ** Type % to find a matching ),], or } . ** + + 1. Place the cursor on any (, [, or { in the line below marked --->. + + 2. Now type the % character. + + 3. The cursor will move to the matching parenthesis or bracket. + + 4. Type % to move the cursor to the other matching bracket. + + 5. Move the cursor to another (,),[,],{ or } and see what % does. + +---> This ( is a test line with ('s, ['s ] and {'s } in it. )) + + +NOTE: This is very useful in debugging a program with unmatched parentheses! + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 4.4: THE SUBSTITUTE COMMAND + + + ** Type :s/old/new/g to substitute 'new' for 'old'. ** + + 1. Move the cursor to the line below marked --->. + + 2. Type :s/thee/the . Note that this command only changes the + first occurrence of "thee" in the line. + + 3. Now type :s/thee/the/g . Adding the g flag means to substitute + globally in the line, change all occurrences of "thee" in the line. + +---> thee best time to see thee flowers is in thee spring. + + 4. To change every occurrence of a character string between two lines, + type :#,#s/old/new/g where #,# are the line numbers of the range + of lines where the substitution is to be done. + Type :%s/old/new/g to change every occurrence in the whole file. + Type :%s/old/new/gc to find every occurrence in the whole file, + with a prompt whether to substitute or not. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 4 SUMMARY + + + 1. CTRL-G displays your location in the file and the file status. + G moves to the end of the file. + number G moves to that line number. + gg moves to the first line. + + 2. Typing / followed by a phrase searches FORWARD for the phrase. + Typing ? followed by a phrase searches BACKWARD for the phrase. + After a search type n to find the next occurrence in the same direction + or N to search in the opposite direction. + CTRL-O takes you back to older positions, CTRL-I to newer positions. + + 3. Typing % while the cursor is on a (,),[,],{, or } goes to its match. + + 4. To substitute new for the first old in a line type :s/old/new + To substitute new for all 'old's on a line type :s/old/new/g + To substitute phrases between two line #'s type :#,#s/old/new/g + To substitute all occurrences in the file type :%s/old/new/g + To ask for confirmation each time add 'c' :%s/old/new/gc + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 5.1: HOW TO EXECUTE AN EXTERNAL COMMAND + + + ** Type :! followed by an external command to execute that command. ** + + 1. Type the familiar command : to set the cursor at the bottom of the + screen. This allows you to enter a command-line command. + + 2. Now type the ! (exclamation point) character. This allows you to + execute any external shell command. + + 3. As an example type ls following the ! and then hit . This + will show you a listing of your directory, just as if you were at the + shell prompt. Or use :!dir if ls doesn't work. + +NOTE: It is possible to execute any external command this way, also with + arguments. + +NOTE: All : commands must be finished by hitting + From here on we will not always mention it. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 5.2: MORE ON WRITING FILES + + + ** To save the changes made to the text, type :w FILENAME ** + + 1. Type :!dir or :!ls to get a listing of your directory. + You already know you must hit after this. + + 2. Choose a filename that does not exist yet, such as TEST. + + 3. Now type: :w TEST (where TEST is the filename you chose.) + + 4. This saves the whole file (the Vim Tutor) under the name TEST. + To verify this, type :!dir or :!ls again to see your directory. + +NOTE: If you were to exit Vim and start it again with vim TEST , the file + would be an exact copy of the tutor when you saved it. + + 5. Now remove the file by typing (Windows): :!del TEST + or (Unix): :!rm TEST + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 5.3: SELECTING TEXT TO WRITE + + + ** To save part of the file, type v motion :w FILENAME ** + + 1. Move the cursor to this line. + + 2. Press v and move the cursor to the fifth item below. Notice that the + text is highlighted. + + 3. Press the : character. At the bottom of the screen :'<,'> will appear. + + 4. Type w TEST , where TEST is a filename that does not exist yet. Verify + that you see :'<,'>w TEST before you press . + + 5. Vim will write the selected lines to the file TEST. Use :!dir or :!ls + to see it. Do not remove it yet! We will use it in the next lesson. + +NOTE: Pressing v starts Visual selection. You can move the cursor around + to make the selection bigger or smaller. Then you can use an operator + to do something with the text. For example, d deletes the text. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 5.4: RETRIEVING AND MERGING FILES + + + ** To insert the contents of a file, type :r FILENAME ** + + 1. Place the cursor just above this line. + +NOTE: After executing Step 2 you will see text from lesson 5.3. Then move + DOWN to see this lesson again. + + 2. Now retrieve your TEST file using the command :r TEST where TEST is + the name of the file you used. + The file you retrieve is placed below the cursor line. + + 3. To verify that a file was retrieved, cursor back and notice that there + are now two copies of lesson 5.3, the original and the file version. + +NOTE: You can also read the output of an external command. For example, + :r !ls reads the output of the ls command and puts it below the + cursor. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 5 SUMMARY + + + 1. :!command executes an external command. + + Some useful examples are: + (Windows) (Unix) + :!dir :!ls - shows a directory listing. + :!del FILENAME :!rm FILENAME - removes file FILENAME. + + 2. :w FILENAME writes the current Vim file to disk with name FILENAME. + + 3. v motion :w FILENAME saves the Visually selected lines in file + FILENAME. + + 4. :r FILENAME retrieves disk file FILENAME and puts it below the + cursor position. + + 5. :r !dir reads the output of the dir command and puts it below the + cursor position. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 6.1: THE OPEN COMMAND + + + ** Type o to open a line below the cursor and place you in Insert mode. ** + + 1. Move the cursor to the first line below marked --->. + + 2. Type the lowercase letter o to open up a line BELOW the cursor and place + you in Insert mode. + + 3. Now type some text and press to exit Insert mode. + +---> After typing o the cursor is placed on the open line in Insert mode. + + 4. To open up a line ABOVE the cursor, simply type a capital O , rather + than a lowercase o. Try this on the line below. + +---> Open up a line above this by typing O while the cursor is on this line. + + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 6.2: THE APPEND COMMAND + + + ** Type a to insert text AFTER the cursor. ** + + 1. Move the cursor to the start of the first line below marked --->. + + 2. Press e until the cursor is on the end of li . + + 3. Type an a (lowercase) to append text AFTER the cursor. + + 4. Complete the word like the line below it. Press to exit Insert + mode. + + 5. Use e to move to the next incomplete word and repeat steps 3 and 4. + +---> This li will allow you to pract appendi text to a line. +---> This line will allow you to practice appending text to a line. + +NOTE: a, i and A all go to the same Insert mode, the only difference is where + the characters are inserted. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 6.3: ANOTHER WAY TO REPLACE + + + ** Type a capital R to replace more than one character. ** + + 1. Move the cursor to the first line below marked --->. Move the cursor to + the beginning of the first xxx . + + 2. Now press R and type the number below it in the second line, so that it + replaces the xxx . + + 3. Press to leave Replace mode. Notice that the rest of the line + remains unmodified. + + 4. Repeat the steps to replace the remaining xxx. + +---> Adding 123 to xxx gives you xxx. +---> Adding 123 to 456 gives you 579. + +NOTE: Replace mode is like Insert mode, but every typed character deletes an + existing character. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 6.4: COPY AND PASTE TEXT + + + ** Use the y operator to copy text and p to paste it ** + + 1. Move to the line below marked ---> and place the cursor after "a)". + + 2. Start Visual mode with v and move the cursor to just before "first". + + 3. Type y to yank (copy) the highlighted text. + + 4. Move the cursor to the end of the next line: j$ + + 5. Type p to put (paste) the text. Then type: a second . + + 6. Use Visual mode to select " item.", yank it with y , move to the end of + the next line with j$ and put the text there with p . + +---> a) this is the first item. + b) + + NOTE: You can also use y as an operator; yw yanks one word. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 6.5: SET OPTION + + + ** Set an option so a search or substitute ignores case ** + + 1. Search for 'ignore' by entering: /ignore + Repeat several times by pressing n . + + 2. Set the 'ic' (Ignore case) option by entering: :set ic + + 3. Now search for 'ignore' again by pressing n + Notice that Ignore and IGNORE are now also found. + + 4. Set the 'hlsearch' and 'incsearch' options: :set hls is + + 5. Now type the search command again and see what happens: /ignore + + 6. To disable ignoring case enter: :set noic + +NOTE: To remove the highlighting of matches enter: :nohlsearch +NOTE: If you want to ignore case for just one search command, use \c + in the phrase: /ignore\c +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 6 SUMMARY + + 1. Type o to open a line BELOW the cursor and start Insert mode. + Type O to open a line ABOVE the cursor. + + 2. Type a to insert text AFTER the cursor. + Type A to insert text after the end of the line. + + 3. The e command moves to the end of a word. + + 4. The y operator yanks (copies) text, p puts (pastes) it. + + 5. Typing a capital R enters Replace mode until is pressed. + + 6. Typing ":set xxx" sets the option "xxx". Some options are: + 'ic' 'ignorecase' ignore upper/lower case when searching + 'is' 'incsearch' show partial matches for a search phrase + 'hls' 'hlsearch' highlight all matching phrases + You can either use the long or the short option name. + + 7. Prepend "no" to switch an option off: :set noic + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 7.1: GETTING HELP + + + ** Use the on-line help system ** + + Vim has a comprehensive on-line help system. To get started, try one of + these three: + - press the key (if you have one) + - press the key (if you have one) + - type :help + + Read the text in the help window to find out how the help works. + Type CTRL-W CTRL-W to jump from one window to another. + Type :q to close the help window. + + You can find help on just about any subject, by giving an argument to the + ":help" command. Try these (don't forget pressing ): + + :help w + :help c_CTRL-D + :help insert-index + :help user-manual +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 7.2: CREATE A STARTUP SCRIPT + + + ** Enable Vim features ** + + Vim has many more features than Vi, but most of them are disabled by + default. To start using more features you have to create a "vimrc" file. + + 1. Start editing the "vimrc" file. This depends on your system: + :e ~/.vimrc for Unix + :e $VIM/_vimrc for Windows + + 2. Now read the example "vimrc" file contents: + :r $VIMRUNTIME/vimrc_example.vim + + 3. Write the file with: + :w + + The next time you start Vim it will use syntax highlighting. + You can add all your preferred settings to this "vimrc" file. + For more information type :help vimrc-intro + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 7.3: COMPLETION + + + ** Command line completion with CTRL-D and ** + + 1. Make sure Vim is not in compatible mode: :set nocp + + 2. Look what files exist in the directory: :!ls or :!dir + + 3. Type the start of a command: :e + + 4. Press CTRL-D and Vim will show a list of commands that start with "e". + + 5. Type d and Vim will complete the command name to ":edit". + + 6. Now add a space and the start of an existing file name: :edit FIL + + 7. Press . Vim will complete the name (if it is unique). + +NOTE: Completion works for many commands. Just try pressing CTRL-D and + . It is especially useful for :help . + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Lesson 7 SUMMARY + + + 1. Type :help or press or to open a help window. + + 2. Type :help cmd to find help on cmd . + + 3. Type CTRL-W CTRL-W to jump to another window. + + 4. Type :q to close the help window. + + 5. Create a vimrc startup script to keep your preferred settings. + + 6. When typing a : command, press CTRL-D to see possible completions. + Press to use one completion. + + + + + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This concludes the Vim Tutor. It was intended to give a brief overview of + the Vim editor, just enough to allow you to use the editor fairly easily. + It is far from complete as Vim has many many more commands. Read the user + manual next: ":help user-manual". + + For further reading and studying, this book is recommended: + Vim - Vi Improved - by Steve Oualline + Publisher: New Riders + The first book completely dedicated to Vim. Especially useful for beginners. + There are many examples and pictures. + See http://iccf-holland.org/click5.html + + This book is older and more about Vi than Vim, but also recommended: + Learning the Vi Editor - by Linda Lamb + Publisher: O'Reilly & Associates Inc. + It is a good book to get to know almost anything you want to do with Vi. + The sixth edition also includes information on Vim. + + This tutorial was written by Michael C. Pierce and Robert K. Ware, + Colorado School of Mines using ideas supplied by Charles Smith, + Colorado State University. E-mail: bware@mines.colorado.edu. + + Modified for Vim by Bram Moolenaar. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cdd577b --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# User-defined +*.txt +*.csv +*.yaml +config.py +training_images/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + diff --git a/Shoe Classifier-Copy1.ipynb b/Shoe Classifier-Copy1.ipynb new file mode 100644 index 0000000..41c374f --- /dev/null +++ b/Shoe Classifier-Copy1.ipynb @@ -0,0 +1,471 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "572dc7fb", + "metadata": {}, + "outputs": [], + "source": [ + "from matplotlib import pyplot as plt\n", + "from matplotlib.image import imread\n", + "import pandas as pd\n", + "from collections import Counter\n", + "import json\n", + "import os\n", + "import re\n", + "import tempfile\n", + "import numpy as np\n", + "from os.path import exists\n", + "from imblearn.under_sampling import RandomUnderSampler\n", + "from PIL import ImageFile\n", + "import sklearn as sk\n", + "from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n", + "import tensorflow as tf\n", + "import tensorflow.keras\n", + "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", + "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.optimizers import Adam\n", + "# custom modules\n", + "import image_faults\n", + "\n", + "ImageFile.LOAD_TRUNCATED_IMAGES = True" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def add_regularization(model, regularizer=tf.keras.regularizers.l2(0.0001)):\n", + "\n", + " if not isinstance(regularizer, tf.keras.regularizers.Regularizer):\n", + " print(\"Regularizer must be a subclass of tf.keras.regularizers.Regularizer\")\n", + " return model\n", + "\n", + " for layer in model.layers:\n", + " for attr in ['kernel_regularizer']:\n", + " if hasattr(layer, attr):\n", + " setattr(layer, attr, regularizer)\n", + "\n", + " # When we change the layers attributes, the change only happens in the model config file\n", + " model_json = model.to_json()\n", + "\n", + " # Save the weights before reloading the model.\n", + " tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')\n", + " model.save_weights(tmp_weights_path)\n", + "\n", + " # load the model from the config\n", + " model = tf.keras.models.model_from_json(model_json)\n", + " \n", + " # Reload the model weights\n", + " model.load_weights(tmp_weights_path, by_name=True)\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a5c72863", + "metadata": {}, + "outputs": [], + "source": [ + "# image_faults.faulty_images() # removes faulty images\n", + "df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1057a442", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{source:target} dictionary created @ /tf/training_images\n" + ] + } + ], + "source": [ + "def dict_pics():\n", + " target_dir = os.getcwd() + os.sep + \"training_images\"\n", + " with open('temp_pics_source_list.txt') as f:\n", + " temp_pics_source_list = json.load(f)\n", + " dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n", + " print(\"{source:target} dictionary created @ \" + target_dir)\n", + " return dict_pics\n", + "\n", + "dict_pics = dict_pics()\n", + "blah = pd.Series(df.PictureURL)\n", + "df = df.drop(labels=['PictureURL'], axis=1)\n", + "blah = blah.apply(lambda x: dict_pics[x])\n", + "df = pd.concat([blah, df],axis=1)\n", + "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n", + "# removes non-existent image paths" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7a6146e6", + "metadata": {}, + "outputs": [], + "source": [ + "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", + "\n", + "df=df.sample(frac=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "undersample = RandomUnderSampler(sampling_strategy='auto')\n", + "train, y_under = undersample.fit_resample(df, df['PrimaryCategoryID'])\n", + "# print(Counter(train['PrimaryCategoryID']))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "506aa5cf", + "metadata": {}, + "outputs": [], + "source": [ + "train, test = train_test_split(train, test_size=0.1, random_state=42)\n", + "# stratify=train['PrimaryCategoryID']\n", + "# train['PrimaryCategoryID'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4d72eb90", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 5110 validated image filenames belonging to 13 classes.\n", + "Found 1277 validated image filenames belonging to 13 classes.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "datagen = ImageDataGenerator(rescale=1./255., \n", + " validation_split=.2,\n", + " #samplewise_std_normalization=True,\n", + " #horizontal_flip= True,\n", + " #vertical_flip= True,\n", + " #width_shift_range= 0.2,\n", + " #height_shift_range= 0.2,\n", + " #rotation_range= 90,\n", + " preprocessing_function=tf.keras.applications.vgg16.preprocess_input)\n", + "train_generator=datagen.flow_from_dataframe(\n", + " dataframe=train[:len(train)],\n", + " directory='./training_images',\n", + " x_col='PictureURL',\n", + " y_col='PrimaryCategoryID',\n", + " batch_size=32,\n", + " seed=42,\n", + " shuffle=True,\n", + " target_size=(224,224),\n", + " subset='training'\n", + " )\n", + "validation_generator=datagen.flow_from_dataframe(\n", + " dataframe=train[:len(train)], # is using train right?\n", + " directory='./training_images',\n", + " x_col='PictureURL',\n", + " y_col='PrimaryCategoryID',\n", + " batch_size=32,\n", + " seed=42,\n", + " shuffle=True,\n", + " target_size=(224,224),\n", + " subset='validation'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "7b70f37f", + "metadata": {}, + "outputs": [], + "source": [ + "imgs, labels = next(train_generator)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1ed54bf5", + "metadata": {}, + "outputs": [], + "source": [ + "def plotImages(images_arr):\n", + " fig, axes = plt.subplots(1, 10, figsize=(20,20))\n", + " axes = axes.flatten()\n", + " for img, ax in zip( images_arr, axes):\n", + " ax.imshow(img)\n", + " ax.axis('off')\n", + " plt.tight_layout()\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "85934565", + "metadata": {}, + "outputs": [], + "source": [ + "#plotImages(imgs)\n", + "#print(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "6322bcad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n" + ] + } + ], + "source": [ + "physical_devices = tf.config.list_physical_devices('GPU')\n", + "print(len(physical_devices))\n", + "tf.config.experimental.set_memory_growth(physical_devices[0], True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "07fd25c6", + "metadata": {}, + "outputs": [], + "source": [ + "# see https://www.kaggle.com/dmitrypukhov/cnn-with-imagedatagenerator-flow-from-dataframe for train/test/val split \n", + "# example\n", + "\n", + "# may need to either create a test dataset from the original dataset or just download a new one" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "b31af79e", + "metadata": {}, + "outputs": [], + "source": [ + "vgg19_model = tf.keras.applications.vgg16.VGG16(weights='imagenet')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "fe06f2bf", + "metadata": {}, + "outputs": [], + "source": [ + "model = Sequential()\n", + "for layer in vgg19_model.layers[:-1]:\n", + " model.add(layer)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "7d3cc82c", + "metadata": {}, + "outputs": [], + "source": [ + "for layer in model.layers:\n", + " layer.trainable = True" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "ea620129", + "metadata": {}, + "outputs": [], + "source": [ + "#model.add(Dropout(.5))\n", + "#model.add(Dense(64, activation='softmax'))\n", + "# model.add(Dropout(.25))\n", + "model.add(Dense(units=13, activation='softmax'))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "c774d787", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "model = add_regularization(model)\n", + "#model.summary()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "fd5d1246", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer=Adam(learning_rate=1e-5), loss='categorical_crossentropy',\n", + " metrics=['accuracy'])\n", + "# sparse_categorical_crossentropy" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9cd2ba27", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/30\n", + "160/160 [==============================] - 59s 360ms/step - loss: 2.7627 - accuracy: 0.1125 - val_loss: 2.7406 - val_accuracy: 0.1237\n", + "Epoch 2/30\n", + "160/160 [==============================] - 56s 351ms/step - loss: 2.7151 - accuracy: 0.1399 - val_loss: 2.7219 - val_accuracy: 0.1402\n", + "Epoch 3/30\n", + "160/160 [==============================] - 56s 351ms/step - loss: 2.6875 - accuracy: 0.1566 - val_loss: 2.6897 - val_accuracy: 0.1629\n", + "Epoch 4/30\n", + "160/160 [==============================] - 56s 353ms/step - loss: 2.6820 - accuracy: 0.1726 - val_loss: 2.6867 - val_accuracy: 0.1684\n", + "Epoch 5/30\n", + "160/160 [==============================] - 57s 355ms/step - loss: 2.6579 - accuracy: 0.1771 - val_loss: 2.6919 - val_accuracy: 0.1558\n", + "Epoch 6/30\n", + "160/160 [==============================] - 56s 353ms/step - loss: 2.6361 - accuracy: 0.1994 - val_loss: 2.6813 - val_accuracy: 0.1832\n", + "Epoch 7/30\n", + "160/160 [==============================] - 56s 352ms/step - loss: 2.6196 - accuracy: 0.2084 - val_loss: 2.6592 - val_accuracy: 0.1950\n", + "Epoch 8/30\n", + "160/160 [==============================] - 57s 353ms/step - loss: 2.6031 - accuracy: 0.2172 - val_loss: 2.6693 - val_accuracy: 0.1770\n", + "Epoch 9/30\n", + "160/160 [==============================] - 57s 355ms/step - loss: 2.5878 - accuracy: 0.2274 - val_loss: 2.6543 - val_accuracy: 0.2091\n", + "Epoch 10/30\n", + "160/160 [==============================] - 56s 350ms/step - loss: 2.5687 - accuracy: 0.2450 - val_loss: 2.6551 - val_accuracy: 0.1942\n", + "Epoch 11/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.5543 - accuracy: 0.2568 - val_loss: 2.6591 - val_accuracy: 0.2020\n", + "Epoch 12/30\n", + "160/160 [==============================] - 56s 352ms/step - loss: 2.5403 - accuracy: 0.2685 - val_loss: 2.6513 - val_accuracy: 0.1973\n", + "Epoch 13/30\n", + "160/160 [==============================] - 56s 352ms/step - loss: 2.5311 - accuracy: 0.2695 - val_loss: 2.6445 - val_accuracy: 0.2060\n", + "Epoch 14/30\n", + "160/160 [==============================] - 56s 351ms/step - loss: 2.5217 - accuracy: 0.2775 - val_loss: 2.6476 - val_accuracy: 0.2044\n", + "Epoch 15/30\n", + "160/160 [==============================] - 56s 351ms/step - loss: 2.5147 - accuracy: 0.2830 - val_loss: 2.6419 - val_accuracy: 0.2036\n", + "Epoch 16/30\n", + "160/160 [==============================] - 56s 351ms/step - loss: 2.5084 - accuracy: 0.2851 - val_loss: 2.6396 - val_accuracy: 0.2200\n", + "Epoch 17/30\n", + "160/160 [==============================] - 56s 348ms/step - loss: 2.5025 - accuracy: 0.2879 - val_loss: 2.6463 - val_accuracy: 0.2302\n", + "Epoch 18/30\n", + "160/160 [==============================] - 56s 350ms/step - loss: 2.4971 - accuracy: 0.2918 - val_loss: 2.6346 - val_accuracy: 0.2208\n", + "Epoch 19/30\n", + "160/160 [==============================] - 56s 353ms/step - loss: 2.4924 - accuracy: 0.2967 - val_loss: 2.6366 - val_accuracy: 0.2208\n", + "Epoch 20/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.4882 - accuracy: 0.2988 - val_loss: 2.6317 - val_accuracy: 0.2271\n", + "Epoch 21/30\n", + "160/160 [==============================] - 56s 349ms/step - loss: 2.4854 - accuracy: 0.3004 - val_loss: 2.6431 - val_accuracy: 0.2240\n", + "Epoch 22/30\n", + "160/160 [==============================] - 56s 352ms/step - loss: 2.4784 - accuracy: 0.3068 - val_loss: 2.6345 - val_accuracy: 0.2114\n", + "Epoch 23/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.4722 - accuracy: 0.3106 - val_loss: 2.6276 - val_accuracy: 0.2294\n", + "Epoch 24/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.4687 - accuracy: 0.3100 - val_loss: 2.6383 - val_accuracy: 0.2177\n", + "Epoch 25/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.4649 - accuracy: 0.3108 - val_loss: 2.6322 - val_accuracy: 0.2122\n", + "Epoch 26/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.4644 - accuracy: 0.3141 - val_loss: 2.6243 - val_accuracy: 0.2247\n", + "Epoch 27/30\n", + "160/160 [==============================] - 56s 352ms/step - loss: 2.4599 - accuracy: 0.3188 - val_loss: 2.6332 - val_accuracy: 0.2138\n", + "Epoch 28/30\n", + "160/160 [==============================] - 57s 353ms/step - loss: 2.4550 - accuracy: 0.3229 - val_loss: 2.6287 - val_accuracy: 0.2232\n", + "Epoch 29/30\n", + "160/160 [==============================] - 57s 354ms/step - loss: 2.4502 - accuracy: 0.3217 - val_loss: 2.6216 - val_accuracy: 0.2287\n", + "Epoch 30/30\n", + "160/160 [==============================] - 56s 351ms/step - loss: 2.4506 - accuracy: 0.3190 - val_loss: 2.6329 - val_accuracy: 0.1793\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.fit(x=train_generator,\n", + " steps_per_epoch=len(train_generator),\n", + " validation_data=validation_generator,\n", + " validation_steps=len(validation_generator),\n", + " epochs=30,\n", + " verbose=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63f791af", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Shoe Classifier.ipynb b/Shoe Classifier.ipynb new file mode 100644 index 0000000..e3b82fe --- /dev/null +++ b/Shoe Classifier.ipynb @@ -0,0 +1,449 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "572dc7fb", + "metadata": {}, + "outputs": [], + "source": [ + "from matplotlib import pyplot as plt\n", + "from matplotlib.image import imread\n", + "import pandas as pd\n", + "import json\n", + "import os\n", + "import re\n", + "import numpy as np\n", + "from os.path import exists\n", + "from PIL import ImageFile\n", + "#import sklearn as sk\n", + "import tensorflow as tf\n", + "import tensorflow.keras\n", + "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", + "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.optimizers import Adam\n", + "# custom modules\n", + "import image_faults\n", + "\n", + "ImageFile.LOAD_TRUNCATED_IMAGES = True" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a5c72863", + "metadata": {}, + "outputs": [], + "source": [ + "# image_faults.faulty_images() # removes faulty images\n", + "df = pd.read_csv('expanded_class.csv', index_col=[0], low_memory=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1057a442", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{source:target} dictionary created @ /tf/training_images\n" + ] + } + ], + "source": [ + "def dict_pics():\n", + " target_dir = os.getcwd() + os.sep + \"training_images\"\n", + " with open('temp_pics_source_list.txt') as f:\n", + " temp_pics_source_list = json.load(f)\n", + " dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}\n", + " print(\"{source:target} dictionary created @ \" + target_dir)\n", + " return dict_pics\n", + "\n", + "dict_pics = dict_pics()\n", + "blah = pd.Series(df.PictureURL)\n", + "df = df.drop(labels=['PictureURL'], axis=1)\n", + "blah = blah.apply(lambda x: dict_pics[x])\n", + "df = pd.concat([blah, df],axis=1)\n", + "df = df.groupby('PrimaryCategoryID').filter(lambda x: len(x)>25) # removes cat outliers\n", + "# removes non-existent image paths" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7a6146e6", + "metadata": {}, + "outputs": [], + "source": [ + "df['PrimaryCategoryID'] = df['PrimaryCategoryID'].astype(str) # pandas thinks ids are ints\n", + "\n", + "df=df.sample(frac=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4d72eb90", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 6217 validated image filenames belonging to 13 classes.\n", + "Found 2664 validated image filenames belonging to 13 classes.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.8/dist-packages/keras_preprocessing/image/dataframe_iterator.py:279: UserWarning: Found 1 invalid image filename(s) in x_col=\"PictureURL\". These filename(s) will be ignored.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "datagen = ImageDataGenerator(rescale=1./255., \n", + " validation_split=.3,\n", + " #featurewise_std_normalization=True,\n", + " #horizontal_flip= True,\n", + " #vertical_flip= True,\n", + " #width_shift_range= 0.2,\n", + " #height_shift_range= 0.2,\n", + " #rotation_range= 180,\n", + " preprocessing_function=tf.keras.applications.vgg16.preprocess_input)\n", + "train_generator=datagen.flow_from_dataframe(\n", + " dataframe=df[:len(df)],\n", + " directory='./training_images',\n", + " x_col='PictureURL',\n", + " y_col='PrimaryCategoryID',\n", + " batch_size=32,\n", + " seed=42,\n", + " shuffle=True,\n", + " target_size=(224,224),\n", + " subset='training'\n", + " )\n", + "validation_generator=datagen.flow_from_dataframe(\n", + " dataframe=df[:len(df)],\n", + " directory='./training_images',\n", + " x_col='PictureURL',\n", + " y_col='PrimaryCategoryID',\n", + " batch_size=32,\n", + " seed=42,\n", + " shuffle=True,\n", + " target_size=(224,224),\n", + " subset='validation'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7b70f37f", + "metadata": {}, + "outputs": [], + "source": [ + "imgs, labels = next(train_generator)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "1ed54bf5", + "metadata": {}, + "outputs": [], + "source": [ + "def plotImages(images_arr):\n", + " fig, axes = plt.subplots(1, 10, figsize=(20,20))\n", + " axes = axes.flatten()\n", + " for img, ax in zip( images_arr, axes):\n", + " ax.imshow(img)\n", + " ax.axis('off')\n", + " plt.tight_layout()\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "85934565", + "metadata": {}, + "outputs": [], + "source": [ + "#plotImages(imgs)\n", + "#print(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "6322bcad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n" + ] + } + ], + "source": [ + "physical_devices = tf.config.list_physical_devices('GPU')\n", + "print(len(physical_devices))\n", + "tf.config.experimental.set_memory_growth(physical_devices[0], True)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "07fd25c6", + "metadata": {}, + "outputs": [], + "source": [ + "# see https://www.kaggle.com/dmitrypukhov/cnn-with-imagedatagenerator-flow-from-dataframe for train/test/val split \n", + "# example\n", + "\n", + "# may need to either create a test dataset from the original dataset or just download a new one" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "b31af79e", + "metadata": {}, + "outputs": [], + "source": [ + "vgg16_model = tf.keras.applications.vgg16.VGG16(weights='imagenet')\n", + "#weights='imagenet'" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "fe06f2bf", + "metadata": {}, + "outputs": [], + "source": [ + "model = Sequential()\n", + "for layer in vgg16_model.layers[:-1]:\n", + " model.add(layer)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7d3cc82c", + "metadata": {}, + "outputs": [], + "source": [ + "for layer in model.layers:\n", + " layer.trainable = True" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "ea620129", + "metadata": {}, + "outputs": [], + "source": [ + "model.add(Dense(units=13, activation='softmax'))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c774d787", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "#model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "fd5d1246", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer=Adam(learning_rate=0.0001), loss='categorical_crossentropy',\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "9cd2ba27", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/100\n", + "195/195 - 83s - loss: 2.5928 - accuracy: 0.1139 - val_loss: 2.3657 - val_accuracy: 0.1674 - 83s/epoch - 426ms/step\n", + "Epoch 2/100\n", + "195/195 - 77s - loss: 2.1473 - accuracy: 0.2582 - val_loss: 1.9276 - val_accuracy: 0.3281 - 77s/epoch - 394ms/step\n", + "Epoch 3/100\n", + "195/195 - 78s - loss: 1.7234 - accuracy: 0.3973 - val_loss: 1.6724 - val_accuracy: 0.4050 - 78s/epoch - 400ms/step\n", + "Epoch 4/100\n", + "195/195 - 78s - loss: 1.4692 - accuracy: 0.4843 - val_loss: 1.5583 - val_accuracy: 0.4662 - 78s/epoch - 402ms/step\n", + "Epoch 5/100\n", + "195/195 - 79s - loss: 1.2598 - accuracy: 0.5477 - val_loss: 1.5135 - val_accuracy: 0.4944 - 79s/epoch - 403ms/step\n", + "Epoch 6/100\n", + "195/195 - 79s - loss: 1.0220 - accuracy: 0.6376 - val_loss: 1.5566 - val_accuracy: 0.4962 - 79s/epoch - 404ms/step\n", + "Epoch 7/100\n", + "195/195 - 78s - loss: 0.8021 - accuracy: 0.7084 - val_loss: 1.7647 - val_accuracy: 0.4711 - 78s/epoch - 398ms/step\n", + "Epoch 8/100\n", + "195/195 - 78s - loss: 0.5998 - accuracy: 0.7804 - val_loss: 1.8439 - val_accuracy: 0.4869 - 78s/epoch - 400ms/step\n", + "Epoch 9/100\n", + "195/195 - 77s - loss: 0.3910 - accuracy: 0.8631 - val_loss: 2.1197 - val_accuracy: 0.4872 - 77s/epoch - 397ms/step\n", + "Epoch 10/100\n", + "195/195 - 78s - loss: 0.2600 - accuracy: 0.9094 - val_loss: 2.3586 - val_accuracy: 0.4703 - 78s/epoch - 402ms/step\n", + "Epoch 11/100\n", + "195/195 - 77s - loss: 0.2066 - accuracy: 0.9279 - val_loss: 2.5012 - val_accuracy: 0.4632 - 77s/epoch - 397ms/step\n", + "Epoch 12/100\n", + "195/195 - 77s - loss: 0.1266 - accuracy: 0.9571 - val_loss: 2.5961 - val_accuracy: 0.4854 - 77s/epoch - 395ms/step\n", + "Epoch 13/100\n", + "195/195 - 77s - loss: 0.1098 - accuracy: 0.9648 - val_loss: 2.9123 - val_accuracy: 0.4602 - 77s/epoch - 395ms/step\n", + "Epoch 14/100\n", + "195/195 - 77s - loss: 0.0794 - accuracy: 0.9744 - val_loss: 2.9060 - val_accuracy: 0.4752 - 77s/epoch - 395ms/step\n", + "Epoch 15/100\n", + "195/195 - 77s - loss: 0.1061 - accuracy: 0.9656 - val_loss: 2.7269 - val_accuracy: 0.4658 - 77s/epoch - 396ms/step\n", + "Epoch 16/100\n", + "195/195 - 77s - loss: 0.0908 - accuracy: 0.9712 - val_loss: 2.9450 - val_accuracy: 0.4621 - 77s/epoch - 396ms/step\n", + "Epoch 17/100\n", + "195/195 - 77s - loss: 0.0820 - accuracy: 0.9736 - val_loss: 2.8900 - val_accuracy: 0.4703 - 77s/epoch - 396ms/step\n", + "Epoch 18/100\n", + "195/195 - 77s - loss: 0.0827 - accuracy: 0.9722 - val_loss: 2.9304 - val_accuracy: 0.4745 - 77s/epoch - 395ms/step\n", + "Epoch 19/100\n", + "195/195 - 77s - loss: 0.0726 - accuracy: 0.9757 - val_loss: 3.2673 - val_accuracy: 0.4580 - 77s/epoch - 393ms/step\n", + "Epoch 20/100\n", + "195/195 - 77s - loss: 0.0560 - accuracy: 0.9821 - val_loss: 3.0101 - val_accuracy: 0.4670 - 77s/epoch - 394ms/step\n", + "Epoch 21/100\n", + "195/195 - 77s - loss: 0.0355 - accuracy: 0.9907 - val_loss: 3.2575 - val_accuracy: 0.4568 - 77s/epoch - 395ms/step\n", + "Epoch 22/100\n", + "195/195 - 78s - loss: 0.0950 - accuracy: 0.9693 - val_loss: 3.0716 - val_accuracy: 0.4568 - 78s/epoch - 399ms/step\n", + "Epoch 23/100\n", + "195/195 - 78s - loss: 0.0307 - accuracy: 0.9910 - val_loss: 3.6348 - val_accuracy: 0.4741 - 78s/epoch - 401ms/step\n", + "Epoch 24/100\n", + "195/195 - 77s - loss: 0.0315 - accuracy: 0.9903 - val_loss: 3.2422 - val_accuracy: 0.4711 - 77s/epoch - 396ms/step\n", + "Epoch 25/100\n", + "195/195 - 78s - loss: 0.0640 - accuracy: 0.9789 - val_loss: 3.5402 - val_accuracy: 0.4298 - 78s/epoch - 402ms/step\n", + "Epoch 26/100\n", + "195/195 - 78s - loss: 0.0916 - accuracy: 0.9715 - val_loss: 3.1916 - val_accuracy: 0.4520 - 78s/epoch - 399ms/step\n", + "Epoch 27/100\n", + "195/195 - 77s - loss: 0.0634 - accuracy: 0.9799 - val_loss: 3.2234 - val_accuracy: 0.4602 - 77s/epoch - 394ms/step\n", + "Epoch 28/100\n", + "195/195 - 78s - loss: 0.0673 - accuracy: 0.9788 - val_loss: 3.4435 - val_accuracy: 0.4647 - 78s/epoch - 400ms/step\n", + "Epoch 29/100\n", + "195/195 - 78s - loss: 0.0440 - accuracy: 0.9870 - val_loss: 3.3068 - val_accuracy: 0.4598 - 78s/epoch - 400ms/step\n", + "Epoch 30/100\n", + "195/195 - 78s - loss: 0.0165 - accuracy: 0.9950 - val_loss: 3.6233 - val_accuracy: 0.4681 - 78s/epoch - 401ms/step\n", + "Epoch 31/100\n", + "195/195 - 79s - loss: 0.0303 - accuracy: 0.9910 - val_loss: 3.7398 - val_accuracy: 0.4572 - 79s/epoch - 403ms/step\n", + "Epoch 32/100\n", + "195/195 - 78s - loss: 0.0554 - accuracy: 0.9815 - val_loss: 3.5560 - val_accuracy: 0.4369 - 78s/epoch - 399ms/step\n", + "Epoch 33/100\n", + "195/195 - 78s - loss: 0.0556 - accuracy: 0.9829 - val_loss: 3.6555 - val_accuracy: 0.4610 - 78s/epoch - 401ms/step\n", + "Epoch 34/100\n", + "195/195 - 78s - loss: 0.0624 - accuracy: 0.9809 - val_loss: 3.3500 - val_accuracy: 0.4617 - 78s/epoch - 401ms/step\n", + "Epoch 35/100\n", + "195/195 - 77s - loss: 0.0367 - accuracy: 0.9886 - val_loss: 3.5968 - val_accuracy: 0.4625 - 77s/epoch - 394ms/step\n", + "Epoch 36/100\n", + "195/195 - 78s - loss: 0.0301 - accuracy: 0.9912 - val_loss: 3.5188 - val_accuracy: 0.4643 - 78s/epoch - 399ms/step\n", + "Epoch 37/100\n", + "195/195 - 79s - loss: 0.0491 - accuracy: 0.9850 - val_loss: 3.3079 - val_accuracy: 0.4471 - 79s/epoch - 403ms/step\n", + "Epoch 38/100\n", + "195/195 - 78s - loss: 0.0577 - accuracy: 0.9821 - val_loss: 3.4042 - val_accuracy: 0.4381 - 78s/epoch - 400ms/step\n", + "Epoch 39/100\n", + "195/195 - 78s - loss: 0.0431 - accuracy: 0.9878 - val_loss: 3.4389 - val_accuracy: 0.4550 - 78s/epoch - 399ms/step\n", + "Epoch 40/100\n", + "195/195 - 77s - loss: 0.0218 - accuracy: 0.9940 - val_loss: 3.1989 - val_accuracy: 0.4854 - 77s/epoch - 397ms/step\n", + "Epoch 41/100\n", + "195/195 - 77s - loss: 0.0296 - accuracy: 0.9921 - val_loss: 3.2759 - val_accuracy: 0.4651 - 77s/epoch - 394ms/step\n", + "Epoch 42/100\n", + "195/195 - 78s - loss: 0.0176 - accuracy: 0.9947 - val_loss: 3.2391 - val_accuracy: 0.4745 - 78s/epoch - 398ms/step\n", + "Epoch 43/100\n", + "195/195 - 79s - loss: 0.0099 - accuracy: 0.9965 - val_loss: 3.5696 - val_accuracy: 0.4553 - 79s/epoch - 405ms/step\n", + "Epoch 44/100\n", + "195/195 - 78s - loss: 0.0516 - accuracy: 0.9852 - val_loss: 3.3857 - val_accuracy: 0.4482 - 78s/epoch - 402ms/step\n", + "Epoch 45/100\n", + "195/195 - 79s - loss: 0.0412 - accuracy: 0.9860 - val_loss: 3.3717 - val_accuracy: 0.4580 - 79s/epoch - 404ms/step\n", + "Epoch 46/100\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit(x=train_generator,\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1214\u001b[0m _r=1):\n\u001b[1;32m 1215\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1216\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1217\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1218\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 151\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 908\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 909\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 912\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 940\u001b[0m \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 941\u001b[0m \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 942\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 943\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 944\u001b[0m \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 3128\u001b[0m (graph_function,\n\u001b[1;32m 3129\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 3130\u001b[0;31m return graph_function._call_flat(\n\u001b[0m\u001b[1;32m 3131\u001b[0m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 3132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1957\u001b[0m and executing_eagerly):\n\u001b[1;32m 1958\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1959\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1960\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1961\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 597\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 599\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 59\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 60\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "model.fit(x=train_generator,\n", + " steps_per_epoch=len(train_generator),\n", + " validation_data=validation_generator,\n", + " validation_steps=len(validation_generator),\n", + " epochs=100,\n", + " verbose=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63f791af", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ebay_api.py b/ebay_api.py index 39979ef..26e3060 100644 --- a/ebay_api.py +++ b/ebay_api.py @@ -404,7 +404,7 @@ class PreProcessing: splits, etc. ''' - def stt_training(self): + def dict_pics(self): ''' Source to target training. Replaces source image URL with target URL determined by values in dict_pics variable. @@ -413,9 +413,9 @@ class PreProcessing: target_dir = os.getcwd() with open('temp_pics_source_list.txt') as f: temp_pics_source_list = json.load(f) - temp_dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list} + dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list} print("{source:target} dictionary created @ " + os.getcwd() + os.sep + 'training_images') - return temp_dict_pics + return dict_pics # TODO pipeline gameplan: 5 files: dict_pics.txt,raw_json.txt, raw_json.csv, expanded_class.csv, expanded_dropd.csv # cont... open raw_json.txt and append, same with csv --> process new data --> pull out image source+dest and expand new dfs for the additional pictures diff --git a/testing.ipynb b/testing.ipynb new file mode 100644 index 0000000..2961a1c --- /dev/null +++ b/testing.ipynb @@ -0,0 +1,144 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "7eea0d4d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-12-24 22:16:08.715996: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num GPUs Available: 1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-12-24 22:16:11.102972: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", + "2021-12-24 22:16:11.103554: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n", + "2021-12-24 22:16:11.157717: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:11.157972: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: \n", + "pciBusID: 0000:01:00.0 name: NVIDIA GeForce RTX 2060 computeCapability: 7.5\n", + "coreClock: 1.2GHz coreCount: 30 deviceMemorySize: 5.79GiB deviceMemoryBandwidth: 312.97GiB/s\n", + "2021-12-24 22:16:11.157995: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n", + "2021-12-24 22:16:11.191221: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.10\n", + "2021-12-24 22:16:11.191428: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.10\n", + "2021-12-24 22:16:11.222375: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n", + "2021-12-24 22:16:11.226481: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n", + "2021-12-24 22:16:11.258066: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10\n", + "2021-12-24 22:16:11.264224: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.10\n", + "2021-12-24 22:16:11.324727: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.7\n", + "2021-12-24 22:16:11.325101: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:11.325903: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:11.326485: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "33d18ebd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-12-24 22:16:11.339696: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE4.1 SSE4.2 AVX AVX2 FMA\n", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2021-12-24 22:16:11.340741: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", + "2021-12-24 22:16:11.340920: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:11.341179: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: \n", + "pciBusID: 0000:01:00.0 name: NVIDIA GeForce RTX 2060 computeCapability: 7.5\n", + "coreClock: 1.2GHz coreCount: 30 deviceMemorySize: 5.79GiB deviceMemoryBandwidth: 312.97GiB/s\n", + "2021-12-24 22:16:11.341221: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n", + "2021-12-24 22:16:11.341261: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.10\n", + "2021-12-24 22:16:11.341281: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.10\n", + "2021-12-24 22:16:11.341293: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n", + "2021-12-24 22:16:11.341304: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n", + "2021-12-24 22:16:11.341315: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10\n", + "2021-12-24 22:16:11.341326: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.10\n", + "2021-12-24 22:16:11.341336: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.7\n", + "2021-12-24 22:16:11.341433: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:11.341629: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:11.341750: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0\n", + "2021-12-24 22:16:11.342051: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n", + "2021-12-24 22:16:12.482371: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:\n", + "2021-12-24 22:16:12.482394: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 \n", + "2021-12-24 22:16:12.482399: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N \n", + "2021-12-24 22:16:12.482832: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:12.483044: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:12.483236: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:941] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2021-12-24 22:16:12.483356: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 5358 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce RTX 2060, pci bus id: 0000:01:00.0, compute capability: 7.5)\n", + "2021-12-24 22:16:12.487174: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:0\n", + "tf.Tensor(\n", + "[[22. 28.]\n", + " [49. 64.]], shape=(2, 2), dtype=float32)\n" + ] + } + ], + "source": [ + "tf.debugging.set_log_device_placement(True)\n", + "\n", + "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", + "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", + "\n", + "# Run on the GPU\n", + "c = tf.matmul(a, b)\n", + "print(c)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b9ca96e", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/try.py b/try.py new file mode 100644 index 0000000..4e2b3ee --- /dev/null +++ b/try.py @@ -0,0 +1,37 @@ +import ebaysdk +import json +import requests +import concurrent.futures +import config as cfg +from ebaysdk.shopping import Connection as Shopping +from ebaysdk.trading import Connection as Trading +sapi = Shopping(config_file = 'ebay.yaml') +tapi = Trading(config_file='ebay.yaml') + +def get_cat_specs(cat): + + response = tapi.execute('GetCategorySpecifics', + {'CategoryID':cat}) + cat_spacs =[name['Name'] for name in response.dict()['Recommendations']['NameRecommendation']] + + return cat_spacs + +with open('cat_list.txt') as f: + cat_list = json.load(f) + +def threadd_cat_spacs(): + + cat_spacs = [] + + with concurrent.futures.ThreadPoolExecutor() as executor: + for future in executor.map(get_cat_specs, cat_list): + cat_spacs.extend(future) + + cat_spacs = list(set(cat_spacs)) + + return cat_spacs + +if __name__=='__main__': + cat_spacs = threadd_cat_spacs() + with open('cat_spacs.txt', 'w') as f: + json.dump(cat_spacs, f)