scott/ebay-ml-lister
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added dict_pics.txt updating for expanded_dfs method

Browse Source
This commit is contained in:
spbeach46 2021-04-07 15:50:23 -07:00
parent 622b824eaf
commit ef237b8a1b
2 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
curate.py
View File

@ -1,5 +1,4 @@
import ebay_api
import numpy as np
'''
file used to compile methods from ebay_api.py for curating training data
@ -13,8 +12,10 @@ nvl_training = curate.nvl_training(training)
dropd = curate.drop_nvl_cols(nvl_training)
expanded_dfs = curate.expand_nvlclass(class_training, dropd)
expanded_class = expanded_dfs[0]
expanded_dropd = expanded_dfs[1]
dict_pics = expanded_dfs[2]
# TODO # need to replace expanded df's PictureURL col values with destination urls
# TODO # still have the problem of duplicate listings. Possibly take care of this before you run curate

19
ebay_api.py
View File

@ -267,28 +267,33 @@ class CurateData:
instances. Modifies both class training and dropd dfs. Appends custom
image url dict {'source':'destination'}.
'''
expanded_class = class_training.explode('PictureURL').reset_index(drop=True) # TODO DROp rows with nan values in PicturlURL cell
expanded_class = class_training.explode('PictureURL').reset_index(drop=True) # TODO drop duplicates here or before instantiating curate object
expanded_class = expanded_class.dropna(subset=['PictureURL'])
expanded_dropd = dropd.explode('PictureURL').reset_index(drop=True) # TODO Drop rows with nan values in PictureURL cells either here or before somewhere
expanded_class = expanded_class.drop_duplicates(subset=['PictureURL'])
expanded_class.loc[:,'PictureURL'] = expanded_class.loc[:, 'PictureURL'].apply(lambda x: dict_pics[x])
expanded_dropd = dropd.explode('PictureURL').reset_index(drop=True) # TODO Drop duplicates here or before instantiating curate object
expanded_dropd = expanded_dropd.dropna(subset=['PictureURL'])
expanded_dropd = expanded_dropd.drop_duplicates(subset=['PictureURL'])
expanded_dropd.loc[:,'PictureURL'] = expanded_dropd.loc[:, 'PictureURL'].apply(lambda x: dict_pics[x])
expanded_dropd = self.extract_df(expanded_dropd) # convert lists to values
dict_pics_list = list(set(expanded_class.PictureURL.to_list())) # prolly need to create set long before df... immediately after Shopping or trading call
destination = 'your target folder' # decide whether or not you want to set a default folder to have the user define it as input every time. or have this only
# defined in the download function
dict_pics = {k:destination+re.search(r'[^/]+(?=/\$_|.jpg)', k).group()+'.jpg' for k in dict_pics_list} # TODO determine how to implement destination variable
# TODO still getting exceptions such as 'https://i.ebayimg.com/images/g/RG8AAOSwqMtd1esL/s-l1600.jpg'. add conditions to dict comprehension.
dict_pics = {k:destination+re.search(r'[^/]+(?=/\$_|.jpg)', k, re.IGNORECASE).group()+'.jpg' for k in dict_pics_list}
expanded_class = expanded_class
# with open('dict_pics.txt','w+') as f: # TODO open if it exists, or write if not, then extend the dictionary with dict_pics
# re.search(r'\w+(?=/\$_)', a).group()
return expanded_class, expanded_dropd # TODO still need to replace source url to destination url in df cols and create custom dict {<source>, <distination>}
return expanded_class, expanded_dropd, dict_pics # TODO still need to replace source url to destination url in df cols and create custom dict {<source>, <distination>}
def dl_pictures(self):
'''
Downloads pictures from api to local storage using custom master dict
'''
with open('dict_pic.txt', 'w+') as jf: # TODO requires cleaning up
with open('dict_pic.txt', 'w+') as jf: # avoid duplicate logic goes here... I think
dict_pics = json.load(jf)
r = requests.get('<dict_pic>', stream=True)