scott/ebay-ml-lister
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
545bc78421
ebay-ml-lister/ebay_api.py

391 lines
19 KiB
Python
Raw Blame History

import importlib
import pdb
import os
import numpy as np
import concurrent.futures
import json
import requests
import pandas as pd
import config as cfg
import shutil
import re
class FindingApi:
'''Methods for accessing eBays FindingApi services'''
def __init__(self, service, pageNumber):
self.service = [
'findItemsAdvanced', 'findCompletedItems',
'findItemsByKeywords', 'findItemsIneBayStores', 'findItemsByCategory',
'findItemsByProduct'
][service]
self.pageNumber = list(range(1, pageNumber)) # 77 pgs will give equal weights to cats given call constraints
# departments = ["3034","93427"] (womens and mens)
# examples of additional params you may want to add:
# 'itemFilter(0).value':'Used'
# 'itemFilter(1).name':'ListingType'
# 'itemFilter(1).value':'AuctionWithBIN'
def get_data(self, category_id, i):
'''
Gets raw JSON data fom FindingApi service call
Currently being used to get itemIDs from categories
'''
params = {
"OPERATION-NAME":self.service,
"SECURITY-APPNAME":cfg.sec['SECURITY-APPNAME'],
"SERVICE-VERSION":"1.13.0",
"RESPONSE-DATA-FORMAT":"JSON",
"categoryId":category_id,
"paginationInput.entriesPerPage":"100",
"paginationInput.PageNumber":i
}
# TODO add try excepts here
try:
response = requests.get("https://svcs.ebay.com/services/search/FindingService/v1",
params=params, timeout=3)
response.raise_for_status()
except requests.exceptions.RequestException:
print('connection error') #TODO DECIDE HOW TO HANDLE EXCEPTION
data = response.json()
return data
# TODO add some other options to finding call api such as for possibly filtering for used items only. This might give you a better dataset for training. Or maybe a mixture of new and used. Maybe
# try and come up with a way to mathematically determine your odds of maximizing the number of pictures in your training set while reducing the number of useless images. Say for example, if you took a
# random set of 3 of 8 pictures total from each listing you might have a better chance of getting 3 good pictures in addition to increasing your training set. Or maybe you would have better luck with limiting
# it to the first 5 pictures instead of random.
# You may even have more consistency with used shoes since they are "one-off" items without confusing multiple variations and colors. What else you can do is run small training sets on both new and used
# to see which one is more accurate or if a combo of both is more accurate.
def get_ids_from_cats(self):
'''
Creates a 20-itemId list to use for the ShoppingApi
call
'''
pages = self.pageNumber
itemid_results_list = []
with open('cat_list.txt') as jf:
cat_list = json.load(jf)
for category_id in cat_list:
args = [(category_id, i) for i in pages]
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(lambda p: self.get_data(*p), args):
data = future
try: # TODO if conditionals are not working due to each thread checking the same unedited item_id_results list
training = pd.read_csv('training.csv')
for item in data['findItemsByCategoryResponse'][0]['searchResult'][0]['item']:
if (item not in training.values) and (item not in itemid_results_list):
itemid_results_list.append(item['itemId'][0])
except (pd.errors.EmptyDataError, FileNotFoundError):
for item in data['findItemsByCategoryResponse'][0]['searchResult'][0]['item']:
if item not in itemid_results_list:
itemid_results_list.append(item['itemId'][0])
item_id_results = list(set(itemid_results_list))
item_id_results = [','.join(itemid_results_list[n:n+20]) for n in list(range(0,
len(itemid_results_list), 20))]
return item_id_results
# TODO during your try except conditionals just check the csv files. At the end you can create sets. You can creat another condition that says if the final set is smaller than 100k then you can call finding
# service on more pages (but only pages you haven't tried) and repeat the search process.
# TODO instead of running through multiple try except loops try to implement set methods for efficiency and ease. Remember symmetric_difference, difference, intersection, set()
# for category_id in cat_list:
class ShoppingApi:
'''
Creates objects from ShoppingApi service calls that can interact with
pandas dataframes
'''
def get_item_from_findItemsByCategory(self, twenty_id):
'''
Gets raw JSON data from multiple live listings given multiple itemIds
'''
params = {
"callname":"GetMultipleItems",
"appid":cfg.sec['SECURITY-APPNAME'],
"version":"671",
"responseencoding":"JSON",
"ItemID":twenty_id,
"IncludeSelector":"ItemSpecifics",
}
# TODO Add try excepts here
try:
response = requests.get("https://open.api.ebay.com/shopping?", params=params, timeout=1)
response.raise_for_status()
except requests.exceptions.RequestException:
print('connection error')
response = response.json()
response = response['Item']
return response
def conky(self):
'''
For some reason item_id_results can only be passed as argument in executor.map
if the variable is made within function
'''
data = []
finding = FindingApi(4, 2) # TODO replace these test values before production
item_id_results = finding.get_ids_from_cats()
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results):
# print(future)
for item in future:
data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
# data.update(future)
# TODO save data here. You'll use this with your curate data class. SAve this as text file
return data # TODO each future is a list of dictionaries because the output of any multithreader in this method is a list.
# data dictionary can't update from list of dicts unless iterated over. Might need a different way to update.
# TODO It seems like the problem with updating the dictionary/csv file is starting here possibly; I think the item data is getting appended out of order from the item itself.
class CurateData:
'''
Contains functions for curating data for machine learning training sets;
Takes item in data from ShoppingApi request as argument and extracts/ creates key
value pairs that gets updated to custom dataframe used in Ml training sets.
'''
def import_raw(self):
'''
imports raw response json from local file
'''
with open('raw_data.txt') as f:
raw_data = json.load(f)
return raw_data
def raw_df(self, raw_data):
'''
creates pandas df from raw json. Indended to be used inline with direct
data stream from ebay's APIs
'''
to_json = json.dumps(raw_data)
raw_df = pd.read_json(to_json)
return raw_df
def to_training(self, raw_data): # NOTE need to create copies not views
'''
creates first pass of potential labels for training set. This is the base
df used to produce other training sets to use.
'''
raw_df = self.raw_df(raw_data)
interm_df1 = raw_df.loc[:,['ItemID', 'PictureURL', 'PrimaryCategoryID', 'PrimaryCategoryName', 'Title', 'ItemSpecifics']]
interm_df1[['ItemID', 'PrimaryCAegoryID']] = interm_df1.loc[:, ['ItemID', 'PrimaryCategoryID']].astype(str)
training = interm_df1
return training # TODO RENAME THIS FUNC AND RETURN VALUE
def class_training(self, training):
'''Training set for multiclass portion of training set. Used to train
seprately from multilabel portion
'''
class_training = training.loc[:, ['PictureURL', 'PrimaryCategoryID']]
return class_training
def nvl_training(self, training):
'''
Training set for multilabel portion
'''
interm_df1 = pd.Series(training.ItemSpecifics)
interm_df1 = interm_df1.apply(lambda x: x['NameValueList'])
# Necessary for json_normalize():
nvl_dict = interm_df1.apply(lambda x: {k:v for (k, v) in zip([n['Name'] for n in x], [v['Value'] for v in x])})
nvl_df = pd.json_normalize(nvl_dict)
nvl_training = pd.concat([pd.Series(training.PictureURL), nvl_df], axis=1)
return nvl_training
def extract_df(self, df):
'''
converts single-value lists of strings of any df to string if not null
'''
extracted_df = df.applymap(lambda x: ' '.join(x) if isinstance(x, list) else np.nan if pd.isnull(x) else x)
return extracted_df
def drop_nvl_cols(self, nvl_training):
col_drop = [
'Fabric Type', 'Type of Sport', 'Mid Sole', 'Modified Item',
'Modification Description', 'Article Type', 'Customized',
'Character', 'Features', 'Colors', 'Shade', 'Product ID',
'Personalized', 'Platform Height', 'Year Manufactured',
'Trim Material', 'Fashion Element', 'Shaft Material',
'Character Family', 'Heel to Toe Drop', 'Custom Bundle',
'California Prop 65 Warning', 'Manufacturer Color', 'Main Color',
'Collection', 'Midsole Type', 'Signed', 'US Shoe Size (Men#!#s)',
'Calf Circumference', 'Handmade', 'Safety Standards',
'Customised', 'Cleat Type', 'Cushioning Level', 'AU Shoe Size',
'Country/Region of Manufacture', 'Type of Sport', 'Main Colour',
'Look', 'Sole Type', 'Manufacturer Colour', 'Sole Material',
'Toe Material', 'Feature', 'Length', 'Width', 'Size Chart',
'Boot Height', 'Water Resistance Level', 'Material Composition',
'Calf Width', 'Insole Material', 'UPC', 'Size Type'
]
col_keep = [
'PictureURL', 'Style', 'Department', 'Type', 'Gender', 'Closure', 'Performance/Activity',
'Accents', 'Occasion', 'Toe Shape', 'Pattern', 'Activity',
'Heel Style', 'Fastening', 'Heel Type', 'Toe Type', 'Departement',
'Product Type', 'Sub Style', 'Season', 'Theme', 'Upper Material',
]
# May be no difference between Product type and sub style; fastening and
# closure; toe shape and toe type; occasion and performance/activity;
# see if you can combine these somehow (you may not want this though).
# Also consider keeping only cols that have plenty of values
# Run some value_count() analysis to determine frequencies and filter
# user created item specifics, leaving only predefined ebay item specs
user_input = input('drop or keep cols?:')
if 'keep' in user_input:
dropd = nvl_training.loc[:,col_keep]
else:
dropd = nvl_training.drop(col_drop, axis=1)
return dropd
def expand_nvlclass(self, class_training, dropd):
'''
takes image url list from each cell and expands them into separate/duplicate
instances. Modifies both class training and dropd dfs. Appends custom
image url dict {'source':'target'}.
'''
expanded_class = class_training.explode('PictureURL').reset_index(drop=True) # TODO drop duplicates here or before instantiating curate object
expanded_class = expanded_class.dropna(subset=['PictureURL'])
expanded_class = expanded_class.drop_duplicates(subset=['PictureURL']).reset_index(drop=True)
# expanded_class.loc[:,'PictureURL'] = expanded_class.loc[:, 'PictureURL'].apply(lambda x: dict_pics[x])
expanded_dropd = dropd.explode('PictureURL').reset_index(drop=True) # TODO Drop duplicates here or before instantiating curate object
expanded_dropd = expanded_dropd.dropna(subset=['PictureURL'])
expanded_dropd = expanded_dropd.drop_duplicates(subset=['PictureURL']).reset_index(drop=True)
# expanded_dropd.loc[:,'PictureURL'] = expanded_dropd.loc[:, 'PictureURL'].apply(lambda x: dict_pics[x])
expanded_dropd = self.extract_df(expanded_dropd) # convert lists to values
temp_pics_source_list = list(set(expanded_class.PictureURL.to_list())) # prolly need to create set long before df... immediately after Shopping or trading call
# defined in the download function
with open('temp_pics_source_list.txt', 'a+') as f: # Temp iterable for use w/executor
try:
temp_pics_source_list = json.load(f)
temp_pics_source_list.append(temp_pics_source_list)
json.dump(temp_pics_source_list, f) # TODO This creates duplicates incorrectly
except ValueError:
json.dump(temp_pics_source_list, f)
return expanded_class, expanded_dropd
def dl_pictures(self, *args):
'''
Downloads pictures from api to local storage using temp_pics_source_list
and creates custom {source:target} dictionary as dict_pics
'''
try:
with open('target_dirs.txt', 'r+') as f: # TODO you can add option to change directory here, too. Look up how to have optional arguments
target_dir = json.load(f)
except (ValueError, FileNotFoundError): # TODO doesn't seem to create target_dir from cwd. I think it's because ValueError should be FileNotFoundError?
target_dir = input('No target dirctory found. Create One? [y] or [n]:')
if target_dir == ('y' or 'Y'):
target_dir = input('Please provide full URL to destination folder:')
with open('target_dirs.txt','w+') as f:
json.dump(target_dir, f)
else:
target_dir = os.getcwd()
with open('target_dirs.txt','w+') as f:
json.dump(target_dir, f)
print('Creating default folder in current directory, ' + target_dir) # TODO does not concat correctly. Fix this
with open('temp_pics_source_list.txt') as f:
try:
if args:
temp_pics_source_list = args[0]
else:
temp_pics_source_list = json.load(f)
except ValueError:
if args:
temp_pics_sources_list = args[0]
else:
print('url list not found. download aborted')
return
temp_dict_pics = {k:target_dir + os.sep + re.search(r'[^/]+(?=/\$_|.jpg)', k, re.IGNORECASE).group() + '.jpg' for k in temp_pics_source_list}
try:
with open('dict_pics.txt') as f:
dict_pics = json.load(f)
dict_pics.update(temp_dict_pics)
with open('dict_pics.txt', 'w') as f:
json.dump(dict_pics, f)
except (ValueError, FileNotFoundError):
with open('dict_pics.txt', 'w') as f:
json.dump(temp_dict_pics, f)
dict_pics = temp_dict_pics
def dl_pic(dict_pics, pic):
if os.path.exists(dict_pics[pic]): # or call temp_dict_pics[pic] can work
pass
else:
r = requests.get(pic, stream=True)
r.raw.decode_content = True
with open(temp_dict_pics[pic], 'wb') as f: # Or call dict_pics[pic] can work
shutil.copyfileobj(r.raw, f)
breakpoint()
bargs = [(dict_pics, pic) for pic in temp_pics_source_list]
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(lambda p: dl_pic(*p), bargs):
future
with open('temp_pics_source_list.txt','w') as f: # Overwrites old when complete
temp_pics_source_list = []
json.dump(temp_pics_source_list, f)
# TODO pipeline gameplan: 5 files: master img download dict,raw_json.txt, raw_json.csv, master_class_training.csv, master_nvl_training.csv
# cont... open raw_json.txt and append, same with csv --> process new data --> pull out image source+dest and expand new dfs for the additional pictures
# if not exists and append to master img download dict
# --> concat m_class_training df and m_nvl_training dfs with new data. Need to add inclusion tests for all files when opened and appended/concatted
def main():
'''
Main program creates/updates a csv file to use for ML training from live
ebay listings
'''
pass
# main goes here:
if __name__ == "__main__":
main()
# Limited to 5000 calls to shopping api per day, and getMultpileitems service maxes out at 20 items
# per call leaving you 100,000 items per day for you pandas dataframe initially. So you'll have
# to divide these up into the categories. This will leave you with about 6.25K results per cat.
# More than enough data for your dataset.
# for future reference, to deal with inconsistent values in the nvl (due to sellers inputting custom values in the fields) you can drop either listings or k/v pairs that are unique which
# can be determined from applying a function to determine frequency of k/v pairs--> list of unique k/v pairs--> function to determine frequency of unique k/v pairs--> drop those that have 1.
# TODO NEED TO ADD TRY EXCEPT CONDITIONS FOR EVERY CALL MADE TO API SERVICES TO
# TO AVOID HICCUPS WHEN CREATING DATASET
# TODO YOU WILL HAVE TO FIND A WAY OF COLLECTING DATA FOR IMAGES OF Shoe TAGS EITHER USING YOUR OWN TAGS OR SOMEHOW FIND A WAY TO FIND TAGS ON OTHERS LISTINGS. CRUCIAL FOR THE LISTINGS PROCESS. May be as simple as adding a def to one of the apis to extract only the picture if it can identify what a tag looks like. So, it may actually be a good thing to include all the pictures in a training set but then when you're ready to begin training you'll have a data cleaning pipeline specific to training a model to either learn shoe features or information on tags.
# Check the above list of cols I want to keep to see if there are duplicates with diff spelling and phrasing (e.g., Departement and Department, or Fastening and Closure Type)
Reference in New Issue View Git Blame Copy Permalink