215 lines
8.5 KiB
Python
215 lines
8.5 KiB
Python
import concurrent.futures
|
|
import json
|
|
import requests
|
|
import pandas as pd
|
|
|
|
class FindingApi:
|
|
'''Methods for accessing eBays FindingApi services'''
|
|
def __init__(self, service, pageNumber):
|
|
self.service = [
|
|
'findItemsAdvanced', 'findCompletedItems',
|
|
'findItemsByKeywords', 'findItemsIneBayStores', 'findItemsByCategory',
|
|
'findItemsByProduct'
|
|
][service]
|
|
self.pageNumber = list(range(1, pageNumber)) # 64 pages is recommended
|
|
# this will give equal weights to cats given call constraints
|
|
|
|
# departments = ["3034","93427"] (womens and mens)
|
|
|
|
def get_data(self, category_id, i): # TODO you're going to have to use nested functions of lambda functions here somewhere
|
|
|
|
'''
|
|
Gets raw JSON data fom FindingApi service call
|
|
Currently being used to get itemIDs from categories
|
|
'''
|
|
|
|
params = {
|
|
"OPERATION-NAME":self.service,
|
|
"SECURITY-APPNAME":"scottbea-xlister-PRD-6796e0ff6-14862949",
|
|
"SERVICE-VERSION":"1.13.0",
|
|
"RESPONSE-DATA-FORMAT":"JSON",
|
|
"categoryId":category_id,
|
|
"paginationInput.entriesPerPage":"100",
|
|
"paginationInput.PageNumber":i
|
|
}
|
|
|
|
response = requests.get("https://svcs.ebay.com/services/search/FindingService/v1",
|
|
params=params)
|
|
|
|
data = response.json()
|
|
return data
|
|
|
|
|
|
def get_ids_from_cats(self):
|
|
'''
|
|
Creates a 20-itemId list to use for the ShoppingApi
|
|
call
|
|
'''
|
|
pages = self.pageNumber
|
|
itemid_results_list = []
|
|
|
|
with open('cat_list.txt') as jf:
|
|
cat_list = json.load(jf)
|
|
|
|
for category_id in cat_list:
|
|
|
|
args = [(category_id, i) for i in pages]
|
|
|
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
for future in executor.map(lambda p: self.get_data(*p), args):
|
|
data = future
|
|
|
|
try:
|
|
training = pd.read_csv('training.csv')
|
|
for item in data['findItemsByCategoryResponse'][0]['searchResult'][0]['item']:
|
|
if (item not in training.values) and (item not in itemid_results_list):
|
|
itemid_results_list.append(item['itemId'][0])
|
|
|
|
except (pd.errors.EmptyDataError, FileNotFoundError):
|
|
for item in data['findItemsByCategoryResponse'][0]['searchResult'][0]['item']:
|
|
if item not in itemid_results_list:
|
|
itemid_results_list.append(item['itemId'][0])
|
|
|
|
item_id_results = list(set(itemid_results_list))
|
|
item_id_results = [','.join(itemid_results_list[n:n+20]) for n in list(range(0,
|
|
len(itemid_results_list), 20))]
|
|
return item_id_results
|
|
|
|
## TODO instead of running through multiple try except loops try to implement set methods for efficiency and ease. Remember symmetric_difference, difference, intersection, set()
|
|
# for category_id in cat_list:
|
|
|
|
class ShoppingApi:
|
|
'''
|
|
Creates objects from ShoppingApi service calls that can interact with
|
|
pandas dataframes
|
|
'''
|
|
def get_item_from_findItemsByCategory(self, twenty_id):
|
|
'''
|
|
Gets raw JSON data from multiple live listings given multiple itemIds
|
|
'''
|
|
params = {
|
|
"callname":"GetMultipleItems",
|
|
"appid":"scottbea-xlister-PRD-6796e0ff6-14862949",
|
|
"version":"671",
|
|
"responseencoding":"JSON",
|
|
"ItemID":twenty_id,
|
|
"IncludeSelector":"ItemSpecifics",
|
|
}
|
|
|
|
response = requests.get("https://open.api.ebay.com/shopping?", params=params)
|
|
response = response.json()
|
|
response = response['Item']
|
|
return response
|
|
|
|
def conky(self):
|
|
'''
|
|
For some reason item_id_results can only be passed as argument in executor.map
|
|
if the variable is made within function
|
|
'''
|
|
data = {}
|
|
finding = FindingApi(4, 2)
|
|
item_id_results = finding.get_ids_from_cats()
|
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results):
|
|
data.update(future)
|
|
return data
|
|
|
|
class CurateData:
|
|
'''
|
|
Contains functions for curating data for machine learning training sets;
|
|
Takes item in data from ShoppingApi request as argument and extracts/ creates key
|
|
value pairs that gets updated to custom dataframe used in Ml training sets.
|
|
'''
|
|
def extract_itemId(self, item):
|
|
item_id = {'ItemID':item['ItemID']}
|
|
return item_id
|
|
|
|
def extract_catId(self, item):
|
|
catId = {'PrimaryCategoryID':item['PrimaryCategoryID']}
|
|
return catId
|
|
|
|
def extract_prime_cat_name(self, item):
|
|
prime_cat_name= {'PrimaryCategoryName':item['PrimaryCategoryName']}
|
|
return prime_cat_name
|
|
|
|
def extract_picture_url(self, item):
|
|
'''
|
|
Only pulls PictureURL list and does not
|
|
create dictionary
|
|
'''
|
|
picture_url_list = item['PictureURL']
|
|
return picture_url_list
|
|
|
|
def extract_nvl(self, item):
|
|
names = []
|
|
values = []
|
|
nvl = item['itemspecifics']['namevaluelist']
|
|
for nvl_dict in nvl:
|
|
names.append(nvl_dict['name'])
|
|
values.append(nvl_dict['value'])
|
|
nvl_dict = dict(zip(names, values))
|
|
return nvl_dict
|
|
|
|
def update_df(self, data):
|
|
'''
|
|
Creates training instances for dataset. picture_url_list expanded to
|
|
max available pictures with each picture url corresponding to features
|
|
in common with same listing (i.e., because there are multiple pictures
|
|
per listing, each picture will be its own training instance.
|
|
'''
|
|
|
|
training = {}
|
|
for item in data:
|
|
# TODO MAY HAVE TO DISCARD THIS IDEA DUE TO CRAPPY PICTURES OF CLOSEUPDS AND TAGS. may have to settle for first picture which is likely to contain more accurate representation of item.
|
|
picture_url_list = self.extract_picture_url(item)
|
|
'''
|
|
Creates same training instance per photo for
|
|
'''
|
|
for url in picture_url_list:
|
|
remote_url = {'PictureURL':url}
|
|
training.update(remote_url)
|
|
item_id = self.extract_itemId(item)
|
|
training.update(item_id)
|
|
catId = self.extract_catId(item)
|
|
training.update(catId)
|
|
prime_cat_name = self.extract_prime_cat_name(item)
|
|
training.update(prime_cat_name)
|
|
nvl_dict = self.extract_nvl(item)
|
|
training.update(nvl_dict)
|
|
|
|
df = pd.json_normalize(training) # TODO FIX INDENT HERE?
|
|
df.to_csv('training.csv', mode='a')
|
|
|
|
def main():
|
|
'''
|
|
Main program creates/updates a csv file to use for ML training from live
|
|
ebay listings
|
|
'''
|
|
service, pageNumber = input('service and pageNumber:').split()
|
|
service = int(service)
|
|
pageNumber = int(pageNumber)
|
|
# finding = FindingApi(service, pageNumber)
|
|
|
|
# item_id_results = finding.get_ids_from_cats()
|
|
shopping = ShoppingApi()
|
|
data = shopping.conky()
|
|
curate = CurateData()
|
|
curate.update_df(data)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
# Limited to 5000 calls to shopping api per day, and getMultpileitems service maxes out at 20 items
|
|
# per call leaving you 100,000 items per day for you pandas dataframe initially. So you'll have
|
|
# to divide these up into the categories. This will leave you with about 6.25K results per cat.
|
|
# More than enough data for your dataset.
|
|
|
|
# Need to make sure dataframe gets important stuff outside of nvl in order to
|
|
# access values for cross referencing itemIds from calls
|
|
# Need to decide if list gets accessed from df or if you're just going to have
|
|
# list contents extracted and possibly placed into separate cells/labels
|
|
|
|
# TODO NEED TO ADD TRY EXCEPT CONDITIONS FOR EVERY CALL MADE TO API SERVICES TO
|
|
# TO AVOID HICCUPS WHEN CREATING DATASET
|
|
# TODO YOU WILL HAVE TO FIND A WAY OF COLLECTING DATA FOR IMAGES OF TAGS EITHER USING YOUR OWN TAGS OR SOMEHOW FIND A WAY TO FIND TAGS ON OTHERS LISTINGS. CRUCIAL FOR THE LISTINGS PROCESS. May be as simple as adding a def to one of the apis to extract only the picture if it can identify what a tag looks like. So, it may actually be a good thing to include all the pictures in a training set but then when you're ready to begin training you'll have a data cleaning pipeline specific to training a model to either learn shoe features or information on tags.
|