scott/ebay-ml-lister
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added random sleeps before making shopping api call

Browse Source
This commit is contained in:
scott 2021-11-26 22:38:01 -07:00
parent 2cf8270707
commit eaf3480341
1 changed files with 17 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
ebay_api.py
View File

@ -1,4 +1,6 @@
import os
from time import sleep
from random import randint
import scrape_ids
from datetime import datetime, timedelta
import dateutil
@ -211,6 +213,11 @@ class ShoppingApi:
'''
Gets raw JSON data from multiple live listings given multiple itemIds
'''
with open('ids.txt') as f:
ids = json.load(f)
item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
headers = {
"X-EBAY-API-IAF-TOKEN":cfg.sec['X-EBAY-API-IAF-TOKEN'], # TODO implement auto oauth token renewal
"version":"671",
@ -219,11 +226,16 @@ class ShoppingApi:
url = "https://open.api.ebay.com/shopping?&callname=GetMultipleItems&responseencoding=JSON&IncludeSelector=ItemSpecifics&ItemID="+twenty_id
try:
response = requests.get(url, headers=headers,timeout=4)
# random sleep here between 0 and 10 secs?
sleep(randint(1,10))
response = requests.get(url, headers=headers,timeout=24)
response.raise_for_status()
print('index number {}'.format(item_id_results.index(twenty_id)))
except requests.exceptions.RequestException: # TODO need better handling
print('connection error')
print('index number {}'.format(item_id_results.index(twenty_id)))
return
response = response.json()
response = response['Item']
@ -244,36 +256,15 @@ class ShoppingApi:
try:
with open('ids.txt') as f:
ids = json.load(f)
item_id_results = [','.join(ids[n:n+20]) for n in list(range(0,
len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
except (FileNotFoundError, ValueError):
item_id_results = scrape_ids.main()
# service_dict = {
# 0: 'findItemsAdvanced', 1: 'findCompletedItems',
# 2: 'findItemsByKeywords', 3: 'findItemsIneBayStores',
# 4: 'findItemsByCategory', 5:'findItemsByProduct'}
# service_dict
#
# fnd_srvc = input(str(service_dict) + "choose Finding call: (press 'enter' for default(4))")
# target_idspc = int(input('how many ids per cat? (7692 max)'))
#
# optional_params = {
# "itemFilter(0).name":"Condition",
# "itemFilter(0).value":"Used"
# } # NOTE setting as default in get_data() method
#
# if fnd_srvc != '':
# fnd_srvc = int(fnd_srvc)
# finding = FindingApi(fnd_srvc, target_idspc)
# else:
# fnd_srvc = 4
# finding = FindingApi(fnd_srvc, target_idspc)
#
with concurrent.futures.ThreadPoolExecutor() as executor:
with concurrent.futures.ThreadPoolExecutor() as executor: # NOTE may need to include sleeps to avoid connection refusal due to overwhelming servers
for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results):
for item in future:
data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
# data.update(future)
with open('raw_data.txt', 'w') as f:
json.dump(data, f) # TODO maybe write for every future returned to avoid losing data if your accidentally reach