diff --git a/ebay_api.py b/ebay_api.py index 15ed330..7a07998 100644 --- a/ebay_api.py +++ b/ebay_api.py @@ -1,4 +1,6 @@ import os +from time import sleep +from random import randint import scrape_ids from datetime import datetime, timedelta import dateutil @@ -211,6 +213,11 @@ class ShoppingApi: ''' Gets raw JSON data from multiple live listings given multiple itemIds ''' + + with open('ids.txt') as f: + ids = json.load(f) + item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints + headers = { "X-EBAY-API-IAF-TOKEN":cfg.sec['X-EBAY-API-IAF-TOKEN'], # TODO implement auto oauth token renewal "version":"671", @@ -219,11 +226,16 @@ class ShoppingApi: url = "https://open.api.ebay.com/shopping?&callname=GetMultipleItems&responseencoding=JSON&IncludeSelector=ItemSpecifics&ItemID="+twenty_id try: - response = requests.get(url, headers=headers,timeout=4) + # random sleep here between 0 and 10 secs? + sleep(randint(1,10)) + response = requests.get(url, headers=headers,timeout=24) response.raise_for_status() + print('index number {}'.format(item_id_results.index(twenty_id))) except requests.exceptions.RequestException: # TODO need better handling print('connection error') + print('index number {}'.format(item_id_results.index(twenty_id))) + return response = response.json() response = response['Item'] @@ -244,36 +256,15 @@ class ShoppingApi: try: with open('ids.txt') as f: ids = json.load(f) - item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, - len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints + item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints except (FileNotFoundError, ValueError): item_id_results = scrape_ids.main() -# service_dict = { -# 0: 'findItemsAdvanced', 1: 'findCompletedItems', -# 2: 'findItemsByKeywords', 3: 'findItemsIneBayStores', -# 4: 'findItemsByCategory', 5:'findItemsByProduct'} -# service_dict -# -# fnd_srvc = input(str(service_dict) + "choose Finding call: (press 'enter' for default(4))") -# target_idspc = int(input('how many ids per cat? (7692 max)')) -# -# optional_params = { -# "itemFilter(0).name":"Condition", -# "itemFilter(0).value":"Used" -# } # NOTE setting as default in get_data() method -# -# if fnd_srvc != '': -# fnd_srvc = int(fnd_srvc) -# finding = FindingApi(fnd_srvc, target_idspc) -# else: -# fnd_srvc = 4 -# finding = FindingApi(fnd_srvc, target_idspc) -# - with concurrent.futures.ThreadPoolExecutor() as executor: + + with concurrent.futures.ThreadPoolExecutor() as executor: # NOTE may need to include sleeps to avoid connection refusal due to overwhelming servers for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results): for item in future: - data.append(item) # The end result should be a list of dicts where each dict in the list is a listing + data.append(item) # The end result should be a list of dicts where each dict in the list is a listing # data.update(future) with open('raw_data.txt', 'w') as f: json.dump(data, f) # TODO maybe write for every future returned to avoid losing data if your accidentally reach