From eaf3480341a324dbf114847a312338618ccf7258 Mon Sep 17 00:00:00 2001
From: scott <spbeach46@gmail.com>
Date: Fri, 26 Nov 2021 22:38:01 -0700
Subject: [PATCH] added random sleeps before making shopping api call

---
 ebay_api.py | 43 +++++++++++++++++--------------------------
 1 file changed, 17 insertions(+), 26 deletions(-)

diff --git a/ebay_api.py b/ebay_api.py
index 15ed330..7a07998 100644
--- a/ebay_api.py
+++ b/ebay_api.py
@@ -1,4 +1,6 @@
 import os
+from time import sleep
+from random import randint
 import scrape_ids
 from datetime import datetime, timedelta
 import dateutil
@@ -211,6 +213,11 @@ class ShoppingApi:
         '''
         Gets raw JSON data from multiple live listings given multiple itemIds
         '''
+
+        with open('ids.txt') as f:
+            ids = json.load(f)
+            item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
+
         headers = {
             "X-EBAY-API-IAF-TOKEN":cfg.sec['X-EBAY-API-IAF-TOKEN'], # TODO implement auto oauth token renewal
             "version":"671",
@@ -219,11 +226,16 @@ class ShoppingApi:
         url = "https://open.api.ebay.com/shopping?&callname=GetMultipleItems&responseencoding=JSON&IncludeSelector=ItemSpecifics&ItemID="+twenty_id
 
         try:
-            response = requests.get(url, headers=headers,timeout=4)
+            # random sleep here between 0 and 10 secs?
+            sleep(randint(1,10))
+            response = requests.get(url, headers=headers,timeout=24)
             response.raise_for_status()
+            print('index number {}'.format(item_id_results.index(twenty_id)))
 
         except requests.exceptions.RequestException: # TODO need better handling
             print('connection error')
+            print('index number {}'.format(item_id_results.index(twenty_id)))
+            return
 
         response = response.json()
         response = response['Item']
@@ -244,36 +256,15 @@ class ShoppingApi:
         try:
             with open('ids.txt') as f:
                 ids = json.load(f)
-                item_id_results = [','.join(ids[n:n+20]) for n in list(range(0,
-                    len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
+                item_id_results = [','.join(ids[n:n+20]) for n in list(range(0, len(ids), 20))] # 20-ItemID list created to maximize dataset/decrease calls given call constraints
 
         except (FileNotFoundError, ValueError):
             item_id_results = scrape_ids.main()
-#        service_dict = {
-#                0: 'findItemsAdvanced', 1: 'findCompletedItems',
-#                2: 'findItemsByKeywords', 3: 'findItemsIneBayStores',
-#                4: 'findItemsByCategory', 5:'findItemsByProduct'}
-#        service_dict
-#
-#        fnd_srvc = input(str(service_dict) + "choose Finding call: (press 'enter' for default(4))")
-#        target_idspc = int(input('how many ids per cat? (7692 max)'))
-#
-#        optional_params = {
-#                "itemFilter(0).name":"Condition",
-#                "itemFilter(0).value":"Used"
-#                } # NOTE setting as default in get_data() method
-#
-#        if fnd_srvc != '':
-#            fnd_srvc = int(fnd_srvc)
-#            finding = FindingApi(fnd_srvc, target_idspc)
-#        else:
-#            fnd_srvc = 4
-#            finding = FindingApi(fnd_srvc, target_idspc)
-#
-        with concurrent.futures.ThreadPoolExecutor() as executor:
+
+        with concurrent.futures.ThreadPoolExecutor() as executor: # NOTE may need to include sleeps to avoid connection refusal due to overwhelming servers
             for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results):
                 for item in future:
-                        data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
+                    data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
                 # data.update(future)
         with open('raw_data.txt', 'w') as f:
             json.dump(data, f) # TODO maybe write for every future returned to avoid losing data if your accidentally reach