From 752fa7efaa9061ec6c2b412ccf2a629bc6b938d6 Mon Sep 17 00:00:00 2001 From: spbeach46 Date: Sun, 4 Oct 2020 00:48:55 -0700 Subject: [PATCH] removed some linting errors and figured out how to add dicts to pandas df (see shopping_test.py comments --- ebay_api_test.py | 13 ++++++++----- shopping_test.py | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ebay_api_test.py b/ebay_api_test.py index 16b5fe0..95e3697 100644 --- a/ebay_api_test.py +++ b/ebay_api_test.py @@ -7,14 +7,17 @@ import pandas as p with open('cat_list.txt') as jf: cat_list = json.load(jf) -finding_service = ['findItemsAdvanced', 'findCompletedItems', 'findItemsByKeywords', 'findItemsIneBayStores', 'findItemsByCategory', 'findItemsByProduct'] # list(set(re.findall(r'find\w+', response.text))) +finding_service = ['findItemsAdvanced', 'findCompletedItems', 'findItemsByKeywords', 'findItemsIneBayStores', 'findItemsByCategory', 'findItemsByProduct'] + pageNumber = list(range(1,101)) + # departments = ["3034","93427"] -item_results_list = [] -for categoryID in cat_list: + +itemid_results_list = [] +for categoryID in cat_list[0:2]: params = { "OPERATION-NAME":finding_service[4], - "SECURITY-APPNAME":"scottbea-xlister-PRD-6796e0ff6-14862949", + "SECURITY-APPNAME":"scottbea-xlister-PRD-6796e0ff6-14862949", "SERVICE-VERSION":"1.13.0", "RESPONSE-DATA-FORMAT":"JSON", "categoryId":categoryID , @@ -23,7 +26,7 @@ for categoryID in cat_list: } # extract item id here for piping into shopping_test.py - response = requests.get("https://svcs.ebay.com/services/search/FindingService/v1", params=params) # + response = requests.get("https://svcs.ebay.com/services/search/FindingService/v1", params=params) data = response.json() pretty_data = json.dumps(data, indent=2) # can use pandas.json_normalize(custom dict cobbled from respons.json()) diff --git a/shopping_test.py b/shopping_test.py index 904d5a4..fa7ccc8 100644 --- a/shopping_test.py +++ b/shopping_test.py @@ -24,4 +24,9 @@ for nvl_dict in nvl: values.append(nvl_dict['Value']) nvl_dict = dict(zip(names, values)) data.update(nvl_dict) +df = pd.json_normalize(data) +df.to_csv('big_data.csv') +# to append items to the pandas dataframe/csv, use var = df.append(json/dict, ignore_index=True) +# you will have to redefine var as a new dataframe everytime, too for some reason (I'm wondering +# if this is a bug?).