diff --git a/curate.py b/curate.py index 5376e64..6e99c55 100644 --- a/curate.py +++ b/curate.py @@ -21,8 +21,6 @@ expanded_dropd = expanded_dfs[1] # TODO incorrect df. Look at nvl_training func. download = input('download images?: ') if ('y' or 'Y') in download: - with open('temp_pics_source_list.txt') as f: - url_list = json.load(f) curate.dl_pictures() else: pass diff --git a/ebay_api.py b/ebay_api.py index 2d44ac9..47fc166 100644 --- a/ebay_api.py +++ b/ebay_api.py @@ -16,12 +16,49 @@ import pandas as pd import config as cfg import shutil import re +import urllib, base64 from ebaysdk.exception import ConnectionError from ebaysdk.trading import Connection as Trading from ebaysdk.finding import Connection as Finding from ebaysdk.shopping import Connection as Shopping +# renew oauth token for shopping api +def getAuthToken(): + AppSettings = { + 'client_id': cfg.oauth.client_id, + 'client_secret':cfg.oauth.client_secret, + 'ruName':cfg.oauth.RuName + } + + authHeaderData = AppSettings['client_id'] + ':' + AppSettings['client_secret'] + encodedAuthHeader = base64.b64encode(str.encode(authHeaderData)) + encodedAuthHeader = str(encodedAuthHeader)[2:len(str(encodedAuthHeader))-1] + + headers = { + "Content-Type" : "application/x-www-form-urlencoded", # what is this? + "Authorization" : "Basic " + str(encodedAuthHeader) + } + + body= { + "grant_type" : "client_credentials", + "redirect_uri" : AppSettings['ruName'], + "scope" : "https://api.ebay.com/oauth/api_scope" + } + + data = urllib.parse.urlencode(body) + + tokenURL = "https://api.ebay.com/identity/v1/oauth2/token" + + response = requests.post(tokenURL, headers=headers, data=data) + error = response['error_description'] #if errors + access_token = response.json()['access_token'] + + with open('temp_oath_token.txt', 'w') as f: + json.dump(access_token, f) + + return access_token, error + class FindingApi: ''' Methods for accessing eBay's FindingApi services @@ -33,12 +70,6 @@ class FindingApi: 'findItemsByKeywords', 'findItemsIneBayStores', 'findItemsByCategory', 'findItemsByProduct' ][service] # Currently using only index 4, i.e., service = 4 - # examples of additional params you may want to add: - # 'itemFilter(0).value':'Used' consider using this with findCompletedItems call - # 'itemFilter(1).name':'ListingType' - # 'itemFilter(1).value':'AuctionWithBIN' - # 'StartTimeNewest' - # HideDuplicateItems def get_data(self, category_id): @@ -132,6 +163,12 @@ class ShoppingApi: pandas dataframes ''' + def __init__(self): + + # renew oauth token + access_token = getAuthToken()[0] + self.access_token = access_token + def update_cats(self): ''' Updates cat_list.txt @@ -143,7 +180,7 @@ class ShoppingApi: for department in parent_cats: headers = { - "X-EBAY-API-IAF-TOKEN":cfg.sec['X-EBAY-API-IAF-TOKEN'], # TODO implement auto oauth token renewal + "X-EBAY-API-IAF-TOKEN":self.access_token, "version":"671", } @@ -172,16 +209,14 @@ class ShoppingApi: ''' headers = { - "X-EBAY-API-IAF-TOKEN":cfg.sec['X-EBAY-API-IAF-TOKEN'], # TODO implement auto oauth token renewal + "X-EBAY-API-IAF-TOKEN":self.access_token, "version":"671", } url = "https://open.api.ebay.com/shopping?&callname=GetMultipleItems&responseencoding=JSON&IncludeSelector=ItemSpecifics&ItemID="+twenty_id try: - # random sleep here between 0 and 10 secs? -# sleep(randint(1,10)) # may not be necessary response = requests.get(url, headers=headers,timeout=24) response.raise_for_status() response = response.json() @@ -400,17 +435,20 @@ class CurateData: try: - if os.path.exists(dict_pics[pic]): # TODO DOES NOT FIND CURRENT PATHS BECUASE TAGS WILL NOW BE DIFFERENT. YOU WILL END UP RE DOWNLOADING IMAGES + # check if image exists in current working directory. avoids dupes + if os.path.exists(dict_pics[pic]): pass else: + try: + r = requests.get(pic, stream=True) r.raw.decode_content = True with open(dict_pics[pic], 'wb') as f: shutil.copyfileobj(r.raw, f) - except ConnectionError: + except ConnectionError: return except KeyError: @@ -421,12 +459,14 @@ class CurateData: try: with open('target_dirs.txt', 'r+') as f: # TODO you can add option to change directory here, too. Look up how to have optional arguments target_dir = json.load(f) + except (ValueError, FileNotFoundError): target_dir = input('No target dirctory found. Create One? [y] or [n]:') if target_dir == ('y' or 'Y'): target_dir = input('Please provide full URL to destination folder:') # TODO need to catch human syntax errors here with open('target_dirs.txt','w') as f: json.dump(target_dir, f) + else: os.mkdir(os.getcwd()+os.sep+'training_images') target_dir = os.getcwd()+os.sep+'training_images' @@ -434,17 +474,22 @@ class CurateData: json.dump(target_dir, f) print('Creating default folder in current directory @ ' + target_dir) + # open url list in working directory with open('temp_pics_source_list.txt') as f: + try: temp_pics_source_list = json.load(f) + except (ValueError, FileNotFoundError): print('url list not found. aborting') return dict_pics = {} + + # make custom dict, {source:target}, and name images from unique URL patt for k in temp_pics_source_list: - patt_1 = re.search(r'[^/]+(?=/\$_|.(.jpg|.jpeg|.png))', k, re.IGNORECASE) - patt_2 = re.search(r'(.jpg|.jpeg|.png)', k, re.IGNORECASE) + patt_1 = re.search(r'[^/]+(?=/\$_|.(\.jpg|\.jpeg|\.png))', k, re.IGNORECASE) + patt_2 = re.search(r'(\.jpg|\.jpeg|\.png)', k, re.IGNORECASE) if patt_1 and patt_2 is not None: tag = patt_1.group() + patt_2.group().lower() file_name = target_dir + os.sep + tag @@ -453,7 +498,7 @@ class CurateData: with open('dict_pics.txt', 'w') as f: json.dump(dict_pics, f) - return dict_pics # TODO still need to find sol to outliers (i.e., naming scheme for unusual source URLs) + return dict_pics # TODO still need to find sol to outliers (aka, naming scheme for unusual source URLs) def dl_pictures(self, *dict_pics): ''' @@ -462,8 +507,8 @@ class CurateData: ''' if not dict_pics: - with open('dict_pics.txt') as f: - dict_pics = json.load(f) + dict_pics = self.dict_pics() + with open('temp_pics_source_list.txt') as f: try: temp_pics_source_list = json.load(f) diff --git a/oauth_fetch.py b/oauth_fetch.py deleted file mode 100644 index bdc91ed..0000000 --- a/oauth_fetch.py +++ /dev/null @@ -1,6 +0,0 @@ -import json -import config as cfg -import ebay_api -import requests - -req_endpoint = "https://api.ebay.com/identity/v1/oauth2/token"