scott/ebay-ml-lister
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed more dl_pictures bugs

Browse Source
This commit is contained in:
spbeach46 2021-04-20 21:09:35 -07:00
parent 5532a82d8d
commit 2f99c63165
1 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
ebay_api.py
View File

@ -1,4 +1,5 @@
import importlib
import pdb
import os
import numpy as np
import concurrent.futures
@ -293,33 +294,36 @@ class CurateData:
and creates custom {source:target} dictionary as dict_pics
'''
print('shitballs')
with open('target_dirs.txt', 'w+') as f: # TODO you can add option to change directory here, too. Look up how to have optional arguments
try:
try:
with open('target_dirs.txt', 'r+') as f: # TODO you can add option to change directory here, too. Look up how to have optional arguments
target_dir = json.load(f)
except ValueError:
target_dir = input('No target directory found. Create One? [y] or [n]:')
if target_dir == 'y' or 'Y':
target_dir = input('Please provide full URL to destination folder:')
else:
target_dir = os.getcwd()
except (ValueError, FileNotFoundError): # TODO doesn't seem to create target_dir from cwd. I think it's because ValueError should be FileNotFoundError?
target_dir = input('No target dirctory found. Create One? [y] or [n]:')
if target_dir == ('y' or 'Y'):
target_dir = input('Please provide full URL to destination folder:')
with open('target_dirs.txt','w+') as f:
json.dump(target_dir, f)
print('Creating default folder in current directory, ' + target_dir)
else:
target_dir = os.getcwd()
with open('target_dirs.txt','w+') as f:
json.dump(target_dir, f)
print('Creating default folder in current directory, ' + target_dir) # TODO does not concat correctly. Fix this
with open('temp_pics_source_list.txt') as f:
try:
if args:
temp_pics_source_list = args
temp_pics_source_list = args[0]
else:
temp_pics_source_list = json.load(f)
except ValueError:
if args:
temp_pics_sources_list = args
temp_pics_sources_list = args[0]
else:
print('url list not found. download aborted')
return
breakpoint()
temp_dict_pics = {k:target_dir+re.search(r'[^/]+(?=/\$_|.jpg)', k, re.IGNORECASE).group()+'.jpg' for k in temp_pics_source_list}
with open('dict_pics.txt', 'w') as f:
@ -330,9 +334,11 @@ class CurateData:
except ValueError:
json.dump(temp_dict_pics, f)
dict_pics = temp_dict_pics
def dl_pic(pic,dict_pics):
breakpoint()
if os.path.exists(dict_pics[pic]): # or call temp_dict_pics[pic] can work
pass
@ -342,8 +348,9 @@ class CurateData:
with open(temp_dict_pics[pic], 'wb') as f: # Or call dict_pics[pic] can work
shutil.copyfileobj(r.raw, f)
bargs = [(dict_pics, temp_dict_pics) for pic in temp_dict_pics]
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(dl_pic, temp_pics_source_list):
for future in executor.map(lambda p: dl_pic(*p), bargs):
future
with open('temp_pics_source_list.txt','w') as f: # Overwrites old when complete