working on curate_data to remove formatting issues in csv
This commit is contained in:
parent
dcbdb7ced3
commit
b4e1961ace
18
ebay_api.py
18
ebay_api.py
@ -106,12 +106,14 @@ class ShoppingApi:
|
||||
For some reason item_id_results can only be passed as argument in executor.map
|
||||
if the variable is made within function
|
||||
'''
|
||||
data = {} # TODO I think you need to append a list of dictionaries rather than update a dictionary of dictionaries. Training var will require an updated dictionary though
|
||||
data = [] # TODO I think you need to append a list of dictionaries rather than update a dictionary of dictionaries. Training var will require an updated dictionary though
|
||||
finding = FindingApi(4, 2)
|
||||
item_id_results = finding.get_ids_from_cats()
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results):
|
||||
print(future)
|
||||
# print(future)
|
||||
for item in future:
|
||||
data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
|
||||
# data.update(future)
|
||||
return data # TODO each future is a list of dictionaries because the output of any multithreader in this method is a list.
|
||||
# data dictionary can't update from list of dicts unless iterated over. Might need a different way to update.
|
||||
@ -145,10 +147,10 @@ class CurateData:
|
||||
def extract_nvl(self, item):
|
||||
names = []
|
||||
values = []
|
||||
nvl = item['itemspecifics']['namevaluelist']
|
||||
nvl = item['ItemSpecifics']['NameValueList']
|
||||
for nvl_dict in nvl:
|
||||
names.append(nvl_dict['name'])
|
||||
values.append(nvl_dict['value'])
|
||||
names.append(nvl_dict['Name'])
|
||||
values.append(nvl_dict['Value'])
|
||||
nvl_dict = dict(zip(names, values))
|
||||
return nvl_dict
|
||||
|
||||
@ -179,8 +181,9 @@ class CurateData:
|
||||
nvl_dict = self.extract_nvl(item)
|
||||
training.update(nvl_dict)
|
||||
|
||||
df = pd.json_normalize(training) # TODO FIX INDENT HERE?
|
||||
df.to_csv('training.csv', mode='a')
|
||||
df = pd.json_normalize(training) # TODO FIX INDENT HERE?
|
||||
df.to_csv('training.csv', mode='a')
|
||||
return training
|
||||
|
||||
def main():
|
||||
'''
|
||||
@ -197,6 +200,7 @@ def main():
|
||||
data = shopping.conky()
|
||||
curate = CurateData()
|
||||
curate.update_df(data)
|
||||
return data
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
Loading…
Reference in New Issue
Block a user