working on curate_data to remove formatting issues in csv

This commit is contained in:
spbeach46 2020-12-29 01:55:21 -07:00
parent dcbdb7ced3
commit b4e1961ace

View File

@ -106,12 +106,14 @@ class ShoppingApi:
For some reason item_id_results can only be passed as argument in executor.map
if the variable is made within function
'''
data = {} # TODO I think you need to append a list of dictionaries rather than update a dictionary of dictionaries. Training var will require an updated dictionary though
data = [] # TODO I think you need to append a list of dictionaries rather than update a dictionary of dictionaries. Training var will require an updated dictionary though
finding = FindingApi(4, 2)
item_id_results = finding.get_ids_from_cats()
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(self.get_item_from_findItemsByCategory, item_id_results):
print(future)
# print(future)
for item in future:
data.append(item) # The end result should be a list of dicts where each dict in the list is a listing
# data.update(future)
return data # TODO each future is a list of dictionaries because the output of any multithreader in this method is a list.
# data dictionary can't update from list of dicts unless iterated over. Might need a different way to update.
@ -145,10 +147,10 @@ class CurateData:
def extract_nvl(self, item):
names = []
values = []
nvl = item['itemspecifics']['namevaluelist']
nvl = item['ItemSpecifics']['NameValueList']
for nvl_dict in nvl:
names.append(nvl_dict['name'])
values.append(nvl_dict['value'])
names.append(nvl_dict['Name'])
values.append(nvl_dict['Value'])
nvl_dict = dict(zip(names, values))
return nvl_dict
@ -179,8 +181,9 @@ class CurateData:
nvl_dict = self.extract_nvl(item)
training.update(nvl_dict)
df = pd.json_normalize(training) # TODO FIX INDENT HERE?
df.to_csv('training.csv', mode='a')
df = pd.json_normalize(training) # TODO FIX INDENT HERE?
df.to_csv('training.csv', mode='a')
return training
def main():
'''
@ -197,6 +200,7 @@ def main():
data = shopping.conky()
curate = CurateData()
curate.update_df(data)
return data
if __name__ == "__main__":
main()