import requests import re import os import json import concurrent.futures def playerArchives(username): # Create archive list. This is a list of pages containing lists of games used for futher downloading url = "https://api.chess.com/pub/player/{}/games/archives".format(username) archive = requests.get(url).json()['archives'] file_path = '/home/unknown/Documents/projects/chess/archives' #TODO come up with general saving scheme # TODO maybe path should be getcwd() instead? file_name = os.path.join(file_path, username+'_archive.txt') with open(file_name, 'w') as f: json.dump(archive, f) return archive # Provide a URL to the archive including from the above function, playerArchives() # or else manually provide chess.com's required GET request parameters # SAVE NESTED PGN TO PGN DATABASE FILE: #pgn_games = [] #with open('username-database.pgn', 'w') as f: # for game in pgn_games: # output.write(game) # output.write("\n\n") # def playerMonthly(url=None): #TODO you now need to provide username and email in for the user agent: # check https://www.chess.com/clubs/forum/view/error-403-in-member-profile?page=2 # session = requests.session() # session.headers["User-Agent"] = "username:hipposcottimus, email:spbeach46@gmail.com" # session.get(url).json() if url: url=url else: username=input("username: ") YYYY=input("year in YYYY: ") MM=input("month in MM: ") url = "https://api.chess.com/pub/player/{}/games/{YYYY}/{MM}".format(username, YYYY, MM) # get and save games list in .pgn format data = requests.get(url)#.json() file_path = '/home/unknown/Documents/projects/chess/games' # for game in data['games']: # TODO just append each game to a single text file somehow and save as pgn. Can't load as pgn from json/dict # uuid = game['uuid'] # filename = os.path.join(file_path, uuid+".pgn") # with open(filename, 'w') as f: # f.write(game['pgn']) # writes a single game to .pgn format return data # return games_list # Multithreaded games download def threddGames(username=None, archive=None): path = '/home/unknown/Documents/projects/chess/games/{}'.format(username) # TODO maybe path should be getcwd() instead? try: os.makedirs(path) except OSError: pass if archive: with open(archive) as f: archive = json.load(f) else: archive = playerArchives(username) # async download games with concurrent.futures.ThreadPoolExecutor() as executor: for future in executor.map(playerMonthly, archive): future #TODO incomplete as is. # TODO Gameplan: use playerArchives to produce list, concurrent futures to send out multithreaded requests, # either append to another empty list or write directly to a file immediately after download and parse # vars in concurrent.futures: url_list from playerArchives, response_to_parse, empty