86 lines
3.0 KiB
Python
86 lines
3.0 KiB
Python
import requests
|
|
import re
|
|
import os
|
|
import json
|
|
import concurrent.futures
|
|
|
|
def playerArchives(username):
|
|
# Create archive list. This is a list of pages containing lists of games used for futher downloading
|
|
url = "https://api.chess.com/pub/player/{}/games/archives".format(username)
|
|
archive = requests.get(url).json()['archives']
|
|
|
|
file_path = '/home/unknown/Documents/projects/chess/archives' #TODO come up with general saving scheme
|
|
# TODO maybe path should be getcwd() instead?
|
|
|
|
file_name = os.path.join(file_path, username+'_archive.txt')
|
|
with open(file_name, 'w') as f:
|
|
json.dump(archive, f)
|
|
|
|
return archive
|
|
|
|
|
|
# Provide a URL to the archive including from the above function, playerArchives()
|
|
# or else manually provide chess.com's required GET request parameters
|
|
|
|
# SAVE NESTED PGN TO PGN DATABASE FILE:
|
|
|
|
#pgn_games = []
|
|
#with open('username-database.pgn', 'w') as f:
|
|
# for game in pgn_games:
|
|
# output.write(game)
|
|
# output.write("\n\n")
|
|
#
|
|
|
|
def playerMonthly(url=None): #TODO you now need to provide username and email in for the user agent:
|
|
# check https://www.chess.com/clubs/forum/view/error-403-in-member-profile?page=2
|
|
# session = requests.session()
|
|
# session.headers["User-Agent"] = "username:hipposcottimus, email:spbeach46@gmail.com"
|
|
# session.get(url).json()
|
|
if url:
|
|
url=url
|
|
else:
|
|
username=input("username: ")
|
|
YYYY=input("year in YYYY: ")
|
|
MM=input("month in MM: ")
|
|
url = "https://api.chess.com/pub/player/{}/games/{YYYY}/{MM}".format(username, YYYY, MM)
|
|
|
|
# get and save games list in .pgn format
|
|
data = requests.get(url)#.json()
|
|
file_path = '/home/unknown/Documents/projects/chess/games'
|
|
|
|
# for game in data['games']: # TODO just append each game to a single text file somehow and save as pgn. Can't load as pgn from json/dict
|
|
# uuid = game['uuid']
|
|
# filename = os.path.join(file_path, uuid+".pgn")
|
|
# with open(filename, 'w') as f:
|
|
# f.write(game['pgn']) # writes a single game to .pgn format
|
|
return data
|
|
# return games_list
|
|
|
|
|
|
# Multithreaded games download
|
|
|
|
def threddGames(username=None, archive=None):
|
|
|
|
path = '/home/unknown/Documents/projects/chess/games/{}'.format(username)
|
|
# TODO maybe path should be getcwd() instead?
|
|
|
|
try:
|
|
os.makedirs(path)
|
|
except OSError:
|
|
pass
|
|
|
|
if archive:
|
|
with open(archive) as f:
|
|
archive = json.load(f)
|
|
else:
|
|
archive = playerArchives(username)
|
|
|
|
# async download games
|
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
for future in executor.map(playerMonthly, archive):
|
|
future #TODO incomplete as is.
|
|
|
|
# TODO Gameplan: use playerArchives to produce list, concurrent futures to send out multithreaded requests,
|
|
# either append to another empty list or write directly to a file immediately after download and parse
|
|
# vars in concurrent.futures: url_list from playerArchives, response_to_parse, empty
|