diff --git a/player_games.py b/player_games.py index f780af2..b584a68 100644 --- a/player_games.py +++ b/player_games.py @@ -1,74 +1,59 @@ +import user_agents +import random import requests import re import os import json import concurrent.futures + def playerArchives(username): # Create archive list. This is a list of pages containing lists of games used for futher downloading - url = "https://api.chess.com/pub/player/{}/games/archives".format(username) - archive = requests.get(url).json()['archives'] - file_path = '/home/unknown/Documents/projects/chess/archives' #TODO come up with general saving scheme - # TODO maybe path should be getcwd() instead? + user_agent = random.choice(user_agents.user_agents) # load up random user agent from list + headers = { + 'User-Agent':user_agent + } + url = f"https://api.chess.com/pub/player/{username}/games/archives" + archive = requests.get(url, headers=headers).json()['archives'] - file_name = os.path.join(file_path, username+'_archive.txt') + try: + cwd = os.getcwd() + path = os.path.join(cwd, 'archives') + os.makedirs(path) + except OSError: + pass + + file_name = os.path.join(path, username+'_archive.txt') with open(file_name, 'w') as f: json.dump(archive, f) return archive -# Provide a URL to the archive including from the above function, playerArchives() -# or else manually provide chess.com's required GET request parameters +def playerMonthly(url=None): -# SAVE NESTED PGN TO PGN DATABASE FILE: + user_agent = random.choice(user_agents.user_agents) # load up random user agent from list + headers = { + 'User-Agent':user_agent + } -#pgn_games = [] -#with open('username-database.pgn', 'w') as f: -# for game in pgn_games: -# output.write(game) -# output.write("\n\n") -# - -def playerMonthly(url=None): #TODO you now need to provide username and email in for the user agent: - # check https://www.chess.com/clubs/forum/view/error-403-in-member-profile?page=2 - # session = requests.session() - # session.headers["User-Agent"] = "username:hipposcottimus, email:spbeach46@gmail.com" - # session.get(url).json() if url: - url=url + url=url+'/pgn' # connect to multi-game pgn download endpoint by appending a "pgn" to the url provided by url in archive list else: username=input("username: ") YYYY=input("year in YYYY: ") MM=input("month in MM: ") - url = "https://api.chess.com/pub/player/{}/games/{YYYY}/{MM}".format(username, YYYY, MM) + url = f"https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM}/pgn" # get and save games list in .pgn format - data = requests.get(url)#.json() - file_path = '/home/unknown/Documents/projects/chess/games' - -# for game in data['games']: # TODO just append each game to a single text file somehow and save as pgn. Can't load as pgn from json/dict -# uuid = game['uuid'] -# filename = os.path.join(file_path, uuid+".pgn") -# with open(filename, 'w') as f: -# f.write(game['pgn']) # writes a single game to .pgn format - return data -# return games_list + games = requests.get(url, headers=headers).content.decode("utf-8") + return games # Multithreaded games download - def threddGames(username=None, archive=None): - path = '/home/unknown/Documents/projects/chess/games/{}'.format(username) - # TODO maybe path should be getcwd() instead? - - try: - os.makedirs(path) - except OSError: - pass - if archive: with open(archive) as f: archive = json.load(f) @@ -76,10 +61,13 @@ def threddGames(username=None, archive=None): archive = playerArchives(username) # async download games + games_list = [] with concurrent.futures.ThreadPoolExecutor() as executor: for future in executor.map(playerMonthly, archive): - future #TODO incomplete as is. + games_list.extend(future) + pgn_db = "\n\n".join(games_list) + with open(username+'.pgn', 'w') as f: + f.write(pgn_db) + + return pgn_db -# TODO Gameplan: use playerArchives to produce list, concurrent futures to send out multithreaded requests, -# either append to another empty list or write directly to a file immediately after download and parse -# vars in concurrent.futures: url_list from playerArchives, response_to_parse, empty diff --git a/user_agents.py b/user_agents.py new file mode 100644 index 0000000..b3b779c --- /dev/null +++ b/user_agents.py @@ -0,0 +1,8 @@ +user_agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35", + "Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/113.0", + "Mozilla/5.0 (Android 12; Mobile; rv:109.0) Gecko/113.0 Firefox/113.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0", +]