jsonp attempts

This commit is contained in:
spbeach46
2023-11-25 11:44:31 -07:00
parent d90f87efe3
commit 3462cd3101
23 changed files with 37259 additions and 7 deletions
@@ -0,0 +1 @@
["https://api.chess.com/pub/player/carlosattack/games/2012/10", "https://api.chess.com/pub/player/carlosattack/games/2013/01", "https://api.chess.com/pub/player/carlosattack/games/2013/02", "https://api.chess.com/pub/player/carlosattack/games/2013/03", "https://api.chess.com/pub/player/carlosattack/games/2013/04", "https://api.chess.com/pub/player/carlosattack/games/2013/07", "https://api.chess.com/pub/player/carlosattack/games/2014/08", "https://api.chess.com/pub/player/carlosattack/games/2015/09", "https://api.chess.com/pub/player/carlosattack/games/2015/10", "https://api.chess.com/pub/player/carlosattack/games/2015/11", "https://api.chess.com/pub/player/carlosattack/games/2015/12", "https://api.chess.com/pub/player/carlosattack/games/2016/01", "https://api.chess.com/pub/player/carlosattack/games/2016/04", "https://api.chess.com/pub/player/carlosattack/games/2016/08", "https://api.chess.com/pub/player/carlosattack/games/2016/10", "https://api.chess.com/pub/player/carlosattack/games/2017/10", "https://api.chess.com/pub/player/carlosattack/games/2018/02", "https://api.chess.com/pub/player/carlosattack/games/2018/03", "https://api.chess.com/pub/player/carlosattack/games/2018/04", "https://api.chess.com/pub/player/carlosattack/games/2019/02", "https://api.chess.com/pub/player/carlosattack/games/2019/06", "https://api.chess.com/pub/player/carlosattack/games/2019/07", "https://api.chess.com/pub/player/carlosattack/games/2019/08", "https://api.chess.com/pub/player/carlosattack/games/2019/09", "https://api.chess.com/pub/player/carlosattack/games/2019/10", "https://api.chess.com/pub/player/carlosattack/games/2019/11", "https://api.chess.com/pub/player/carlosattack/games/2019/12", "https://api.chess.com/pub/player/carlosattack/games/2020/02", "https://api.chess.com/pub/player/carlosattack/games/2020/03", "https://api.chess.com/pub/player/carlosattack/games/2020/04", "https://api.chess.com/pub/player/carlosattack/games/2020/05", "https://api.chess.com/pub/player/carlosattack/games/2020/06", "https://api.chess.com/pub/player/carlosattack/games/2020/07", "https://api.chess.com/pub/player/carlosattack/games/2020/08", "https://api.chess.com/pub/player/carlosattack/games/2020/09", "https://api.chess.com/pub/player/carlosattack/games/2020/10", "https://api.chess.com/pub/player/carlosattack/games/2020/11", "https://api.chess.com/pub/player/carlosattack/games/2020/12", "https://api.chess.com/pub/player/carlosattack/games/2021/01", "https://api.chess.com/pub/player/carlosattack/games/2021/02", "https://api.chess.com/pub/player/carlosattack/games/2021/03", "https://api.chess.com/pub/player/carlosattack/games/2021/04", "https://api.chess.com/pub/player/carlosattack/games/2021/05", "https://api.chess.com/pub/player/carlosattack/games/2021/06", "https://api.chess.com/pub/player/carlosattack/games/2021/07", "https://api.chess.com/pub/player/carlosattack/games/2021/08", "https://api.chess.com/pub/player/carlosattack/games/2021/09", "https://api.chess.com/pub/player/carlosattack/games/2021/10", "https://api.chess.com/pub/player/carlosattack/games/2021/11", "https://api.chess.com/pub/player/carlosattack/games/2021/12", "https://api.chess.com/pub/player/carlosattack/games/2022/01", "https://api.chess.com/pub/player/carlosattack/games/2022/02", "https://api.chess.com/pub/player/carlosattack/games/2022/03", "https://api.chess.com/pub/player/carlosattack/games/2022/04", "https://api.chess.com/pub/player/carlosattack/games/2022/05", "https://api.chess.com/pub/player/carlosattack/games/2022/06", "https://api.chess.com/pub/player/carlosattack/games/2022/07", "https://api.chess.com/pub/player/carlosattack/games/2022/08", "https://api.chess.com/pub/player/carlosattack/games/2022/09", "https://api.chess.com/pub/player/carlosattack/games/2023/01", "https://api.chess.com/pub/player/carlosattack/games/2023/02", "https://api.chess.com/pub/player/carlosattack/games/2023/03", "https://api.chess.com/pub/player/carlosattack/games/2023/04", "https://api.chess.com/pub/player/carlosattack/games/2023/05", "https://api.chess.com/pub/player/carlosattack/games/2023/06", "https://api.chess.com/pub/player/carlosattack/games/2023/07"]
+1
View File
@@ -0,0 +1 @@
["https://api.chess.com/pub/player/hippodrunkimus/games/2023/03", "https://api.chess.com/pub/player/hippodrunkimus/games/2023/04", "https://api.chess.com/pub/player/hippodrunkimus/games/2023/05", "https://api.chess.com/pub/player/hippodrunkimus/games/2023/06", "https://api.chess.com/pub/player/hippodrunkimus/games/2023/07"]
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+73
View File
@@ -0,0 +1,73 @@
import user_agents
import random
import requests
import re
import os
import json
import concurrent.futures
def playerArchives(username):
# Create archive list. This is a list of pages containing lists of games used for futher downloading
user_agent = random.choice(user_agents.user_agents) # load up random user agent from list
headers = {
'User-Agent':user_agent
}
url = f"https://api.chess.com/pub/player/{username}/games/archives"
archive = requests.get(url, headers=headers).json()['archives']
try:
cwd = os.getcwd()
path = os.path.join(cwd, 'archives')
os.makedirs(path)
except OSError:
pass
file_name = os.path.join(path, username+'_archive.txt')
with open(file_name, 'w') as f:
json.dump(archive, f)
return archive
def playerMonthly(url=None):
user_agent = random.choice(user_agents.user_agents) # load up random user agent from list
headers = {
'User-Agent':user_agent
}
if url:
url=url+'/pgn' # connect to multi-game pgn download endpoint by appending a "pgn" to the url provided by url in archive list
else:
username=input("username: ")
YYYY=input("year in YYYY: ")
MM=input("month in MM: ")
url = f"https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM}/pgn"
# get and save games list in .pgn format
games = requests.get(url, headers=headers).content.decode("utf-8")
return games
# Multithreaded games download
def threddGames(username=None, archive=None):
if archive:
with open(archive) as f:
archive = json.load(f)
else:
archive = playerArchives(username)
# async download games
games_list = []
with concurrent.futures.ThreadPoolExecutor() as executor:
for future in executor.map(playerMonthly, archive):
games_list.extend(future)
pgn_db = "\n\n".join(games_list)
with open(username+'.pgn', 'w') as f:
f.write(pgn_db)
return pgn_db
+8
View File
@@ -0,0 +1,8 @@
user_agents = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35",
"Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/113.0",
"Mozilla/5.0 (Android 12; Mobile; rv:109.0) Gecko/113.0 Firefox/113.0",
"mozilla/5.0 (macintosh; intel mac os x 10.15; rv:109.0) gecko/20100101 firefox/113.0",
]