diff --git a/api/love3.py b/api/love3.py index bc7543c..eff6e95 100644 --- a/api/love3.py +++ b/api/love3.py @@ -8,11 +8,12 @@ from Cryptodome.Cipher import AES import Cryptodome.Cipher.AES from binascii import hexlify, unhexlify -import requests, io, math, json, os +import asyncio, io, math, json, os from PIL import Image +from server import client -def getTitleData(titleID:hex): - data = requests.get('https://idbe-ctr.cdn.nintendo.net/icondata/10/%s.idbe' % titleID.zfill(16), verify = False).content +async def getTitleData(titleID:hex): + data = (await client.get('https://idbe-ctr.cdn.nintendo.net/icondata/10/%s.idbe' % titleID.zfill(16), verify = False)).content IV = unhexlify('A46987AE47D82BB4FA8ABC0450285FA4') @@ -29,7 +30,7 @@ def getTitleData(titleID:hex): return decipher.decrypt(data[2:]) -def getTitleInfo(titleID:hex): +async def getTitleInfo(titleID:hex): titleID = str(titleID) try: int(titleID, 16) # Errors if not HEX @@ -48,7 +49,7 @@ def getTitleInfo(titleID:hex): return json.loads(file.read()) try: - data = getTitleData(titleID) + data = await getTitleData(titleID) except: with open('cache/homebrew' + titleID + '.txt', 'w') as file: file.write('') diff --git a/api/util.py b/api/util.py index 6d480db..50c002b 100644 --- a/api/util.py +++ b/api/util.py @@ -30,7 +30,7 @@ def __init__(self, width:int = terminalSize): self.progress = 0 self.close = True - def update(self, fraction:float): + def update(self, fraction:float): # TODO: Async this fraction = int(fraction * self.width) self.progress += fraction def loop(self): @@ -43,7 +43,7 @@ def loop(self): self.close = True threading.Thread(target = loop, args = (self,)).start() - def end(self): # Can take up time on main thread to finish + def end(self): # Can take up time on main thread to finish TODO: Async this for n in range(self.width - self.progress): sys.stdout.write('#') sys.stdout.flush() @@ -53,7 +53,7 @@ def end(self): # Can take up time on main thread to finish sys.stdout.write(']\n') # Get image url from title ID -def getTitle(titleID, titlesToUID, titleDatabase): +async def getTitle(titleID, titlesToUID, titleDatabase): _pass = None uid = None @@ -104,7 +104,7 @@ def getTitle(titleID, titlesToUID, titleDatabase): game[key] = _template[key] if game == _template: - response = getTitleInfo(titleID) + response = await getTitleInfo(titleID) if response: game['name'] = response['short'] game['publisher']['name'] = response['publisher'] diff --git a/backend.py b/backend.py index e60c814..b72afbd 100644 --- a/backend.py +++ b/backend.py @@ -47,7 +47,7 @@ async def main(): session = Session(engine) while True: - time.sleep(1) + await asyncio.sleep(1) print('Grabbing new friends...') queried_friends = session.scalars(select(Friend).where(Friend.network == network)).all() @@ -104,7 +104,7 @@ async def main(): except Exception as e: print('An error occurred!\n%s' % e) print(traceback.format_exc()) - time.sleep(2) + await asyncio.sleep(2) if scrape_only: print('Done scraping.') @@ -114,7 +114,7 @@ async def main(): async def main_friends_loop(friends_client: friends.FriendsClientV1, session: Session, current_rotation: list[QueriedFriend]): # If we recently started, update our comment, and remove existing friends. if time.time() - backend_start_time < 30: - time.sleep(delay) + await asyncio.sleep(delay) await friends_client.update_comment('3dsrpc.com') # Synchronize our current roster of friends. @@ -129,12 +129,12 @@ async def main_friends_loop(friends_client: friends.FriendsClientV1, session: Se # Clear our current, registered friends. removables = await friends_client.get_all_friends() for friend in removables: - time.sleep(delay) + await asyncio.sleep(delay) await friends_client.remove_friend_by_principal_id(friend.pid) # Individually add all pending friend PIDs. for friend_pid in all_friend_pids: - time.sleep(delay) + await asyncio.sleep(delay) await friends_client.add_friend_by_principal_id(0, friend_pid) else: # We expect the remote NEX implementation to remove all existing @@ -142,7 +142,7 @@ async def main_friends_loop(friends_client: friends.FriendsClientV1, session: Se # This path is currently only for Nintendo. await friends_client.sync_friend(0, all_friend_pids, []) - time.sleep(delay) + await asyncio.sleep(delay) # Query all successful friends. current_friends_list: [friends.FriendRelationship] = await friends_client.get_all_friends() @@ -170,7 +170,7 @@ async def main_friends_loop(friends_client: friends.FriendsClientV1, session: Se # All of our friends removed us, so there's no more work to be done. return - time.sleep(delay) + await asyncio.sleep(delay) # Query the presences of all of our added friends. # Only online users will have their presence returned. @@ -229,7 +229,7 @@ async def main_friends_loop(friends_client: friends.FriendsClientV1, session: Se if not work: continue - time.sleep(delay) + await asyncio.sleep(delay) try: current_info = await friends_client.get_friend_persistent_info([current_friend.pid,]) diff --git a/discord.py b/discord.py index d8cf6a8..933673a 100644 --- a/discord.py +++ b/discord.py @@ -1,3 +1,4 @@ +import asyncio import sys, pickle from typing import Optional @@ -7,17 +8,17 @@ from sqlalchemy import create_engine, select, update, delete from sqlalchemy.orm import Session -from database import get_db_url, DiscordFriends, Friend +from database import get_db_url, DiscordFriends, Friend, Discord from database import Discord as DiscordTable from dataclasses import dataclass -from requests.exceptions import HTTPError +from httpx import HTTPStatusError API_ENDPOINT: str = 'https://discord.com/api/v10' with open('./cache/databases.dat', 'rb') as file: - t = pickle.loads(file.read()) - titleDatabase = t[0] - titlesToUID = t[1] + t = pickle.loads(file.read()) + titleDatabase = t[0] + titlesToUID = t[1] engine = create_engine(get_db_url()) @@ -25,293 +26,306 @@ @dataclass class UserData: - """Represents information about the current Discord user's game.""" - friend_code: str - online: bool - game: dict - game_description: str - username: str - mii_urls: Optional[dict] - last_accessed: int + """Represents information about the current Discord user's game.""" + friend_code: str + online: bool + game: dict + game_description: str + username: str + mii_urls: Optional[dict] + last_accessed: int class DiscordSession: - def retire(self, refresh_token: str): - session.execute( - update(DiscordTable) - .where(DiscordTable.refresh_token == refresh_token) - .values( - rpc_session_token=None, - last_accessed=time.time() - ) - ) - session.commit() - - def create(self, refresh_token: str, session_token: Optional[str]): - session.execute( - update(DiscordTable) - .where(DiscordTable.refresh_token == refresh_token) - .values( - rpc_session_token=session_token, - last_accessed=time.time() - ) - ) - session.commit() + def retire(self, refresh_token: str): + session.execute( + update(DiscordTable) + .where(DiscordTable.refresh_token == refresh_token) + .values( + rpc_session_token=None, + last_accessed=time.time() + ) + ) + session.commit() + + def create(self, refresh_token: str, session_token: Optional[str]): + session.execute( + update(DiscordTable) + .where(DiscordTable.refresh_token == refresh_token) + .values( + rpc_session_token=session_token, + last_accessed=time.time() + ) + ) + session.commit() class APIClient: - current_user: DiscordTable - - def __init__(self, current_user: DiscordTable): - self.current_user = current_user - - - def update_presence(self, user_data: UserData, network: NetworkType): - last_accessed = user_data.last_accessed - if time.time() - last_accessed <= 30: - print('[MANUAL RATE LIMITED]') - return False - - game = user_data.game - - # This ends up in an array of activities - see `data` below. - activity_data = { - 'type': 0, - 'application_id': CLIENT_ID, - 'assets': {}, - 'name': game['name'] + ' (3DS)', - 'platform': 'desktop' - } - - if game['icon_url']: - activity_data['assets']['large_image'] = game['icon_url'].replace('/cdn/', HOST + '/cdn/') - activity_data['assets']['large_text'] = game['name'] - if user_data.game_description: - activity_data['details'] = user_data.game_description - - # Only add a profile button if the user has enabled it. - if user_data.username and self.current_user.show_profile_button: - profile_url = HOST + '/user/' + user_data.friend_code + '/?network=' + network.lower_name() - activity_data['buttons'] = [{ - 'label': 'Profile', - 'url': profile_url - }] - - # Similarly, only show the user's Mii if enabled. - if user_data.username and game['icon_url'] and self.current_user.show_small_image: - # Format as a human-readable friend code (XXXX-XXXX-XXXX). - user_friend_code = '-'.join(user_data.friend_code[i:i+4] for i in range(0, 12, 4)) - user_network_name = network.lower_name().capitalize() - small_text_detail = f"{user_friend_code} on {user_network_name}" - - activity_data['assets']['small_image'] = user_data.mii_urls['face'] - activity_data['assets']['small_text'] = small_text_detail - - # Quickly sanitize our activity data by truncating - # any text exceeding the maximum field limit, 128 characters. - for key_name in list(activity_data): - # However, don't modify image assets as they can go over 128. - if 'image' in key_name: - continue - - if isinstance(activity_data[key_name], str): - if len(activity_data[key_name]) > 128: - activity_data[key_name] = activity_data[key_name][:128] - - data = {'activities': [activity_data]} - if discord_user.rpc_session_token: - data['token'] = discord_user.rpc_session_token - - headers = { - 'Authorization': 'Bearer %s' % self.current_user.bearer_token, - 'Content-Type': 'application/json', - } - - r = requests.post('%s/users/@me/headless-sessions' % API_ENDPOINT, data=json.dumps(data), headers=headers) - r.raise_for_status() - - response = r.json() - DiscordSession().create(self.current_user.refresh_token, response['token']) - - - def reset_presence(self): - if not self.current_user.rpc_session_token: - print('[NO SESSION TO RESET]') - return False - elif time.time() - self.current_user.last_accessed <= 30: - print('[MANUAL RATE LIMITED]') - return False - - headers = { - 'Authorization': 'Bearer %s' % self.current_user.bearer_token, - 'Content-Type': 'application/json', - } - data = { - 'token': self.current_user.rpc_session_token, - } - r = requests.post('%s/users/@me/headless-sessions/delete' % API_ENDPOINT, data=json.dumps(data), headers=headers) - - try: - r.raise_for_status() - except HTTPError as e: - # If we encounter 400, we assume that this session has already expired. - # Let's go ahead and reset the session anyway. - if e.response.status_code == 400: - DiscordSession().retire(self.current_user.refresh_token) - else: - raise e - - - def refresh_bearer(self): - print('[REFRESH BEARER %s]' % self.current_user.id) - current_refresh_token = self.current_user.refresh_token - data = { - 'client_id': '%s' % CLIENT_ID, - 'client_secret': '%s' % CLIENT_SECRET, - 'grant_type': 'refresh_token', - 'refresh_token': current_refresh_token, - } - headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - } - json_response = requests.post('%s/oauth2/token' % API_ENDPOINT, data=data, headers=headers) - json_response.raise_for_status() - response = json_response.json() - - session.execute( - update(DiscordTable) - .where(DiscordTable.refresh_token == current_refresh_token) - .values( - refresh_token=response['refresh_token'], - bearer_token=response['access_token'], - generation_date=time.time() - ) - ) - session.commit() - - - def delete_discord_user(self): - user_id = self.current_user.id - print('[DELETING %s]' % user_id) - session.execute(delete(DiscordTable).where(DiscordTable.id == user_id)) - session.execute(delete(DiscordFriends).where(DiscordFriends.id == user_id)) - session.commit() + current_user: DiscordTable + + def __init__(self, current_user: DiscordTable): + self.current_user = current_user + + + async def update_presence(self, user_data: UserData, network: NetworkType): + last_accessed = user_data.last_accessed + if time.time() - last_accessed <= 30: + print('[MANUAL RATE LIMITED]') + return False + + game = user_data.game + + # This ends up in an array of activities - see `data` below. + activity_data = { + 'type': 0, + 'application_id': CLIENT_ID, + 'assets': {}, + 'name': game['name'] + ' (3DS)', + 'platform': 'desktop' + } + + if game['icon_url']: + activity_data['assets']['large_image'] = game['icon_url'].replace('/cdn/', HOST + '/cdn/') + activity_data['assets']['large_text'] = game['name'] + if user_data.game_description: + activity_data['details'] = user_data.game_description + + # Only add a profile button if the user has enabled it. + if user_data.username and self.current_user.show_profile_button: + profile_url = HOST + '/user/' + user_data.friend_code + '/?network=' + network.lower_name() + activity_data['buttons'] = [{ + 'label': 'Profile', + 'url': profile_url + }] + + # Similarly, only show the user's Mii if enabled. + if user_data.username and game['icon_url'] and self.current_user.show_small_image: + # Format as a human-readable friend code (XXXX-XXXX-XXXX). + user_friend_code = '-'.join(user_data.friend_code[i:i+4] for i in range(0, 12, 4)) + user_network_name = network.lower_name().capitalize() + small_text_detail = f"{user_friend_code} on {user_network_name}" + + activity_data['assets']['small_image'] = user_data.mii_urls['face'] + activity_data['assets']['small_text'] = small_text_detail + + # Quickly sanitize our activity data by truncating + # any text exceeding the maximum field limit, 128 characters. + for key_name in list(activity_data): + # However, don't modify image assets as they can go over 128. + if 'image' in key_name: + continue + + if isinstance(activity_data[key_name], str): + if len(activity_data[key_name]) > 128: + activity_data[key_name] = activity_data[key_name][:128] + + data = {'activities': [activity_data]} + if self.current_user.rpc_session_token: + data['token'] = self.current_user.rpc_session_token + + headers = { + 'Authorization': 'Bearer %s' % self.current_user.bearer_token, + 'Content-Type': 'application/json', + } + + r = await client.post('%s/users/@me/headless-sessions' % API_ENDPOINT, data=json.dumps(data), headers=headers) + r.raise_for_status() + + response = r.json() + DiscordSession().create(self.current_user.refresh_token, response['token']) + return None + + async def reset_presence(self): + if not self.current_user.rpc_session_token: + print('[NO SESSION TO RESET]') + return False + elif time.time() - self.current_user.last_accessed <= 30: + print('[MANUAL RATE LIMITED]') + return False + + headers = { + 'Authorization': 'Bearer %s' % self.current_user.bearer_token, + 'Content-Type': 'application/json', + } + data = { + 'token': self.current_user.rpc_session_token, + } + r = await client.post('%s/users/@me/headless-sessions/delete' % API_ENDPOINT, data=json.dumps(data), headers=headers) + + try: + r.raise_for_status() + except HTTPStatusError as e: + # If we encounter 400, we assume that this session has already expired. + # Let's go ahead and reset the session anyway. + if e.response.status_code == 400: + DiscordSession().retire(self.current_user.refresh_token) + else: + raise e + + + async def refresh_bearer(self): + print('[REFRESH BEARER %s]' % self.current_user.id) + current_refresh_token = self.current_user.refresh_token + data = { + 'client_id': '%s' % CLIENT_ID, + 'client_secret': '%s' % CLIENT_SECRET, + 'grant_type': 'refresh_token', + 'refresh_token': current_refresh_token, + } + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + } + json_response = await client.post('%s/oauth2/token' % API_ENDPOINT, data=data, headers=headers) + json_response.raise_for_status() + response = json_response.json() + + session.execute( + update(DiscordTable) + .where(DiscordTable.refresh_token == current_refresh_token) + .values( + refresh_token=response['refresh_token'], + bearer_token=response['access_token'], + generation_date=time.time() + ) + ) + session.commit() + + + def delete_discord_user(self): + user_id = self.current_user.id + print('[DELETING %s]' % user_id) + session.execute(delete(DiscordTable).where(DiscordTable.id == user_id)) + session.execute(delete(DiscordFriends).where(DiscordFriends.id == user_id)) + session.commit() delay = 2 -while True: - # First, refresh all OAuth2 bearer tokens if necessary. - all_users = session.scalars(select(DiscordTable)).all() - for oauth_user in all_users: - # We only need to refresh 30 minutes before the token expires. - if time.time() - oauth_user.generation_date < 604800 - 1800: - continue - - # Any HTTP error expected here is a 403. - # This would mean that the refresh token is now invalid, - # likely due to the user removing access via Discord. - api_client = APIClient(oauth_user) - try: - api_client.refresh_bearer() - time.sleep(delay * 2) - except HTTPError: - api_client.delete_discord_user() - - # Inactive users have removed our bot: the backend removed them - # from both `friends` and `discord_friends`, but they still - # have an account (i.e. they exist with credentials in `discord`). - # - # Find these users with ongoing sessions and reset their presence. - inactive_query = ( - select(DiscordTable) - .outerjoin(DiscordFriends, DiscordFriends.id == DiscordTable.id) - .filter(DiscordFriends.id == None) - .filter(DiscordTable.rpc_session_token != None) - ) - inactive_users = session.scalars(inactive_query).all() - - if len(inactive_users) > 0: - print('[INACTIVES] Handling %s' % len(inactive_users)) - - for inactive_user in inactive_users: - api_client = APIClient(inactive_user) - try: - print('[INACTIVES] Resetting %s' % inactive_user.id) - api_client.reset_presence() - time.sleep(delay) - except HTTPError as e: - print(f"[INACTIVE RESET FAILURE] {e}") - # api_client.delete_discord_user() - - time.sleep(delay) - - # Finally, we'll refresh presences for all remaining users. - discord_friends = session.scalars(select(DiscordFriends).where(DiscordFriends.active)).all() - - if len(discord_friends) < 1: - time.sleep(delay) - continue - - for discord_friend in discord_friends: - # If we've updated this user within the past minute, there's no need to update again. - discord_user = session.scalar(select(DiscordTable).where(DiscordTable.id == discord_friend.id)) - if time.time() - discord_user.last_accessed < 60: - continue - - # If this user has no friend data, we cannot process them. - friend_data: Friend = session.scalar( - select(Friend) - .where(Friend.friend_code == discord_friend.friend_code) - .where(Friend.network == discord_friend.network) - ) - if not friend_data: - continue - - api_client = APIClient(discord_user) - - if not friend_data.online: - # If the user is offline, and they lack an RPC session, - # there's nothing for us to do. - if not discord_user.rpc_session_token: - continue - - # Remove our presence for this now-offline user. - try: - print('[FRIENDS] Resetting presence for %s on %s' % (friend_data.friend_code, friend_data.network.lower_name())) - api_client.reset_presence() - time.sleep(delay) - except HTTPError as e: - print(f"[FRIEND RESET FAILURE] {e}") - # api_client.delete_discord_user() - continue - - print('[FRIENDS] Creating RPC for Discord ID %s - %s on %s]' % (discord_friend.id, discord_friend.friend_code, discord_friend.network.lower_name())) - principal_id = friend_code_to_principal_id(friend_data.friend_code) - mii = friend_data.mii - if mii: - mii = MiiData().mii_studio_url(mii) - - try: - friend_code = str(principal_id_to_friend_code(principal_id)).zfill(12) - title_data = getTitle(friend_data.title_id, titlesToUID, titleDatabase) - - discord_user_data = UserData( - friend_code=friend_code, - online=friend_data.online, - game=title_data, - game_description=friend_data.game_description, - username=friend_data.username, - mii_urls=mii, - last_accessed=friend_data.last_accessed - ) - - api_client.update_presence(discord_user_data, discord_friend.network) - time.sleep(delay) - except HTTPError as e: - print(f"[FRIEND PRESENCE FAILURE] {e}") - # api_client.delete_discord_user() - time.sleep(delay) - - # Sleep for 5x our delay. - time.sleep(delay * 5) +async def main(): + while True: + async def refresh_user(oauth_user: Discord): + # We only need to refresh 30 minutes before the token expires. + if time.time() - oauth_user.generation_date < 604800 - 1800: + return + + # Any HTTP error expected here is a 403. + # This would mean that the refresh token is now invalid, + # likely due to the user removing access via Discord. + api_client = APIClient(oauth_user) + try: + await api_client.refresh_bearer() + await asyncio.sleep(delay * 2) + except HTTPStatusError: + api_client.delete_discord_user() + + # First, asynchronously refresh all OAuth2 bearer tokens if necessary. + all_users = session.scalars(select(DiscordTable)) + await asyncio.gather(*[refresh_user(oauth_user) for oauth_user in all_users]) + + # Inactive users have removed our bot: the backend removed them + # from both `friends` and `discord_friends`, but they still + # have an account (i.e. they exist with credentials in `discord`). + # + # Find these users with ongoing sessions and reset their presence. + inactive_query = ( + select(DiscordTable) + .outerjoin(DiscordFriends, DiscordFriends.id == DiscordTable.id) + .filter(DiscordFriends.id == None) + .filter(DiscordTable.rpc_session_token != None) + ) + inactive_users = session.scalars(inactive_query) + + if inactive_users.one_or_none() is not None: + print('[INACTIVES] Handling inactive users') + + async def reset_inactive_presence(inactive_user: Discord): + api_client = APIClient(inactive_user) + try: + print('[INACTIVES] Resetting %s' % inactive_user.id) + await api_client.reset_presence() + await asyncio.sleep(delay) + except HTTPStatusError as e: + print(f"[INACTIVE RESET FAILURE] {e}") + # api_client.delete_discord_user() + + # asynchronously update inactive users before continuing + await asyncio.gather(*[reset_inactive_presence(inactive_user) for inactive_user in inactive_users]) + await asyncio.sleep(delay) + + # Finally, we'll refresh presences for all remaining users. + discord_friends = session.scalars(select(DiscordFriends).where(DiscordFriends.active)) + + if discord_friends.one_or_none() is not None: + await asyncio.sleep(delay) + continue + + async def refresh_active_users_presence(discord_friend): + # If we've updated this user within the past minute, there's no need to update again. + discord_user = session.scalar(select(DiscordTable).where(DiscordTable.id == discord_friend.id)) + if time.time() - discord_user.last_accessed < 60: + return + + # If this user has no friend data, we cannot process them. + friend_data: Friend = session.scalar( + select(Friend) + .where(Friend.friend_code == discord_friend.friend_code) + .where(Friend.network == discord_friend.network) + ) + if not friend_data: + return + + api_client = APIClient(discord_user) + + if not friend_data.online: + # If the user is offline, and they lack an RPC session, + # there's nothing for us to do. + if not discord_user.rpc_session_token: + return + + # Remove our presence for this now-offline user. + try: + print('[FRIENDS] Resetting presence for %s on %s' % (friend_data.friend_code, + friend_data.network.lower_name())) + await api_client.reset_presence() + await asyncio.sleep(delay) + except HTTPStatusError as e: + print(f"[FRIEND RESET FAILURE] {e}") + # api_client.delete_discord_user() + return + + print( + '[FRIENDS] Creating RPC for Discord ID %s - %s on %s]' % (discord_friend.id, discord_friend.friend_code, + discord_friend.network.lower_name())) + principal_id = friend_code_to_principal_id(friend_data.friend_code) + mii = friend_data.mii + if mii: + mii = MiiData().mii_studio_url(mii) + + try: + friend_code = str(principal_id_to_friend_code(principal_id)).zfill(12) + title_data = await getTitle(friend_data.title_id, titlesToUID, titleDatabase) + + discord_user_data = UserData( + friend_code=friend_code, + online=friend_data.online, + game=title_data, + game_description=friend_data.game_description, + username=friend_data.username, + mii_urls=mii, + last_accessed=friend_data.last_accessed + ) + + await api_client.update_presence(discord_user_data, discord_friend.network) + await asyncio.sleep(delay) + except HTTPStatusError as e: + print(f"[FRIEND PRESENCE FAILURE] {e}") + # api_client.delete_discord_user() + await asyncio.sleep(delay) + + # Asynchronously update every presence of active users + await asyncio.gather(*[refresh_active_users_presence(discord_friend) for discord_friend in discord_friends]) + + # Sleep for 5x our delay. + await asyncio.sleep(delay * 5) + +asyncio.run(main()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 18e6638..5a68b02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests +requests # eventually preferably removed for complete async solutions flask nintendoclients>=2.1.0 flask-limiter>=3.12 @@ -9,4 +9,6 @@ Flask-Migrate>=4.0 xmltodict Pillow pycryptodomex -pymysql \ No newline at end of file +pymysql +asyncio +httpx \ No newline at end of file diff --git a/server.py b/server.py index d7a9457..edfeb81 100644 --- a/server.py +++ b/server.py @@ -1,10 +1,14 @@ # Created by Deltaion Lee (MCMi460) on Github +import asyncio + +import httpx from flask import Flask, make_response, request, redirect, render_template, send_file from flask_limiter import Limiter from flask_limiter.util import get_remote_address from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy -import sys, datetime, xmltodict, pickle, secrets +import sys, datetime, xmltodict, pickle, secrets, os + from sqlalchemy import select, update, insert, delete @@ -43,6 +47,8 @@ start_db_time(None, NetworkType.NINTENDO) start_db_time(None, NetworkType.PRETENDO) +client = httpx.AsyncClient(verify=False) +"Httpx client for asynchronous requests (Previously handled synchronously with Requests)" @app.errorhandler(404) def handler404(e): @@ -69,11 +75,8 @@ def handler404(e): title_database = [] titles_to_uid = [] -requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) - - # Create title cache -def cache_titles(): +async def cache_titles(): global title_database, titles_to_uid # Pull databases @@ -96,12 +99,12 @@ def cache_titles(): for region in ['US', 'JP', 'GB', 'KR', 'TW']: title_database.append( - xmltodict.parse(requests.get('https://samurai.ctr.shop.nintendo.net/samurai/ws/%s/titles?shop_id=1&limit=5000&offset=0' % region, verify = False).text) + xmltodict.parse((await client.get('https://samurai.ctr.shop.nintendo.net/samurai/ws/%s/titles?shop_id=1&limit=5000&offset=0' % region, ver)).text) ) # Update progress bar as database requests complete bar.update(.5 / 5) - titles_to_uid += requests.get('https://raw.githubusercontent.com/hax0kartik/3dsdb/master/jsons/list_%s.json' % region).json() + titles_to_uid += (await client.get('https://raw.githubusercontent.com/hax0kartik/3dsdb/master/jsons/list_%s.json' % region)).json() bar.update(.5 / 5) bar.end() # End the progress bar @@ -202,7 +205,7 @@ def create_user(friend_code: int, network: NetworkType, add_new_instance: bool): db.session.commit() -def fetch_bearer_token(code: str): +async def fetch_bearer_token(code: str): data = { 'client_id': '%s' % CLIENT_ID, 'client_secret': '%s' % CLIENT_SECRET, @@ -213,12 +216,12 @@ def fetch_bearer_token(code: str): headers = { 'Content-Type': 'application/x-www-form-urlencoded', } - r = requests.post('%s/oauth2/token' % API_ENDPOINT, data=data, headers=headers) + r = await client.post('%s/oauth2/token' % API_ENDPOINT, data=data, headers=headers) r.raise_for_status() return r.json() -def refresh_bearer(token: str): +async def refresh_bearer(token: str): user = user_from_token(token) data = { 'client_id': '%s' % CLIENT_ID, @@ -229,9 +232,9 @@ def refresh_bearer(token: str): headers = { 'Content-Type': 'application/x-www-form-urlencoded', } - r = requests.post('%s/oauth2/token' % API_ENDPOINT, data=data, headers=headers) + r = await client.post('%s/oauth2/token' % API_ENDPOINT, data=data, headers=headers) r.raise_for_status() - token, user, pfp = create_discord_user('', r.json()) + token, user, pfp = await create_discord_user('', r.json()) return token, user, pfp @@ -249,13 +252,13 @@ def user_from_token(token: str) -> Discord: return result -def create_discord_user(code: str, response: dict = None): +async def create_discord_user(code: str, response: dict = None): if not response: - response = fetch_bearer_token(code) + response = await fetch_bearer_token(code) headers = { 'Authorization': 'Bearer %s' % response['access_token'], } - new = requests.get('https://discord.com/api/users/@me', headers=headers) + new = await client.get('https://discord.com/api/users/@me', headers=headers) user = new.json() token = secrets.token_hex(20) try: @@ -355,7 +358,7 @@ def user_agent_check(): raise Exception('this client is invalid') -def get_presence(friend_code: int, network: NetworkType, is_api: bool): +async def get_presence(friend_code: int, network: NetworkType, is_api: bool): try: if is_api: # First, run 3DS-RPC client checks. @@ -386,7 +389,7 @@ def get_presence(friend_code: int, network: NetworkType, is_api: bool): 'updateID': result.upd_id, 'joinable': result.joinable, 'gameDescription': result.game_description, - 'game': getTitle(result.title_id, titles_to_uid, title_database), + 'game': await getTitle(result.title_id, titles_to_uid, title_database), 'disclaimer': 'all information regarding the title (User/Presence/game) is downloaded from Nintendo APIs', } else: @@ -424,7 +427,7 @@ def get_presence(friend_code: int, network: NetworkType, is_api: bool): # Index page @app.route('/') -def index(): +async def index(): stmt = ( select(Friend) .where(Friend.online == True) @@ -434,14 +437,19 @@ def index(): results = db.session.scalars(stmt).all() num = len(results) data = sidenav() - data['active'] = [({ - 'mii': MiiData().mii_studio_url(user.mii), - 'username': user.username, - 'game': getTitle(user.title_id, titles_to_uid, title_database), - 'friendCode': user.friend_code.zfill(12), - 'joinable': user.joinable, - 'network': user.network.lower_name(), - }) for user in results if user.username] + for user in results: + if user.username: + game = await getTitle(user.title_id, titles_to_uid, title_database) + + data['active'] += { + 'mii': MiiData().mii_studio_url(user.mii), + 'username': user.username, + 'game': game, + 'friendCode': user.friend_code.zfill(12), + 'joinable': user.joinable, + 'network': user.network.lower_name(), + } + data['active'] = data['active'][:2] stmt = ( @@ -450,16 +458,21 @@ def index(): .order_by(Friend.account_creation.desc()) .limit(6) ) - results = db.session.scalars(stmt).all() + results = db.session.scalars(stmt) + + for user in results: + if user.username: + game = await getTitle(user.title_id, titles_to_uid, title_database) if user.online and user.title_id != 0 else '' + + data['new'] += { + 'mii': MiiData().mii_studio_url(user.mii), + 'username': user.username, + 'game': game, + 'friendCode': user.friend_code.zfill(12), + 'joinable': user.joinable, + 'network': user.network.lower_name(), + } - data['new'] = [({ - 'mii': MiiData().mii_studio_url(user.mii), - 'username': user.username, - 'game': getTitle(user.title_id, titles_to_uid, title_database) if user.online and user.title_id != 0 else '', - 'friendCode': user.friend_code.zfill(12), - 'joinable': user.joinable, - 'network': user.network.lower_name(), - }) for user in results if user.username] data['new'] = data['new'][:2] data['num'] = num @@ -518,7 +531,7 @@ def settings_redirect(): # Roster page @app.route('/roster') -def roster(): +async def roster(): stmt = ( select(Friend) .where(Friend.username != None) @@ -530,14 +543,18 @@ def roster(): data = sidenav() data['title'] = 'New Users' - data['users'] = [({ - 'mii': MiiData().mii_studio_url(user.mii), - 'username': user.username, - 'game': getTitle(user.title_id, titles_to_uid, title_database), - 'friendCode': user.friend_code.zfill(12), - 'joinable': user.joinable, - 'network': user.network.lower_name(), - }) for user in results if user.username] + for user in results: + if user.username: + game = await getTitle(user.title_id, titles_to_uid, title_database) + + data['users'] += { + 'mii': MiiData().mii_studio_url(user.mii), + 'username': user.username, + 'game': game, + 'friendCode': user.friend_code.zfill(12), + 'joinable': user.joinable, + 'network': user.network.lower_name(), + } response = make_response(render_template('dist/users.html', data=data)) return response @@ -545,7 +562,7 @@ def roster(): # Active page @app.route('/active') -def active(): +async def active(): stmt = ( select(Friend) .where(Friend.username != None) @@ -557,14 +574,18 @@ def active(): data = sidenav() data['title'] = 'Active Users' - data['users'] = [({ - 'mii': MiiData().mii_studio_url(user.mii), - 'username': user.username, - 'game': getTitle(user.title_id, titles_to_uid, title_database), - 'friendCode': user.friend_code.zfill(12), - 'joinable': user.joinable, - 'network': user.network.lower_name(), - }) for user in results if user.username] + for user in results: + if user.username: + game = await getTitle(user.title_id, titles_to_uid, title_database) + + data['users'] += { + 'mii': MiiData().mii_studio_url(user.mii), + 'username': user.username, + 'game': game, + 'friendCode': user.friend_code.zfill(12), + 'joinable': user.joinable, + 'network': user.network.lower_name(), + } response = make_response(render_template('dist/users.html', data=data)) return response @@ -660,14 +681,14 @@ def consoles(): @app.route('/user//') -def user_page(friend_code: str): +async def user_page(friend_code: str): network: NetworkType try: network = name_to_network_type(request.args.get('network')) friend_code_int = int(friend_code.replace('-', '')) - user_data = get_presence(friend_code_int, network, False) + user_data = await get_presence(friend_code_int, network, False) if user_data['Exception'] or not user_data['User']['username']: raise Exception(user_data['Exception']) except: @@ -675,7 +696,7 @@ def user_page(friend_code: str): if not user_data['User']['online'] or not user_data['User']['Presence']: user_data['User']['Presence']['game'] = None - user_data['User']['favoriteGame'] = getTitle(user_data['User']['favoriteGame'], titles_to_uid, title_database) + user_data['User']['favoriteGame'] = await getTitle(user_data['User']['favoriteGame'], titles_to_uid, title_database) user_data['User']['network'] = network.lower_name() if user_data['User']['favoriteGame']['name'] == 'Home Screen': user_data['User']['favoriteGame'] = None @@ -708,7 +729,7 @@ def terms(): # Create entry in database with friendCode @app.route('/api/user/create//', methods=['POST']) @limiter.limit(new_user_limit) -def new_user(friend_code: int, network: int = -1, user_check: bool = True): +def new_user(friend_code: int, network: NetworkType = -1, user_check: bool = True): try: if user_check: user_agent_check() @@ -869,8 +890,8 @@ def settings_toggler(which: str): # Make Nintendo's cert a 'secure' cert @app.route('/cdn/i//', methods=['GET']) @limiter.limit(cdn_limit) -def cdn_image(file: str): - response = make_response(requests.get('https://kanzashi-ctr.cdn.nintendo.net/i/%s' % file, verify=False).content) +async def cdn_image(file: str): + response = make_response((await client.get('https://kanzashi-ctr.cdn.nintendo.net/i/%s' % file)).content) response.headers['Content-Type'] = 'image/jpeg' return response @@ -902,10 +923,10 @@ def login(): # Discord route @app.route('/authorize') @limiter.limit(new_user_limit) -def authorize(): +async def authorize(): if not request.args.get('code'): return render_template('dist/404.html') - token, user, pfp = create_discord_user(request.args['code']) + token, user, pfp = await create_discord_user(request.args['code']) response = make_response(redirect('/consoles')) response.set_cookie('token', token, expires=datetime.datetime.now() + datetime.timedelta(days=30)) response.set_cookie('user', user, expires=datetime.datetime.now() + datetime.timedelta(days=30)) @@ -914,10 +935,10 @@ def authorize(): @app.route('/refresh') -def refresh(): +async def refresh(): if local: try: - token, user, pfp = refresh_bearer(request.cookies['token']) + token, user, pfp = await refresh_bearer(request.cookies['token']) response = make_response(redirect('/consoles')) response.set_cookie('token', token, expires=datetime.datetime.now() + datetime.timedelta(days=30)) response.set_cookie('user', user, expires=datetime.datetime.now() + datetime.timedelta(days=30)) @@ -927,12 +948,14 @@ def refresh(): delete_discord_user(user_from_token(request.cookies['token']).id) return redirect('/404.html') - -if __name__ == '__main__': - cache_titles() +async def main(): + await cache_titles() if local: app.run(host='0.0.0.0', port=port) else: import gevent.pywsgi server = gevent.pywsgi.WSGIServer(('0.0.0.0', port), app) server.serve_forever() + +if __name__ == '__main__': + asyncio.run(main())