From c4d25da3fa514ada2125b795dc95181064d69f30 Mon Sep 17 00:00:00 2001 From: EvanLi <1939065721@qq.com> Date: Tue, 15 Sep 2020 11:41:02 +0800 Subject: [PATCH 1/4] feat: use Github OAuth2 token with Authorization header --- LICENSE | 21 ++++++++++ source/.gitignore | 3 ++ source/common.py | 31 +++++++++++++++ source/contents.md | 32 +++++++++++++++ source/process.py | 96 +++++++++++++++++++++++++++++++++++++++++++++ source/run.py | 20 ++++++++++ source/writefile.py | 44 +++++++++++++++++++++ 7 files changed, 247 insertions(+) create mode 100644 LICENSE create mode 100644 source/.gitignore create mode 100644 source/common.py create mode 100644 source/contents.md create mode 100644 source/process.py create mode 100644 source/run.py create mode 100644 source/writefile.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..d78b2d267b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Evan Li + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/source/.gitignore b/source/.gitignore new file mode 100644 index 0000000000..909d692e5d --- /dev/null +++ b/source/.gitignore @@ -0,0 +1,3 @@ +access_token.txt +.ipynb_checkpoints +__pycache__ \ No newline at end of file diff --git a/source/common.py b/source/common.py new file mode 100644 index 0000000000..781dbad36b --- /dev/null +++ b/source/common.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +import json +import requests +import time + +def get_access_token(): + with open('access_token.txt', 'r') as f: + access_token = f.read().strip() + return access_token + +def get_api_repos(API_URL): + ''' + get repos of api, return repos list + ''' + access_token = get_access_token() + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Authorization': 'token {}'.format(access_token), + } + s = requests.session() + s.keep_alive = False # don't keep the session + time.sleep(3) # not get so fast + # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, + r = requests.get(API_URL,headers=headers) + if r.status_code != 200: + raise ValueError('Can not retrieve from {}'.format(api)) + repos_dict = json.loads(r.content) + repos = repos_dict['items'] + return repos \ No newline at end of file diff --git a/source/contents.md b/source/contents.md new file mode 100644 index 0000000000..579410c05f --- /dev/null +++ b/source/contents.md @@ -0,0 +1,32 @@ +[Github Ranking](./README.md) +========== +**A list of the most github stars and forks repositories.** + +## Table of Contents +* [Most Stars](#most-stars) +* [Most Forks](#most-forks) +* [ActionScript](#actionscript) +* [C](#c) +* [C\#](#c-1) +* [C\+\+](#c-2) +* [Clojure](#clojure) +* [CoffeeScript](#coffeescript) +* [CSS](#css) +* [Go](#go) +* [Haskell](#haskell) +* [HTML](#html) +* [Java](#java) +* [JavaScript](#javascript) +* [Lua](#lua) +* [MATLAB](#matlab) +* [Objective\-C](#objective-c) +* [Perl](#perl) +* [PHP](#php) +* [Python](#python) +* [R](#r) +* [Ruby](#ruby) +* [Scala](#scala) +* [Shell](#shell) +* [Swift](#swift) +* [TeX](#tex) +* [Vim script](#vim-script) diff --git a/source/process.py b/source/process.py new file mode 100644 index 0000000000..89840bba03 --- /dev/null +++ b/source/process.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import os +import pandas as pd +from common import get_api_repos +from writefile import write_text, write_head_contents, write_ranking_repo + +class Processor(object): + + def __init__(self): + # self.languages = ['Python'] # For test + # self.languages_md = ['Python'] # For test + self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] + self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] + # search limit 10 times per minute, with token is 30 per minute + # check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit + self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100' + self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100' + self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100' + + self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] + self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() + + def get_all_repos(self): + # get all repos of most stars and forks, and different languages + + print("Get repos of most stars...") + repos_stars = get_api_repos(self.api_repo_stars) + + print("Get repos of most forks...") + repos_forks = get_api_repos(self.api_repo_forks) + + repos_languages = {} + for lang in self.languages: + print("Get most stars repos of {}...".format(lang)) + repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang = lang)) + return repos_stars, repos_forks, repos_languages + + def write_head_contents(self): + write_head_contents('../README.md') + + def write_readme_lang_md(self): + # Most stars save + write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') + write_ranking_repo('../README.md','a',self.repos_stars[0:10]) + print("Save most stars in README.md!") + os.makedirs('../Top100',exist_ok=True) + write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') + write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) + print("Save most stars in Top100/Top-100-stars.md!\n") + + # Most forks save + write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") + write_ranking_repo('../README.md','a',self.repos_forks[0:10]) + print("Save most forks in README.md!") + write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') + write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) + print("Save most forks in Top100/Top-100-forks.md!\n") + + # Most stars in language save + for i in range(len(self.languages)): + lang = self.languages[i] + write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) + write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) + print("Save most stars of {} in README.md!".format(lang)) + write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) + write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) + print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) + + def repo_to_df(self,repos,item): + # prepare for saving data to csv file + repos_list = [] + for idx, repo in enumerate(repos): + repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] + repos_list.append(repo_info) + return pd.DataFrame(repos_list,columns = self.col) + + def save_to_csv(self): + # save top100 repos info to csv file in Data/github-ranking-year-month-day.md + df_all = pd.DataFrame(columns=self.col) + df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') + df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') + df_all = df_all.append(df_repos_stars,ignore_index = True) + df_all = df_all.append(df_repos_forks,ignore_index = True) + for lang in self.repos_languages.keys(): + df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) + df_all = df_all.append(df_repos_lang,ignore_index = True) + + save_date = datetime.utcnow().strftime("%Y-%m-%d") + os.makedirs('../Data',exist_ok=True) + df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') + print('Save data to Data/github-ranking-'+save_date+'.csv') \ No newline at end of file diff --git a/source/run.py b/source/run.py new file mode 100644 index 0000000000..87cebcb36c --- /dev/null +++ b/source/run.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import os +from process import Processor + +if __name__=="__main__": + t1 = datetime.now() + + ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../")) + os.chdir(os.path.join(ROOT_PATH,'source')) + + processor = Processor() + print("Write head and contents of README.md!") + processor.write_head_contents() + print("write to readme and languages.md") + processor.write_readme_lang_md() + print("Save data to csv file") + processor.save_to_csv() + + print("Total time: {}s".format((datetime.now()-t1).total_seconds())) \ No newline at end of file diff --git a/source/writefile.py b/source/writefile.py new file mode 100644 index 0000000000..aeb50470c7 --- /dev/null +++ b/source/writefile.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from datetime import datetime + +def write_text(file_name,method,text): + ''' + write text to file + + method: 'a'-append, 'w'-overwrite + ''' + with open(file_name, method, encoding='utf-8') as f: + f.write(text) + +def write_head_contents(file_name): + # write the head and contents of README.md + # write_head_contents('README.md') + write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + head = "*Last Automatic Update Time: {}*\n\n".format(write_time) + with open('contents.md','r') as f: + contents = f.read() + write_text(file_name,'w',head+contents) + +def write_ranking_repo(file_name,method,repos): + # method: 'a'-append or 'w'-overwrite + table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\ +| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n" + with open(file_name, method, encoding='utf-8') as f: + f.write(table_head) + for idx, repo in enumerate(repos): + repo_description = repo['description'] + if repo_description is not None: + repo_description = repo_description.replace('|','\|') #in case there is '|' in description + f.write('| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format( + idx + 1, + repo['name'], + repo['html_url'], + repo['stargazers_count'], + repo['forks_count'], + repo['language'], + repo['open_issues_count'], + repo_description, + repo['pushed_at'] + ) + ) + f.write('\n') \ No newline at end of file From 6dfb27f7ee974eeff0a6a338d841cbd2c5626817 Mon Sep 17 00:00:00 2001 From: EvanLi <1939065721@qq.com> Date: Wed, 16 Sep 2020 18:48:59 +0800 Subject: [PATCH 2/4] feat: use Github GraphQL API v4 to get data --- source/common.py | 25 +++++++- source/process.py | 144 +++++++++++++++++++++++++++++++++++++++++++++- source/run.py | 5 +- 3 files changed, 170 insertions(+), 4 deletions(-) diff --git a/source/common.py b/source/common.py index 781dbad36b..c7457fec54 100644 --- a/source/common.py +++ b/source/common.py @@ -28,4 +28,27 @@ def get_api_repos(API_URL): raise ValueError('Can not retrieve from {}'.format(api)) repos_dict = json.loads(r.content) repos = repos_dict['items'] - return repos \ No newline at end of file + return repos + +def get_graphql_data(GQL): + ''' + use graphql to get data + ''' + access_token = get_access_token() + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Authorization': 'bearer {}'.format(access_token), + } + s = requests.session() + s.keep_alive = False # don't keep the session + graphql_api = "https://api.github.com/graphql" + time.sleep(5) # not get so fast + + # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, + r = requests.post(url = graphql_api, json={"query": GQL}, headers=headers) + + if r.status_code != 200: + raise ValueError('Can not retrieve from {}'.format(GQL)) + return r.json() \ No newline at end of file diff --git a/source/process.py b/source/process.py index 89840bba03..e5962b5bdf 100644 --- a/source/process.py +++ b/source/process.py @@ -2,7 +2,7 @@ from datetime import datetime import os import pandas as pd -from common import get_api_repos +from common import get_api_repos, get_graphql_data from writefile import write_text, write_head_contents, write_ranking_repo class Processor(object): @@ -79,6 +79,148 @@ def repo_to_df(self,repos,item): repos_list.append(repo_info) return pd.DataFrame(repos_list,columns = self.col) + def save_to_csv(self): + # save top100 repos info to csv file in Data/github-ranking-year-month-day.md + df_all = pd.DataFrame(columns=self.col) + df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') + df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') + df_all = df_all.append(df_repos_stars,ignore_index = True) + df_all = df_all.append(df_repos_forks,ignore_index = True) + for lang in self.repos_languages.keys(): + df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) + df_all = df_all.append(df_repos_lang,ignore_index = True) + + save_date = datetime.utcnow().strftime("%Y-%m-%d") + os.makedirs('../Data',exist_ok=True) + df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') + print('Save data to Data/github-ranking-'+save_date+'.csv') + +class ProcessorGQL(object): + + def __init__(self): + # self.languages = ["ActionScript","C","CSharp","CPP"] # For test + # self.languages_md = ["ActionScript","C","C\#","CPP"] # For test + self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] + self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] + # use graphql to get data, limit 5000 points per hour + # check rate_limit with : + # curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql + self.gql_format = """query{ + search(query: "%s", type: REPOSITORY, first: 100) { + edges { + node { + ...on Repository { + id + name + url + forkCount + stargazers { + totalCount + } + owner { + login + } + issues(states: OPEN) { + totalCount + } + description + pushedAt + primaryLanguage { + name + } + } + } + } + } + } + """ + self.gql_stars = self.gql_format % ("stars:>1000 sort:stars") + self.gql_forks = self.gql_format % ("forks:>1000 sort:forks") + self.gql_stars_lang = self.gql_format % ("language:%s stars:>0 sort:stars") + + self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] + self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() + + def parse_gql_result(self,result): + res = [] + for repo in result["data"]["search"]["edges"]: + repo_data = repo['node'] + res.append({ + 'name': repo_data['name'], + 'stargazers_count': repo_data['stargazers']['totalCount'], + 'forks_count': repo_data['forkCount'], + 'language': repo_data['primaryLanguage']['name'] if repo_data['primaryLanguage'] is not None else None, + 'html_url': repo_data['url'], + 'owner': { + 'login': repo_data['owner']['login'], + }, + 'open_issues_count': repo_data['issues']['totalCount'], + 'pushed_at': repo_data['pushedAt'], + 'description': repo_data['description'], + }) + return res + + def get_all_repos(self): + # get all repos of most stars and forks, and different languages + print("Get repos of most stars...") + repos_stars_gql = get_graphql_data(self.gql_stars) + repos_stars = self.parse_gql_result(repos_stars_gql) + + print("Get repos of most forks...") + repos_forks_gql = get_graphql_data(self.gql_forks) + repos_forks = self.parse_gql_result(repos_forks_gql) + + repos_languages = {} + for lang in self.languages: + print("Get most stars repos of {}...".format(lang)) + repos_languages[lang] = self.parse_gql_result( + get_graphql_data(self.gql_stars_lang % (lang)) + ) + return repos_stars, repos_forks, repos_languages + + def write_head_contents(self): + write_head_contents('../README.md') + + def write_readme_lang_md(self): + # Most stars save + write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') + write_ranking_repo('../README.md','a',self.repos_stars[0:10]) + print("Save most stars in README.md!") + os.makedirs('../Top100',exist_ok=True) + write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') + write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) + print("Save most stars in Top100/Top-100-stars.md!\n") + + # Most forks save + write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") + write_ranking_repo('../README.md','a',self.repos_forks[0:10]) + print("Save most forks in README.md!") + write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') + write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) + print("Save most forks in Top100/Top-100-forks.md!\n") + + # Most stars in language save + for i in range(len(self.languages)): + lang = self.languages[i] + write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) + write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) + print("Save most stars of {} in README.md!".format(lang)) + write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) + write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) + print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) + + def repo_to_df(self,repos,item): + # prepare for saving data to csv file + repos_list = [] + for idx, repo in enumerate(repos): + repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] + repos_list.append(repo_info) + return pd.DataFrame(repos_list,columns = self.col) + def save_to_csv(self): # save top100 repos info to csv file in Data/github-ranking-year-month-day.md df_all = pd.DataFrame(columns=self.col) diff --git a/source/run.py b/source/run.py index 87cebcb36c..2003ba6cef 100644 --- a/source/run.py +++ b/source/run.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from datetime import datetime import os -from process import Processor +from process import Processor,ProcessorGQL if __name__=="__main__": t1 = datetime.now() @@ -9,7 +9,8 @@ ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../")) os.chdir(os.path.join(ROOT_PATH,'source')) - processor = Processor() + # processor = Processor() # use Github REST API v3 + processor = ProcessorGQL() # use Github GraphQL API v4 print("Write head and contents of README.md!") processor.write_head_contents() print("write to readme and languages.md") From f7a032664d34ce96fab1a09546befd5680ac8869 Mon Sep 17 00:00:00 2001 From: EvanLi Date: Thu, 19 Aug 2021 18:57:31 +0800 Subject: [PATCH 3/4] refactor code and use Github GraphQL --- source/.gitignore => .gitignore | 3 +- source/common.py | 51 ++++-- source/contents.md | 32 ---- source/process.py | 309 ++++++++++++++++++-------------- source/run.py | 21 --- source/writefile.py | 44 ----- 6 files changed, 218 insertions(+), 242 deletions(-) rename source/.gitignore => .gitignore (65%) delete mode 100644 source/contents.md delete mode 100644 source/run.py delete mode 100644 source/writefile.py diff --git a/source/.gitignore b/.gitignore similarity index 65% rename from source/.gitignore rename to .gitignore index 909d692e5d..4fad37e937 100644 --- a/source/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ access_token.txt .ipynb_checkpoints -__pycache__ \ No newline at end of file +__pycache__ +.idea/ diff --git a/source/common.py b/source/common.py index c7457fec54..f4c53892cb 100644 --- a/source/common.py +++ b/source/common.py @@ -3,15 +3,43 @@ import requests import time + def get_access_token(): - with open('access_token.txt', 'r') as f: + with open('../access_token.txt', 'r') as f: access_token = f.read().strip() return access_token + +def write_text(file_name, method, text): + """ + write text to file + method: 'a'-append, 'w'-overwrite + """ + with open(file_name, method, encoding='utf-8') as f: + f.write(text) + + +def write_ranking_repo(file_name, method, repos): + # method: 'a'-append or 'w'-overwrite + table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\ +| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n" + with open(file_name, method, encoding='utf-8') as f: + f.write(table_head) + for idx, repo in enumerate(repos): + repo_description = repo['description'] + if repo_description is not None: + repo_description = repo_description.replace('|', '\|') # in case there is '|' in description + f.write("| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n".format( + idx + 1, repo['name'], repo['html_url'], repo['stargazers_count'], repo['forks_count'], + repo['language'], repo['open_issues_count'], repo_description, repo['pushed_at'] + )) + f.write('\n') + + def get_api_repos(API_URL): - ''' + """ get repos of api, return repos list - ''' + """ access_token = get_access_token() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', @@ -21,19 +49,20 @@ def get_api_repos(API_URL): } s = requests.session() s.keep_alive = False # don't keep the session - time.sleep(3) # not get so fast + time.sleep(3) # not get so fast # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, - r = requests.get(API_URL,headers=headers) + r = requests.get(API_URL, headers=headers) if r.status_code != 200: - raise ValueError('Can not retrieve from {}'.format(api)) + raise ValueError('Can not retrieve from {}'.format(API_URL)) repos_dict = json.loads(r.content) repos = repos_dict['items'] return repos + def get_graphql_data(GQL): - ''' + """ use graphql to get data - ''' + """ access_token = get_access_token() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', @@ -44,11 +73,11 @@ def get_graphql_data(GQL): s = requests.session() s.keep_alive = False # don't keep the session graphql_api = "https://api.github.com/graphql" - time.sleep(5) # not get so fast + time.sleep(5) # not get so fast # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, - r = requests.post(url = graphql_api, json={"query": GQL}, headers=headers) + r = requests.post(url=graphql_api, json={"query": GQL}, headers=headers) if r.status_code != 200: raise ValueError('Can not retrieve from {}'.format(GQL)) - return r.json() \ No newline at end of file + return r.json() diff --git a/source/contents.md b/source/contents.md deleted file mode 100644 index 579410c05f..0000000000 --- a/source/contents.md +++ /dev/null @@ -1,32 +0,0 @@ -[Github Ranking](./README.md) -========== -**A list of the most github stars and forks repositories.** - -## Table of Contents -* [Most Stars](#most-stars) -* [Most Forks](#most-forks) -* [ActionScript](#actionscript) -* [C](#c) -* [C\#](#c-1) -* [C\+\+](#c-2) -* [Clojure](#clojure) -* [CoffeeScript](#coffeescript) -* [CSS](#css) -* [Go](#go) -* [Haskell](#haskell) -* [HTML](#html) -* [Java](#java) -* [JavaScript](#javascript) -* [Lua](#lua) -* [MATLAB](#matlab) -* [Objective\-C](#objective-c) -* [Perl](#perl) -* [PHP](#php) -* [Python](#python) -* [R](#r) -* [Ruby](#ruby) -* [Scala](#scala) -* [Shell](#shell) -* [Swift](#swift) -* [TeX](#tex) -* [Vim script](#vim-script) diff --git a/source/process.py b/source/process.py index e5962b5bdf..17228b26bb 100644 --- a/source/process.py +++ b/source/process.py @@ -2,27 +2,66 @@ from datetime import datetime import os import pandas as pd -from common import get_api_repos, get_graphql_data -from writefile import write_text, write_head_contents, write_ranking_repo +from common import get_api_repos, get_graphql_data, write_text, write_ranking_repo +import inspect -class Processor(object): +# languages = ['Python'] # For test +# languages_md = ['Python'] # For test +# table_of_contents = """ +# * [Python](#python)""" # For test +languages = ["ActionScript", "C", "CSharp", "CPP", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java", + "JavaScript", "Lua", "MATLAB", "Objective-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell", + "Swift", "TeX", "Vim-script"] +# Escape characters in markdown like # + - etc +languages_md = ["ActionScript", "C", "C\#", "C\+\+", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java", + "JavaScript", "Lua", "MATLAB", "Objective\-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell", + "Swift", "TeX", "Vim script"] +table_of_contents = """ +* [ActionScript](#actionscript) +* [C](#c) +* [C\#](#c-1) +* [C\+\+](#c-2) +* [Clojure](#clojure) +* [CoffeeScript](#coffeescript) +* [CSS](#css) +* [Go](#go) +* [Haskell](#haskell) +* [HTML](#html) +* [Java](#java) +* [Kotlin](#kotlin) +* [JavaScript](#javascript) +* [Lua](#lua) +* [MATLAB](#matlab) +* [Objective\-C](#objective-c) +* [Perl](#perl) +* [PHP](#php) +* [Python](#python) +* [R](#r) +* [Ruby](#ruby) +* [Scala](#scala) +* [Shell](#shell) +* [Swift](#swift) +* [TeX](#tex) +* [Vim script](#vim-script)""" + + +class ProcessorREST(object): + """ + Github REST API v3 + Deprecating API authentication through query parameters, should send the token in the header + ref: https://developer.github.com/changes/2020-02-10-deprecating-auth-through-query-param/ + search limit 10 times per minute, with token is 30 per minute + check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit + """ def __init__(self): - # self.languages = ['Python'] # For test - # self.languages_md = ['Python'] # For test - self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] - self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] - # search limit 10 times per minute, with token is 30 per minute - # check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit + self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100' self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100' self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100' - self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] + self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues', + 'last_commit', 'description'] self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() def get_all_repos(self): @@ -35,80 +74,22 @@ def get_all_repos(self): repos_forks = get_api_repos(self.api_repo_forks) repos_languages = {} - for lang in self.languages: + for lang in languages: print("Get most stars repos of {}...".format(lang)) - repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang = lang)) + repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang=lang)) return repos_stars, repos_forks, repos_languages - def write_head_contents(self): - write_head_contents('../README.md') - - def write_readme_lang_md(self): - # Most stars save - write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') - write_ranking_repo('../README.md','a',self.repos_stars[0:10]) - print("Save most stars in README.md!") - os.makedirs('../Top100',exist_ok=True) - write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') - write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) - print("Save most stars in Top100/Top-100-stars.md!\n") - - # Most forks save - write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") - write_ranking_repo('../README.md','a',self.repos_forks[0:10]) - print("Save most forks in README.md!") - write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') - write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) - print("Save most forks in Top100/Top-100-forks.md!\n") - - # Most stars in language save - for i in range(len(self.languages)): - lang = self.languages[i] - write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) - write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) - print("Save most stars of {} in README.md!".format(lang)) - write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) - write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) - print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) - - def repo_to_df(self,repos,item): - # prepare for saving data to csv file - repos_list = [] - for idx, repo in enumerate(repos): - repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] - repos_list.append(repo_info) - return pd.DataFrame(repos_list,columns = self.col) - - def save_to_csv(self): - # save top100 repos info to csv file in Data/github-ranking-year-month-day.md - df_all = pd.DataFrame(columns=self.col) - df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') - df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') - df_all = df_all.append(df_repos_stars,ignore_index = True) - df_all = df_all.append(df_repos_forks,ignore_index = True) - for lang in self.repos_languages.keys(): - df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) - df_all = df_all.append(df_repos_lang,ignore_index = True) - - save_date = datetime.utcnow().strftime("%Y-%m-%d") - os.makedirs('../Data',exist_ok=True) - df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') - print('Save data to Data/github-ranking-'+save_date+'.csv') class ProcessorGQL(object): + """ + Github GraphQL API v4 + ref: https://docs.github.com/en/graphql + use graphql to get data, limit 5000 points per hour + check rate_limit with : + curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql + """ def __init__(self): - # self.languages = ["ActionScript","C","CSharp","CPP"] # For test - # self.languages_md = ["ActionScript","C","C\#","CPP"] # For test - self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] - self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] - # use graphql to get data, limit 5000 points per hour - # check rate_limit with : - # curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql self.gql_format = """query{ search(query: "%s", type: REPOSITORY, first: 100) { edges { @@ -138,14 +119,15 @@ def __init__(self): } } """ - self.gql_stars = self.gql_format % ("stars:>1000 sort:stars") - self.gql_forks = self.gql_format % ("forks:>1000 sort:forks") - self.gql_stars_lang = self.gql_format % ("language:%s stars:>0 sort:stars") + self.gql_stars = self.gql_format % "stars:>1000 sort:stars" + self.gql_forks = self.gql_format % "forks:>1000 sort:forks" + self.gql_stars_lang = self.gql_format % "language:%s stars:>0 sort:stars" - self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] - self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() + self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues', + 'last_commit', 'description'] - def parse_gql_result(self,result): + @staticmethod + def parse_gql_result(result): res = [] for repo in result["data"]["search"]["edges"]: repo_data = repo['node'] @@ -169,70 +151,131 @@ def get_all_repos(self): print("Get repos of most stars...") repos_stars_gql = get_graphql_data(self.gql_stars) repos_stars = self.parse_gql_result(repos_stars_gql) + print("Get repos of most stars success!") print("Get repos of most forks...") repos_forks_gql = get_graphql_data(self.gql_forks) repos_forks = self.parse_gql_result(repos_forks_gql) + print("Get repos of most forks success!") repos_languages = {} - for lang in self.languages: + for lang in languages: print("Get most stars repos of {}...".format(lang)) repos_languages[lang] = self.parse_gql_result( - get_graphql_data(self.gql_stars_lang % (lang)) + get_graphql_data(self.gql_stars_lang % lang) ) + print("Get most stars repos of {} success!".format(lang)) return repos_stars, repos_forks, repos_languages - def write_head_contents(self): - write_head_contents('../README.md') + +class WriteFile(object): + def __init__(self, repos_stars, repos_forks, repos_languages): + self.repos_stars = repos_stars + self.repos_forks = repos_forks + self.repos_languages = repos_languages + self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues', + 'last_commit', 'description'] + self.repo_list = [] + self.repo_list.extend([{ + "desc": "Stars", + "desc_md": "Stars", + "title_readme": "Most Stars", + "title_100": "Top 100 Stars", + "file_100": "Top-100-stars.md", + "data": repos_stars, + "item": "top-100-stars", + }, { + "desc": "Forks", + "desc_md": "Forks", + "title_readme": "Most Forks", + "title_100": "Top 100 Forks", + "file_100": "Top-100-forks.md", + "data": repos_forks, + "item": "top-100-forks", + }]) + for i in range(len(languages)): + lang = languages[i] + lang_md = languages_md[i] + self.repo_list.append({ + "desc": "Forks", + "desc_md": "Forks", + "title_readme": lang_md, + "title_100": f"Top 100 Stars in {lang_md}", + "file_100": f"{lang}.md", + "data": repos_languages[lang], + "item": lang, + }) + + @staticmethod + def write_head_contents(): + # write the head and contents of README.md + write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + head_contents = inspect.cleandoc("""[Github Ranking](./README.md) + ========== + + **A list of the most github stars and forks repositories.** + + *Last Automatic Update Time: {write_time}* + + ## Table of Contents + + * [Most Stars](#most-stars) + * [Most Forks](#most-forks)""".format(write_time=write_time)) + table_of_contents + write_text("../README.md", 'w', head_contents) def write_readme_lang_md(self): - # Most stars save - write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') - write_ranking_repo('../README.md','a',self.repos_stars[0:10]) - print("Save most stars in README.md!") - os.makedirs('../Top100',exist_ok=True) - write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') - write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) - print("Save most stars in Top100/Top-100-stars.md!\n") - - # Most forks save - write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") - write_ranking_repo('../README.md','a',self.repos_forks[0:10]) - print("Save most forks in README.md!") - write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') - write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) - print("Save most forks in Top100/Top-100-forks.md!\n") - - # Most stars in language save - for i in range(len(self.languages)): - lang = self.languages[i] - write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) - write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) - print("Save most stars of {} in README.md!".format(lang)) - write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) - write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) - print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) - - def repo_to_df(self,repos,item): - # prepare for saving data to csv file + os.makedirs('../Top100', exist_ok=True) + for repo in self.repo_list: + # README.md + title_readme, title_100, file_100, data = repo["title_readme"], repo["title_100"], repo["file_100"], repo["data"] + write_text('../README.md', 'a', + f"\n## {title_readme}\n\nThis is top 10, for more click **[{title_100}](Top100/{file_100})**\n\n") + write_ranking_repo('../README.md', 'a', data[:10]) + print(f"Save {title_readme} in README.md!") + + # Top 100 file + write_text(f"../Top100/{file_100}", "w", + f"[Github Ranking](../README.md)\n==========\n\n## {title_100}\n\n") + write_ranking_repo(f"../Top100/{file_100}", 'a', data) + print(f"Save {title_100} in Top100/{file_100}!\n") + + def repo_to_df(self, repos, item): + # prepare for saving data to csv file repos_list = [] for idx, repo in enumerate(repos): - repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] + repo_info = [idx + 1, item, repo['name'], repo['stargazers_count'], repo['forks_count'], repo['language'], + repo['html_url'], repo['owner']['login'], repo['open_issues_count'], repo['pushed_at'], + repo['description']] repos_list.append(repo_info) - return pd.DataFrame(repos_list,columns = self.col) + return pd.DataFrame(repos_list, columns=self.col) def save_to_csv(self): - # save top100 repos info to csv file in Data/github-ranking-year-month-day.md + # save top100 repos info to csv file in Data/github-ranking-year-month-day.md df_all = pd.DataFrame(columns=self.col) - df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') - df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') - df_all = df_all.append(df_repos_stars,ignore_index = True) - df_all = df_all.append(df_repos_forks,ignore_index = True) - for lang in self.repos_languages.keys(): - df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) - df_all = df_all.append(df_repos_lang,ignore_index = True) + for repo in self.repo_list: + df_repos = self.repo_to_df(repos=repo["data"], item=repo["item"]) + df_all = df_all.append(df_repos, ignore_index=True) save_date = datetime.utcnow().strftime("%Y-%m-%d") - os.makedirs('../Data',exist_ok=True) - df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') - print('Save data to Data/github-ranking-'+save_date+'.csv') \ No newline at end of file + os.makedirs('../Data', exist_ok=True) + df_all.to_csv('../Data/github-ranking-' + save_date + '.csv', index=False, encoding='utf-8') + print('Save data to Data/github-ranking-' + save_date + '.csv') + + +def run_by_gql(): + ROOT_PATH = os.path.abspath(os.path.join(__file__, "../../")) + os.chdir(os.path.join(ROOT_PATH, 'source')) + + # processor = ProcessorREST() # use Github REST API v3 + processor = ProcessorGQL() # use Github GraphQL API v4 + repos_stars, repos_forks, repos_languages = processor.get_all_repos() + wt_obj = WriteFile(repos_stars, repos_forks, repos_languages) + wt_obj.write_head_contents() + wt_obj.write_readme_lang_md() + wt_obj.save_to_csv() + + +if __name__ == "__main__": + t1 = datetime.now() + run_by_gql() + print("Total time: {}s".format((datetime.now() - t1).total_seconds())) diff --git a/source/run.py b/source/run.py deleted file mode 100644 index 2003ba6cef..0000000000 --- a/source/run.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime -import os -from process import Processor,ProcessorGQL - -if __name__=="__main__": - t1 = datetime.now() - - ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../")) - os.chdir(os.path.join(ROOT_PATH,'source')) - - # processor = Processor() # use Github REST API v3 - processor = ProcessorGQL() # use Github GraphQL API v4 - print("Write head and contents of README.md!") - processor.write_head_contents() - print("write to readme and languages.md") - processor.write_readme_lang_md() - print("Save data to csv file") - processor.save_to_csv() - - print("Total time: {}s".format((datetime.now()-t1).total_seconds())) \ No newline at end of file diff --git a/source/writefile.py b/source/writefile.py deleted file mode 100644 index aeb50470c7..0000000000 --- a/source/writefile.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime - -def write_text(file_name,method,text): - ''' - write text to file - - method: 'a'-append, 'w'-overwrite - ''' - with open(file_name, method, encoding='utf-8') as f: - f.write(text) - -def write_head_contents(file_name): - # write the head and contents of README.md - # write_head_contents('README.md') - write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - head = "*Last Automatic Update Time: {}*\n\n".format(write_time) - with open('contents.md','r') as f: - contents = f.read() - write_text(file_name,'w',head+contents) - -def write_ranking_repo(file_name,method,repos): - # method: 'a'-append or 'w'-overwrite - table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\ -| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n" - with open(file_name, method, encoding='utf-8') as f: - f.write(table_head) - for idx, repo in enumerate(repos): - repo_description = repo['description'] - if repo_description is not None: - repo_description = repo_description.replace('|','\|') #in case there is '|' in description - f.write('| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format( - idx + 1, - repo['name'], - repo['html_url'], - repo['stargazers_count'], - repo['forks_count'], - repo['language'], - repo['open_issues_count'], - repo_description, - repo['pushed_at'] - ) - ) - f.write('\n') \ No newline at end of file From aad59863647aea7a79724ed74d686d1db1ad8cbd Mon Sep 17 00:00:00 2001 From: EvanLi Date: Thu, 19 Aug 2021 19:58:22 +0800 Subject: [PATCH 4/4] add languages --- source/process.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/source/process.py b/source/process.py index 17228b26bb..89a16220e1 100644 --- a/source/process.py +++ b/source/process.py @@ -9,13 +9,13 @@ # languages_md = ['Python'] # For test # table_of_contents = """ # * [Python](#python)""" # For test -languages = ["ActionScript", "C", "CSharp", "CPP", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java", - "JavaScript", "Lua", "MATLAB", "Objective-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell", - "Swift", "TeX", "Vim-script"] +languages = ["ActionScript", "C", "CSharp", "CPP", "Clojure", "CoffeeScript", "CSS", "Dart", "DM", "Elixir", "Go", "Groovy", "Haskell", "HTML", "Java", + "JavaScript", "Julia", "Kotlin", "Lua", "MATLAB", "Objective-C", "Perl", "PHP", "PowerShell", "Python", "R", "Ruby", "Rust", "Scala", "Shell", + "Swift", "TeX", "TypeScript", "Vim-script"] # Escape characters in markdown like # + - etc -languages_md = ["ActionScript", "C", "C\#", "C\+\+", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java", - "JavaScript", "Lua", "MATLAB", "Objective\-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell", - "Swift", "TeX", "Vim script"] +languages_md = ["ActionScript", "C", "C\#", "C\+\+", "Clojure", "CoffeeScript", "CSS", "Dart", "DM", "Elixir", "Go", "Groovy", "Haskell", "HTML", "Java", + "JavaScript", "Julia", "Kotlin", "Lua", "MATLAB", "Objective\-C", "Perl", "PHP", "PowerShell", "Python", "R", "Ruby", "Rust", "Scala", "Shell", + "Swift", "TeX", "TypeScript", "Vim script"] table_of_contents = """ * [ActionScript](#actionscript) * [C](#c) @@ -24,24 +24,32 @@ * [Clojure](#clojure) * [CoffeeScript](#coffeescript) * [CSS](#css) +* [Dart](#dart) +* [DM](#dm) +* [Elixir](#elixir) * [Go](#go) +* [Groovy](#groovy) * [Haskell](#haskell) * [HTML](#html) * [Java](#java) -* [Kotlin](#kotlin) * [JavaScript](#javascript) +* [Julia](#julia) +* [Kotlin](#kotlin) * [Lua](#lua) * [MATLAB](#matlab) * [Objective\-C](#objective-c) * [Perl](#perl) * [PHP](#php) +* [PowerShell](#powershell) * [Python](#python) * [R](#r) * [Ruby](#ruby) +* [Rust](#rust) * [Scala](#scala) * [Shell](#shell) * [Swift](#swift) * [TeX](#tex) +* [TypeScript](#typeScript) * [Vim script](#vim-script)"""