github公开项目爬取

import requestsdef search_github_repositories(keyword, token=None, language=None, max_results=1000):"""通过 GitHub API 搜索仓库，支持分页获取所有结果（最多 1000 条）:param keyword: 搜索关键词:param token: GitHub Token（可选，但建议使用以提高速率限制）:param language: 过滤语言:param max_results: 最大结果数（GitHub 允许最多 1000 条）:return: 仓库列表"""url = "https://api.github.com/search/repositories"headers = {"Accept": "application/vnd.github.v3+json"}if token:headers["Authorization"] = f"Bearer {token}"query = keywordif language:query += f" language:{language}"repositories = []page = 1per_page = 100  # GitHub 单页最大允许 100 条while len(repositories) < max_results:params = {"q": query,"sort": "stars","order": "desc","page": page,"per_page": per_page}try:response = requests.get(url, headers=headers, params=params)response.raise_for_status()data = response.json()# 检查是否还有更多结果if not data.get("items"):breakfor item in data["items"]:repo_info = {"name": item["name"],"owner": item["owner"]["login"],"url": item["html_url"],"clone_url": item["clone_url"],"description": item["description"],"language": item["language"],"stars": item["stargazers_count"]}repositories.append(repo_info)# 达到用户指定的最大数量时停止if len(repositories) >= max_results:breakpage += 1# GitHub 最多允许 10 页（即 10 * 100=1000 条）if page > 10:breakexcept requests.exceptions.RequestException as e:print(f"请求失败: {e}")breakreturn repositoriesdef save_to_txt(results, filename="github_results.txt"):"""将结果保存到文本文件:param results: 仓库列表:param filename: 保存文件名"""with open(filename, "w", encoding="utf-8") as f:for repo in results:# 格式化单行文本（用 | 分隔关键信息）line = (f"仓库: {repo['owner']}/{repo['name']} | "f"URL: {repo['url']} | "f"语言: {repo['language']} | "f"星数: {repo['stars']} | "f"克隆地址: {repo['clone_url']}\n")f.write(line)print(f"结果已保存至 {filename}")# 示例用法
if __name__ == "__main__":keyword = "Aerospace Control"language = "Python"token = "********"  # 强烈建议使用 Token# 获取最多 1000 条结果（实际数量取决于搜索匹配的总数）results = search_github_repositories(keyword, token, language=language, max_results=1000)if results:print(f"找到 {len(results)} 个 {language} 相关仓库:")for idx, repo in enumerate(results, 1):print(f"\n{idx}. {repo['owner']}/{repo['name']}")print(f"   URL: {repo['url']}")print(f"   语言: {repo['language']}")  # 显示语言print(f"   克隆地址: {repo['clone_url']}")else:print("未找到结果")if results:save_to_txt(results)print(f"实际获取 {len(results)} 条结果")else:print("未找到结果")