go并发请求url
sync.WaitGroup写法
package mainimport ("database/sql""fmt""net/http""sync""time"_ "github.com/go-sql-driver/mysql"
)func main() {//开始计时start := time.Now()//链接数据库，用户名：密码@tcp(地址:端口)/数据库名dsn := "gin:gin@tcp(42.1.25.18:3306)/gin"// 连接到MySQL数据库db, err := sql.Open("mysql", dsn)if err != nil {fmt.Println("数据库连接错误: ", err)return}defer db.Close()urls := []string{"https://www.baidu.com","https://www.taobao.com","https://www.tmall.com","https://www.jd.com","https://www.sina.com.cn","https://www.sohu.com","https://www.qq.com","https://www.163.com","https://www.zhihu.com","https://www.weibo.com","https://www.bilibili.com","https://www.youku.com","https://www.iqiyi.com","https://www.alipay.com","https://www.dingtalk.com","https://www.wechat.com","https://www.360.cn","https://www.58.com","https://www.liepin.com","https://www.zhaopin.com","https://www.douban.com","https://www.ifeng.com","https://www.ctrip.com","https://www.qunar.com","https://www.meituan.com","https://www.dianping.com","https://www.ele.me","https://www.vip.com","https://www.suning.com","https://www.mi.com","https://www.huawei.com","https://www.vivo.com.cn","https://www.oppo.com","https://www.le.com","https://www.ganji.com","https://www.kuaidi100.com","https://www.17173.com","https://www.4399.com","https://www.tgbus.com","https://www.10010.com","https://www.10086.cn","https://www.189.cn","https://www.21cn.com","https://www.chinadaily.com.cn","https://www.xinhuanet.com","https://www.people.com.cn","https://www.thepaper.cn","https://www.guokr.com","https://www.ftchinese.com","https://www.cbnweek.com","https://www.163.com","https://www.sina.com.cn","https://www.sohu.com","https://www.qq.com","https://e.baidu.com","https://hezuo.baidu.com","https://yiyan.baidu.com",}//设置3s超时，避免请求时间过长client := http.Client{Timeout: 3 * time.Second,}//定义chan结构体，url，statustype Web struct {url stringstatus string}//并发请求，定义一个WaitGroup，用于等待所有请求完成var wg sync.WaitGroupfor _, url := range urls {//增加一个等待wg.Add(1)//启动一个goroutinego func(url string) {//协程结束时调用Done通知main函数工作已经完成defer wg.Done()//请求resp, err := client.Get(url)if err != nil {fmt.Println("请求失败", err)_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, "请求失败")if err != nil {return}} else {// 写入数据库_, err1 := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, resp.StatusCode)if err1 != nil {fmt.Println("写入数据库错误:", err1)return}}}(url)}//等待所有请求完成wg.Wait()//输出耗时elapsed := time.Since(start)fmt.Println("耗时：", elapsed)
}
sync.WaitGroup+channel
package mainimport ("database/sql""fmt""net/http""strconv""sync""time"_ "github.com/go-sql-driver/mysql"
)func main() {//开始计时start := time.Now()//链接数据库，用户名：密码@tcp(地址:端口)/数据库名dsn := "gin:gin@tcp(4.19.21.17:3306)/gin"// 连接到MySQL数据库db, err := sql.Open("mysql", dsn)if err != nil {fmt.Println("数据库连接错误: ", err)return}//关闭数据库defer db.Close()urls := []string{"https://www.baidu.com","https://www.taobao.com","https://www.tmall.com","https://www.jd.com","https://www.sina.com.cn","https://www.sohu.com","https://www.qq.com","https://www.163.com","https://www.zhihu.com","https://www.weibo.com","https://www.bilibili.com","https://www.youku.com","https://www.iqiyi.com","https://www.alipay.com","https://www.dingtalk.com","https://www.wechat.com","https://www.360.cn","https://www.58.com","https://www.liepin.com","https://www.zhaopin.com","https://www.douban.com","https://www.ifeng.com","https://www.ctrip.com","https://www.qunar.com","https://www.meituan.com","https://www.dianping.com","https://www.ele.me","https://www.vip.com","https://www.suning.com","https://www.mi.com","https://www.huawei.com","https://www.vivo.com.cn","https://www.oppo.com","https://www.le.com","https://www.ganji.com","https://www.kuaidi100.com","https://www.17173.com","https://www.4399.com","https://www.tgbus.com","https://www.10010.com","https://www.10086.cn","https://www.189.cn","https://www.21cn.com","https://www.chinadaily.com.cn","https://www.xinhuanet.com","https://www.people.com.cn","https://www.thepaper.cn","https://www.guokr.com","https://www.ftchinese.com","https://www.cbnweek.com","https://www.163.com","https://www.sina.com.cn","https://www.sohu.com","https://www.qq.com","https://e.baidu.com","https://hezuo.baidu.com","https://yiyan.baidu.com",}//设置3s超时，避免请求时间过长client := http.Client{Timeout: 3 * time.Second,}//定义chan结构体，url，statustype Web struct {url stringstatus string}//并发请求var wg sync.WaitGroup//创建一个并发请求的channel,缓冲区大小为urls的长度results := make(chan *Web, len(urls))//遍历urls，k为索引，url为值for k, url := range urls {//增加一个等待wg.Add(1)fmt.Println("开始请求：", k, url)//启动一个goroutine，传入k和urlgo func(k int, url string) {//协程结束时调用Done通知main函数工作已经完成defer wg.Done()//请求resp, err := client.Get(url)if err != nil {fmt.Println("请求失败", err)//设置状态为请求失败results <- &Web{url: url, status: "请求失败"}return}//关闭请求defer resp.Body.Close()fmt.Println("请求完成：", k, url)//写入channelresults <- &Web{url: url, status: strconv.Itoa(resp.StatusCode)}}(k, url)}//统计有多少个请求完成了fmt.Println("等待所有请求完成")//等待所有请求完成wg.Wait()//关闭channelclose(results)//统计results的长度//fmt.Println("results长度：", len(results))fmt.Println("开始写入数据库")//遍历channelfor resp := range results {// 写入数据库_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", resp.url, resp.status)if err != nil {fmt.Println("写入数据库错误:", err)}}fmt.Println("写入数据库完成")//输出耗时elapsed := time.Since(start)fmt.Println("耗时：", elapsed)
}