add go-rod/stealth for chrome
add httpproxy, useragent and some chrome config
This commit is contained in:
parent
a07bbb7412
commit
f24a55fba1
176
chrome.go
176
chrome.go
@ -1,7 +1,11 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"apigo.cc/gojs"
|
||||
@ -10,6 +14,7 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/launcher/flags"
|
||||
"github.com/ssgo/u"
|
||||
"github.com/ysmood/fetchup"
|
||||
)
|
||||
|
||||
type Chrome struct {
|
||||
@ -63,10 +68,14 @@ type ChromeInfo struct {
|
||||
}
|
||||
|
||||
type ChromeOption struct {
|
||||
ChromeURL string
|
||||
ChromeOption []string
|
||||
ShowWindow bool
|
||||
ChromeHtpProxy string
|
||||
ChromeURL string
|
||||
ChromePath string
|
||||
ChromeDownloadMirror string
|
||||
ChromeDownloadPath string
|
||||
ChromeOption []string
|
||||
ShowWindow bool
|
||||
HttpProxy string
|
||||
UserAgent string
|
||||
}
|
||||
|
||||
func (ch *Chrome) Info(showWindow *bool, vm *goja.Runtime) (ChromeInfo, error) {
|
||||
@ -88,6 +97,17 @@ func (ch *Chrome) Info(showWindow *bool, vm *goja.Runtime) (ChromeInfo, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
var hostConf = map[string]struct {
|
||||
urlPrefix string
|
||||
zipName string
|
||||
}{
|
||||
"darwin_amd64": {"Mac", "chrome-mac.zip"},
|
||||
"darwin_arm64": {"Mac_Arm", "chrome-mac.zip"},
|
||||
"linux_amd64": {"Linux_x64", "chrome-linux.zip"},
|
||||
"windows_386": {"Win", "chrome-win.zip"},
|
||||
"windows_amd64": {"Win_x64", "chrome-win.zip"},
|
||||
}[runtime.GOOS+"_"+runtime.GOARCH]
|
||||
|
||||
func StartChrome(opt *ChromeOption, vm *goja.Runtime) (*Chrome, error) {
|
||||
if opt == nil {
|
||||
opt = &ChromeOption{}
|
||||
@ -95,8 +115,20 @@ func StartChrome(opt *ChromeOption, vm *goja.Runtime) (*Chrome, error) {
|
||||
if opt.ChromeURL == "" {
|
||||
opt.ChromeURL = conf.ChromeURL
|
||||
}
|
||||
if opt.ChromeHtpProxy == "" {
|
||||
opt.ChromeHtpProxy = conf.ChromeHtpProxy
|
||||
if opt.ChromePath == "" {
|
||||
opt.ChromePath = conf.ChromePath
|
||||
}
|
||||
if opt.ChromeDownloadMirror == "" {
|
||||
opt.ChromeDownloadMirror = conf.ChromeDownloadMirror
|
||||
}
|
||||
if opt.ChromeDownloadPath == "" {
|
||||
opt.ChromeDownloadPath = conf.ChromeDownloadPath
|
||||
}
|
||||
if opt.HttpProxy == "" {
|
||||
opt.HttpProxy = conf.HttpProxy
|
||||
}
|
||||
if opt.UserAgent == "" {
|
||||
opt.UserAgent = conf.UserAgent
|
||||
}
|
||||
if opt.ChromeOption == nil {
|
||||
opt.ChromeOption = conf.ChromeOption
|
||||
@ -109,22 +141,137 @@ func StartChrome(opt *ChromeOption, vm *goja.Runtime) (*Chrome, error) {
|
||||
if opt.ChromeURL == "" {
|
||||
// 使用本地Chrome
|
||||
ch.launcher = launcher.New()
|
||||
if localBrowserPath, hasLocalBrowser := launcher.LookPath(); hasLocalBrowser {
|
||||
ch.launcher.Bin(localBrowserPath)
|
||||
ch.chromePath = localBrowserPath
|
||||
if opt.ChromePath != "" {
|
||||
ch.launcher.Bin(opt.ChromePath)
|
||||
ch.chromePath = opt.ChromePath
|
||||
} else if path, has := launcher.LookPath(); has {
|
||||
ch.launcher.Bin(path)
|
||||
ch.chromePath = path
|
||||
} else {
|
||||
// 自动下载Chrome
|
||||
lb := launcher.NewBrowser()
|
||||
|
||||
// 默认优先级：淘宝(NPM) -> Playwright -> Google
|
||||
lb.Hosts = []launcher.Host{launcher.HostNPM, launcher.HostPlaywright, launcher.HostGoogle}
|
||||
|
||||
if opt.ChromeDownloadMirror != "" {
|
||||
// 定义私有镜像 Host 函数
|
||||
var customHost launcher.Host = func(rev int) string {
|
||||
// 构造逻辑完全对齐 rod 官方：Prefix/OS_Arch/Revision/ZipName
|
||||
return fmt.Sprintf("%s/%s/%d/%s",
|
||||
strings.TrimSuffix(opt.ChromeDownloadMirror, "/"), // 去掉末尾的斜杠，防止拼接出双斜杠
|
||||
hostConf.urlPrefix,
|
||||
rev,
|
||||
hostConf.zipName,
|
||||
)
|
||||
}
|
||||
// 将自定义镜像插入到最前面
|
||||
lb.Hosts = append([]launcher.Host{customHost}, lb.Hosts...)
|
||||
}
|
||||
|
||||
var path string
|
||||
var err error
|
||||
|
||||
// --- 核心修改区：只有指定了路径才走手动逻辑 ---
|
||||
if opt.ChromeDownloadPath != "" {
|
||||
rev := lb.Revision
|
||||
|
||||
// 构造存放目录和最终二进制路径
|
||||
destDir := filepath.Join(opt.ChromeDownloadPath, fmt.Sprintf("chromium-%d", rev))
|
||||
var binPath string
|
||||
switch runtime.GOOS {
|
||||
case "darwin": // macOS
|
||||
binPath = filepath.Join(destDir, "Chromium.app/Contents/MacOS/Chromium")
|
||||
case "windows": // Windows
|
||||
binPath = filepath.Join(destDir, "chrome.exe")
|
||||
default: // linux
|
||||
binPath = filepath.Join(destDir, "chrome")
|
||||
}
|
||||
|
||||
// --- 检查本地是否存在 (实现跳过下载) ---
|
||||
if _, err = os.Stat(binPath); err == nil {
|
||||
path = binPath
|
||||
} else {
|
||||
// --- 手动触发 fetchup 下载流程 ---
|
||||
urls := []string{}
|
||||
for _, host := range lb.Hosts {
|
||||
urls = append(urls, host(rev))
|
||||
}
|
||||
|
||||
// 使用你发现的 fetchup 逻辑，直接下载到指定的 destDir
|
||||
fu := fetchup.New(destDir, urls...)
|
||||
// 如果你设置了代理，也可以传给 fu.HttpClient
|
||||
err = fu.Fetch()
|
||||
if err == nil {
|
||||
// 自动处理解压后的目录层级缩减
|
||||
_ = fetchup.StripFirstDir(destDir)
|
||||
path = binPath
|
||||
// 修正 Linux 权限
|
||||
if runtime.GOOS != "windows" {
|
||||
_ = os.Chmod(path, 0755)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 如果没有指定自定义路径，直接走 Rod 默认逻辑（会走 ENV 和默认缓存）
|
||||
path, err = lb.Get()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
ch.Close(vm)
|
||||
return nil, gojs.Err(fmt.Errorf("自动下载浏览器失败: %w", err))
|
||||
}
|
||||
|
||||
ch.launcher.Bin(path)
|
||||
ch.chromePath = path
|
||||
}
|
||||
// "--headless=new", "--no-sandbox", "--disable-dev-shm-usage", "--hide-scrollbars", "--font-render-hinting=none", "--disable-blink-features=AutomationControlled", "--disable-infobars", "--lang=zh-CN,zh", "--disable-extensions", "--disable-gpu", "--use-gl=swiftshader", "--ignore-gpu-blocklist", "--use-angle=swiftshader", "--disable-features=Translate"
|
||||
ch.launcher.Headless(!opt.ShowWindow).Set("disable-dev-shm-usage").Set("single-process").Set("disable-blink-features", "AutomationControlled")
|
||||
ch.launcher.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0")
|
||||
if opt.ChromeHtpProxy != "" {
|
||||
ch.launcher.Proxy(opt.ChromeHtpProxy)
|
||||
|
||||
// 基础防御参数
|
||||
ch.launcher.
|
||||
Set("disable-dev-shm-usage").
|
||||
Set("single-process"). // 保留你原有的逻辑
|
||||
Set("disable-blink-features", "AutomationControlled").
|
||||
Set("no-sandbox").
|
||||
Set("disable-infobars").
|
||||
Delete("enable-automation") // 彻底删除自动化控制标志
|
||||
|
||||
// --- 动态 Headless 逻辑 ---
|
||||
if opt.ShowWindow {
|
||||
ch.launcher.Headless(false)
|
||||
} else {
|
||||
// 使用新版无头模式，不要调用 .Headless(true)
|
||||
ch.launcher.Set("headless", "new")
|
||||
}
|
||||
|
||||
// --- 动态 User-Agent 优化 ---
|
||||
if opt.UserAgent != "" {
|
||||
ch.launcher.Set("user-agent", opt.UserAgent)
|
||||
} else {
|
||||
// 建议版本号保持在 130+，目前 146 有点过于领先（当前稳定版大约在 134 左右）
|
||||
const chromeVersion = "134.0.0.0"
|
||||
switch runtime.GOOS {
|
||||
case "darwin": // macOS
|
||||
ch.launcher.Set("user-agent", fmt.Sprintf("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", chromeVersion))
|
||||
case "linux":
|
||||
// 建议 Linux 下还是用 Linux 的 UA，减少被识别为“伪装者”的风险
|
||||
ch.launcher.Set("user-agent", fmt.Sprintf("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", chromeVersion))
|
||||
default: // windows
|
||||
ch.launcher.Set("user-agent", fmt.Sprintf("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", chromeVersion))
|
||||
}
|
||||
}
|
||||
|
||||
if opt.HttpProxy != "" {
|
||||
ch.launcher.Proxy(opt.HttpProxy)
|
||||
}
|
||||
|
||||
// 操作系统特定兼容
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
ch.launcher.Set("disable-setuid-sandbox")
|
||||
case "windows":
|
||||
ch.launcher.Set("disable-features=RendererCodeIntegrity")
|
||||
}
|
||||
|
||||
if opt.ChromeOption != nil {
|
||||
for _, opt := range opt.ChromeOption {
|
||||
a := u.SplitTrimN(opt, "=", 2)
|
||||
@ -157,5 +304,4 @@ func StartChrome(opt *ChromeOption, vm *goja.Runtime) (*Chrome, error) {
|
||||
chromes[ch.id] = ch
|
||||
chromesLock.Unlock()
|
||||
return ch, nil
|
||||
|
||||
}
|
||||
|
||||
33
go.mod
33
go.mod
@ -1,32 +1,31 @@
|
||||
module apigo.cc/gojs/http
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.3
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
apigo.cc/gojs v0.0.32
|
||||
apigo.cc/gojs v0.0.34
|
||||
apigo.cc/gojs/console v0.0.4
|
||||
apigo.cc/gojs/file v0.0.8
|
||||
apigo.cc/gojs/util v0.0.17
|
||||
apigo.cc/gojs/util v0.0.18
|
||||
github.com/go-rod/rod v0.116.2
|
||||
github.com/go-rod/stealth v0.4.9
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/ssgo/config v1.7.10
|
||||
github.com/ssgo/httpclient v1.7.8
|
||||
github.com/ssgo/log v1.7.10
|
||||
github.com/ssgo/s v1.7.25
|
||||
github.com/ssgo/u v1.7.23
|
||||
github.com/ssgo/httpclient v1.7.9
|
||||
github.com/ssgo/log v1.7.11
|
||||
github.com/ssgo/s v1.7.26
|
||||
github.com/ssgo/u v1.7.24
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ZZMarquis/gm v1.3.2 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/emmansun/gmsm v0.40.0 // indirect
|
||||
github.com/emmansun/gmsm v0.41.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/gomodule/redigo v1.9.2 // indirect
|
||||
github.com/google/pprof v0.0.0-20250903194437-c28834ac2320 // indirect
|
||||
github.com/gomodule/redigo v1.9.3 // indirect
|
||||
github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
|
||||
github.com/obscuren/ecies v0.0.0-20150213224233-7c0f4a9b18d9 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
@ -34,7 +33,7 @@ require (
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/ssgo/discover v1.7.10 // indirect
|
||||
github.com/ssgo/redis v1.7.8 // indirect
|
||||
github.com/ssgo/standard v1.7.7 // indirect
|
||||
github.com/ssgo/standard v1.7.8 // indirect
|
||||
github.com/ssgo/tool v0.4.29 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.15 // indirect
|
||||
github.com/tklauser/numcpus v0.10.0 // indirect
|
||||
@ -44,9 +43,9 @@ require (
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.9.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
26
http.go
26
http.go
@ -4,6 +4,7 @@ import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -33,10 +34,14 @@ var defaultHttp = &Http{
|
||||
}
|
||||
|
||||
var conf = struct {
|
||||
Timeout int
|
||||
ChromeURL string
|
||||
ChromeOption []string
|
||||
ChromeHtpProxy string
|
||||
Timeout int
|
||||
ChromeURL string
|
||||
ChromePath string
|
||||
ChromeDownloadMirror string
|
||||
ChromeDownloadPath string
|
||||
ChromeOption []string
|
||||
HttpProxy string
|
||||
UserAgent string
|
||||
}{}
|
||||
|
||||
func init() {
|
||||
@ -99,10 +104,23 @@ func newClient(portal string, argsIn goja.FunctionCall, vm *goja.Runtime) goja.V
|
||||
} else {
|
||||
client = httpclient.GetClient(timeout)
|
||||
}
|
||||
if conf.HttpProxy != "" {
|
||||
if hpUrl, err := url.Parse(conf.HttpProxy); err == nil {
|
||||
rc := client.GetRawClient()
|
||||
if rc.Transport == nil {
|
||||
rc.Transport = &http.Transport{Proxy: http.ProxyURL(hpUrl)}
|
||||
} else if tp, ok := rc.Transport.(*http.Transport); ok {
|
||||
tp.Proxy = http.ProxyURL(hpUrl)
|
||||
}
|
||||
}
|
||||
}
|
||||
cli := &Http{
|
||||
client: client,
|
||||
globalHeaders: make(map[string]string),
|
||||
}
|
||||
if conf.UserAgent != "" {
|
||||
cli.globalHeaders["User-Agent"] = conf.UserAgent
|
||||
}
|
||||
setConfig(cli, opt)
|
||||
return vm.ToValue(gojs.MakeMap(cli))
|
||||
}
|
||||
|
||||
9
page.go
9
page.go
@ -11,6 +11,7 @@ import (
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/input"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/go-rod/stealth"
|
||||
"github.com/ssgo/u"
|
||||
)
|
||||
|
||||
@ -43,7 +44,13 @@ type Position struct {
|
||||
type Rect struct{ X, Y, Width, Height float64 }
|
||||
|
||||
func (ch *Chrome) Open(url string) (*Page, error) {
|
||||
page, err := ch.browser.Page(proto.TargetCreateTarget{URL: url})
|
||||
// page, err := ch.browser.Page(proto.TargetCreateTarget{URL: url})
|
||||
page, err := stealth.Page(ch.browser)
|
||||
if err != nil {
|
||||
return nil, gojs.Err(err)
|
||||
}
|
||||
|
||||
err = page.Navigate(url)
|
||||
if err != nil {
|
||||
return nil, gojs.Err(err)
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user