From b7cb6774bf4d07bfb0bd23d8070447573f3386ef Mon Sep 17 00:00:00 2001 From: wwqgtxx Date: Sun, 22 Sep 2024 13:57:57 +0800 Subject: [PATCH] chore: support ETag for update geo --- adapter/provider/parser.go | 2 +- component/resource/fetcher.go | 20 +-------- component/resource/vehicle.go | 50 +++++++++++++++++----- component/updater/update_geo.go | 75 +++++++++++++++++++++++++++------ component/updater/update_ui.go | 4 +- component/updater/utils.go | 4 +- config/config.go | 4 +- constant/provider/interface.go | 1 + hub/executor/executor.go | 4 +- hub/route/upgrade.go | 2 +- rules/provider/parse.go | 2 +- 11 files changed, 117 insertions(+), 51 deletions(-) diff --git a/adapter/provider/parser.go b/adapter/provider/parser.go index 078e8695c..b0db5db34 100644 --- a/adapter/provider/parser.go +++ b/adapter/provider/parser.go @@ -111,7 +111,7 @@ func ParseProxyProvider(name string, mapping map[string]any) (types.ProxyProvide return nil, fmt.Errorf("%w: %s", errSubPath, path) } } - vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header) + vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header, resource.DefaultHttpTimeout) default: return nil, fmt.Errorf("%w: %s", errVehicleType, schema.Type) } diff --git a/component/resource/fetcher.go b/component/resource/fetcher.go index 0b15e6c32..3e2ec2399 100644 --- a/component/resource/fetcher.go +++ b/component/resource/fetcher.go @@ -3,7 +3,6 @@ package resource import ( "context" "os" - "path/filepath" "time" types "github.com/metacubex/mihomo/constant/provider" @@ -13,11 +12,6 @@ import ( "github.com/samber/lo" ) -var ( - fileMode os.FileMode = 0o666 - dirMode os.FileMode = 0o755 -) - type Parser[V any] func([]byte) (V, error) type Fetcher[V any] struct { @@ -118,7 +112,7 @@ func (f *Fetcher[V]) loadBuf(buf []byte, hash types.HashType, updateFile bool) ( } if updateFile { - if err = safeWrite(f.vehicle.Path(), buf); err != nil { + if err = f.vehicle.Write(buf); err != nil { return lo.Empty[V](), false, err } } @@ -205,18 +199,6 @@ func (f *Fetcher[V]) updateWithLog() { return } -func safeWrite(path string, buf []byte) error { - dir := filepath.Dir(path) - - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err := os.MkdirAll(dir, dirMode); err != nil { - return err - } - } - - return os.WriteFile(path, buf, fileMode) -} - func NewFetcher[V any](name string, interval time.Duration, vehicle types.Vehicle, parser Parser[V], onUpdate func(V)) *Fetcher[V] { ctx, cancel := context.WithCancel(context.Background()) return &Fetcher[V]{ diff --git a/component/resource/vehicle.go b/component/resource/vehicle.go index ccc59aece..200b950e0 100644 --- a/component/resource/vehicle.go +++ b/component/resource/vehicle.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "path/filepath" "time" mihomoHttp "github.com/metacubex/mihomo/component/http" @@ -13,6 +14,25 @@ import ( types "github.com/metacubex/mihomo/constant/provider" ) +const ( + DefaultHttpTimeout = time.Second * 20 + + fileMode os.FileMode = 0o666 + dirMode os.FileMode = 0o755 +) + +func safeWrite(path string, buf []byte) error { + dir := filepath.Dir(path) + + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err := os.MkdirAll(dir, dirMode); err != nil { + return err + } + } + + return os.WriteFile(path, buf, fileMode) +} + type FileVehicle struct { path string } @@ -42,15 +62,20 @@ func (f *FileVehicle) Proxy() string { return "" } +func (f *FileVehicle) Write(buf []byte) error { + return safeWrite(f.path, buf) +} + func NewFileVehicle(path string) *FileVehicle { return &FileVehicle{path: path} } type HTTPVehicle struct { - url string - path string - proxy string - header http.Header + url string + path string + proxy string + header http.Header + timeout time.Duration } func (h *HTTPVehicle) Url() string { @@ -69,8 +94,12 @@ func (h *HTTPVehicle) Proxy() string { return h.proxy } +func (h *HTTPVehicle) Write(buf []byte) error { + return safeWrite(h.path, buf) +} + func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []byte, hash types.HashType, err error) { - ctx, cancel := context.WithTimeout(ctx, time.Second*20) + ctx, cancel := context.WithTimeout(ctx, h.timeout) defer cancel() header := h.header setIfNoneMatch := false @@ -107,11 +136,12 @@ func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []b return } -func NewHTTPVehicle(url string, path string, proxy string, header http.Header) *HTTPVehicle { +func NewHTTPVehicle(url string, path string, proxy string, header http.Header, timeout time.Duration) *HTTPVehicle { return &HTTPVehicle{ - url: url, - path: path, - proxy: proxy, - header: header, + url: url, + path: path, + proxy: proxy, + header: header, + timeout: timeout, } } diff --git a/component/updater/update_geo.go b/component/updater/update_geo.go index 6246b1c66..454cd84de 100644 --- a/component/updater/update_geo.go +++ b/component/updater/update_geo.go @@ -13,7 +13,9 @@ import ( "github.com/metacubex/mihomo/component/geodata" _ "github.com/metacubex/mihomo/component/geodata/standard" "github.com/metacubex/mihomo/component/mmdb" + "github.com/metacubex/mihomo/component/resource" C "github.com/metacubex/mihomo/constant" + P "github.com/metacubex/mihomo/constant/provider" "github.com/metacubex/mihomo/log" "github.com/oschwald/maxminddb-golang" @@ -43,30 +45,52 @@ func SetGeoUpdateInterval(newGeoUpdateInterval int) { } func UpdateMMDB() (err error) { - defer mmdb.ReloadIP() - data, err := downloadForBytes(geodata.MmdbUrl()) + vehicle := resource.NewHTTPVehicle(geodata.MmdbUrl(), C.Path.MMDB(), "", nil, defaultHttpTimeout) + var oldHash P.HashType + if buf, err := os.ReadFile(vehicle.Path()); err == nil { + oldHash = P.MakeHash(buf) + } + data, hash, err := vehicle.Read(context.Background(), oldHash) if err != nil { return fmt.Errorf("can't download MMDB database file: %w", err) } + if oldHash.Equal(hash) { // same hash, ignored + return nil + } + if len(data) == 0 { + return fmt.Errorf("can't download MMDB database file: no data") + } + instance, err := maxminddb.FromBytes(data) if err != nil { return fmt.Errorf("invalid MMDB database file: %s", err) } _ = instance.Close() + defer mmdb.ReloadIP() mmdb.IPInstance().Reader.Close() // mmdb is loaded with mmap, so it needs to be closed before overwriting the file - if err = saveFile(data, C.Path.MMDB()); err != nil { + if err = vehicle.Write(data); err != nil { return fmt.Errorf("can't save MMDB database file: %w", err) } return nil } func UpdateASN() (err error) { - defer mmdb.ReloadASN() - data, err := downloadForBytes(geodata.ASNUrl()) + vehicle := resource.NewHTTPVehicle(geodata.ASNUrl(), C.Path.ASN(), "", nil, defaultHttpTimeout) + var oldHash P.HashType + if buf, err := os.ReadFile(vehicle.Path()); err == nil { + oldHash = P.MakeHash(buf) + } + data, hash, err := vehicle.Read(context.Background(), oldHash) if err != nil { return fmt.Errorf("can't download ASN database file: %w", err) } + if oldHash.Equal(hash) { // same hash, ignored + return nil + } + if len(data) == 0 { + return fmt.Errorf("can't download ASN database file: no data") + } instance, err := maxminddb.FromBytes(data) if err != nil { @@ -74,42 +98,69 @@ func UpdateASN() (err error) { } _ = instance.Close() + defer mmdb.ReloadASN() mmdb.ASNInstance().Reader.Close() // mmdb is loaded with mmap, so it needs to be closed before overwriting the file - if err = saveFile(data, C.Path.ASN()); err != nil { + if err = vehicle.Write(data); err != nil { return fmt.Errorf("can't save ASN database file: %w", err) } return nil } func UpdateGeoIp() (err error) { - defer geodata.ClearGeoIPCache() geoLoader, err := geodata.GetGeoDataLoader("standard") - data, err := downloadForBytes(geodata.GeoIpUrl()) + + vehicle := resource.NewHTTPVehicle(geodata.GeoIpUrl(), C.Path.GeoIP(), "", nil, defaultHttpTimeout) + var oldHash P.HashType + if buf, err := os.ReadFile(vehicle.Path()); err == nil { + oldHash = P.MakeHash(buf) + } + data, hash, err := vehicle.Read(context.Background(), oldHash) if err != nil { return fmt.Errorf("can't download GeoIP database file: %w", err) } + if oldHash.Equal(hash) { // same hash, ignored + return nil + } + if len(data) == 0 { + return fmt.Errorf("can't download GeoIP database file: no data") + } + if _, err = geoLoader.LoadIPByBytes(data, "cn"); err != nil { return fmt.Errorf("invalid GeoIP database file: %s", err) } - if err = saveFile(data, C.Path.GeoIP()); err != nil { + + defer geodata.ClearGeoIPCache() + if err = vehicle.Write(data); err != nil { return fmt.Errorf("can't save GeoIP database file: %w", err) } return nil } func UpdateGeoSite() (err error) { - defer geodata.ClearGeoSiteCache() geoLoader, err := geodata.GetGeoDataLoader("standard") - data, err := downloadForBytes(geodata.GeoSiteUrl()) + + vehicle := resource.NewHTTPVehicle(geodata.GeoSiteUrl(), C.Path.GeoSite(), "", nil, defaultHttpTimeout) + var oldHash P.HashType + if buf, err := os.ReadFile(vehicle.Path()); err == nil { + oldHash = P.MakeHash(buf) + } + data, hash, err := vehicle.Read(context.Background(), oldHash) if err != nil { return fmt.Errorf("can't download GeoSite database file: %w", err) } + if oldHash.Equal(hash) { // same hash, ignored + return nil + } + if len(data) == 0 { + return fmt.Errorf("can't download GeoSite database file: no data") + } if _, err = geoLoader.LoadSiteByBytes(data, "cn"); err != nil { return fmt.Errorf("invalid GeoSite database file: %s", err) } - if err = saveFile(data, C.Path.GeoSite()); err != nil { + defer geodata.ClearGeoSiteCache() + if err = vehicle.Write(data); err != nil { return fmt.Errorf("can't save GeoSite database file: %w", err) } return nil diff --git a/component/updater/update_ui.go b/component/updater/update_ui.go index b29bee9f9..29081761a 100644 --- a/component/updater/update_ui.go +++ b/component/updater/update_ui.go @@ -17,12 +17,12 @@ import ( var ( ExternalUIURL string ExternalUIPath string - AutoUpdateUI bool + AutoDownloadUI bool ) var xdMutex sync.Mutex -func UpdateUI() error { +func DownloadUI() error { xdMutex.Lock() defer xdMutex.Unlock() diff --git a/component/updater/utils.go b/component/updater/utils.go index d9da48c31..b5c694ff4 100644 --- a/component/updater/utils.go +++ b/component/updater/utils.go @@ -13,8 +13,10 @@ import ( "golang.org/x/exp/constraints" ) +const defaultHttpTimeout = time.Second * 90 + func downloadForBytes(url string) ([]byte, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*90) + ctx, cancel := context.WithTimeout(context.Background(), defaultHttpTimeout) defer cancel() resp, err := mihomoHttp.HttpRequest(ctx, url, http.MethodGet, nil, nil) if err != nil { diff --git a/config/config.go b/config/config.go index 5a95de79d..8fde168c3 100644 --- a/config/config.go +++ b/config/config.go @@ -701,7 +701,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) { // checkout externalUI exist if cfg.ExternalUI != "" { - updater.AutoUpdateUI = true + updater.AutoDownloadUI = true updater.ExternalUIPath = C.Path.Resolve(cfg.ExternalUI) } else { // default externalUI path @@ -710,7 +710,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) { // checkout UIpath/name exist if cfg.ExternalUIName != "" { - updater.AutoUpdateUI = true + updater.AutoDownloadUI = true updater.ExternalUIPath = path.Join(updater.ExternalUIPath, cfg.ExternalUIName) } diff --git a/constant/provider/interface.go b/constant/provider/interface.go index 2c83a1b88..511e8f18e 100644 --- a/constant/provider/interface.go +++ b/constant/provider/interface.go @@ -33,6 +33,7 @@ func (v VehicleType) String() string { type Vehicle interface { Read(ctx context.Context, oldHash HashType) (buf []byte, hash HashType, err error) + Write(buf []byte) error Path() string Url() string Proxy() string diff --git a/hub/executor/executor.go b/hub/executor/executor.go index fddcc42ef..07b3b0909 100644 --- a/hub/executor/executor.go +++ b/hub/executor/executor.go @@ -381,13 +381,13 @@ func updateTunnels(tunnels []LC.Tunnel) { } func initExternalUI() { - if updater.AutoUpdateUI { + if updater.AutoDownloadUI { dirEntries, _ := os.ReadDir(updater.ExternalUIPath) if len(dirEntries) > 0 { log.Infoln("UI already exists, skip downloading") } else { log.Infoln("External UI downloading ...") - updater.UpdateUI() + updater.DownloadUI() } } } diff --git a/hub/route/upgrade.go b/hub/route/upgrade.go index 76ac26a10..25c326ddd 100644 --- a/hub/route/upgrade.go +++ b/hub/route/upgrade.go @@ -47,7 +47,7 @@ func upgradeCore(w http.ResponseWriter, r *http.Request) { } func updateUI(w http.ResponseWriter, r *http.Request) { - err := updater.UpdateUI() + err := updater.DownloadUI() if err != nil { log.Warnln("%s", err) render.Status(r, http.StatusInternalServerError) diff --git a/rules/provider/parse.go b/rules/provider/parse.go index b3c6af3cb..3bd8f54c8 100644 --- a/rules/provider/parse.go +++ b/rules/provider/parse.go @@ -53,7 +53,7 @@ func ParseRuleProvider(name string, mapping map[string]interface{}, parse func(t return nil, fmt.Errorf("%w: %s", errSubPath, path) } } - vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil) + vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil, resource.DefaultHttpTimeout) default: return nil, fmt.Errorf("unsupported vehicle type: %s", schema.Type) }