From 50b0e5a4b00d3c57a8f560cd7ec75ee42d42a06d Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Feb 2023 01:04:09 +0700 Subject: [PATCH 01/37] cmd/ctrld: use proper exit codes for status command While at it, disable sort commands, so help output will be in order. Updates #48 --- cmd/ctrld/cli.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index df96bde..cd18ee1 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -56,6 +56,7 @@ func initCLI() { // Enable opening via explorer.exe on Windows. // See: https://github.com/spf13/cobra/issues/844. cobra.MousetrapHelpText = "" + cobra.EnableCommandSorting = false rootCmd := &cobra.Command{ Use: "ctrld", @@ -177,7 +178,7 @@ func initCLI() { startCmd := &cobra.Command{ PreRun: checkHasElevatedPrivilege, Use: "start", - Short: "Start the ctrld service", + Short: "Install and start the ctrld service", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { sc := &service.Config{} @@ -293,15 +294,18 @@ func initCLI() { status, err := s.Status() if err != nil { stderrMsg(err.Error()) - return + os.Exit(1) } switch status { case service.StatusUnknown: stdoutMsg("Unknown status") + os.Exit(2) case service.StatusRunning: stdoutMsg("Service is running") + os.Exit(0) case service.StatusStopped: stdoutMsg("Service is stopped") + os.Exit(1) } }, } @@ -309,7 +313,7 @@ func initCLI() { uninstallCmd := &cobra.Command{ PreRun: checkHasElevatedPrivilege, Use: "uninstall", - Short: "Uninstall the ctrld service", + Short: "Stop and uninstall the ctrld service", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { prog := &prog{} From df514d15a5811289f6c95c2171b13315e56690c4 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Feb 2023 01:13:01 +0700 Subject: [PATCH 02/37] Update quic-go to v0.32.0 Updates #51 --- config.go | 4 ++-- doh.go | 2 +- doq.go | 2 +- go.mod | 22 +++++++++---------- go.sum | 65 +++++++++++++++++++------------------------------------ 5 files changed, 37 insertions(+), 58 deletions(-) diff --git a/config.go b/config.go index 8916cb4..f4fec96 100644 --- a/config.go +++ b/config.go @@ -12,9 +12,9 @@ import ( "github.com/Control-D-Inc/ctrld/internal/dnsrcode" "github.com/go-playground/validator/v10" - "github.com/lucas-clemente/quic-go" - "github.com/lucas-clemente/quic-go/http3" "github.com/miekg/dns" + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" "github.com/spf13/viper" ) diff --git a/doh.go b/doh.go index 2c68512..c31a3e1 100644 --- a/doh.go +++ b/doh.go @@ -7,8 +7,8 @@ import ( "io" "net/http" - "github.com/lucas-clemente/quic-go/http3" "github.com/miekg/dns" + "github.com/quic-go/quic-go/http3" ) func newDohResolver(uc *UpstreamConfig) *dohResolver { diff --git a/doq.go b/doq.go index 9f498f4..21ca301 100644 --- a/doq.go +++ b/doq.go @@ -7,8 +7,8 @@ import ( "net" "time" - "github.com/lucas-clemente/quic-go" "github.com/miekg/dns" + "github.com/quic-go/quic-go" ) type doqResolver struct { diff --git a/go.mod b/go.mod index 5cb1fde..af47c82 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,9 @@ require ( github.com/illarion/gonotify v1.0.1 github.com/insomniacslk/dhcp v0.0.0-20211209223715-7d93572ebe8e github.com/kardianos/service v1.2.1 - github.com/lucas-clemente/quic-go v0.29.1 github.com/miekg/dns v1.1.50 github.com/pelletier/go-toml/v2 v2.0.6 + github.com/quic-go/quic-go v0.32.0 github.com/rs/zerolog v1.28.0 github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.14.0 @@ -32,6 +32,7 @@ require ( github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/golang/mock v1.6.0 // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/native v1.0.0 // indirect @@ -40,9 +41,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/marten-seemann/qpack v0.2.1 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 // indirect @@ -50,10 +48,13 @@ require ( github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 // indirect github.com/mdlayher/socket v0.2.3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/ginkgo/v2 v2.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/qtls-go1-18 v0.2.0 // indirect + github.com/quic-go/qtls-go1-19 v0.2.0 // indirect + github.com/quic-go/qtls-go1-20 v0.1.0 // indirect github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect @@ -62,15 +63,14 @@ require ( github.com/subosito/gotenv v1.4.1 // indirect github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 // indirect go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect - golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect - golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/crypto v0.4.0 // indirect + golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect + golang.org/x/mod v0.6.0 // indirect golang.org/x/net v0.5.1-0.20230105164244-f8411da775a6 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/text v0.6.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/tools v0.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 58871ab..59f965b 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,6 @@ github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:Pjfxu github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -143,6 +141,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -155,7 +155,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnx github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -191,16 +190,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lucas-clemente/quic-go v0.29.1 h1:Z+WMJ++qMLhvpFkRZA+jl3BTxUjm415YBmWanXB8zP0= -github.com/lucas-clemente/quic-go v0.29.1/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= -github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= -github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -223,16 +214,8 @@ github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= +github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -244,6 +227,16 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U= +github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= +github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk= +github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI= +github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA= +github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= @@ -306,8 +299,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -318,8 +311,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw= -golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= +golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -344,11 +337,10 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -372,7 +364,6 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -414,7 +405,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -427,11 +417,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -445,7 +432,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -456,7 +442,6 @@ golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -534,14 +519,13 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -642,14 +626,9 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 3218b5fac19c9668e42ae07e830fb683ea986111 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Feb 2023 01:35:16 +0700 Subject: [PATCH 03/37] Add quic-free binaries in build pipeline Updates #51 --- config.go | 46 ++++++++------------------------------------- config_quic.go | 40 +++++++++++++++++++++++++++++++++++++++ config_quic_free.go | 5 +++++ doh.go | 11 ++++++++--- doq.go | 2 ++ doq_quic_free.go | 18 ++++++++++++++++++ 6 files changed, 81 insertions(+), 41 deletions(-) create mode 100644 config_quic.go create mode 100644 config_quic_free.go create mode 100644 doq_quic_free.go diff --git a/config.go b/config.go index f4fec96..1364b4e 100644 --- a/config.go +++ b/config.go @@ -2,7 +2,6 @@ package ctrld import ( "context" - "crypto/tls" "net" "net/http" "net/url" @@ -13,8 +12,6 @@ import ( "github.com/Control-D-Inc/ctrld/internal/dnsrcode" "github.com/go-playground/validator/v10" "github.com/miekg/dns" - "github.com/quic-go/quic-go" - "github.com/quic-go/quic-go/http3" "github.com/spf13/viper" ) @@ -95,14 +92,14 @@ type NetworkConfig struct { // UpstreamConfig specifies configuration for upstreams that ctrld will forward requests to. type UpstreamConfig struct { - Name string `mapstructure:"name" toml:"name,omitempty"` - Type string `mapstructure:"type" toml:"type,omitempty" validate:"oneof=doh doh3 dot doq os legacy"` - Endpoint string `mapstructure:"endpoint" toml:"endpoint,omitempty" validate:"required_unless=Type os"` - BootstrapIP string `mapstructure:"bootstrap_ip" toml:"bootstrap_ip,omitempty"` - Domain string `mapstructure:"-" toml:"-"` - Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` - transport *http.Transport `mapstructure:"-" toml:"-"` - http3RoundTripper *http3.RoundTripper `mapstructure:"-" toml:"-"` + Name string `mapstructure:"name" toml:"name,omitempty"` + Type string `mapstructure:"type" toml:"type,omitempty" validate:"oneof=doh doh3 dot doq os legacy"` + Endpoint string `mapstructure:"endpoint" toml:"endpoint,omitempty" validate:"required_unless=Type os"` + BootstrapIP string `mapstructure:"bootstrap_ip" toml:"bootstrap_ip,omitempty"` + Domain string `mapstructure:"-" toml:"-"` + Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` + transport *http.Transport `mapstructure:"-" toml:"-"` + http3RoundTripper http.RoundTripper `mapstructure:"-" toml:"-"` } // ListenerConfig specifies the networks configuration that ctrld will run on. @@ -179,33 +176,6 @@ func (uc *UpstreamConfig) setupDOHTransport() { uc.pingUpstream() } -func (uc *UpstreamConfig) setupDOH3Transport() { - uc.http3RoundTripper = &http3.RoundTripper{} - uc.http3RoundTripper.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) { - host := addr - ProxyLog.Debug().Msgf("debug dial context D0H3 %s - %s", addr, bootstrapDNS) - // if we have a bootstrap ip set, use it to avoid DNS lookup - if uc.BootstrapIP != "" { - if _, port, _ := net.SplitHostPort(addr); port != "" { - addr = net.JoinHostPort(uc.BootstrapIP, port) - } - ProxyLog.Debug().Msgf("sending doh3 request to: %s", addr) - } - remoteAddr, err := net.ResolveUDPAddr("udp", addr) - if err != nil { - return nil, err - } - - udpConn, err := net.ListenUDP("udp", nil) - if err != nil { - return nil, err - } - return quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg) - } - - uc.pingUpstream() -} - func (uc *UpstreamConfig) pingUpstream() { // Warming up the transport by querying a test packet. dnsResolver, err := NewResolver(uc) diff --git a/config_quic.go b/config_quic.go new file mode 100644 index 0000000..72ce351 --- /dev/null +++ b/config_quic.go @@ -0,0 +1,40 @@ +//go:build !qf + +package ctrld + +import ( + "context" + "crypto/tls" + "net" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +func (uc *UpstreamConfig) setupDOH3Transport() { + rt := &http3.RoundTripper{} + rt.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) { + host := addr + ProxyLog.Debug().Msgf("debug dial context D0H3 %s - %s", addr, bootstrapDNS) + // if we have a bootstrap ip set, use it to avoid DNS lookup + if uc.BootstrapIP != "" { + if _, port, _ := net.SplitHostPort(addr); port != "" { + addr = net.JoinHostPort(uc.BootstrapIP, port) + } + ProxyLog.Debug().Msgf("sending doh3 request to: %s", addr) + } + remoteAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return nil, err + } + + udpConn, err := net.ListenUDP("udp", nil) + if err != nil { + return nil, err + } + return quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg) + } + + uc.http3RoundTripper = rt + uc.pingUpstream() +} diff --git a/config_quic_free.go b/config_quic_free.go new file mode 100644 index 0000000..5f39bc6 --- /dev/null +++ b/config_quic_free.go @@ -0,0 +1,5 @@ +//go:build qf + +package ctrld + +func (uc *UpstreamConfig) setupDOH3Transport() {} diff --git a/doh.go b/doh.go index c31a3e1..433cf41 100644 --- a/doh.go +++ b/doh.go @@ -3,12 +3,12 @@ package ctrld import ( "context" "encoding/base64" + "errors" "fmt" "io" "net/http" "github.com/miekg/dns" - "github.com/quic-go/quic-go/http3" ) func newDohResolver(uc *UpstreamConfig) *dohResolver { @@ -25,7 +25,7 @@ type dohResolver struct { endpoint string isDoH3 bool transport *http.Transport - http3RoundTripper *http3.RoundTripper + http3RoundTripper http.RoundTripper } func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { @@ -44,12 +44,17 @@ func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro c := http.Client{Transport: r.transport} if r.isDoH3 { + if r.http3RoundTripper == nil { + return nil, errors.New("DoH3 is not supported") + } c.Transport = r.http3RoundTripper } resp, err := c.Do(req) if err != nil { if r.isDoH3 { - r.http3RoundTripper.Close() + if closer, ok := r.http3RoundTripper.(io.Closer); ok { + closer.Close() + } } return nil, fmt.Errorf("could not perform request: %w", err) } diff --git a/doq.go b/doq.go index 21ca301..20919e3 100644 --- a/doq.go +++ b/doq.go @@ -1,3 +1,5 @@ +//go:build !qf + package ctrld import ( diff --git a/doq_quic_free.go b/doq_quic_free.go new file mode 100644 index 0000000..36fd63c --- /dev/null +++ b/doq_quic_free.go @@ -0,0 +1,18 @@ +//go:build qf + +package ctrld + +import ( + "context" + "errors" + + "github.com/miekg/dns" +) + +type doqResolver struct { + uc *UpstreamConfig +} + +func (r *doqResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + return nil, errors.New("DoQ is not supported") +} From 45f827a2c568f512bd8001e46e08e59cd845756e Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Feb 2023 00:06:32 +0700 Subject: [PATCH 04/37] internal/controld: connect to API using ipv4 only Connecting to API using ipv6 sometimes hang at TLS handshake, using ipv4 only so we can fetch the config more reliably. Fixed #53 --- internal/controld/config.go | 2 +- internal/controld/config_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/controld/config.go b/internal/controld/config.go index db01292..88cfd97 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -70,7 +70,7 @@ func FetchResolverConfig(uid string) (*ResolverConfig, error) { req.Header.Add("Content-Type", "application/json") transport := http.DefaultTransport.(*http.Transport).Clone() transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - return Dialer.DialContext(ctx, network, addr) + return Dialer.DialContext(ctx, "tcp4", addr) } client := http.Client{ Timeout: 10 * time.Second, diff --git a/internal/controld/config_test.go b/internal/controld/config_test.go index 3c09ed7..cd6ea06 100644 --- a/internal/controld/config_test.go +++ b/internal/controld/config_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const utilityURL = "https://api.controld.com/utility" @@ -24,7 +25,7 @@ func TestFetchResolverConfig(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() got, err := FetchResolverConfig(tc.uid) - assert.False(t, (err != nil) != tc.wantErr) + require.False(t, (err != nil) != tc.wantErr, err) if !tc.wantErr { assert.NotEmpty(t, got.DOH) } From 83b551fb2dbf507971ff47ea4ec4d0ff1ffa4cd2 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Feb 2023 01:19:53 +0700 Subject: [PATCH 05/37] internal/controld: check if ipv4 is available before connect to API Updates #53 --- cmd/ctrld/cli.go | 3 +- cmd/ctrld/dns_proxy.go | 3 +- cmd/ctrld/net.go | 65 ---------------------------- cmd/ctrld/os_linux.go | 3 +- cmd/ctrld/os_windows.go | 8 ++-- cmd/ctrld/prog.go | 3 +- internal/controld/config.go | 24 ++++------- internal/net/net.go | 86 +++++++++++++++++++++++++++++++++++++ 8 files changed, 108 insertions(+), 87 deletions(-) delete mode 100644 cmd/ctrld/net.go create mode 100644 internal/net/net.go diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index cd18ee1..d09e777 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -24,6 +24,7 @@ import ( "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/controld" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) var ( @@ -105,7 +106,7 @@ func initCLI() { log.Fatalf("failed to unmarshal config: %v", err) } // Wait for network up. - if !netUp() { + if !ctrldnet.Up() { log.Fatal("network is not up yet") } processLogAndCacheFlags() diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 4cdfab0..0f3d085 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -15,6 +15,7 @@ import ( "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/dnscache" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) const staleTTL = 60 * time.Second @@ -55,7 +56,7 @@ func (p *prog) serveUDP(listenerNum string) error { // On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can // listen on ::1, then spawn a listener for receiving DNS requests. - if runtime.GOOS == "windows" && supportsIPv6ListenLocal() { + if runtime.GOOS == "windows" && ctrldnet.SupportsIPv6() { go func() { s := &dns.Server{ Addr: net.JoinHostPort("::1", strconv.Itoa(listenerConfig.Port)), diff --git a/cmd/ctrld/net.go b/cmd/ctrld/net.go deleted file mode 100644 index 595f03f..0000000 --- a/cmd/ctrld/net.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "context" - "net" - "sync" - "time" - - "tailscale.com/logtail/backoff" - - "github.com/Control-D-Inc/ctrld/internal/controld" -) - -const ( - controldIPv6Test = "ipv6.controld.io" -) - -var ( - stackOnce sync.Once - ipv6Enabled bool - canListenIPv6Local bool - hasNetworkUp bool -) - -func probeStack() { - b := backoff.NewBackoff("probeStack", func(format string, args ...any) {}, time.Minute) - for { - if _, err := controld.Dialer.Dial("udp", net.JoinHostPort(bootstrapDNS, "53")); err == nil { - hasNetworkUp = true - break - } else { - b.BackOff(context.Background(), err) - } - } - if _, err := controld.Dialer.Dial("tcp6", net.JoinHostPort(controldIPv6Test, "80")); err == nil { - ipv6Enabled = true - } - if ln, err := net.Listen("tcp6", "[::1]:53"); err == nil { - ln.Close() - canListenIPv6Local = true - } -} - -func netUp() bool { - stackOnce.Do(probeStack) - return hasNetworkUp -} - -func supportsIPv6() bool { - stackOnce.Do(probeStack) - return ipv6Enabled -} - -func supportsIPv6ListenLocal() bool { - stackOnce.Do(probeStack) - return canListenIPv6Local -} - -// isIPv6 checks if the provided IP is v6. -// -//lint:ignore U1000 use in os_windows.go -func isIPv6(ip string) bool { - parsedIP := net.ParseIP(ip) - return parsedIP != nil && parsedIP.To4() == nil && parsedIP.To16() != nil -} diff --git a/cmd/ctrld/os_linux.go b/cmd/ctrld/os_linux.go index 50ff469..22a469e 100644 --- a/cmd/ctrld/os_linux.go +++ b/cmd/ctrld/os_linux.go @@ -19,6 +19,7 @@ import ( "tailscale.com/util/dnsname" "github.com/Control-D-Inc/ctrld/internal/dns" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) @@ -111,7 +112,7 @@ func resetDNS(iface *net.Interface) error { } // TODO(cuonglm): handle DHCPv6 properly. - if supportsIPv6() { + if ctrldnet.SupportsIPv6() { c := client6.NewClient() conversation, err := c.Exchange(iface.Name) if err != nil { diff --git a/cmd/ctrld/os_windows.go b/cmd/ctrld/os_windows.go index 213c104..bb1631f 100644 --- a/cmd/ctrld/os_windows.go +++ b/cmd/ctrld/os_windows.go @@ -10,6 +10,8 @@ import ( "strconv" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" + + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) // TODO(cuonglm): implement. @@ -39,7 +41,7 @@ func setDNS(iface *net.Interface, nameservers []string) error { // TODO(cuonglm): should we use system API? func resetDNS(iface *net.Interface) error { - if supportsIPv6ListenLocal() { + if ctrldnet.SupportsIPv6ListenLocal() { if output, err := netsh("interface", "ipv6", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp"); err != nil { mainLog.Warn().Err(err).Msgf("failed to reset ipv6 DNS: %s", string(output)) } @@ -54,7 +56,7 @@ func resetDNS(iface *net.Interface) error { func setPrimaryDNS(iface *net.Interface, dns string) error { ipVer := "ipv4" - if isIPv6(dns) { + if ctrldnet.IsIPv6(dns) { ipVer = "ipv6" } idx := strconv.Itoa(iface.Index) @@ -73,7 +75,7 @@ func setPrimaryDNS(iface *net.Interface, dns string) error { func addSecondaryDNS(iface *net.Interface, dns string) error { ipVer := "ipv4" - if isIPv6(dns) { + if ctrldnet.IsIPv6(dns) { ipVer = "ipv6" } output, err := netsh("interface", ipVer, "add", "dns", strconv.Itoa(iface.Index), dns, "index=2") diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index b8e22bd..d99449c 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -13,6 +13,7 @@ import ( "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/dnscache" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) var errWindowsAddrInUse = syscall.Errno(0x2740) @@ -64,7 +65,7 @@ func (p *prog) run() { // resolve it manually and set the bootstrap ip c := new(dns.Client) for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} { - if !supportsIPv6() && dnsType == dns.TypeAAAA { + if !ctrldnet.SupportsIPv6() && dnsType == dns.TypeAAAA { continue } m := new(dns.Msg) diff --git a/internal/controld/config.go b/internal/controld/config.go index 88cfd97..4f16b88 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -8,6 +8,8 @@ import ( "net" "net/http" "time" + + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) const ( @@ -15,20 +17,6 @@ const ( InvalidConfigCode = 40401 ) -const bootstrapDNS = "76.76.2.0:53" - -var Dialer = &net.Dialer{ - Resolver: &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{ - Timeout: 10 * time.Second, - } - return d.DialContext(ctx, "udp", bootstrapDNS) - }, - }, -} - // ResolverConfig represents Control D resolver data. type ResolverConfig struct { DOH string `json:"doh"` @@ -70,7 +58,13 @@ func FetchResolverConfig(uid string) (*ResolverConfig, error) { req.Header.Add("Content-Type", "application/json") transport := http.DefaultTransport.(*http.Transport).Clone() transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - return Dialer.DialContext(ctx, "tcp4", addr) + // We experiment hanging in TLS handshake when connecting to ControlD API + // with ipv6. So prefer ipv4 if available. + network = "tcp6" + if ctrldnet.SupportsIPv4() { + network = "tcp4" + } + return ctrldnet.Dialer.DialContext(ctx, network, addr) } client := http.Client{ Timeout: 10 * time.Second, diff --git a/internal/net/net.go b/internal/net/net.go new file mode 100644 index 0000000..b553c13 --- /dev/null +++ b/internal/net/net.go @@ -0,0 +1,86 @@ +package net + +import ( + "context" + "net" + "sync" + "time" + + "tailscale.com/logtail/backoff" +) + +const ( + controldIPv6Test = "ipv6.controld.io" + controldIPv4Test = "ipv4.controld.io" + bootstrapDNS = "76.76.2.0:53" +) + +var Dialer = &net.Dialer{ + Resolver: &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: 10 * time.Second, + } + return d.DialContext(ctx, "udp", bootstrapDNS) + }, + }, +} + +var ( + stackOnce sync.Once + ipv4Enabled bool + ipv6Enabled bool + canListenIPv6Local bool + hasNetworkUp bool +) + +func probeStack() { + b := backoff.NewBackoff("probeStack", func(format string, args ...any) {}, time.Minute) + for { + if _, err := Dialer.Dial("udp", net.JoinHostPort(bootstrapDNS, "53")); err == nil { + hasNetworkUp = true + break + } else { + b.BackOff(context.Background(), err) + } + } + if _, err := Dialer.Dial("tcp4", net.JoinHostPort(controldIPv4Test, "80")); err == nil { + ipv4Enabled = true + } + if _, err := Dialer.Dial("tcp6", net.JoinHostPort(controldIPv6Test, "80")); err == nil { + ipv6Enabled = true + } + if ln, err := net.Listen("tcp6", "[::1]:53"); err == nil { + ln.Close() + canListenIPv6Local = true + } +} + +func Up() bool { + stackOnce.Do(probeStack) + return hasNetworkUp +} + +func SupportsIPv4() bool { + stackOnce.Do(probeStack) + return ipv4Enabled +} + +func SupportsIPv6() bool { + stackOnce.Do(probeStack) + return ipv6Enabled +} + +func SupportsIPv6ListenLocal() bool { + stackOnce.Do(probeStack) + return canListenIPv6Local +} + +// IsIPv6 checks if the provided IP is v6. +// +//lint:ignore U1000 use in os_windows.go +func IsIPv6(ip string) bool { + parsedIP := net.ParseIP(ip) + return parsedIP != nil && parsedIP.To4() == nil && parsedIP.To16() != nil +} From e38554746105a44efde37ee39cce3e7c0025ed89 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Feb 2023 19:49:15 +0700 Subject: [PATCH 06/37] internal/net: fix wrong address when testing network up --- internal/net/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/net/net.go b/internal/net/net.go index b553c13..4360fc2 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -38,7 +38,7 @@ var ( func probeStack() { b := backoff.NewBackoff("probeStack", func(format string, args ...any) {}, time.Minute) for { - if _, err := Dialer.Dial("udp", net.JoinHostPort(bootstrapDNS, "53")); err == nil { + if _, err := Dialer.Dial("udp", bootstrapDNS); err == nil { hasNetworkUp = true break } else { From 997ec342e0d6484a1b05d68f9eb64fa8e2008eb2 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Feb 2023 21:50:08 +0700 Subject: [PATCH 07/37] cmd/ctrld,internal/dns: support systemd-networkd dbus For interface managed by systemd-networkd, systemd-resolved can not reset DNS. To fix this, attempting to check before the run loop and set the suitable manager for the system. Updates #55 --- cmd/ctrld/os_linux.go | 20 ++++--- internal/dns/resolved.go | 111 +++++++++++++++++++++++++++++++-------- 2 files changed, 102 insertions(+), 29 deletions(-) diff --git a/cmd/ctrld/os_linux.go b/cmd/ctrld/os_linux.go index 22a469e..970de78 100644 --- a/cmd/ctrld/os_linux.go +++ b/cmd/ctrld/os_linux.go @@ -80,16 +80,20 @@ func setDNS(iface *net.Interface, nameservers []string) error { return nil } -func resetDNS(iface *net.Interface) error { - if r, err := dns.NewOSConfigurator(logf, iface.Name); err == nil { - if err := r.Close(); err != nil { - mainLog.Error().Err(err).Msg("failed to rollback DNS setting") - return err +func resetDNS(iface *net.Interface) (err error) { + defer func() { + if err == nil { + return } - if r.Mode() == "direct" { - return nil + if r, oerr := dns.NewOSConfigurator(logf, iface.Name); oerr == nil { + _ = r.SetDNS(dns.OSConfig{}) + if err := r.Close(); err != nil { + mainLog.Error().Err(err).Msg("failed to rollback DNS setting") + return + } + err = nil } - } + }() var ns []string c, err := nclient4.New(iface.Name) diff --git a/internal/dns/resolved.go b/internal/dns/resolved.go index 6c0b1de..8d03249 100644 --- a/internal/dns/resolved.go +++ b/internal/dns/resolved.go @@ -37,14 +37,45 @@ const reconfigTimeout = time.Second // Clients connect to the bus and walk that same hierarchy to invoke // RPCs, get/set properties, or listen for signals. const ( - dbusResolvedObject = "org.freedesktop.resolve1" - dbusResolvedPath dbus.ObjectPath = "/org/freedesktop/resolve1" - dbusResolvedInterface = "org.freedesktop.resolve1.Manager" - dbusPath dbus.ObjectPath = "/org/freedesktop/DBus" - dbusInterface = "org.freedesktop.DBus" - dbusOwnerSignal = "NameOwnerChanged" // broadcast when a well-known name's owning process changes. + dbusResolvedObject = "org.freedesktop.resolve1" + dbusNetworkdObject = "org.freedesktop.network1" + dbusResolvedPath dbus.ObjectPath = "/org/freedesktop/resolve1" + dbusNetworkdPath dbus.ObjectPath = "/org/freedesktop/network1" + dbusResolvedInterface = "org.freedesktop.resolve1.Manager" + dbusNetworkdInterface = "org.freedesktop.network1.Manager" + dbusPath dbus.ObjectPath = "/org/freedesktop/DBus" + dbusInterface = "org.freedesktop.DBus" + dbusOwnerSignal = "NameOwnerChanged" // broadcast when a well-known name's owning process changes. + dbusResolvedErrorLinkBusy = "org.freedesktop.resolve1.LinkBusy" ) +var ( + dbusSetLinkDNS string + dbusSetLinkDomains string + dbusSetLinkDefaultRoute string + dbusSetLinkLLMNR string + dbusSetLinkMulticastDNS string + dbusSetLinkDNSSEC string + dbusSetLinkDNSOverTLS string + dbusFlushCaches string + dbusRevertLink string +) + +func setDbusMethods(dbusInterface string) { + dbusSetLinkDNS = dbusInterface + ".SetLinkDNS" + dbusSetLinkDomains = dbusInterface + ".SetLinkDomains" + dbusSetLinkDefaultRoute = dbusInterface + ".SetLinkDefaultRoute" + dbusSetLinkLLMNR = dbusInterface + ".SetLinkLLMNR" + dbusSetLinkMulticastDNS = dbusInterface + ".SetLinkMulticastDNS" + dbusSetLinkDNSSEC = dbusInterface + ".SetLinkDNSSEC" + dbusSetLinkDNSOverTLS = dbusInterface + ".SetLinkDNSOverTLS" + dbusFlushCaches = dbusInterface + ".FlushCaches" + dbusRevertLink = dbusInterface + ".RevertLink" + if dbusInterface == dbusNetworkdInterface { + dbusRevertLink = dbusInterface + ".RevertLinkDNS" + } +} + type resolvedLinkNameserver struct { Family int32 Address []byte @@ -69,7 +100,9 @@ type resolvedManager struct { logf logger.Logf ifidx int - configCR chan changeRequest // tracks OSConfigs changes and error responses + configCR chan changeRequest // tracks OSConfigs changes and error responses + revertCh chan struct{} + newManager func(conn *dbus.Conn) dbus.BusObject } func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) { @@ -89,6 +122,7 @@ func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManage ifidx: iface.Index, configCR: make(chan changeRequest), + revertCh: make(chan struct{}), } go mgr.run(ctx) @@ -117,6 +151,16 @@ func (m *resolvedManager) SetDNS(config OSConfig) error { } } +func newResolvedObject(conn *dbus.Conn) dbus.BusObject { + setDbusMethods(dbusResolvedInterface) + return conn.Object(dbusResolvedObject, dbusResolvedPath) +} + +func newNetworkdObject(conn *dbus.Conn) dbus.BusObject { + setDbusMethods(dbusNetworkdInterface) + return conn.Object(dbusNetworkdObject, dbusNetworkdPath) +} + func (m *resolvedManager) run(ctx context.Context) { var ( conn *dbus.Conn @@ -131,6 +175,22 @@ func (m *resolvedManager) run(ctx context.Context) { } }() + newManager := newResolvedObject + func() { + conn, err := dbus.SystemBus() + if err != nil { + m.logf("dbus connection error: %v", err) + return + } + rManager = newManager(conn) + if call := rManager.CallWithContext(ctx, dbusRevertLink, 0, m.ifidx); call.Err != nil { + if dbusErr, ok := call.Err.(dbus.Error); ok && dbusErr.Name == dbusResolvedErrorLinkBusy { + m.logf("[v1] Using %s as manager", dbusNetworkdObject) + newManager = newNetworkdObject + } + } + }() + // Reconnect the systemBus if disconnected. reconnect := func() error { var err error @@ -151,7 +211,7 @@ func (m *resolvedManager) run(ctx context.Context) { return err } - rManager = conn.Object(dbusResolvedObject, dbus.ObjectPath(dbusResolvedPath)) + rManager = newManager(conn) // Only receive the DBus signals we need to resync our config on // resolved restart. Failure to set filters isn't a fatal error, @@ -160,6 +220,9 @@ func (m *resolvedManager) run(ctx context.Context) { if err = conn.AddMatchSignal(dbus.WithMatchObjectPath(dbusPath), dbus.WithMatchInterface(dbusInterface), dbus.WithMatchMember(dbusOwnerSignal), dbus.WithMatchArg(0, dbusResolvedObject)); err != nil { m.logf("[v1] Setting DBus signal filter failed: %v", err) } + if err = conn.AddMatchSignal(dbus.WithMatchObjectPath(dbusPath), dbus.WithMatchInterface(dbusInterface), dbus.WithMatchMember(dbusOwnerSignal), dbus.WithMatchArg(0, dbusNetworkdObject)); err != nil { + m.logf("[v1] Setting DBus signal filter failed: %v", err) + } conn.Signal(signals) // Reset backoff and SetNSOSHealth after successful on reconnect. @@ -179,13 +242,15 @@ func (m *resolvedManager) run(ctx context.Context) { if rManager == nil { return } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) // RevertLink resets all per-interface settings on systemd-resolved to defaults. // When ctx goes away systemd-resolved auto reverts. // Keeping for potential use in future refactor. - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".RevertLink", 0, m.ifidx); call.Err != nil { + if call := rManager.CallWithContext(ctx, dbusRevertLink, 0, m.ifidx); call.Err != nil { m.logf("[v1] RevertLink: %v", call.Err) - return } + cancel() + close(m.revertCh) return case configCR := <-m.configCR: // Track and update sync with latest config change. @@ -223,7 +288,7 @@ func (m *resolvedManager) run(ctx context.Context) { if len(signal.Body) != 3 { m.logf("[unexpected] DBus NameOwnerChanged len(Body) = %d, want 3") } - if name, ok := signal.Body[0].(string); !ok || name != dbusResolvedObject { + if name, ok := signal.Body[0].(string); !ok || (name != dbusResolvedObject && name != dbusNetworkdObject) { continue } newOwner, ok := signal.Body[2].(string) @@ -271,7 +336,7 @@ func (m *resolvedManager) setConfigOverDBus(ctx context.Context, rManager dbus.B } } err := rManager.CallWithContext( - ctx, dbusResolvedInterface+".SetLinkDNS", 0, + ctx, dbusSetLinkDNS, 0, m.ifidx, linkNameservers, ).Store() if err != nil { @@ -311,14 +376,14 @@ func (m *resolvedManager) setConfigOverDBus(ctx context.Context, rManager dbus.B } err = rManager.CallWithContext( - ctx, dbusResolvedInterface+".SetLinkDomains", 0, + ctx, dbusSetLinkDomains, 0, m.ifidx, linkDomains, ).Store() if err != nil && err.Error() == "Argument list too long" { // TODO: better error match // Issue 3188: older systemd-resolved had argument length limits. // Trim out the *.arpa. entries and try again. err = rManager.CallWithContext( - ctx, dbusResolvedInterface+".SetLinkDomains", 0, + ctx, dbusSetLinkDomains, 0, m.ifidx, linkDomainsWithoutReverseDNS(linkDomains), ).Store() } @@ -326,7 +391,7 @@ func (m *resolvedManager) setConfigOverDBus(ctx context.Context, rManager dbus.B return fmt.Errorf("setLinkDomains: %w", err) } - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".SetLinkDefaultRoute", 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil { + if call := rManager.CallWithContext(ctx, dbusSetLinkDefaultRoute, 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil { if dbusErr, ok := call.Err.(dbus.Error); ok && dbusErr.Name == dbus.ErrMsgUnknownMethod.Name { // on some older systems like Kubuntu 18.04.6 with systemd 237 method SetLinkDefaultRoute is absent, // but otherwise it's working good @@ -341,33 +406,37 @@ func (m *resolvedManager) setConfigOverDBus(ctx context.Context, rManager dbus.B // or something). // Disable LLMNR, we don't do multicast. - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".SetLinkLLMNR", 0, m.ifidx, "no"); call.Err != nil { + if call := rManager.CallWithContext(ctx, dbusSetLinkLLMNR, 0, m.ifidx, "no"); call.Err != nil { m.logf("[v1] failed to disable LLMNR: %v", call.Err) } // Disable mdns. - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".SetLinkMulticastDNS", 0, m.ifidx, "no"); call.Err != nil { + if call := rManager.CallWithContext(ctx, dbusSetLinkMulticastDNS, 0, m.ifidx, "no"); call.Err != nil { m.logf("[v1] failed to disable mdns: %v", call.Err) } // We don't support dnssec consistently right now, force it off to // avoid partial failures when we split DNS internally. - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".SetLinkDNSSEC", 0, m.ifidx, "no"); call.Err != nil { + if call := rManager.CallWithContext(ctx, dbusSetLinkDNSSEC, 0, m.ifidx, "no"); call.Err != nil { m.logf("[v1] failed to disable DNSSEC: %v", call.Err) } - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".SetLinkDNSOverTLS", 0, m.ifidx, "no"); call.Err != nil { + if call := rManager.CallWithContext(ctx, dbusSetLinkDNSOverTLS, 0, m.ifidx, "no"); call.Err != nil { m.logf("[v1] failed to disable DoT: %v", call.Err) } - if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".FlushCaches", 0); call.Err != nil { - m.logf("failed to flush resolved DNS cache: %v", call.Err) + if rManager.Path() == dbusResolvedPath { + if call := rManager.CallWithContext(ctx, dbusFlushCaches, 0); call.Err != nil { + m.logf("failed to flush resolved DNS cache: %v", call.Err) + } } + return nil } func (m *resolvedManager) Close() error { m.cancel() // stops the 'run' method goroutine + <-m.revertCh return nil } From bac6810956b237c32b2b006ded2fbad2377dc9fc Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Feb 2023 22:36:44 +0700 Subject: [PATCH 08/37] cmd/ctrld: fix missing unmarshalling config without --cd Otherwise, DNS won't be set in non-Linux systems. Updates #54 --- cmd/ctrld/cli.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index d09e777..9e557f4 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -203,6 +203,10 @@ func initCLI() { sc.Arguments = append(sc.Arguments, "--homedir="+dir) } + if err := v.Unmarshal(&cfg); err != nil { + log.Fatalf("failed to unmarshal config: %v", err) + } + initLogging() processCDFlags() // On Windows, the service will be run as SYSTEM, so if ctrld start as Admin, From 35c890048ba62c13bd547285f5a38da0f985c7cd Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Feb 2023 23:49:12 +0700 Subject: [PATCH 09/37] cmd/ctrld: remove prefix main field While at it, also make init logging with empty log path when running start command. Updates #55 --- cmd/ctrld/cli.go | 4 ++++ cmd/ctrld/main.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index 9e557f4..a5fecf3 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -207,7 +207,11 @@ func initCLI() { log.Fatalf("failed to unmarshal config: %v", err) } + logPath := cfg.Service.LogPath + cfg.Service.LogPath = "" initLogging() + cfg.Service.LogPath = logPath + processCDFlags() // On Windows, the service will be run as SYSTEM, so if ctrld start as Admin, // the user home dir is different, so pass specific arguments that relevant here. diff --git a/cmd/ctrld/main.go b/cmd/ctrld/main.go index c336a2b..5c55123 100644 --- a/cmd/ctrld/main.go +++ b/cmd/ctrld/main.go @@ -80,7 +80,7 @@ func initLogging() { }) writers = append(writers, consoleWriter) multi := zerolog.MultiLevelWriter(writers...) - mainLog = mainLog.Output(multi).With().Timestamp().Str("prefix", "main").Logger() + mainLog = mainLog.Output(multi).With().Timestamp().Logger() if verbose > 0 || isLog { proxyLog = proxyLog.Output(multi).With().Timestamp().Logger() // TODO: find a better way. From 71b1b324dbf06c12301e25f35ca4bd27cd4cff42 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 14 Feb 2023 00:30:36 +0700 Subject: [PATCH 10/37] cmd/ctrld: honor configPath when writing config file Updates #58 --- cmd/ctrld/cli.go | 2 ++ cmd/ctrld/cli_test.go | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 cmd/ctrld/cli_test.go diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index a5fecf3..22497a7 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -445,6 +445,8 @@ func initCLI() { func writeConfigFile() error { if cfu := v.ConfigFileUsed(); cfu != "" { defaultConfigFile = cfu + } else if configPath != "" { + defaultConfigFile = configPath } f, err := os.OpenFile(defaultConfigFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0o644)) if err != nil { diff --git a/cmd/ctrld/cli_test.go b/cmd/ctrld/cli_test.go new file mode 100644 index 0000000..23746b7 --- /dev/null +++ b/cmd/ctrld/cli_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_writeConfigFile(t *testing.T) { + tmpdir := t.TempDir() + // simulate --config CLI flag by setting configPath manually. + configPath = filepath.Join(tmpdir, "ctrld.toml") + _, err := os.Stat(configPath) + assert.True(t, os.IsNotExist(err)) + + assert.NoError(t, writeConfigFile()) + + _, err = os.Stat(configPath) + require.NoError(t, err) +} From d9b699501df0c88a84047742304a8b2285f95964 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 15 Feb 2023 09:27:11 +0700 Subject: [PATCH 11/37] cmd/ctrld: merge proxy log to main log There's no reason to separate those two loggers anymore, and making them separated may lead to inconsistent logging behavior. Updates #54 --- cmd/ctrld/dns_proxy.go | 24 ++++++++++++------------ cmd/ctrld/main.go | 10 ++-------- cmd/ctrld/prog.go | 24 ++++++++++++------------ 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 0f3d085..8ece9c1 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -37,7 +37,7 @@ func (p *prog) serveUDP(listenerNum string) error { fmtSrcToDest := fmtRemoteToLocal(listenerNum, w.RemoteAddr().String(), w.LocalAddr().String()) t := time.Now() ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqId) - ctrld.Log(ctx, proxyLog.Debug(), "%s received query: %s", fmtSrcToDest, domain) + ctrld.Log(ctx, mainLog.Debug(), "%s received query: %s", fmtSrcToDest, domain) upstreams, matched := p.upstreamFor(ctx, listenerNum, listenerConfig, w.RemoteAddr(), domain) var answer *dns.Msg if !matched && listenerConfig.Restricted { @@ -47,7 +47,7 @@ func (p *prog) serveUDP(listenerNum string) error { } else { answer = p.proxy(ctx, upstreams, failoverRcodes, m) rtt := time.Since(t) - ctrld.Log(ctx, proxyLog.Debug(), "received response of %d bytes in %s", answer.Len(), rtt) + ctrld.Log(ctx, mainLog.Debug(), "received response of %d bytes in %s", answer.Len(), rtt) } if err := w.WriteMsg(answer); err != nil { ctrld.Log(ctx, mainLog.Error().Err(err), "serveUDP: failed to send DNS response to client") @@ -84,10 +84,10 @@ func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *c defer func() { if !matched && lc.Restricted { - ctrld.Log(ctx, proxyLog.Info(), "query refused, %s does not match any network policy", addr.String()) + ctrld.Log(ctx, mainLog.Info(), "query refused, %s does not match any network policy", addr.String()) return } - ctrld.Log(ctx, proxyLog.Info(), "%s, %s, %s -> %v", matchedPolicy, matchedNetwork, matchedRule, upstreams) + ctrld.Log(ctx, mainLog.Info(), "%s, %s, %s -> %v", matchedPolicy, matchedNetwork, matchedRule, upstreams) }() if lc.Policy == nil { @@ -160,7 +160,7 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i answer.SetRcode(msg, answer.Rcode) now := time.Now() if cachedValue.Expire.After(now) { - ctrld.Log(ctx, proxyLog.Debug(), "hit cached response") + ctrld.Log(ctx, mainLog.Debug(), "hit cached response") setCachedAnswerTTL(answer, now, cachedValue.Expire) return answer } @@ -168,10 +168,10 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i } } resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg { - ctrld.Log(ctx, proxyLog.Debug(), "sending query to %s: %s", upstreams[n], upstreamConfig.Name) + ctrld.Log(ctx, mainLog.Debug(), "sending query to %s: %s", upstreams[n], upstreamConfig.Name) dnsResolver, err := ctrld.NewResolver(upstreamConfig) if err != nil { - ctrld.Log(ctx, proxyLog.Error().Err(err), "failed to create resolver") + ctrld.Log(ctx, mainLog.Error().Err(err), "failed to create resolver") return nil } resolveCtx, cancel := context.WithCancel(ctx) @@ -183,7 +183,7 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i } answer, err := dnsResolver.Resolve(resolveCtx, msg) if err != nil { - ctrld.Log(ctx, proxyLog.Error().Err(err), "failed to resolve query") + ctrld.Log(ctx, mainLog.Error().Err(err), "failed to resolve query") return nil } return answer @@ -192,7 +192,7 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i answer := resolve(n, upstreamConfig, msg) if answer == nil { if serveStaleCache && staleAnswer != nil { - ctrld.Log(ctx, proxyLog.Debug(), "serving stale cached response") + ctrld.Log(ctx, mainLog.Debug(), "serving stale cached response") now := time.Now() setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) return staleAnswer @@ -200,7 +200,7 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i continue } if answer.Rcode != dns.RcodeSuccess && len(upstreamConfigs) > 1 && containRcode(failoverRcodes, answer.Rcode) { - ctrld.Log(ctx, proxyLog.Debug(), "failover rcode matched, process to next upstream") + ctrld.Log(ctx, mainLog.Debug(), "failover rcode matched, process to next upstream") continue } if p.cache != nil { @@ -212,11 +212,11 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i } setCachedAnswerTTL(answer, now, expired) p.cache.Add(dnscache.NewKey(msg, upstreams[n]), dnscache.NewValue(answer, expired)) - ctrld.Log(ctx, proxyLog.Debug(), "add cached response") + ctrld.Log(ctx, mainLog.Debug(), "add cached response") } return answer } - ctrld.Log(ctx, proxyLog.Error(), "all upstreams failed") + ctrld.Log(ctx, mainLog.Error(), "all upstreams failed") answer := new(dns.Msg) answer.SetRcode(msg, dns.RcodeServerFailure) return answer diff --git a/cmd/ctrld/main.go b/cmd/ctrld/main.go index 5c55123..033fd2e 100644 --- a/cmd/ctrld/main.go +++ b/cmd/ctrld/main.go @@ -31,7 +31,6 @@ var ( rootLogger = zerolog.New(io.Discard) mainLog = rootLogger - proxyLog = rootLogger cdUID string iface string @@ -59,7 +58,6 @@ func normalizeLogFilePath(logFilePath string) string { func initLogging() { writers := []io.Writer{io.Discard} - isLog := cfg.Service.LogLevel != "" if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" { // Create parent directory if necessary. if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil { @@ -71,7 +69,6 @@ func initLogging() { fmt.Fprintf(os.Stderr, "failed to create log file: %v", err) os.Exit(1) } - isLog = true writers = append(writers, logFile) } zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs @@ -81,11 +78,8 @@ func initLogging() { writers = append(writers, consoleWriter) multi := zerolog.MultiLevelWriter(writers...) mainLog = mainLog.Output(multi).With().Timestamp().Logger() - if verbose > 0 || isLog { - proxyLog = proxyLog.Output(multi).With().Timestamp().Logger() - // TODO: find a better way. - ctrld.ProxyLog = proxyLog - } + // TODO: find a better way. + ctrld.ProxyLog = mainLog zerolog.SetGlobalLevel(zerolog.InfoLevel) logLevel := cfg.Service.LogLevel diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index d99449c..2046be4 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -52,7 +52,7 @@ func (p *prog) run() { for _, cidr := range nc.Cidrs { _, ipNet, err := net.ParseCIDR(cidr) if err != nil { - proxyLog.Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("invalid cidr") + mainLog.Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("invalid cidr") continue } nc.IPNets = append(nc.IPNets, ipNet) @@ -73,11 +73,11 @@ func (p *prog) run() { m.RecursionDesired = true r, _, err := c.Exchange(m, net.JoinHostPort(bootstrapDNS, "53")) if err != nil { - proxyLog.Error().Err(err).Msgf("could not resolve domain %s for upstream.%s", uc.Domain, n) + mainLog.Error().Err(err).Msgf("could not resolve domain %s for upstream.%s", uc.Domain, n) continue } if r.Rcode != dns.RcodeSuccess { - proxyLog.Error().Msgf("could not resolve domain return code: %d, upstream.%s", r.Rcode, n) + mainLog.Error().Msgf("could not resolve domain return code: %d, upstream.%s", r.Rcode, n) continue } if len(r.Answer) == 0 { @@ -110,14 +110,14 @@ func (p *prog) run() { listenerConfig := p.cfg.Listener[listenerNum] upstreamConfig := p.cfg.Upstream[listenerNum] if upstreamConfig == nil { - proxyLog.Error().Msgf("missing upstream config for: [listener.%s]", listenerNum) + mainLog.Error().Msgf("missing upstream config for: [listener.%s]", listenerNum) return } addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)) mainLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, addr) err := p.serveUDP(listenerNum) if err != nil && !defaultConfigWritten { - proxyLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) + mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) return } if err == nil { @@ -126,16 +126,16 @@ func (p *prog) run() { if opErr, ok := err.(*net.OpError); ok { if sErr, ok := opErr.Err.(*os.SyscallError); ok && errors.Is(opErr.Err, syscall.EADDRINUSE) || errors.Is(sErr.Err, errWindowsAddrInUse) { - proxyLog.Warn().Msgf("Address %s already in used, pick a random one", addr) + mainLog.Warn().Msgf("Address %s already in used, pick a random one", addr) pc, err := net.ListenPacket("udp", net.JoinHostPort(listenerConfig.IP, "0")) if err != nil { - proxyLog.Fatal().Err(err).Msg("failed to listen packet") + mainLog.Fatal().Err(err).Msg("failed to listen packet") return } _, portStr, _ := net.SplitHostPort(pc.LocalAddr().String()) port, err := strconv.Atoi(portStr) if err != nil { - proxyLog.Fatal().Err(err).Msg("malformed port") + mainLog.Fatal().Err(err).Msg("malformed port") return } listenerConfig.Port = port @@ -146,7 +146,7 @@ func (p *prog) run() { }, }) if err := writeConfigFile(); err != nil { - proxyLog.Fatal().Err(err).Msg("failed to write config file") + mainLog.Fatal().Err(err).Msg("failed to write config file") } else { mainLog.Info().Msg("writing config file to: " + defaultConfigFile) } @@ -154,16 +154,16 @@ func (p *prog) run() { // There can be a race between closing the listener and start our own UDP server, but it's // rare, and we only do this once, so let conservative here. if err := pc.Close(); err != nil { - proxyLog.Fatal().Err(err).Msg("failed to close packet conn") + mainLog.Fatal().Err(err).Msg("failed to close packet conn") return } if err := p.serveUDP(listenerNum); err != nil { - proxyLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) + mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) return } } } - proxyLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) + mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) }(listenerNum) } From 4172fc09d074365eb7a96bf9228cc78dbd731b91 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 14 Feb 2023 09:26:12 +0700 Subject: [PATCH 12/37] cmd/ctrld: add self check for better error message reported After telling service manager to start ctrld, performing self check status by sending DNS query to ctrld listener. So if ctrld could not start for any reason, an error message will be reported to user instead of simply telling service started. Updates #56 --- cmd/ctrld/cli.go | 50 +++++++++++++++++++++++++++++++++++++++++-- cmd/ctrld/os_linux.go | 4 ---- cmd/ctrld/prog.go | 4 ++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index 22497a7..1ba562e 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/base64" "errors" "fmt" @@ -14,12 +15,15 @@ import ( "runtime" "strconv" "strings" + "time" "github.com/go-playground/validator/v10" "github.com/kardianos/service" + "github.com/miekg/dns" "github.com/pelletier/go-toml/v2" "github.com/spf13/cobra" "github.com/spf13/viper" + "tailscale.com/logtail/backoff" "tailscale.com/net/interfaces" "github.com/Control-D-Inc/ctrld" @@ -27,6 +31,8 @@ import ( ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) +const selfCheckFQDN = "verify.controld.com" + var ( v = viper.NewWithOptions(viper.KeyDelimiter("::")) defaultConfigWritten = false @@ -234,8 +240,24 @@ func initCLI() { {s.Start, true}, } if doTasks(tasks) { + status, err := s.Status() + if err != nil { + mainLog.Warn().Err(err).Msg("could not get service status") + return + } + + status = selfCheckStatus(status) + switch status { + case service.StatusRunning: + mainLog.Info().Msg("Service started") + default: + mainLog.Error().Msg("Service did not start, please check system/service log for details error") + if runtime.GOOS == "linux" { + prog.resetDNS() + } + os.Exit(1) + } prog.setDNS() - mainLog.Info().Msg("Service started") } }, } @@ -549,7 +571,7 @@ func processCDFlags() { iface = "auto" } logger := mainLog.With().Str("mode", "cd").Logger() - logger.Info().Msg("fetching Controld-D configuration") + logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID) resolverConfig, err := controld.FetchResolverConfig(cdUID) if uer, ok := err.(*controld.UtilityErrorResponse); ok && uer.ErrorField.Code == controld.InvalidConfigCode { s, err := service.New(&prog{}, svcConfig) @@ -681,3 +703,27 @@ func defaultIfaceName() string { } return dri } + +func selfCheckStatus(status service.Status) service.Status { + c := new(dns.Client) + lc := cfg.Listener["0"] + bo := backoff.NewBackoff("self-check", logf, 10*time.Second) + bo.LogLongerThan = 500 * time.Millisecond + ctx := context.Background() + err := errors.New("query failed") + maxAttempts := 10 + mainLog.Debug().Msg("Performing self-check") + for i := 0; i < maxAttempts; i++ { + m := new(dns.Msg) + m.SetQuestion(selfCheckFQDN+".", dns.TypeA) + m.RecursionDesired = true + r, _, _ := c.ExchangeContext(ctx, m, net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port))) + if r != nil && r.Rcode == dns.RcodeSuccess && len(r.Answer) > 0 { + mainLog.Debug().Msgf("self-check against %q succeeded", selfCheckFQDN) + return status + } + bo.BackOff(ctx, err) + } + mainLog.Debug().Msgf("self-check against %q failed", selfCheckFQDN) + return service.StatusUnknown +} diff --git a/cmd/ctrld/os_linux.go b/cmd/ctrld/os_linux.go index 970de78..a29c7f4 100644 --- a/cmd/ctrld/os_linux.go +++ b/cmd/ctrld/os_linux.go @@ -23,10 +23,6 @@ import ( "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) -var logf = func(format string, args ...any) { - mainLog.Debug().Msgf(format, args...) -} - // allocate loopback ip // sudo ip a add 127.0.0.2/24 dev lo func allocateIP(ip string) error { diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index 2046be4..e3d6239 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -16,6 +16,10 @@ import ( ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) +var logf = func(format string, args ...any) { + mainLog.Debug().Msgf(format, args...) +} + var errWindowsAddrInUse = syscall.Errno(0x2740) var svcConfig = &service.Config{ From 4c2d21a8f8486b05e8618fe744f05d6b2ed028e5 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 15 Feb 2023 22:42:33 +0700 Subject: [PATCH 13/37] all: add freebsd supports This commit add support for ctrld to run on freebsd, supported platforms are amd64/arm64/armv6/armv7,386. Supporting freebsd also requires adding debian and openresolv resolvconf. Updates #47 --- cmd/ctrld/cli.go | 3 +- ...rk_manager.go => network_manager_linux.go} | 9 -- cmd/ctrld/network_manager_others.go | 13 ++ cmd/ctrld/{os_mac.go => os_darwin.go} | 3 - cmd/ctrld/os_freebsd.go | 47 ++++++ cmd/ctrld/os_others.go | 13 ++ cmd/ctrld/os_windows.go | 13 -- cmd/ctrld/prog_freebsd.go | 20 +++ cmd/ctrld/prog_linux.go | 4 + cmd/ctrld/prog_others.go | 8 +- internal/dns/debian_resolvconf.go | 153 ++++++++++++++++++ internal/dns/direct.go | 4 + internal/dns/manager_freebsd.go | 39 +++++ internal/dns/nm.go | 2 + internal/dns/openresolv.go | 57 +++++++ internal/dns/osconfig.go | 2 + internal/dns/resolvconf-workaround.sh | 63 ++++++++ internal/dns/resolved.go | 2 + 18 files changed, 426 insertions(+), 29 deletions(-) rename cmd/ctrld/{network_manager.go => network_manager_linux.go} (88%) create mode 100644 cmd/ctrld/network_manager_others.go rename cmd/ctrld/{os_mac.go => os_darwin.go} (97%) create mode 100644 cmd/ctrld/os_freebsd.go create mode 100644 cmd/ctrld/os_others.go create mode 100644 cmd/ctrld/prog_freebsd.go create mode 100644 internal/dns/debian_resolvconf.go create mode 100644 internal/dns/manager_freebsd.go create mode 100644 internal/dns/openresolv.go create mode 100644 internal/dns/resolvconf-workaround.sh diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index 1ba562e..e3ca198 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -197,8 +197,7 @@ func initCLI() { setDependencies(sc) sc.Arguments = append([]string{"run"}, osArgs...) if dir, err := os.UserHomeDir(); err == nil { - // WorkingDirectory is not supported on Windows. - sc.WorkingDirectory = dir + setWorkingDirectory(sc, dir) // No config path, generating config in HOME directory. noConfigStart := isNoConfigStart(cmd) writeDefaultConfig := !noConfigStart && configBase64 == "" diff --git a/cmd/ctrld/network_manager.go b/cmd/ctrld/network_manager_linux.go similarity index 88% rename from cmd/ctrld/network_manager.go rename to cmd/ctrld/network_manager_linux.go index 670fe9c..fe00f3a 100644 --- a/cmd/ctrld/network_manager.go +++ b/cmd/ctrld/network_manager_linux.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "runtime" "time" "github.com/coreos/go-systemd/v22/dbus" @@ -24,10 +23,6 @@ systemd-resolved=false var networkManagerCtrldConfFile = filepath.Join(nmConfDir, nmCtrldConfFilename) func setupNetworkManager() error { - if runtime.GOOS != "linux" { - mainLog.Debug().Msg("skipping NetworkManager setup, not on Linux") - return nil - } if content, _ := os.ReadFile(nmCtrldConfContent); string(content) == nmCtrldConfContent { mainLog.Debug().Msg("NetworkManager already setup, nothing to do") return nil @@ -48,10 +43,6 @@ func setupNetworkManager() error { } func restoreNetworkManager() error { - if runtime.GOOS != "linux" { - mainLog.Debug().Msg("skipping NetworkManager restoring, not on Linux") - return nil - } err := os.Remove(networkManagerCtrldConfFile) if os.IsNotExist(err) { mainLog.Debug().Msg("NetworkManager is not available") diff --git a/cmd/ctrld/network_manager_others.go b/cmd/ctrld/network_manager_others.go new file mode 100644 index 0000000..3cdb762 --- /dev/null +++ b/cmd/ctrld/network_manager_others.go @@ -0,0 +1,13 @@ +//go:build !linux + +package main + +func setupNetworkManager() error { + return nil +} + +func restoreNetworkManager() error { + return nil +} + +func reloadNetworkManager() {} diff --git a/cmd/ctrld/os_mac.go b/cmd/ctrld/os_darwin.go similarity index 97% rename from cmd/ctrld/os_mac.go rename to cmd/ctrld/os_darwin.go index 95786f3..04bc66b 100644 --- a/cmd/ctrld/os_mac.go +++ b/cmd/ctrld/os_darwin.go @@ -1,6 +1,3 @@ -//go:build darwin -// +build darwin - package main import ( diff --git a/cmd/ctrld/os_freebsd.go b/cmd/ctrld/os_freebsd.go new file mode 100644 index 0000000..b65de54 --- /dev/null +++ b/cmd/ctrld/os_freebsd.go @@ -0,0 +1,47 @@ +package main + +import ( + "net" + "net/netip" + + "github.com/Control-D-Inc/ctrld/internal/dns" + "github.com/Control-D-Inc/ctrld/internal/resolvconffile" +) + +// set the dns server for the provided network interface +func setDNS(iface *net.Interface, nameservers []string) error { + r, err := dns.NewOSConfigurator(logf, iface.Name) + if err != nil { + mainLog.Error().Err(err).Msg("failed to create DNS OS configurator") + return err + } + + ns := make([]netip.Addr, 0, len(nameservers)) + for _, nameserver := range nameservers { + ns = append(ns, netip.MustParseAddr(nameserver)) + } + + if err := r.SetDNS(dns.OSConfig{Nameservers: ns}); err != nil { + mainLog.Error().Err(err).Msg("failed to set DNS") + return err + } + return nil +} + +func resetDNS(iface *net.Interface) error { + r, err := dns.NewOSConfigurator(logf, iface.Name) + if err != nil { + mainLog.Error().Err(err).Msg("failed to create DNS OS configurator") + return err + } + + if err := r.Close(); err != nil { + mainLog.Error().Err(err).Msg("failed to rollback DNS setting") + return err + } + return nil +} + +func currentDNS(_ *net.Interface) []string { + return resolvconffile.NameServers("") +} diff --git a/cmd/ctrld/os_others.go b/cmd/ctrld/os_others.go new file mode 100644 index 0000000..e9f9d61 --- /dev/null +++ b/cmd/ctrld/os_others.go @@ -0,0 +1,13 @@ +//go:build !linux && !darwin + +package main + +// TODO(cuonglm): implement. +func allocateIP(ip string) error { + return nil +} + +// TODO(cuonglm): implement. +func deAllocateIP(ip string) error { + return nil +} diff --git a/cmd/ctrld/os_windows.go b/cmd/ctrld/os_windows.go index bb1631f..0bd7358 100644 --- a/cmd/ctrld/os_windows.go +++ b/cmd/ctrld/os_windows.go @@ -1,6 +1,3 @@ -//go:build windows -// +build windows - package main import ( @@ -14,16 +11,6 @@ import ( ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) -// TODO(cuonglm): implement. -func allocateIP(ip string) error { - return nil -} - -// TODO(cuonglm): implement. -func deAllocateIP(ip string) error { - return nil -} - func setDNS(iface *net.Interface, nameservers []string) error { if len(nameservers) == 0 { return errors.New("empty DNS nameservers") diff --git a/cmd/ctrld/prog_freebsd.go b/cmd/ctrld/prog_freebsd.go new file mode 100644 index 0000000..63d8179 --- /dev/null +++ b/cmd/ctrld/prog_freebsd.go @@ -0,0 +1,20 @@ +package main + +import ( + "os" + + "github.com/kardianos/service" +) + +func (p *prog) preRun() { + if !service.Interactive() { + p.setDNS() + } +} + +func setDependencies(svc *service.Config) { + // TODO(cuonglm): remove once https://github.com/kardianos/service/issues/359 fixed. + _ = os.MkdirAll("/usr/local/etc/rc.d", 0755) +} + +func setWorkingDirectory(svc *service.Config, dir string) {} diff --git a/cmd/ctrld/prog_linux.go b/cmd/ctrld/prog_linux.go index 7d4f87a..4ec9416 100644 --- a/cmd/ctrld/prog_linux.go +++ b/cmd/ctrld/prog_linux.go @@ -18,3 +18,7 @@ func setDependencies(svc *service.Config) { "After=NetworkManager-wait-online.service", } } + +func setWorkingDirectory(svc *service.Config, dir string) { + svc.WorkingDirectory = dir +} diff --git a/cmd/ctrld/prog_others.go b/cmd/ctrld/prog_others.go index 9d72f91..d790438 100644 --- a/cmd/ctrld/prog_others.go +++ b/cmd/ctrld/prog_others.go @@ -1,5 +1,4 @@ -//go:build !linux -// +build !linux +//go:build !linux && !freebsd package main @@ -8,3 +7,8 @@ import "github.com/kardianos/service" func (p *prog) preRun() {} func setDependencies(svc *service.Config) {} + +func setWorkingDirectory(svc *service.Config, dir string) { + // WorkingDirectory is not supported on Windows. + svc.WorkingDirectory = dir +} diff --git a/internal/dns/debian_resolvconf.go b/internal/dns/debian_resolvconf.go new file mode 100644 index 0000000..f3d736d --- /dev/null +++ b/internal/dns/debian_resolvconf.go @@ -0,0 +1,153 @@ +// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux || freebsd || openbsd + +package dns + +import ( + "bytes" + _ "embed" + "fmt" + "os" + "os/exec" + "path/filepath" + + "tailscale.com/atomicfile" + "tailscale.com/types/logger" +) + +//go:embed resolvconf-workaround.sh +var workaroundScript []byte + +// resolvconfConfigName is the name of the config submitted to +// resolvconf. +// The name starts with 'tun' in order to match the hardcoded +// interface order in debian resolvconf, which will place this +// configuration ahead of regular network links. In theory, this +// doesn't matter because we then fix things up to ensure our config +// is the only one in use, but in case that fails, this will make our +// configuration slightly preferred. +// The 'inet' suffix has no specific meaning, but conventionally +// resolvconf implementations encourage adding a suffix roughly +// indicating where the config came from, and "inet" is the "none of +// the above" value (rather than, say, "ppp" or "dhcp"). +const resolvconfConfigName = "ctrld.inet" + +// resolvconfLibcHookPath is the directory containing libc update +// scripts, which are run by Debian resolvconf when /etc/resolv.conf +// has been updated. +const resolvconfLibcHookPath = "/etc/resolvconf/update-libc.d" + +// resolvconfHookPath is the name of the libc hook script we install +// to force Ctrld's DNS config to take effect. +var resolvconfHookPath = filepath.Join(resolvconfLibcHookPath, "ctrld") + +// resolvconfManager manages DNS configuration using the Debian +// implementation of the `resolvconf` program, written by Thomas Hood. +type resolvconfManager struct { + logf logger.Logf + listRecordsPath string + interfacesDir string + scriptInstalled bool // libc update script has been installed +} + +var _ OSConfigurator = (*resolvconfManager)(nil) + +func newDebianResolvconfManager(logf logger.Logf) (*resolvconfManager, error) { + ret := &resolvconfManager{ + logf: logf, + listRecordsPath: "/lib/resolvconf/list-records", + interfacesDir: "/etc/resolvconf/run/interface", // panic fallback if nothing seems to work + } + + if _, err := os.Stat(ret.listRecordsPath); os.IsNotExist(err) { + // This might be a Debian system from before the big /usr + // merge, try /usr instead. + ret.listRecordsPath = "/usr" + ret.listRecordsPath + } + // The runtime directory is currently (2020-04) canonically + // /etc/resolvconf/run, but the manpage is making noise about + // switching to /run/resolvconf and dropping the /etc path. So, + // let's probe the possible directories and use the first one + // that works. + for _, path := range []string{ + "/etc/resolvconf/run/interface", + "/run/resolvconf/interface", + "/var/run/resolvconf/interface", + } { + if _, err := os.Stat(path); err == nil { + ret.interfacesDir = path + break + } + } + if ret.interfacesDir == "" { + // None of the paths seem to work, use the canonical location + // that the current manpage says to use. + ret.interfacesDir = "/etc/resolvconf/run/interfaces" + } + + return ret, nil +} + +func (m *resolvconfManager) deleteCtrldConfig() error { + cmd := exec.Command("resolvconf", "-d", resolvconfConfigName) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("running %s: %s", cmd, out) + } + return nil +} + +func (m *resolvconfManager) SetDNS(config OSConfig) error { + if !m.scriptInstalled { + m.logf("injecting resolvconf workaround script") + if err := os.MkdirAll(resolvconfLibcHookPath, 0755); err != nil { + return err + } + if err := atomicfile.WriteFile(resolvconfHookPath, workaroundScript, 0755); err != nil { + return err + } + m.scriptInstalled = true + } + + if config.IsZero() { + if err := m.deleteCtrldConfig(); err != nil { + return err + } + } else { + stdin := new(bytes.Buffer) + writeResolvConf(stdin, config.Nameservers, config.SearchDomains) // dns_direct.go + + // This resolvconf implementation doesn't support exclusive + // mode or interface priorities, so it will end up blending + // our configuration with other sources. However, this will + // get fixed up by the script we injected above. + cmd := exec.Command("resolvconf", "-a", resolvconfConfigName) + cmd.Stdin = stdin + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("running %s: %s", cmd, out) + } + } + + return nil +} + +func (m *resolvconfManager) Close() error { + if err := m.deleteCtrldConfig(); err != nil { + return err + } + + if m.scriptInstalled { + m.logf("removing resolvconf workaround script") + os.Remove(resolvconfHookPath) // Best-effort + } + + return nil +} + +func (m *resolvconfManager) Mode() string { + return "resolvconf" +} diff --git a/internal/dns/direct.go b/internal/dns/direct.go index 7258649..e11be05 100644 --- a/internal/dns/direct.go +++ b/internal/dns/direct.go @@ -144,6 +144,10 @@ type directManager struct { lastWarnContents []byte // last resolv.conf contents that we warned about } +func newDirectManager(logf logger.Logf) *directManager { + return newDirectManagerOnFS(logf, directFS{}) +} + func newDirectManagerOnFS(logf logger.Logf, fs wholeFileFS) *directManager { ctx, cancel := context.WithCancel(context.Background()) m := &directManager{ diff --git a/internal/dns/manager_freebsd.go b/internal/dns/manager_freebsd.go new file mode 100644 index 0000000..27a4e7f --- /dev/null +++ b/internal/dns/manager_freebsd.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dns + +import ( + "fmt" + "os" + + "tailscale.com/types/logger" +) + +func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) { + bs, err := os.ReadFile("/etc/resolv.conf") + if os.IsNotExist(err) { + return newDirectManager(logf), nil + } + if err != nil { + return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err) + } + + switch resolvOwner(bs) { + case "resolvconf": + switch resolvconfStyle() { + case "": + return newDirectManager(logf), nil + case "debian": + return newDebianResolvconfManager(logf) + case "openresolv": + return newOpenresolvManager() + default: + logf("[unexpected] got unknown flavor of resolvconf %q, falling back to direct manager", resolvconfStyle()) + return newDirectManager(logf), nil + } + default: + return newDirectManager(logf), nil + } +} diff --git a/internal/dns/nm.go b/internal/dns/nm.go index 68ce71b..03e6f4a 100644 --- a/internal/dns/nm.go +++ b/internal/dns/nm.go @@ -31,6 +31,8 @@ type nmManager struct { dnsManager dbus.BusObject } +var _ OSConfigurator = (*nmManager)(nil) + func newNMManager(interfaceName string) (*nmManager, error) { conn, err := dbus.SystemBus() if err != nil { diff --git a/internal/dns/openresolv.go b/internal/dns/openresolv.go new file mode 100644 index 0000000..8c53d87 --- /dev/null +++ b/internal/dns/openresolv.go @@ -0,0 +1,57 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux || freebsd || openbsd + +package dns + +import ( + "bytes" + "fmt" + "os/exec" +) + +// openresolvManager manages DNS configuration using the openresolv +// implementation of the `resolvconf` program. +type openresolvManager struct{} + +var _ OSConfigurator = (*openresolvManager)(nil) + +func newOpenresolvManager() (openresolvManager, error) { + return openresolvManager{}, nil +} + +func (m openresolvManager) deleteTailscaleConfig() error { + cmd := exec.Command("resolvconf", "-f", "-d", "ctrld") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("running %s: %s", cmd, out) + } + return nil +} + +func (m openresolvManager) SetDNS(config OSConfig) error { + if config.IsZero() { + return m.deleteTailscaleConfig() + } + + var stdin bytes.Buffer + writeResolvConf(&stdin, config.Nameservers, config.SearchDomains) + + cmd := exec.Command("resolvconf", "-m", "0", "-x", "-a", "ctrld") + cmd.Stdin = &stdin + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("running %s: %s", cmd, out) + } + return nil +} + +func (m openresolvManager) Close() error { + return m.deleteTailscaleConfig() +} + +func (m openresolvManager) Mode() string { + return "resolvconf" +} diff --git a/internal/dns/osconfig.go b/internal/dns/osconfig.go index 0f5e91d..36fcaec 100644 --- a/internal/dns/osconfig.go +++ b/internal/dns/osconfig.go @@ -13,6 +13,8 @@ import ( "tailscale.com/util/dnsname" ) +var _ OSConfigurator = (*directManager)(nil) + // An OSConfigurator applies DNS settings to the operating system. type OSConfigurator interface { // SetDNS updates the OS's DNS configuration to match cfg. diff --git a/internal/dns/resolvconf-workaround.sh b/internal/dns/resolvconf-workaround.sh new file mode 100644 index 0000000..d04c723 --- /dev/null +++ b/internal/dns/resolvconf-workaround.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +# +# This script is a workaround for a vpn-unfriendly behavior of the +# original resolvconf by Thomas Hood. Unlike the `openresolv` +# implementation (whose binary is also called resolvconf, +# confusingly), the original resolvconf lacks a way to specify +# "exclusive mode" for a provider configuration. In practice, this +# means that if Ctrld wants to install a DNS configuration, that +# config will get "blended" with the configs from other sources, +# rather than override those other sources. +# +# This script gets installed at /etc/resolvconf/update-libc.d, which +# is a directory of hook scripts that get run after resolvconf's libc +# helper has finished rewriting /etc/resolv.conf. It's meant to notify +# consumers of resolv.conf of a new configuration. +# +# Instead, we use that hook mechanism to reach into resolvconf's +# stuff, and rewrite the libc-generated resolv.conf to exclusively +# contain Ctrld's configuration - effectively implementing +# exclusive mode ourselves in post-production. + +set -e + +if [ -n "$CTRLD_RESOLVCONF_HOOK_LOOP" ]; then + # Hook script being invoked by itself, skip. + exit 0 +fi + +if [ ! -f ctrld.inet ]; then + # Ctrld isn't trying to manage DNS, do nothing. + exit 0 +fi + +if ! grep resolvconf /etc/resolv.conf >/dev/null; then + # resolvconf isn't managing /etc/resolv.conf, do nothing. + exit 0 +fi + +# Write out a modified /etc/resolv.conf containing just our config. +( + if [ -f /etc/resolvconf/resolv.conf.d/head ]; then + cat /etc/resolvconf/resolv.conf.d/head + fi + echo "# Ctrld workaround applied to set exclusive DNS configuration." + cat tun-tailscale.inet + if [ -f /etc/resolvconf/resolv.conf.d/base ]; then + # Keep options and sortlist, discard other base things since + # they're the things we're trying to override. + grep -e 'sortlist ' -e 'options ' /etc/resolvconf/resolv.conf.d/base || true + fi + if [ -f /etc/resolvconf/resolv.conf.d/tail ]; then + cat /etc/resolvconf/resolv.conf.d/tail + fi +) >/etc/resolv.conf + +if [ -d /etc/resolvconf/update-libc.d ] ; then + # Re-notify libc watchers that we've changed resolv.conf again. + export CTRLD_RESOLVCONF_HOOK_LOOP=1 + exec run-parts /etc/resolvconf/update-libc.d +fi \ No newline at end of file diff --git a/internal/dns/resolved.go b/internal/dns/resolved.go index 8d03249..02455b5 100644 --- a/internal/dns/resolved.go +++ b/internal/dns/resolved.go @@ -105,6 +105,8 @@ type resolvedManager struct { newManager func(conn *dbus.Conn) dbus.BusObject } +var _ OSConfigurator = (*resolvedManager)(nil) + func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) { iface, err := net.InterfaceByName(interfaceName) if err != nil { From 64f2dcb25bbe5503e9b2be5738c75f5b7e71ab10 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 15 Feb 2023 23:44:22 +0700 Subject: [PATCH 14/37] Fix parsing network service name on darwin The network service name appears on the previous line, not the same line with "Device" name. Updates #57 --- cmd/ctrld/net_darwin.go | 24 ++++++++++----- cmd/ctrld/net_darwin_test.go | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 cmd/ctrld/net_darwin_test.go diff --git a/cmd/ctrld/net_darwin.go b/cmd/ctrld/net_darwin.go index 223cc75..0939c85 100644 --- a/cmd/ctrld/net_darwin.go +++ b/cmd/ctrld/net_darwin.go @@ -3,6 +3,7 @@ package main import ( "bufio" "bytes" + "io" "net" "os/exec" "strings" @@ -14,21 +15,30 @@ func patchNetIfaceName(iface *net.Interface) error { return err } - scanner := bufio.NewScanner(bytes.NewReader(b)) + if name := networkServiceName(iface.Name, bytes.NewReader(b)); name != "" { + iface.Name = name + mainLog.Debug().Str("network_service", name).Msg("found network service name for interface") + } + return nil +} + +func networkServiceName(ifaceName string, r io.Reader) string { + scanner := bufio.NewScanner(r) + prevLine := "" for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "*") { // Network services is disabled. continue } - if !strings.Contains(line, "Device: "+iface.Name) { + if !strings.Contains(line, "Device: "+ifaceName) { + prevLine = line continue } - parts := strings.Split(line, ",") - if _, networkServiceName, ok := strings.Cut(parts[0], "(Hardware Port: "); ok { - mainLog.Debug().Str("network_service", networkServiceName).Msg("found network service name for interface") - iface.Name = networkServiceName + parts := strings.SplitN(prevLine, " ", 2) + if len(parts) == 2 { + return strings.TrimSpace(parts[1]) } } - return nil + return "" } diff --git a/cmd/ctrld/net_darwin_test.go b/cmd/ctrld/net_darwin_test.go new file mode 100644 index 0000000..7110d15 --- /dev/null +++ b/cmd/ctrld/net_darwin_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const listnetworkserviceorderOutput = ` +(1) USB 10/100/1000 LAN 2 +(Hardware Port: USB 10/100/1000 LAN, Device: en7) + +(2) Ethernet +(Hardware Port: Ethernet, Device: en0) + +(3) Wi-Fi +(Hardware Port: Wi-Fi, Device: en1) + +(4) Bluetooth PAN +(Hardware Port: Bluetooth PAN, Device: en4) + +(5) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(6) kernal +(Hardware Port: com.wireguard.macos, Device: ) + +(7) WS BT +(Hardware Port: com.wireguard.macos, Device: ) + +(8) ca-001-stg +(Hardware Port: com.wireguard.macos, Device: ) + +(9) ca-001-stg-2 +(Hardware Port: com.wireguard.macos, Device: ) + +` + +func Test_networkServiceName(t *testing.T) { + tests := []struct { + ifaceName string + networkServiceName string + }{ + {"en7", "USB 10/100/1000 LAN 2"}, + {"en0", "Ethernet"}, + {"en1", "Wi-Fi"}, + {"en4", "Bluetooth PAN"}, + {"bridge0", "Thunderbolt Bridge"}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.ifaceName, func(t *testing.T) { + t.Parallel() + name := networkServiceName(tc.ifaceName, strings.NewReader(listnetworkserviceorderOutput)) + assert.Equal(t, tc.networkServiceName, name) + }) + } +} From 84fca06c6290b0b41321d5c9559bf1a656413f59 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 21 Feb 2023 20:39:45 +0700 Subject: [PATCH 15/37] cmd/ctrld: implement allocate/deallocate ip on freebsd Updates #56 --- cmd/ctrld/os_freebsd.go | 21 +++++++++++++++++++++ cmd/ctrld/os_others.go | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cmd/ctrld/os_freebsd.go b/cmd/ctrld/os_freebsd.go index b65de54..da1a05a 100644 --- a/cmd/ctrld/os_freebsd.go +++ b/cmd/ctrld/os_freebsd.go @@ -3,11 +3,32 @@ package main import ( "net" "net/netip" + "os/exec" "github.com/Control-D-Inc/ctrld/internal/dns" "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) +// allocate loopback ip +// sudo ifconfig lo0 127.0.0.53 alias +func allocateIP(ip string) error { + cmd := exec.Command("ifconfig", "lo0", ip, "alias") + if err := cmd.Run(); err != nil { + mainLog.Error().Err(err).Msg("allocateIP failed") + return err + } + return nil +} + +func deAllocateIP(ip string) error { + cmd := exec.Command("ifconfig", "lo0", ip, "-alias") + if err := cmd.Run(); err != nil { + mainLog.Error().Err(err).Msg("deAllocateIP failed") + return err + } + return nil +} + // set the dns server for the provided network interface func setDNS(iface *net.Interface, nameservers []string) error { r, err := dns.NewOSConfigurator(logf, iface.Name) diff --git a/cmd/ctrld/os_others.go b/cmd/ctrld/os_others.go index e9f9d61..3807bcc 100644 --- a/cmd/ctrld/os_others.go +++ b/cmd/ctrld/os_others.go @@ -1,4 +1,4 @@ -//go:build !linux && !darwin +//go:build !linux && !darwin && !freebsd package main From 82900eeca6bf415d1799cff174510d700ae7c47a Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 23 Feb 2023 09:16:34 +0700 Subject: [PATCH 16/37] cmd/ctrld: move log file if existed on app start Updates #59 --- cmd/ctrld/main.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/ctrld/main.go b/cmd/ctrld/main.go index 033fd2e..ff7d3d5 100644 --- a/cmd/ctrld/main.go +++ b/cmd/ctrld/main.go @@ -1,8 +1,8 @@ package main import ( - "fmt" "io" + "log" "os" "path/filepath" "time" @@ -61,12 +61,16 @@ func initLogging() { if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" { // Create parent directory if necessary. if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil { - fmt.Fprintf(os.Stderr, "failed to create log path: %v", err) + log.Printf("failed to create log path: %v", err) os.Exit(1) } - logFile, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_RDWR, os.FileMode(0o600)) + // Backup old log file with .1 suffix. + if err := os.Rename(logFilePath, logFilePath+".1"); err != nil && !os.IsNotExist(err) { + log.Printf("could not backup old log file: %v", err) + } + logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_RDWR, os.FileMode(0o600)) if err != nil { - fmt.Fprintf(os.Stderr, "failed to create log file: %v", err) + log.Printf("failed to create log file: %v", err) os.Exit(1) } writers = append(writers, logFile) From cad71997aa03bcf42ea2b60012caaa464a98c48e Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 21 Feb 2023 23:52:50 +0700 Subject: [PATCH 17/37] cmd/ctrld: allocate new ip instead of port So the alternative listener address can still be used as system resolver. --- cmd/ctrld/cli.go | 33 +++++++++++++++++++++------------ cmd/ctrld/os_linux.go | 4 ++-- cmd/ctrld/prog.go | 43 ++++++++++++++++--------------------------- config.go | 2 +- go.mod | 2 +- 5 files changed, 41 insertions(+), 43 deletions(-) diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index e3ca198..f419813 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -17,6 +17,8 @@ import ( "strings" "time" + "github.com/fsnotify/fsnotify" + "github.com/go-playground/validator/v10" "github.com/kardianos/service" "github.com/miekg/dns" @@ -87,7 +89,6 @@ func initCLI() { if daemon && runtime.GOOS == "windows" { log.Fatal("Cannot run in daemon mode. Please install a Windows service.") } - noConfigStart := isNoConfigStart(cmd) writeDefaultConfig := !noConfigStart && configBase64 == "" configs := []struct { @@ -196,18 +197,23 @@ func initCLI() { } setDependencies(sc) sc.Arguments = append([]string{"run"}, osArgs...) + + // No config path, generating config in HOME directory. + noConfigStart := isNoConfigStart(cmd) + writeDefaultConfig := !noConfigStart && configBase64 == "" + if configPath != "" { + v.SetConfigFile(configPath) + } if dir, err := os.UserHomeDir(); err == nil { setWorkingDirectory(sc, dir) - // No config path, generating config in HOME directory. - noConfigStart := isNoConfigStart(cmd) - writeDefaultConfig := !noConfigStart && configBase64 == "" if configPath == "" && writeDefaultConfig { defaultConfigFile = filepath.Join(dir, defaultConfigFile) - readConfigFile(writeDefaultConfig && cdUID == "") + v.SetConfigFile(defaultConfigFile) } sc.Arguments = append(sc.Arguments, "--homedir="+dir) } + readConfigFile(writeDefaultConfig && cdUID == "") if err := v.Unmarshal(&cfg); err != nil { log.Fatalf("failed to unmarshal config: %v", err) } @@ -480,7 +486,7 @@ func writeConfigFile() error { } } enc := toml.NewEncoder(f).SetIndentTables(true) - if err := enc.Encode(v.AllSettings()); err != nil { + if err := enc.Encode(&cfg); err != nil { return err } if err := f.Close(); err != nil { @@ -495,6 +501,13 @@ func readConfigFile(writeDefaultConfig bool) bool { if err == nil { fmt.Println("loading config file from:", v.ConfigFileUsed()) defaultConfigFile = v.ConfigFileUsed() + v.OnConfigChange(func(in fsnotify.Event) { + if err := v.UnmarshalKey("listener", &cfg.Listener); err != nil { + log.Printf("failed to unmarshal listener config: %v", err) + return + } + }) + v.WatchConfig() return true } @@ -630,10 +643,6 @@ func processCDFlags() { }, } - v = viper.NewWithOptions(viper.KeyDelimiter("::")) - v.Set("network", cfg.Network) - v.Set("upstream", cfg.Upstream) - v.Set("listener", cfg.Listener) processLogAndCacheFlags() if err := writeConfigFile(); err != nil { logger.Fatal().Err(err).Msg("failed to write config file") @@ -705,14 +714,14 @@ func defaultIfaceName() string { func selfCheckStatus(status service.Status) service.Status { c := new(dns.Client) - lc := cfg.Listener["0"] bo := backoff.NewBackoff("self-check", logf, 10*time.Second) bo.LogLongerThan = 500 * time.Millisecond ctx := context.Background() err := errors.New("query failed") - maxAttempts := 10 + maxAttempts := 20 mainLog.Debug().Msg("Performing self-check") for i := 0; i < maxAttempts; i++ { + lc := cfg.Listener["0"] m := new(dns.Msg) m.SetQuestion(selfCheckFQDN+".", dns.TypeA) m.RecursionDesired = true diff --git a/cmd/ctrld/os_linux.go b/cmd/ctrld/os_linux.go index a29c7f4..839d99d 100644 --- a/cmd/ctrld/os_linux.go +++ b/cmd/ctrld/os_linux.go @@ -27,8 +27,8 @@ import ( // sudo ip a add 127.0.0.2/24 dev lo func allocateIP(ip string) error { cmd := exec.Command("ip", "a", "add", ip+"/24", "dev", "lo") - if err := cmd.Run(); err != nil { - mainLog.Error().Err(err).Msg("allocateIP failed") + if out, err := cmd.CombinedOutput(); err != nil { + mainLog.Error().Err(err).Msgf("allocateIP failed: %s", string(out)) return err } return nil diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index e3d6239..edb43bf 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -2,6 +2,8 @@ package main import ( "errors" + "fmt" + "math/rand" "net" "os" "strconv" @@ -120,7 +122,7 @@ func (p *prog) run() { addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)) mainLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, addr) err := p.serveUDP(listenerNum) - if err != nil && !defaultConfigWritten { + if err != nil && !defaultConfigWritten && cdUID == "" { mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) return } @@ -128,39 +130,21 @@ func (p *prog) run() { return } - if opErr, ok := err.(*net.OpError); ok { + if opErr, ok := err.(*net.OpError); ok && listenerNum == "0" { if sErr, ok := opErr.Err.(*os.SyscallError); ok && errors.Is(opErr.Err, syscall.EADDRINUSE) || errors.Is(sErr.Err, errWindowsAddrInUse) { mainLog.Warn().Msgf("Address %s already in used, pick a random one", addr) - pc, err := net.ListenPacket("udp", net.JoinHostPort(listenerConfig.IP, "0")) - if err != nil { - mainLog.Fatal().Err(err).Msg("failed to listen packet") - return - } - _, portStr, _ := net.SplitHostPort(pc.LocalAddr().String()) - port, err := strconv.Atoi(portStr) - if err != nil { - mainLog.Fatal().Err(err).Msg("malformed port") - return - } - listenerConfig.Port = port - v.Set("listener", map[string]*ctrld.ListenerConfig{ - "0": { - IP: "127.0.0.1", - Port: port, - }, - }) + ip := randomLocalIP() + listenerConfig.IP = ip + port := listenerConfig.Port + cfg.Upstream = map[string]*ctrld.UpstreamConfig{"0": cfg.Upstream["0"]} if err := writeConfigFile(); err != nil { mainLog.Fatal().Err(err).Msg("failed to write config file") } else { mainLog.Info().Msg("writing config file to: " + defaultConfigFile) } - mainLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, pc.LocalAddr()) - // There can be a race between closing the listener and start our own UDP server, but it's - // rare, and we only do this once, so let conservative here. - if err := pc.Close(); err != nil { - mainLog.Fatal().Err(err).Msg("failed to close packet conn") - return - } + p.cfg.Service.AllocateIP = true + p.preRun() + mainLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, net.JoinHostPort(ip, strconv.Itoa(port))) if err := p.serveUDP(listenerNum); err != nil { mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum) return @@ -254,3 +238,8 @@ func (p *prog) resetDNS() { } logger.Debug().Msg("Restoring DNS successfully") } + +func randomLocalIP() string { + n := rand.Intn(254-2) + 2 + return fmt.Sprintf("127.0.0.%d", n) +} diff --git a/config.go b/config.go index 1364b4e..f8d7736 100644 --- a/config.go +++ b/config.go @@ -66,9 +66,9 @@ func InitConfig(v *viper.Viper, name string) { // Config represents ctrld supported configuration. type Config struct { Service ServiceConfig `mapstructure:"service" toml:"service,omitempty"` + Listener map[string]*ListenerConfig `mapstructure:"listener" toml:"listener" validate:"min=1,dive"` Network map[string]*NetworkConfig `mapstructure:"network" toml:"network" validate:"min=1,dive"` Upstream map[string]*UpstreamConfig `mapstructure:"upstream" toml:"upstream" validate:"min=1,dive"` - Listener map[string]*ListenerConfig `mapstructure:"listener" toml:"listener" validate:"min=1,dive"` } // ServiceConfig specifies the general ctrld config. diff --git a/go.mod b/go.mod index af47c82..99ef4e0 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 github.com/frankban/quicktest v1.14.3 + github.com/fsnotify/fsnotify v1.6.0 github.com/go-playground/validator/v10 v10.11.1 github.com/godbus/dbus/v5 v5.0.6 github.com/hashicorp/golang-lru/v2 v2.0.1 @@ -26,7 +27,6 @@ require ( require ( github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect From 6d3c82d38df0d51b6f0c176ea7e368302eb14ee0 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Feb 2023 21:41:20 +0700 Subject: [PATCH 18/37] internal/dns: add debian/openresolv to linux manager --- internal/dns/manager_linux.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/dns/manager_linux.go b/internal/dns/manager_linux.go index 20ccf7e..1fa1650 100644 --- a/internal/dns/manager_linux.go +++ b/internal/dns/manager_linux.go @@ -62,6 +62,10 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat return newResolvedManager(logf, interfaceName) case "network-manager": return newNMManager(interfaceName) + case "debian-resolvconf": + return newDebianResolvconfManager(logf) + case "openresolv": + return newOpenresolvManager() default: logf("[unexpected] detected unknown DNS mode %q, using direct manager as last resort", mode) return newDirectManagerOnFS(logf, env.fs), nil From 2e1b3f9d072774066bf009a8028d25f52a1ee38c Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Feb 2023 22:01:16 +0700 Subject: [PATCH 19/37] Upgrade golang.org/x/net to v0.7.0 For pulling CVE-2022-41723 fix. --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 99ef4e0..1b09384 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.14.0 github.com/stretchr/testify v1.8.1 - golang.org/x/sys v0.4.0 + golang.org/x/sys v0.5.0 golang.zx2c4.com/wireguard/windows v0.5.3 tailscale.com v1.34.1 ) @@ -66,9 +66,9 @@ require ( golang.org/x/crypto v0.4.0 // indirect golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.5.1-0.20230105164244-f8411da775a6 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 59f965b..a158354 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.5.1-0.20230105164244-f8411da775a6 h1:pKt/LWZC6+FwNujj5E7DdVyWcbtQvKqPuN0GPKWMyB8= -golang.org/x/net v0.5.1-0.20230105164244-f8411da775a6/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -457,8 +457,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -469,8 +469,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 8852f60ccba0ba8fea910c3d46a9a6d48cea08c1 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 28 Feb 2023 09:29:57 +0700 Subject: [PATCH 20/37] Add idle conn timeout for HTTP transport Allowing the connection to be re-new once it becomes un-usable. --- config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config.go b/config.go index f8d7736..f6a132c 100644 --- a/config.go +++ b/config.go @@ -157,6 +157,7 @@ func (uc *UpstreamConfig) SetupTransport() { func (uc *UpstreamConfig) setupDOHTransport() { uc.transport = http.DefaultTransport.(*http.Transport).Clone() + uc.transport.IdleConnTimeout = 5 * time.Second uc.transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{ Timeout: 10 * time.Second, From 930a5ad439e0f375e1efc8083e6fa28c5571fee6 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 1 Mar 2023 09:02:10 +0700 Subject: [PATCH 21/37] cmd/ctrld: only set ::1 as DNS server on Windows if ipv6 available --- cmd/ctrld/os_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ctrld/os_windows.go b/cmd/ctrld/os_windows.go index 0bd7358..8858027 100644 --- a/cmd/ctrld/os_windows.go +++ b/cmd/ctrld/os_windows.go @@ -52,7 +52,7 @@ func setPrimaryDNS(iface *net.Interface, dns string) error { mainLog.Error().Err(err).Msgf("failed to set primary DNS: %s", string(output)) return err } - if ipVer == "ipv4" { + if ipVer == "ipv4" && ctrldnet.SupportsIPv6ListenLocal() { // Disable IPv6 DNS, so the query will be fallback to IPv4. _, _ = netsh("interface", "ipv6", "set", "dnsserver", idx, "static", "::1", "primary") } From 8b08cc8a6edd9bdd08b8a09bf7ce7edba0a2f2ae Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 1 Mar 2023 11:14:10 +0700 Subject: [PATCH 22/37] all: rework bootstrap IP discovering At startup, ctrld gathers bootstrap IP information and use this bootstrap IP for connecting to upstream. However, in case the network stack changed, for example, dues to VPN connection, ctrld will still use this old (maybe invalid) bootstrap IP for the current network stack. This commit rework the discovering process, and re-initializing the bootstrap IP if connecting to upstream failed. --- cmd/ctrld/dns_proxy.go | 10 +++++++ cmd/ctrld/prog.go | 44 ++--------------------------- config.go | 63 ++++++++++++++++++++++++++++++++++++++---- config_quic.go | 7 ++++- doq.go | 2 +- internal/net/net.go | 19 +++++++++---- resolver.go | 3 ++ 7 files changed, 95 insertions(+), 53 deletions(-) diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 8ece9c1..4a428a4 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "errors" "fmt" "net" "runtime" @@ -182,6 +183,15 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i resolveCtx = timeoutCtx } answer, err := dnsResolver.Resolve(resolveCtx, msg) + if errors.Is(err, ctrld.ErrUpstreamFailed) { + ctrldnet.Reset() + if err := upstreamConfig.SetupBootstrapIP(); err != nil { + mainLog.Error().Err(err).Msg("could not re-initialize bootstrap IP") + } else { + mainLog.Debug().Msg("re-initialize bootstrap IP done") + } + return nil + } if err != nil { ctrld.Log(ctx, mainLog.Error().Err(err), "failed to resolve query") return nil diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index edb43bf..8a1dd6e 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -11,11 +11,9 @@ import ( "syscall" "github.com/kardianos/service" - "github.com/miekg/dns" "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/dnscache" - ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) var logf = func(format string, args ...any) { @@ -37,7 +35,6 @@ type prog struct { func (p *prog) Start(s service.Service) error { p.cfg = &cfg go p.run() - mainLog.Info().Msg("Service started") return nil } @@ -67,45 +64,10 @@ func (p *prog) run() { for n := range p.cfg.Upstream { uc := p.cfg.Upstream[n] uc.Init() - if uc.BootstrapIP == "" { - // resolve it manually and set the bootstrap ip - c := new(dns.Client) - for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} { - if !ctrldnet.SupportsIPv6() && dnsType == dns.TypeAAAA { - continue - } - m := new(dns.Msg) - m.SetQuestion(uc.Domain+".", dnsType) - m.RecursionDesired = true - r, _, err := c.Exchange(m, net.JoinHostPort(bootstrapDNS, "53")) - if err != nil { - mainLog.Error().Err(err).Msgf("could not resolve domain %s for upstream.%s", uc.Domain, n) - continue - } - if r.Rcode != dns.RcodeSuccess { - mainLog.Error().Msgf("could not resolve domain return code: %d, upstream.%s", r.Rcode, n) - continue - } - if len(r.Answer) == 0 { - continue - } - for _, a := range r.Answer { - switch ar := a.(type) { - case *dns.A: - uc.BootstrapIP = ar.A.String() - case *dns.AAAA: - uc.BootstrapIP = ar.AAAA.String() - default: - continue - } - mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Setting bootstrap IP for upstream.%s", n) - // Stop if we reached here, because we got the bootstrap IP from r.Answer. - break - } - // If we reached here, uc.BootstrapIP was set, nothing to do anymore. - break - } + if err := uc.SetupBootstrapIP(); err != nil { + mainLog.Fatal().Err(err).Msgf("failed to setup bootstrap IP for upstream.%s", n) } + mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Setting bootstrap IP for upstream.%s", n) uc.SetupTransport() } diff --git a/config.go b/config.go index f6a132c..9082e01 100644 --- a/config.go +++ b/config.go @@ -2,19 +2,26 @@ package ctrld import ( "context" + "errors" "net" "net/http" "net/url" "os" "strings" + "sync" "time" - "github.com/Control-D-Inc/ctrld/internal/dnsrcode" "github.com/go-playground/validator/v10" "github.com/miekg/dns" "github.com/spf13/viper" + + "github.com/Control-D-Inc/ctrld/internal/dnsrcode" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) +// ErrUpstreamFailed indicates that ctrld failed to connect to an upstream. +var ErrUpstreamFailed = errors.New("could not connect to upstream") + // SetConfigName set the config name that ctrld will look for. func SetConfigName(v *viper.Viper, name string) { v.SetConfigName(name) @@ -100,6 +107,9 @@ type UpstreamConfig struct { Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` transport *http.Transport `mapstructure:"-" toml:"-"` http3RoundTripper http.RoundTripper `mapstructure:"-" toml:"-"` + + // guard BootstrapIP + mu sync.Mutex } // ListenerConfig specifies the networks configuration that ctrld will run on. @@ -155,13 +165,51 @@ func (uc *UpstreamConfig) SetupTransport() { } } +// SetupBootstrapIP manually find all available IPs of the upstream. +func (uc *UpstreamConfig) SetupBootstrapIP() error { + uc.mu.Lock() + defer uc.mu.Unlock() + + c := new(dns.Client) + m := new(dns.Msg) + dnsType := dns.TypeA + if ctrldnet.SupportsIPv6() { + dnsType = dns.TypeAAAA + } + m.SetQuestion(uc.Domain+".", dnsType) + m.RecursionDesired = true + r, _, err := c.Exchange(m, net.JoinHostPort(bootstrapDNS, "53")) + if err != nil { + ProxyLog.Error().Err(err).Msgf("could not resolve domain %s for upstream", uc.Domain) + return err + } + if r.Rcode != dns.RcodeSuccess { + ProxyLog.Error().Msgf("could not resolve domain return code: %d, upstream", r.Rcode) + return errors.New(dns.RcodeToString[r.Rcode]) + } + if len(r.Answer) == 0 { + return errors.New("no answer from bootstrap DNS server") + } + for _, a := range r.Answer { + switch ar := a.(type) { + case *dns.A: + uc.BootstrapIP = ar.A.String() + break + case *dns.AAAA: + uc.BootstrapIP = ar.AAAA.String() + break + } + } + return nil +} + func (uc *UpstreamConfig) setupDOHTransport() { uc.transport = http.DefaultTransport.(*http.Transport).Clone() uc.transport.IdleConnTimeout = 5 * time.Second uc.transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 10 * time.Second, + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, } Log(ctx, ProxyLog.Debug(), "debug dial context %s - %s - %s", addr, network, bootstrapDNS) // if we have a bootstrap ip set, use it to avoid DNS lookup @@ -169,9 +217,14 @@ func (uc *UpstreamConfig) setupDOHTransport() { if _, port, _ := net.SplitHostPort(addr); port != "" { addr = net.JoinHostPort(uc.BootstrapIP, port) } - Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr) } - return dialer.DialContext(ctx, network, addr) + Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr) + conn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + Log(ctx, ProxyLog.Debug().Err(err), "could not dial to upstream") + return nil, ErrUpstreamFailed + } + return conn, nil } uc.pingUpstream() diff --git a/config_quic.go b/config_quic.go index 72ce351..fb00655 100644 --- a/config_quic.go +++ b/config_quic.go @@ -32,7 +32,12 @@ func (uc *UpstreamConfig) setupDOH3Transport() { if err != nil { return nil, err } - return quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg) + conn, err := quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg) + if err != nil { + Log(ctx, ProxyLog.Debug().Err(err), "could not dial early to upstream") + return nil, ErrUpstreamFailed + } + return conn, nil } uc.http3RoundTripper = rt diff --git a/doq.go b/doq.go index 20919e3..ab3fbb6 100644 --- a/doq.go +++ b/doq.go @@ -47,7 +47,7 @@ func resolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls. func doResolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.Config) (*dns.Msg, error) { session, err := quic.DialAddr(endpoint, tlsConfig, nil) if err != nil { - return nil, err + return nil, ErrUpstreamFailed } defer session.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") diff --git a/internal/net/net.go b/internal/net/net.go index 4360fc2..1488da9 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -4,6 +4,7 @@ import ( "context" "net" "sync" + "sync/atomic" "time" "tailscale.com/logtail/backoff" @@ -28,13 +29,17 @@ var Dialer = &net.Dialer{ } var ( - stackOnce sync.Once + stackOnce atomic.Pointer[sync.Once] ipv4Enabled bool ipv6Enabled bool canListenIPv6Local bool hasNetworkUp bool ) +func init() { + stackOnce.Store(new(sync.Once)) +} + func probeStack() { b := backoff.NewBackoff("probeStack", func(format string, args ...any) {}, time.Minute) for { @@ -57,23 +62,27 @@ func probeStack() { } } +func Reset() { + stackOnce.Store(new(sync.Once)) +} + func Up() bool { - stackOnce.Do(probeStack) + stackOnce.Load().Do(probeStack) return hasNetworkUp } func SupportsIPv4() bool { - stackOnce.Do(probeStack) + stackOnce.Load().Do(probeStack) return ipv4Enabled } func SupportsIPv6() bool { - stackOnce.Do(probeStack) + stackOnce.Load().Do(probeStack) return ipv6Enabled } func SupportsIPv6ListenLocal() bool { - stackOnce.Do(probeStack) + stackOnce.Load().Do(probeStack) return canListenIPv6Local } diff --git a/resolver.go b/resolver.go index 5c04f37..bb23627 100644 --- a/resolver.go +++ b/resolver.go @@ -93,5 +93,8 @@ func (r *legacyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, e Dialer: dialer, } answer, _, err := dnsClient.ExchangeContext(ctx, msg, r.endpoint) + if _, ok := err.(*net.OpError); ok { + return answer, ErrUpstreamFailed + } return answer, err } From 262dcb1dffbdf90cbc958f242eb96da9cd52bdb1 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 2 Mar 2023 09:16:51 +0700 Subject: [PATCH 23/37] cmd/ctrld: check for ipv6 listen local Since when the machine may not have external ipv6 capability, but still can do ipv6 network on local network. --- cmd/ctrld/dns_proxy.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 4a428a4..4e67df6 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -57,14 +57,16 @@ func (p *prog) serveUDP(listenerNum string) error { // On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can // listen on ::1, then spawn a listener for receiving DNS requests. - if runtime.GOOS == "windows" && ctrldnet.SupportsIPv6() { + if runtime.GOOS == "windows" && ctrldnet.SupportsIPv6ListenLocal() { go func() { s := &dns.Server{ Addr: net.JoinHostPort("::1", strconv.Itoa(listenerConfig.Port)), Net: "udp", Handler: handler, } - _ = s.ListenAndServe() + if err := s.ListenAndServe(); err != nil { + mainLog.Error().Err(err).Msg("could not serving on ::1") + } }() } From fb20d443c184b90b2d550703e84318f631abfafd Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 3 Mar 2023 00:47:21 +0700 Subject: [PATCH 24/37] all: retry the request more agressively For better recovery and dealing with network stack changes, this commit change the request flow to: failure of any kind -> recreate transport/re-bootstrap -> retry once That would make ctrld recover from all scenarios in theory. --- cmd/ctrld/dns_proxy.go | 25 ++++++++++++------------- config.go | 32 +++++++++++++++++++++----------- config_quic.go | 7 +------ doq.go | 2 +- go.mod | 2 +- resolver.go | 3 --- 6 files changed, 36 insertions(+), 35 deletions(-) diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 4e67df6..bbe84f1 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -4,7 +4,6 @@ import ( "context" "crypto/rand" "encoding/hex" - "errors" "fmt" "net" "runtime" @@ -170,12 +169,12 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i staleAnswer = answer } } - resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg { + resolve1 := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) (*dns.Msg, error) { ctrld.Log(ctx, mainLog.Debug(), "sending query to %s: %s", upstreams[n], upstreamConfig.Name) dnsResolver, err := ctrld.NewResolver(upstreamConfig) if err != nil { ctrld.Log(ctx, mainLog.Error().Err(err), "failed to create resolver") - return nil + return nil, err } resolveCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -184,17 +183,17 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i defer cancel() resolveCtx = timeoutCtx } - answer, err := dnsResolver.Resolve(resolveCtx, msg) - if errors.Is(err, ctrld.ErrUpstreamFailed) { - ctrldnet.Reset() - if err := upstreamConfig.SetupBootstrapIP(); err != nil { - mainLog.Error().Err(err).Msg("could not re-initialize bootstrap IP") - } else { - mainLog.Debug().Msg("re-initialize bootstrap IP done") - } - return nil - } + return dnsResolver.Resolve(resolveCtx, msg) + } + resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg { + answer, err := resolve1(n, upstreamConfig, msg) if err != nil { + // If any error occurred, re-bootstrap transport/ip, retry the request. + upstreamConfig.ReBootstrap() + answer, err = resolve1(n, upstreamConfig, msg) + if err == nil { + return answer + } ctrld.Log(ctx, mainLog.Error().Err(err), "failed to resolve query") return nil } diff --git a/config.go b/config.go index 9082e01..507c914 100644 --- a/config.go +++ b/config.go @@ -14,14 +14,12 @@ import ( "github.com/go-playground/validator/v10" "github.com/miekg/dns" "github.com/spf13/viper" + "golang.org/x/sync/singleflight" "github.com/Control-D-Inc/ctrld/internal/dnsrcode" ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) -// ErrUpstreamFailed indicates that ctrld failed to connect to an upstream. -var ErrUpstreamFailed = errors.New("could not connect to upstream") - // SetConfigName set the config name that ctrld will look for. func SetConfigName(v *viper.Viper, name string) { v.SetConfigName(name) @@ -108,6 +106,7 @@ type UpstreamConfig struct { transport *http.Transport `mapstructure:"-" toml:"-"` http3RoundTripper http.RoundTripper `mapstructure:"-" toml:"-"` + g singleflight.Group // guard BootstrapIP mu sync.Mutex } @@ -154,6 +153,22 @@ func (uc *UpstreamConfig) Init() { } } +// ReBootstrap re-setup the bootstrap IP and the transport. +func (uc *UpstreamConfig) ReBootstrap() { + _, _, _ = uc.g.Do("rebootstrap", func() (any, error) { + ProxyLog.Debug().Msg("re-bootstrapping upstream ip") + ctrldnet.Reset() + err := uc.SetupBootstrapIP() + if err != nil { + ProxyLog.Error().Err(err).Msg("re-bootstrapping failed") + } else { + ProxyLog.Debug().Msgf("bootstrap ip set to: %s", uc.BootstrapIP) + } + uc.SetupTransport() + return err == nil, err + }) +} + // SetupTransport initializes the network transport used to connect to upstream server. // For now, only DoH upstream is supported. func (uc *UpstreamConfig) SetupTransport() { @@ -208,8 +223,8 @@ func (uc *UpstreamConfig) setupDOHTransport() { uc.transport.IdleConnTimeout = 5 * time.Second uc.transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, + Timeout: 2 * time.Second, + KeepAlive: 2 * time.Second, } Log(ctx, ProxyLog.Debug(), "debug dial context %s - %s - %s", addr, network, bootstrapDNS) // if we have a bootstrap ip set, use it to avoid DNS lookup @@ -219,12 +234,7 @@ func (uc *UpstreamConfig) setupDOHTransport() { } } Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr) - conn, err := dialer.DialContext(ctx, network, addr) - if err != nil { - Log(ctx, ProxyLog.Debug().Err(err), "could not dial to upstream") - return nil, ErrUpstreamFailed - } - return conn, nil + return dialer.DialContext(ctx, network, addr) } uc.pingUpstream() diff --git a/config_quic.go b/config_quic.go index fb00655..72ce351 100644 --- a/config_quic.go +++ b/config_quic.go @@ -32,12 +32,7 @@ func (uc *UpstreamConfig) setupDOH3Transport() { if err != nil { return nil, err } - conn, err := quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg) - if err != nil { - Log(ctx, ProxyLog.Debug().Err(err), "could not dial early to upstream") - return nil, ErrUpstreamFailed - } - return conn, nil + return quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg) } uc.http3RoundTripper = rt diff --git a/doq.go b/doq.go index ab3fbb6..20919e3 100644 --- a/doq.go +++ b/doq.go @@ -47,7 +47,7 @@ func resolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls. func doResolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.Config) (*dns.Msg, error) { session, err := quic.DialAddr(endpoint, tlsConfig, nil) if err != nil { - return nil, ErrUpstreamFailed + return nil, err } defer session.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") diff --git a/go.mod b/go.mod index 1b09384..7193372 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.14.0 github.com/stretchr/testify v1.8.1 + golang.org/x/sync v0.1.0 golang.org/x/sys v0.5.0 golang.zx2c4.com/wireguard/windows v0.5.3 tailscale.com v1.34.1 @@ -67,7 +68,6 @@ require ( golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.6.0 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/resolver.go b/resolver.go index bb23627..5c04f37 100644 --- a/resolver.go +++ b/resolver.go @@ -93,8 +93,5 @@ func (r *legacyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, e Dialer: dialer, } answer, _, err := dnsClient.ExchangeContext(ctx, msg, r.endpoint) - if _, ok := err.(*net.OpError); ok { - return answer, ErrUpstreamFailed - } return answer, err } From b0114dfaebd64ac47a11d6f09a067bd11db0e4c8 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Feb 2023 21:43:05 +0700 Subject: [PATCH 25/37] cmd/ctrld: make staticcheck happy --- cmd/ctrld/main.go | 2 -- cmd/ctrld/network_manager_others.go | 2 ++ config.go | 2 ++ internal/controld/config.go | 6 +++--- internal/dns/resolved.go | 5 ++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/ctrld/main.go b/cmd/ctrld/main.go index ff7d3d5..e53f7fd 100644 --- a/cmd/ctrld/main.go +++ b/cmd/ctrld/main.go @@ -27,8 +27,6 @@ var ( cfg ctrld.Config verbose int - bootstrapDNS = "76.76.2.0" - rootLogger = zerolog.New(io.Discard) mainLog = rootLogger diff --git a/cmd/ctrld/network_manager_others.go b/cmd/ctrld/network_manager_others.go index 3cdb762..cd43bbc 100644 --- a/cmd/ctrld/network_manager_others.go +++ b/cmd/ctrld/network_manager_others.go @@ -3,10 +3,12 @@ package main func setupNetworkManager() error { + reloadNetworkManager() return nil } func restoreNetworkManager() error { + reloadNetworkManager() return nil } diff --git a/config.go b/config.go index 507c914..73f648c 100644 --- a/config.go +++ b/config.go @@ -209,9 +209,11 @@ func (uc *UpstreamConfig) SetupBootstrapIP() error { switch ar := a.(type) { case *dns.A: uc.BootstrapIP = ar.A.String() + //lint:ignore S1023 false alarm? break case *dns.AAAA: uc.BootstrapIP = ar.AAAA.String() + //lint:ignore S1023 false alarm? break } } diff --git a/internal/controld/config.go b/internal/controld/config.go index 4f16b88..3994837 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -60,11 +60,11 @@ func FetchResolverConfig(uid string) (*ResolverConfig, error) { transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { // We experiment hanging in TLS handshake when connecting to ControlD API // with ipv6. So prefer ipv4 if available. - network = "tcp6" + proto := "tcp6" if ctrldnet.SupportsIPv4() { - network = "tcp4" + proto = "tcp4" } - return ctrldnet.Dialer.DialContext(ctx, network, addr) + return ctrldnet.Dialer.DialContext(ctx, proto, addr) } client := http.Client{ Timeout: 10 * time.Second, diff --git a/internal/dns/resolved.go b/internal/dns/resolved.go index 02455b5..a9bf911 100644 --- a/internal/dns/resolved.go +++ b/internal/dns/resolved.go @@ -100,9 +100,8 @@ type resolvedManager struct { logf logger.Logf ifidx int - configCR chan changeRequest // tracks OSConfigs changes and error responses - revertCh chan struct{} - newManager func(conn *dbus.Conn) dbus.BusObject + configCR chan changeRequest // tracks OSConfigs changes and error responses + revertCh chan struct{} } var _ OSConfigurator = (*resolvedManager)(nil) From 12512a60dae0ecf4a70712076b1bf1ddc8a329a7 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 7 Mar 2023 10:41:14 +0700 Subject: [PATCH 26/37] Always use first record from DNS response --- config.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/config.go b/config.go index 73f648c..45563fe 100644 --- a/config.go +++ b/config.go @@ -205,15 +205,19 @@ func (uc *UpstreamConfig) SetupBootstrapIP() error { if len(r.Answer) == 0 { return errors.New("no answer from bootstrap DNS server") } - for _, a := range r.Answer { - switch ar := a.(type) { + + bootstrapIP := func(record dns.RR) string { + switch ar := record.(type) { case *dns.A: - uc.BootstrapIP = ar.A.String() - //lint:ignore S1023 false alarm? - break + return ar.A.String() case *dns.AAAA: - uc.BootstrapIP = ar.AAAA.String() - //lint:ignore S1023 false alarm? + return ar.AAAA.String() + } + return "" + } + for _, a := range r.Answer { + if ip := bootstrapIP(a); ip != "" { + uc.BootstrapIP = ip break } } From 1a40767cb7f5e435ef6a9812276cafed195ab6c3 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 8 Mar 2023 09:01:42 +0700 Subject: [PATCH 27/37] Use upstream timeout when querying bootstrap IP --- config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index 45563fe..1c9094f 100644 --- a/config.go +++ b/config.go @@ -182,6 +182,9 @@ func (uc *UpstreamConfig) SetupTransport() { // SetupBootstrapIP manually find all available IPs of the upstream. func (uc *UpstreamConfig) SetupBootstrapIP() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Second) + defer cancel() + uc.mu.Lock() defer uc.mu.Unlock() @@ -193,7 +196,7 @@ func (uc *UpstreamConfig) SetupBootstrapIP() error { } m.SetQuestion(uc.Domain+".", dnsType) m.RecursionDesired = true - r, _, err := c.Exchange(m, net.JoinHostPort(bootstrapDNS, "53")) + r, _, err := c.ExchangeContext(ctx, m, net.JoinHostPort(bootstrapDNS, "53")) if err != nil { ProxyLog.Error().Err(err).Msgf("could not resolve domain %s for upstream", uc.Domain) return err From 018f6651c17838ee1e7bcc4a186a17eef2432701 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 8 Mar 2023 10:15:34 +0700 Subject: [PATCH 28/37] Fix wrong time precision in bootstrapping timeout The timeout is in millisecond, not second. --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 1c9094f..f35a7ff 100644 --- a/config.go +++ b/config.go @@ -182,7 +182,7 @@ func (uc *UpstreamConfig) SetupTransport() { // SetupBootstrapIP manually find all available IPs of the upstream. func (uc *UpstreamConfig) SetupBootstrapIP() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond) defer cancel() uc.mu.Lock() From fa50cd4df4bb2b83227b1f0cde3780962c6c7c35 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 8 Mar 2023 11:38:46 +0700 Subject: [PATCH 29/37] all: another rework on discovering bootstrap IPs Instead of re-query DNS record for upstream when re-bootstrapping, just query all records on startup, then selecting the next bootstrap ip depends on the current network stack. --- cmd/ctrld/prog.go | 8 ++- config.go | 136 +++++++++++++++++++++++++------------------- internal/net/net.go | 40 ++++++++----- 3 files changed, 108 insertions(+), 76 deletions(-) diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index 8a1dd6e..6b58116 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -64,10 +64,12 @@ func (p *prog) run() { for n := range p.cfg.Upstream { uc := p.cfg.Upstream[n] uc.Init() - if err := uc.SetupBootstrapIP(); err != nil { - mainLog.Fatal().Err(err).Msgf("failed to setup bootstrap IP for upstream.%s", n) + if uc.BootstrapIP == "" { + uc.SetupBootstrapIP() + mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Setting bootstrap IP for upstream.%s", n) + } else { + mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Using bootstrap IP for upstream.%s", n) } - mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Setting bootstrap IP for upstream.%s", n) uc.SetupTransport() } diff --git a/config.go b/config.go index f35a7ff..84e2f40 100644 --- a/config.go +++ b/config.go @@ -2,13 +2,12 @@ package ctrld import ( "context" - "errors" "net" "net/http" "net/url" "os" "strings" - "sync" + "sync/atomic" "time" "github.com/go-playground/validator/v10" @@ -106,9 +105,9 @@ type UpstreamConfig struct { transport *http.Transport `mapstructure:"-" toml:"-"` http3RoundTripper http.RoundTripper `mapstructure:"-" toml:"-"` - g singleflight.Group - // guard BootstrapIP - mu sync.Mutex + g singleflight.Group + bootstrapIPs []string + nextBootstrapIP atomic.Uint32 } // ListenerConfig specifies the networks configuration that ctrld will run on. @@ -153,19 +152,85 @@ func (uc *UpstreamConfig) Init() { } } +// SetupBootstrapIP manually find all available IPs of the upstream. +// The first usable IP will be used as bootstrap IP of the upstream. +func (uc *UpstreamConfig) SetupBootstrapIP() { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond) + defer cancel() + + c := new(dns.Client) + bootstrapIP := func(record dns.RR) string { + switch ar := record.(type) { + case *dns.A: + return ar.A.String() + case *dns.AAAA: + return ar.AAAA.String() + } + return "" + } + + // Find all A, AAAA records of the upstream. + for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} { + m := new(dns.Msg) + m.SetQuestion(uc.Domain+".", dnsType) + m.RecursionDesired = true + r, _, err := c.ExchangeContext(ctx, m, net.JoinHostPort(bootstrapDNS, "53")) + if err != nil { + ProxyLog.Error().Err(err).Str("type", dns.TypeToString[dnsType]).Msgf("could not resolve domain %s for upstream", uc.Domain) + continue + } + if r.Rcode != dns.RcodeSuccess { + ProxyLog.Error().Msgf("could not resolve domain return code: %d, upstream", r.Rcode) + continue + } + if len(r.Answer) == 0 { + ProxyLog.Error().Msg("no answer from bootstrap DNS server") + continue + } + for _, a := range r.Answer { + ip := bootstrapIP(a) + if ip == "" { + continue + } + + // Storing the ip to uc.bootstrapIPs list, so it can be selected later + // when retrying failed request due to network stack changed. + uc.bootstrapIPs = append(uc.bootstrapIPs, ip) + if uc.BootstrapIP == "" { + // Remember what's the current IP in bootstrap IPs list, + // so we can select next one upon re-bootstrapping. + uc.nextBootstrapIP.Add(1) + + // If this is an ipv6, and ipv6 is not available, don't use it as bootstrap ip. + if !ctrldnet.IPv6Available() && ctrldnet.IsIPv6(ip) { + continue + } + uc.BootstrapIP = ip + } + } + } + ProxyLog.Debug().Msgf("Bootstrap IPs: %v", uc.bootstrapIPs) +} + // ReBootstrap re-setup the bootstrap IP and the transport. func (uc *UpstreamConfig) ReBootstrap() { _, _, _ = uc.g.Do("rebootstrap", func() (any, error) { ProxyLog.Debug().Msg("re-bootstrapping upstream ip") - ctrldnet.Reset() - err := uc.SetupBootstrapIP() - if err != nil { - ProxyLog.Error().Err(err).Msg("re-bootstrapping failed") - } else { - ProxyLog.Debug().Msgf("bootstrap ip set to: %s", uc.BootstrapIP) + n := uint32(len(uc.bootstrapIPs)) + // Only attempt n times, because if there's no usable ip, + // the bootstrap ip will be kept as-is. + for i := uint32(0); i < n; i++ { + // Select the next ip in bootstrap ip list. + next := uc.nextBootstrapIP.Add(1) + ip := uc.bootstrapIPs[(next-1)%n] + if !ctrldnet.IPv6Available() && ctrldnet.IsIPv6(ip) { + continue + } + uc.BootstrapIP = ip + break } uc.SetupTransport() - return err == nil, err + return true, nil }) } @@ -180,53 +245,6 @@ func (uc *UpstreamConfig) SetupTransport() { } } -// SetupBootstrapIP manually find all available IPs of the upstream. -func (uc *UpstreamConfig) SetupBootstrapIP() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond) - defer cancel() - - uc.mu.Lock() - defer uc.mu.Unlock() - - c := new(dns.Client) - m := new(dns.Msg) - dnsType := dns.TypeA - if ctrldnet.SupportsIPv6() { - dnsType = dns.TypeAAAA - } - m.SetQuestion(uc.Domain+".", dnsType) - m.RecursionDesired = true - r, _, err := c.ExchangeContext(ctx, m, net.JoinHostPort(bootstrapDNS, "53")) - if err != nil { - ProxyLog.Error().Err(err).Msgf("could not resolve domain %s for upstream", uc.Domain) - return err - } - if r.Rcode != dns.RcodeSuccess { - ProxyLog.Error().Msgf("could not resolve domain return code: %d, upstream", r.Rcode) - return errors.New(dns.RcodeToString[r.Rcode]) - } - if len(r.Answer) == 0 { - return errors.New("no answer from bootstrap DNS server") - } - - bootstrapIP := func(record dns.RR) string { - switch ar := record.(type) { - case *dns.A: - return ar.A.String() - case *dns.AAAA: - return ar.AAAA.String() - } - return "" - } - for _, a := range r.Answer { - if ip := bootstrapIP(a); ip != "" { - uc.BootstrapIP = ip - break - } - } - return nil -} - func (uc *UpstreamConfig) setupDOHTransport() { uc.transport = http.DefaultTransport.(*http.Transport).Clone() uc.transport.IdleConnTimeout = 5 * time.Second diff --git a/internal/net/net.go b/internal/net/net.go index 1488da9..6c78586 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -40,6 +40,24 @@ func init() { stackOnce.Store(new(sync.Once)) } +func supportIPv4() bool { + _, err := Dialer.Dial("tcp4", net.JoinHostPort(controldIPv4Test, "80")) + return err == nil +} + +func supportIPv6() bool { + _, err := Dialer.Dial("tcp6", net.JoinHostPort(controldIPv6Test, "80")) + return err == nil +} + +func supportListenIPv6Local() bool { + if ln, err := net.Listen("tcp6", "[::1]:0"); err == nil { + ln.Close() + return true + } + return false +} + func probeStack() { b := backoff.NewBackoff("probeStack", func(format string, args ...any) {}, time.Minute) for { @@ -50,20 +68,9 @@ func probeStack() { b.BackOff(context.Background(), err) } } - if _, err := Dialer.Dial("tcp4", net.JoinHostPort(controldIPv4Test, "80")); err == nil { - ipv4Enabled = true - } - if _, err := Dialer.Dial("tcp6", net.JoinHostPort(controldIPv6Test, "80")); err == nil { - ipv6Enabled = true - } - if ln, err := net.Listen("tcp6", "[::1]:53"); err == nil { - ln.Close() - canListenIPv6Local = true - } -} - -func Reset() { - stackOnce.Store(new(sync.Once)) + ipv4Enabled = supportIPv4() + ipv6Enabled = supportIPv6() + canListenIPv6Local = supportListenIPv6Local() } func Up() bool { @@ -86,6 +93,11 @@ func SupportsIPv6ListenLocal() bool { return canListenIPv6Local } +// IPv6Available is like SupportsIPv6, but always do the check without caching. +func IPv6Available() bool { + return supportIPv6() +} + // IsIPv6 checks if the provided IP is v6. // //lint:ignore U1000 use in os_windows.go From 85c95a6a3ade8f1cd59a41b52889a97329d7480e Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Mar 2023 10:27:15 +0700 Subject: [PATCH 30/37] all: set timeout for re-bootstrapping --- config.go | 43 +++++++++++++++++++++++++++++++++++-------- config_quic.go | 6 +++++- config_quic_free.go | 2 ++ internal/net/net.go | 10 +++++----- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/config.go b/config.go index 84e2f40..dfafd91 100644 --- a/config.go +++ b/config.go @@ -169,6 +169,7 @@ func (uc *UpstreamConfig) SetupBootstrapIP() { return "" } + Log(ctx, ProxyLog.Debug(), "Resolving %q using bootstrap DNS %q", uc.Domain, bootstrapDNS) // Find all A, AAAA records of the upstream. for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} { m := new(dns.Msg) @@ -202,7 +203,7 @@ func (uc *UpstreamConfig) SetupBootstrapIP() { uc.nextBootstrapIP.Add(1) // If this is an ipv6, and ipv6 is not available, don't use it as bootstrap ip. - if !ctrldnet.IPv6Available() && ctrldnet.IsIPv6(ip) { + if !ctrldnet.IPv6Available(ctx) && ctrldnet.IsIPv6(ip) { continue } uc.BootstrapIP = ip @@ -217,23 +218,41 @@ func (uc *UpstreamConfig) ReBootstrap() { _, _, _ = uc.g.Do("rebootstrap", func() (any, error) { ProxyLog.Debug().Msg("re-bootstrapping upstream ip") n := uint32(len(uc.bootstrapIPs)) + + timeoutMs := 1000 + if uc.Timeout < timeoutMs { + timeoutMs = uc.Timeout + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMs)*time.Millisecond) + defer cancel() + + hasIPv6 := ctrldnet.IPv6Available(ctx) // Only attempt n times, because if there's no usable ip, // the bootstrap ip will be kept as-is. for i := uint32(0); i < n; i++ { // Select the next ip in bootstrap ip list. next := uc.nextBootstrapIP.Add(1) ip := uc.bootstrapIPs[(next-1)%n] - if !ctrldnet.IPv6Available() && ctrldnet.IsIPv6(ip) { + if !hasIPv6 && ctrldnet.IsIPv6(ip) { continue } uc.BootstrapIP = ip break } - uc.SetupTransport() + uc.setupTransportWithoutPingUpstream() return true, nil }) } +func (uc *UpstreamConfig) setupTransportWithoutPingUpstream() { + switch uc.Type { + case ResolverTypeDOH: + uc.setupDOHTransportWithoutPingUpstream() + case ResolverTypeDOH3: + uc.setupDOH3TransportWithoutPingUpstream() + } +} + // SetupTransport initializes the network transport used to connect to upstream server. // For now, only DoH upstream is supported. func (uc *UpstreamConfig) SetupTransport() { @@ -246,14 +265,24 @@ func (uc *UpstreamConfig) SetupTransport() { } func (uc *UpstreamConfig) setupDOHTransport() { + uc.setupDOHTransportWithoutPingUpstream() + uc.pingUpstream() +} + +func (uc *UpstreamConfig) setupDOHTransportWithoutPingUpstream() { uc.transport = http.DefaultTransport.(*http.Transport).Clone() uc.transport.IdleConnTimeout = 5 * time.Second + + dialerTimeoutMs := 2000 + if uc.Timeout < dialerTimeoutMs { + dialerTimeoutMs = uc.Timeout + } + dialerTimeout := time.Duration(dialerTimeoutMs) * time.Millisecond uc.transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{ - Timeout: 2 * time.Second, - KeepAlive: 2 * time.Second, + Timeout: dialerTimeout, + KeepAlive: dialerTimeout, } - Log(ctx, ProxyLog.Debug(), "debug dial context %s - %s - %s", addr, network, bootstrapDNS) // if we have a bootstrap ip set, use it to avoid DNS lookup if uc.BootstrapIP != "" { if _, port, _ := net.SplitHostPort(addr); port != "" { @@ -263,8 +292,6 @@ func (uc *UpstreamConfig) setupDOHTransport() { Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr) return dialer.DialContext(ctx, network, addr) } - - uc.pingUpstream() } func (uc *UpstreamConfig) pingUpstream() { diff --git a/config_quic.go b/config_quic.go index 72ce351..253fc4e 100644 --- a/config_quic.go +++ b/config_quic.go @@ -12,6 +12,11 @@ import ( ) func (uc *UpstreamConfig) setupDOH3Transport() { + uc.setupDOH3TransportWithoutPingUpstream() + uc.pingUpstream() +} + +func (uc *UpstreamConfig) setupDOH3TransportWithoutPingUpstream() { rt := &http3.RoundTripper{} rt.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) { host := addr @@ -36,5 +41,4 @@ func (uc *UpstreamConfig) setupDOH3Transport() { } uc.http3RoundTripper = rt - uc.pingUpstream() } diff --git a/config_quic_free.go b/config_quic_free.go index 5f39bc6..3817e51 100644 --- a/config_quic_free.go +++ b/config_quic_free.go @@ -3,3 +3,5 @@ package ctrld func (uc *UpstreamConfig) setupDOH3Transport() {} + +func (uc *UpstreamConfig) setupDOH3TransportWithoutPingUpstream() {} diff --git a/internal/net/net.go b/internal/net/net.go index 6c78586..a155f2c 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -45,8 +45,8 @@ func supportIPv4() bool { return err == nil } -func supportIPv6() bool { - _, err := Dialer.Dial("tcp6", net.JoinHostPort(controldIPv6Test, "80")) +func supportIPv6(ctx context.Context) bool { + _, err := Dialer.DialContext(ctx, "tcp6", net.JoinHostPort(controldIPv6Test, "80")) return err == nil } @@ -69,7 +69,7 @@ func probeStack() { } } ipv4Enabled = supportIPv4() - ipv6Enabled = supportIPv6() + ipv6Enabled = supportIPv6(context.Background()) canListenIPv6Local = supportListenIPv6Local() } @@ -94,8 +94,8 @@ func SupportsIPv6ListenLocal() bool { } // IPv6Available is like SupportsIPv6, but always do the check without caching. -func IPv6Available() bool { - return supportIPv6() +func IPv6Available(ctx context.Context) bool { + return supportIPv6(ctx) } // IsIPv6 checks if the provided IP is v6. From d1589bd9d6be9de35bb45a3ac24abe0c3968c677 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Mar 2023 14:12:12 +0700 Subject: [PATCH 31/37] Use separate context when querying upstream ips While at it, also include query type in log, and only honor upstream timeout when it greater than zero. --- cmd/ctrld/dns_proxy.go | 5 +++-- config.go | 29 ++++++++++++++++++----------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index bbe84f1..808cba2 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -32,12 +32,13 @@ func (p *prog) serveUDP(listenerNum string) error { failoverRcodes = listenerConfig.Policy.FailoverRcodeNumbers } handler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { - domain := canonicalName(m.Question[0].Name) + q := m.Question[0] + domain := canonicalName(q.Name) reqId := requestID() fmtSrcToDest := fmtRemoteToLocal(listenerNum, w.RemoteAddr().String(), w.LocalAddr().String()) t := time.Now() ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqId) - ctrld.Log(ctx, mainLog.Debug(), "%s received query: %s", fmtSrcToDest, domain) + ctrld.Log(ctx, mainLog.Debug(), "%s received query: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], domain) upstreams, matched := p.upstreamFor(ctx, listenerNum, listenerConfig, w.RemoteAddr(), domain) var answer *dns.Msg if !matched && listenerConfig.Restricted { diff --git a/config.go b/config.go index dfafd91..2700efb 100644 --- a/config.go +++ b/config.go @@ -155,9 +155,6 @@ func (uc *UpstreamConfig) Init() { // SetupBootstrapIP manually find all available IPs of the upstream. // The first usable IP will be used as bootstrap IP of the upstream. func (uc *UpstreamConfig) SetupBootstrapIP() { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond) - defer cancel() - c := new(dns.Client) bootstrapIP := func(record dns.RR) string { switch ar := record.(type) { @@ -169,24 +166,25 @@ func (uc *UpstreamConfig) SetupBootstrapIP() { return "" } - Log(ctx, ProxyLog.Debug(), "Resolving %q using bootstrap DNS %q", uc.Domain, bootstrapDNS) - // Find all A, AAAA records of the upstream. - for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} { + ProxyLog.Debug().Msgf("Resolving %q using bootstrap DNS %q", uc.Domain, bootstrapDNS) + do := func(dnsType uint16) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond) + defer cancel() m := new(dns.Msg) m.SetQuestion(uc.Domain+".", dnsType) m.RecursionDesired = true r, _, err := c.ExchangeContext(ctx, m, net.JoinHostPort(bootstrapDNS, "53")) if err != nil { ProxyLog.Error().Err(err).Str("type", dns.TypeToString[dnsType]).Msgf("could not resolve domain %s for upstream", uc.Domain) - continue + return } if r.Rcode != dns.RcodeSuccess { ProxyLog.Error().Msgf("could not resolve domain return code: %d, upstream", r.Rcode) - continue + return } if len(r.Answer) == 0 { ProxyLog.Error().Msg("no answer from bootstrap DNS server") - continue + return } for _, a := range r.Answer { ip := bootstrapIP(a) @@ -210,17 +208,26 @@ func (uc *UpstreamConfig) SetupBootstrapIP() { } } } + // Find all A, AAAA records of the upstream. + for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} { + do(dnsType) + } ProxyLog.Debug().Msgf("Bootstrap IPs: %v", uc.bootstrapIPs) } // ReBootstrap re-setup the bootstrap IP and the transport. func (uc *UpstreamConfig) ReBootstrap() { + switch uc.Type { + case ResolverTypeDOH, ResolverTypeDOH3: + default: + return + } _, _, _ = uc.g.Do("rebootstrap", func() (any, error) { ProxyLog.Debug().Msg("re-bootstrapping upstream ip") n := uint32(len(uc.bootstrapIPs)) timeoutMs := 1000 - if uc.Timeout < timeoutMs { + if uc.Timeout > 0 && uc.Timeout < timeoutMs { timeoutMs = uc.Timeout } ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMs)*time.Millisecond) @@ -274,7 +281,7 @@ func (uc *UpstreamConfig) setupDOHTransportWithoutPingUpstream() { uc.transport.IdleConnTimeout = 5 * time.Second dialerTimeoutMs := 2000 - if uc.Timeout < dialerTimeoutMs { + if uc.Timeout > 0 && uc.Timeout < dialerTimeoutMs { dialerTimeoutMs = uc.Timeout } dialerTimeout := time.Duration(dialerTimeoutMs) * time.Millisecond From 4f6c2032a1166b04b86fcb4ee9546705de9da192 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Mar 2023 14:42:40 +0700 Subject: [PATCH 32/37] cmd/ctrld: log reason if first query failed --- cmd/ctrld/dns_proxy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 808cba2..bc52332 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -189,6 +189,7 @@ func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []i resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg { answer, err := resolve1(n, upstreamConfig, msg) if err != nil { + ctrld.Log(ctx, mainLog.Debug().Err(err), "could not resolve query on first attempt, retrying...") // If any error occurred, re-bootstrap transport/ip, retry the request. upstreamConfig.ReBootstrap() answer, err = resolve1(n, upstreamConfig, msg) From e6800fbc82deacc39d9aa69fb9194c40376b9499 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 9 Mar 2023 23:48:53 +0700 Subject: [PATCH 33/37] Query all possible nameservers for os resolver So we don't have to worry about network stack changes causes an upstream to be broken. Just send requests to all nameservers concurrently, and get the first success response. --- errors.go | 43 +++++++++++++++++++++++++++++++++++++++++++ resolver.go | 35 +++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 errors.go diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..8b47c6c --- /dev/null +++ b/errors.go @@ -0,0 +1,43 @@ +package ctrld + +// TODO(cuonglm): use stdlib once we bump minimum version to 1.20 + +func joinErrors(errs ...error) error { + n := 0 + for _, err := range errs { + if err != nil { + n++ + } + } + if n == 0 { + return nil + } + e := &joinError{ + errs: make([]error, 0, n), + } + for _, err := range errs { + if err != nil { + e.errs = append(e.errs, err) + } + } + return e +} + +type joinError struct { + errs []error +} + +func (e *joinError) Error() string { + var b []byte + for i, err := range e.errs { + if i > 0 { + b = append(b, '\n') + } + b = append(b, err.Error()...) + } + return string(b) +} + +func (e *joinError) Unwrap() []error { + return e.errs +} diff --git a/resolver.go b/resolver.go index 5c04f37..a12c700 100644 --- a/resolver.go +++ b/resolver.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net" - "sync/atomic" "github.com/miekg/dns" ) @@ -51,22 +50,42 @@ func NewResolver(uc *UpstreamConfig) (Resolver, error) { type osResolver struct { nameservers []string - next atomic.Uint32 +} + +type osResolverResult struct { + answer *dns.Msg + err error } // Resolve performs DNS resolvers using OS default nameservers. Nameserver is chosen from // available nameservers with a roundrobin algorithm. func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { - numServers := uint32(len(o.nameservers)) + numServers := len(o.nameservers) if numServers == 0 { return nil, errors.New("no nameservers available") } - next := o.next.Add(1) - server := o.nameservers[(next-1)%numServers] - dnsClient := &dns.Client{Net: "udp"} - answer, _, err := dnsClient.ExchangeContext(ctx, msg, server) + ctx, cancel := context.WithCancel(ctx) + defer cancel() - return answer, err + dnsClient := &dns.Client{Net: "udp"} + ch := make(chan *osResolverResult, numServers) + for _, server := range o.nameservers { + go func(server string) { + answer, _, err := dnsClient.ExchangeContext(ctx, msg, server) + ch <- &osResolverResult{answer: answer, err: err} + }(server) + } + + errs := make([]error, 0, numServers) + for res := range ch { + if res.err == nil { + cancel() + return res.answer, res.err + } + errs = append(errs, res.err) + } + + return nil, joinErrors(errs...) } func newDialer(dnsAddress string) *net.Dialer { From 14bc29751fb0a47dd2835159e341810ff283bef6 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 10 Mar 2023 09:08:59 +0700 Subject: [PATCH 34/37] Use both os and bootstrap DNS to resolve bootstrap IP --- config.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config.go b/config.go index 2700efb..76fa51c 100644 --- a/config.go +++ b/config.go @@ -155,7 +155,6 @@ func (uc *UpstreamConfig) Init() { // SetupBootstrapIP manually find all available IPs of the upstream. // The first usable IP will be used as bootstrap IP of the upstream. func (uc *UpstreamConfig) SetupBootstrapIP() { - c := new(dns.Client) bootstrapIP := func(record dns.RR) string { switch ar := record.(type) { case *dns.A: @@ -166,14 +165,17 @@ func (uc *UpstreamConfig) SetupBootstrapIP() { return "" } - ProxyLog.Debug().Msgf("Resolving %q using bootstrap DNS %q", uc.Domain, bootstrapDNS) + resolver := &osResolver{nameservers: nameservers()} + resolver.nameservers = append([]string{net.JoinHostPort(bootstrapDNS, "53")}, resolver.nameservers...) + ProxyLog.Debug().Msgf("Resolving %q using bootstrap DNS %q", uc.Domain, resolver.nameservers) do := func(dnsType uint16) { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond) defer cancel() m := new(dns.Msg) m.SetQuestion(uc.Domain+".", dnsType) m.RecursionDesired = true - r, _, err := c.ExchangeContext(ctx, m, net.JoinHostPort(bootstrapDNS, "53")) + + r, err := resolver.Resolve(ctx, m) if err != nil { ProxyLog.Error().Err(err).Str("type", dns.TypeToString[dnsType]).Msgf("could not resolve domain %s for upstream", uc.Domain) return From 0dfa377e085ba8c425042e8bba2cdc6f168d20a7 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Feb 2023 21:25:07 +0700 Subject: [PATCH 35/37] Add freebsd to goreleaser config While at it, fixed the hook upx script to run per file, and ignore binaries which are not supported. --- .goreleaser.yaml | 7 ++++++- scripts/upx.sh | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index de268bc..f010ff9 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -11,6 +11,7 @@ builds: - -s -w goos: - linux + - freebsd - windows goarch: - 386 @@ -26,7 +27,11 @@ builds: - softfloat main: ./cmd/ctrld hooks: - post: /bin/sh ./scripts/upx.sh + post: /bin/sh ./scripts/upx.sh {{ .Path }} + ignore: + - goos: freebsd + goarch: arm + goarm: 5 archives: - format_overrides: - goos: windows diff --git a/scripts/upx.sh b/scripts/upx.sh index 55c60e7..5d366eb 100755 --- a/scripts/upx.sh +++ b/scripts/upx.sh @@ -2,6 +2,22 @@ set -ex -for dist_dir in ./dist/ctrld*; do - upx --brute "${dist_dir}/ctrld" -done +binary=$1 + +if [ -z "$binary" ]; then + echo >&2 "Usage: $0 " + exit 1 +fi + +case "$binary" in + *_freebsd_*) + echo >&2 "upx does not work with freebsd binary yet" + exit 0 + ;; + *_windows_arm*) + echo >&2 "upx does not work with windows arm/arm64 binary yet" + exit 0 + ;; +esac + +upx -- "$binary" From 9a249c30297d8e2293880008f5482a05854f9796 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Feb 2023 21:55:32 +0700 Subject: [PATCH 36/37] .github/workflows: use go 1.20 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2902211..74f72a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: os: ["windows-latest", "ubuntu-latest", "macOS-latest"] - go: ["1.19.x"] + go: ["1.20.x"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -21,6 +21,6 @@ jobs: - run: "go test -race ./..." - uses: dominikh/staticcheck-action@v1.2.0 with: - version: "2022.1.1" + version: "2023.1.2" install-go: false cache-key: ${{ matrix.go }} From 7cd1f7adda71b671ea078aba35352e9395baf6f6 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Feb 2023 21:26:05 +0700 Subject: [PATCH 37/37] cmd/ctrld: bump version to v1.1.1 --- cmd/ctrld/cli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index f419813..e791e20 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -70,7 +70,7 @@ func initCLI() { rootCmd := &cobra.Command{ Use: "ctrld", Short: strings.TrimLeft(rootShortDesc, "\n"), - Version: "1.1.0", + Version: "1.1.1", } rootCmd.PersistentFlags().CountVarP( &verbose,