From 99f15698332730de73d273d6e810a4bf33d2bdf0 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 8 Oct 2020 13:04:47 +1000 Subject: [PATCH 1/4] dns lookup: Fork unbound process instead of using libunbound --- Makefile | 4 +- dns_util.go | 99 +++-------------------- go.mod | 1 - unbound.go | 222 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 234 insertions(+), 92 deletions(-) create mode 100644 unbound.go diff --git a/Makefile b/Makefile index 3fcbb33..bbfe921 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ test: server-dev: generate LETSDEBUG_WEB_DEBUG=1 \ LETSDEBUG_WEB_DB_DSN="user=letsdebug dbname=letsdebug password=password sslmode=disable" \ - LETSDEBUG_DEBUG=1 go \ - run -race cmd/server/server.go + LETSDEBUG_DEBUG=1 \ + go run -race cmd/server/server.go server-dev-db-up: docker run -d --name letsdebug-db -p 5432:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_USER=letsdebug postgres:10.3-alpine diff --git a/dns_util.go b/dns_util.go index 71577e8..0e1d7a8 100644 --- a/dns_util.go +++ b/dns_util.go @@ -1,58 +1,14 @@ package letsdebug import ( - "fmt" "net" "strings" - - "github.com/miekg/dns" - "github.com/miekg/unbound" ) var ( reservedNets []*net.IPNet ) -func lookup(name string, rrType uint16) ([]dns.RR, error) { - ub := unbound.New() - defer ub.Destroy() - - if err := setUnboundConfig(ub); err != nil { - return nil, fmt.Errorf("Failed to configure Unbound resolver: %v", err) - } - - result, err := ub.Resolve(name, rrType, dns.ClassINET) - if err != nil { - return nil, err - } - - if result.Bogus { - return nil, fmt.Errorf("DNS response for %s had fatal DNSSEC issues: %v", name, result.WhyBogus) - } - - if result.Rcode == dns.RcodeServerFailure || result.Rcode == dns.RcodeRefused { - return nil, fmt.Errorf("DNS response for %s/%s did not have an acceptable response code: %s", - name, dns.TypeToString[rrType], dns.RcodeToString[result.Rcode]) - } - - return result.Rr, nil -} - -func normalizeFqdn(name string) string { - name = strings.TrimSpace(name) - name = strings.TrimSuffix(name, ".") - return strings.ToLower(name) -} - -func isAddressReserved(ip net.IP) bool { - for _, reserved := range reservedNets { - if reserved.Contains(ip) { - return true - } - } - return false -} - func init() { reservedNets = []*net.IPNet{} reservedCIDRs := []string{ @@ -75,52 +31,17 @@ func init() { } } -func setUnboundConfig(ub *unbound.Unbound) error { - // options need the : in the option key according to docs - opts := []struct { - Opt string - Val string - }{ - {"verbosity:", "0"}, - {"use-syslog:", "no"}, - {"do-ip4:", "yes"}, - {"do-ip6:", "yes"}, - {"do-udp:", "yes"}, - {"do-tcp:", "yes"}, - {"tcp-upstream:", "no"}, - {"harden-glue:", "yes"}, - {"harden-dnssec-stripped:", "yes"}, - {"cache-min-ttl:", "0"}, - {"cache-max-ttl:", "0"}, - {"cache-max-negative-ttl:", "0"}, - {"neg-cache-size:", "0"}, - {"prefetch:", "no"}, - {"unwanted-reply-threshold:", "10000"}, - {"do-not-query-localhost:", "yes"}, - {"val-clean-additional:", "yes"}, - {"harden-algo-downgrade:", "yes"}, - {"edns-buffer-size:", "512"}, - {"val-sig-skew-min:", "0"}, - {"val-sig-skew-max:", "0"}, - } - - for _, opt := range opts { - // Can't ignore these because we cant silently have policies being ignored - if err := ub.SetOption(opt.Opt, opt.Val); err != nil { - return fmt.Errorf("Failed to configure unbound with option %s %v", opt.Opt, err) - } - } +func normalizeFqdn(name string) string { + name = strings.TrimSpace(name) + name = strings.TrimSuffix(name, ".") + return strings.ToLower(name) +} - // use-caps-for-id was bugged (no colon) < 1.7.1, try both ways in order to be compatible - // https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=4092 - if err := ub.SetOption("use-caps-for-id:", "yes"); err != nil { - if err = ub.SetOption("use-caps-for-id", "yes"); err != nil { - return fmt.Errorf("Failed to configure unbound with use-caps-for-id: %v", err) +func isAddressReserved(ip net.IP) bool { + for _, reserved := range reservedNets { + if reserved.Contains(ip) { + return true } } - - return ub.AddTa(`. 172800 IN DNSKEY 257 3 8 AwEAAaz/tAm8yTn4Mfeh5eyI96WSVexTBAvkMgJzkKTOiW1vkIbzxeF3+/4RgWOq7HrxRixHlFlExOLAJr5emLvN7SWXgnLh4+B5xQlNVz8Og8kvArMtNROxVQuCaSnIDdD5LKyWbRd2n9WGe2R8PzgCmr3EgVLrjyBxWezF0jLHwVN8efS3rCj/EWgvIWgb9tarpVUDK/b58Da+sqqls3eNbuv7pr+eoZG+SrDK6nWeL3c6H5Apxz7LjVc1uTIdsIXxuOLYA4/ilBmSVIzuDWfdRUfhHdY6+cn8HFRm+2hM8AnXGXws9555KrUB5qihylGa8subX2Nn6UwNR1AkUTV74bU= - . 172800 IN DNSKEY 256 3 8 AwEAAdp440E6Mz7c+Vl4sPd0lTv2Qnc85dTW64j0RDD7sS/zwxWDJ3QRES2VKDO0OXLMqVJSs2YCCSDKuZXpDPuf++YfAu0j7lzYYdWTGwyNZhEaXtMQJIKYB96pW6cRkiG2Dn8S2vvo/PxW9PKQsyLbtd8PcwWglHgReBVp7kEv/Dd+3b3YMukt4jnWgDUddAySg558Zld+c9eGWkgWoOiuhg4rQRkFstMX1pRyOSHcZuH38o1WcsT4y3eT0U/SR6TOSLIB/8Ftirux/h297oS7tCcwSPt0wwry5OFNTlfMo8v7WGurogfk8hPipf7TTKHIi20LWen5RCsvYsQBkYGpF78= - . 172800 IN DNSKEY 257 3 8 AwEAAagAIKlVZrpC6Ia7gEzahOR+9W29euxhJhVVLOyQbSEW0O8gcCjFFVQUTf6v58fLjwBd0YI0EzrAcQqBGCzh/RStIoO8g0NfnfL2MTJRkxoXbfDaUeVPQuYEhg37NZWAJQ9VnMVDxP/VHL496M/QZxkjf5/Efucp2gaDX6RS6CXpoY68LsvPVjR0ZSwzz1apAzvN9dlzEheX7ICJBBtuA6G3LQpzW5hOA2hzCTMjJPJ8LbqF6dsV6DoBQzgul0sGIcGOYl7OyQdXfZ57relSQageu+ipAdTTJ25AsRTAoub8ONGcLmqrAmRLKBP1dfwhYB4N7knNnulqQxA+Uk1ihz0= - . 172800 IN RRSIG DNSKEY 8 0 172800 20181101000000 20181011000000 20326 . M/LTswhCjuJUTvX1CFqC+TiJ4Fez7AROa5mM+1AI2MJ+zLHhr3JaMxyydFLWrBHR0056Hz7hNqQ9i63hGeiR6uMfanF0jIRb9XqgGP8nY37T8ESpS1UiM9rJn4b40RFqDSEvuFdd4hGwK3EX0snOCLdUT8JezxtreXI0RilmqDC2g44TAKyFw+Is9Qwl+k6+fbMQ/atA8adANbYgyuHfiwQCCUtXRaTCpRgQtsAz9izO0VYIGeHIoJta0demAIrLCOHNVH2ogHTqMEQ18VqUNzTd0aGURACBdS7PeP2KogPD7N8Q970O84TFmO4ahPIvqO+milCn5OQTbbgsjHqY6Q==`) + return false } diff --git a/go.mod b/go.mod index db8f0c5..fffb348 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/juju/ratelimit v1.0.1 github.com/lib/pq v1.8.0 github.com/miekg/dns v1.1.31 - github.com/miekg/unbound v0.0.0-20180419064740-e2b53b2dbcba github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/unbound.go b/unbound.go new file mode 100644 index 0000000..9444e87 --- /dev/null +++ b/unbound.go @@ -0,0 +1,222 @@ +package letsdebug + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/miekg/dns" +) + +const ( + rootKeyContents = `. 172800 IN DNSKEY 257 3 8 AwEAAaz/tAm8yTn4Mfeh5eyI96WSVexTBAvkMgJzkKTOiW1vkIbzxeF3+/4RgWOq7HrxRixHlFlExOLAJr5emLvN7SWXgnLh4+B5xQlNVz8Og8kvArMtNROxVQuCaSnIDdD5LKyWbRd2n9WGe2R8PzgCmr3EgVLrjyBxWezF0jLHwVN8efS3rCj/EWgvIWgb9tarpVUDK/b58Da+sqqls3eNbuv7pr+eoZG+SrDK6nWeL3c6H5Apxz7LjVc1uTIdsIXxuOLYA4/ilBmSVIzuDWfdRUfhHdY6+cn8HFRm+2hM8AnXGXws9555KrUB5qihylGa8subX2Nn6UwNR1AkUTV74bU= + . 172800 IN DNSKEY 256 3 8 AwEAAdp440E6Mz7c+Vl4sPd0lTv2Qnc85dTW64j0RDD7sS/zwxWDJ3QRES2VKDO0OXLMqVJSs2YCCSDKuZXpDPuf++YfAu0j7lzYYdWTGwyNZhEaXtMQJIKYB96pW6cRkiG2Dn8S2vvo/PxW9PKQsyLbtd8PcwWglHgReBVp7kEv/Dd+3b3YMukt4jnWgDUddAySg558Zld+c9eGWkgWoOiuhg4rQRkFstMX1pRyOSHcZuH38o1WcsT4y3eT0U/SR6TOSLIB/8Ftirux/h297oS7tCcwSPt0wwry5OFNTlfMo8v7WGurogfk8hPipf7TTKHIi20LWen5RCsvYsQBkYGpF78= + . 172800 IN DNSKEY 257 3 8 AwEAAagAIKlVZrpC6Ia7gEzahOR+9W29euxhJhVVLOyQbSEW0O8gcCjFFVQUTf6v58fLjwBd0YI0EzrAcQqBGCzh/RStIoO8g0NfnfL2MTJRkxoXbfDaUeVPQuYEhg37NZWAJQ9VnMVDxP/VHL496M/QZxkjf5/Efucp2gaDX6RS6CXpoY68LsvPVjR0ZSwzz1apAzvN9dlzEheX7ICJBBtuA6G3LQpzW5hOA2hzCTMjJPJ8LbqF6dsV6DoBQzgul0sGIcGOYl7OyQdXfZ57relSQageu+ipAdTTJ25AsRTAoub8ONGcLmqrAmRLKBP1dfwhYB4N7knNnulqQxA+Uk1ihz0= + . 172800 IN RRSIG DNSKEY 8 0 172800 20181101000000 20181011000000 20326 . M/LTswhCjuJUTvX1CFqC+TiJ4Fez7AROa5mM+1AI2MJ+zLHhr3JaMxyydFLWrBHR0056Hz7hNqQ9i63hGeiR6uMfanF0jIRb9XqgGP8nY37T8ESpS1UiM9rJn4b40RFqDSEvuFdd4hGwK3EX0snOCLdUT8JezxtreXI0RilmqDC2g44TAKyFw+Is9Qwl+k6+fbMQ/atA8adANbYgyuHfiwQCCUtXRaTCpRgQtsAz9izO0VYIGeHIoJta0demAIrLCOHNVH2ogHTqMEQ18VqUNzTd0aGURACBdS7PeP2KogPD7N8Q970O84TFmO4ahPIvqO+milCn5OQTbbgsjHqY6Q==` + + unboundConfContents = `server: + edns-buffer-size: 512 + directory: "." + auto-trust-anchor-file: "%s" + pidfile: "" + logfile: "" + chroot: "" + username: "" + log-replies: yes + log-queries: yes + num-threads: 1 + so-reuseport: yes + verbosity: 2 + use-syslog: no + log-time-ascii: yes + do-ip4: yes + do-ip6: yes + do-udp: yes + do-tcp: yes + tcp-upstream: no + port: %d + private-address: 192.168.0.0/16 + private-address: 172.16.0.0/12 + private-address: 10.0.0.0/8 + private-address: 169.254.0.0/16 + private-address: fd00::/8 + private-address: fe80::/10 + hide-identity: yes + hide-version: yes + harden-glue: yes + harden-dnssec-stripped: yes + use-caps-for-id: yes + cache-min-ttl: 0 + cache-max-ttl: 0 + cache-max-negative-ttl: 0 + neg-cache-size: 0 + prefetch: no + unwanted-reply-threshold: 10000 + do-not-query-localhost: yes + val-clean-additional: yes + val-sig-skew-max: 0 + val-sig-skew-min: 0 + ipsecmod-enabled: no` + + portMin = 20000 + portMax = 25000 +) + +var ( + portChan chan int + configPath string +) + +func init() { + portChan = make(chan int) + go func() { + for { + for i := portMin; i <= portMax; i++ { + portChan <- i + } + } + }() + + var err error + configPath, err = os.UserConfigDir() + if err != nil { + configPath, err = os.UserCacheDir() + if err != nil { + configPath = os.TempDir() + } + } + if configPath == "" { + log.Fatal("unable to find directory for unbound files") + } + + configPath = filepath.Join(configPath, "letsdebug") + + _ = os.Mkdir(configPath, 0755) + + rootKeyFile := filepath.Join(configPath, "root.key") + if !fileExists(rootKeyFile) { + debug("Writing unbound root.key ta to: %s\n", rootKeyFile) + if err := ioutil.WriteFile(rootKeyFile, []byte(rootKeyContents), 0644); err != nil { + log.Fatalf("error writing root key file %q: %v", rootKeyFile, err) + } + } + + printedConfOutput := false + for i := portMin; i <= portMax; i++ { + unboundConfFile := filepath.Join(configPath, fmt.Sprintf("unbound%d.conf", i)) + if !fileExists(unboundConfFile) { + if !printedConfOutput { + printedConfOutput = true + debug("Writing unbound config to: %s\n", unboundConfFile) + } + if err := ioutil.WriteFile(unboundConfFile, []byte(fmt.Sprintf(unboundConfContents, rootKeyFile, i)), 0644); err != nil { + log.Fatalf("error writing conf file %q: %v", unboundConfFile, err) + } + } + } +} + +func fileExists(filename string) bool { + _, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return true +} + +func lookup(name string, rrType uint16) ([]dns.RR, error) { + if !strings.HasSuffix(name, ".") { + name += "." + } + + port := <-portChan + unboundConfFile := filepath.Join(configPath, fmt.Sprintf("unbound%d.conf", port)) + + path := os.Getenv("LETSDEBUG_UNBOUND_PATH") + if path == "" { + path = "unbound" + } else { + path = filepath.Join(path, "unbound") + } + + // TODO: pass through a parent context for this? + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer func() { + cancel() + }() + + cmd := exec.CommandContext(ctx, path, "-p", "-d", "-c", unboundConfFile) + errPipe, _ := cmd.StderrPipe() + + // start the unbound process + debug("[unbound-%d] Starting unbound: %s\n", port, strings.Join(cmd.Args, " ")) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("error starting unbound: %w", err) + } + defer func() { + debug("[unbound-%d] unbound closing\n", port) + cmd.Process.Kill() + cmd.Wait() + debug("[unbound-%d] unbound closed\n", port) + }() + + // listen for the start of service output + readyChan := make(chan bool) + go func() { + scanner := bufio.NewScanner(errPipe) + for scanner.Scan() { + if strings.Contains(scanner.Text(), "start of service") { + readyChan <- true + } + debug("[unbound-%d] output: %s\n", port, scanner.Text()) + } + }() + + // wait for unbound + select { + case <-time.After(1 * time.Second): + break + case <-readyChan: + break + } + + // spin off a go func to exchange a dns request + type dnsResult struct { + r *dns.Msg + err error + } + dnsChan := make(chan dnsResult) + go func() { + c := new(dns.Client) + c.Timeout = time.Second * 30 + m := new(dns.Msg) + m.SetQuestion(name, rrType) + r, _, err := c.Exchange(m, fmt.Sprintf("127.0.0.1:%d", port)) + dnsChan <- dnsResult{r, err} + }() + + // wait for either the dns response or the timeout context to finish + select { + case result := <-dnsChan: + if result.err != nil { + return nil, result.err + } + + if result.r.Rcode == dns.RcodeServerFailure || result.r.Rcode == dns.RcodeRefused { + return nil, fmt.Errorf("DNS response for %s/%s did not have an acceptable response code: %s", + name, dns.TypeToString[rrType], dns.RcodeToString[result.r.Rcode]) + } + + return result.r.Answer, nil + + case <-ctx.Done(): + return nil, ctx.Err() + } +} From 9fb5b718c141a275d129bda99f35793c7804bfe4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 8 Oct 2020 18:43:03 +1000 Subject: [PATCH 2/4] Include unbound logs in debug info --- context.go | 14 ++++++++------ dns01.go | 4 ++-- generic.go | 2 +- http01.go | 16 ++++++++++++---- problem.go | 9 +++++++++ unbound.go | 42 +++++++++++++++++++++++++++++------------- 6 files changed, 61 insertions(+), 26 deletions(-) diff --git a/context.go b/context.go index 48e8725..b270698 100644 --- a/context.go +++ b/context.go @@ -11,6 +11,7 @@ import ( type lookupResult struct { RRs []dns.RR + Logs string Error error } @@ -29,7 +30,7 @@ func newScanContext() *scanContext { } } -func (sc *scanContext) Lookup(name string, rrType uint16) ([]dns.RR, error) { +func (sc *scanContext) Lookup(name string, rrType uint16) ([]dns.RR, string, error) { sc.rrsMutex.Lock() rrMap, ok := sc.rrs[name] if !ok { @@ -40,24 +41,25 @@ func (sc *scanContext) Lookup(name string, rrType uint16) ([]dns.RR, error) { sc.rrsMutex.Unlock() if ok { - return result.RRs, result.Error + return result.RRs, result.Logs, result.Error } - resolved, err := lookup(name, rrType) + resolved, logs, err := lookup(name, rrType) sc.rrsMutex.Lock() rrMap[rrType] = lookupResult{ RRs: resolved, + Logs: logs, Error: err, } sc.rrsMutex.Unlock() - return resolved, err + return resolved, logs, err } // Only slightly random - it will use AAAA over A if possible. func (sc *scanContext) LookupRandomHTTPRecord(name string) (net.IP, error) { - v6RRs, err := sc.Lookup(name, dns.TypeAAAA) + v6RRs, _, err := sc.Lookup(name, dns.TypeAAAA) if err != nil { return net.IP{}, err } @@ -67,7 +69,7 @@ func (sc *scanContext) LookupRandomHTTPRecord(name string) (net.IP, error) { } } - v4RRs, err := sc.Lookup(name, dns.TypeA) + v4RRs, _, err := sc.Lookup(name, dns.TypeA) if err != nil { return net.IP{}, err } diff --git a/dns01.go b/dns01.go index 82deb78..e03a039 100644 --- a/dns01.go +++ b/dns01.go @@ -47,7 +47,7 @@ func (c txtRecordChecker) Check(ctx *scanContext, domain string, method Validati domain = domain[2:] } - if _, err := ctx.Lookup("_acme-challenge."+domain, dns.TypeTXT); err != nil { + if _, _, err := ctx.Lookup("_acme-challenge."+domain, dns.TypeTXT); err != nil { // report this problem as a fatal problem as that is the purpose of this checker return []Problem{txtRecordError(domain, err)}, nil } @@ -93,7 +93,7 @@ func (c txtDoubledLabelChecker) Check(ctx *scanContext, domain string, method Va doQuery := func(q string) ([]string, string) { found := []string{} combined := []string{} - rrs, _ := ctx.Lookup(q, dns.TypeTXT) + rrs, _, _ := ctx.Lookup(q, dns.TypeTXT) for _, rr := range rrs { txt, ok := rr.(*dns.TXT) if !ok { diff --git a/generic.go b/generic.go index 9ab2e4b..61c71cf 100644 --- a/generic.go +++ b/generic.go @@ -114,7 +114,7 @@ func (c caaChecker) Check(ctx *scanContext, domain string, method ValidationMeth domain = domain[2:] } - rrs, err := ctx.Lookup(domain, dns.TypeCAA) + rrs, _, err := ctx.Lookup(domain, dns.TypeCAA) if err != nil { probs = append(probs, dnsLookupFailed(domain, "CAA", err)) return probs, nil diff --git a/http01.go b/http01.go index 01c65c0..984659d 100644 --- a/http01.go +++ b/http01.go @@ -35,6 +35,7 @@ func (c dnsAChecker) Check(ctx *scanContext, domain string, method ValidationMet var probs []Problem var aRRs, aaaaRRs []dns.RR + var aLogs, aaaLogs string var aErr, aaaaErr error var wg sync.WaitGroup @@ -42,12 +43,12 @@ func (c dnsAChecker) Check(ctx *scanContext, domain string, method ValidationMet go func() { defer wg.Done() - aaaaRRs, aaaaErr = ctx.Lookup(domain, dns.TypeAAAA) + aaaaRRs, aaaLogs, aaaaErr = ctx.Lookup(domain, dns.TypeAAAA) }() go func() { defer wg.Done() - aRRs, aErr = ctx.Lookup(domain, dns.TypeA) + aRRs, aLogs, aErr = ctx.Lookup(domain, dns.TypeA) }() wg.Wait() @@ -59,6 +60,13 @@ func (c dnsAChecker) Check(ctx *scanContext, domain string, method ValidationMet probs = append(probs, dnsLookupFailed(domain, "AAAA", aaaaErr)) } + if aLogs != "" { + probs = append(probs, unboundLogs("A record unbound logs for: "+domain, aLogs)) + } + if aaaLogs != "" { + probs = append(probs, unboundLogs("AAAA record unbound logs for: "+domain, aaaLogs)) + } + for _, rr := range aRRs { if aRR, ok := rr.(*dns.A); ok && isAddressReserved(aRR.A) { probs = append(probs, reservedAddress(domain, aRR.A.String())) @@ -101,7 +109,7 @@ func (c httpAccessibilityChecker) Check(ctx *scanContext, domain string, method var ips []net.IP - rrs, _ := ctx.Lookup(domain, dns.TypeAAAA) + rrs, _, _ := ctx.Lookup(domain, dns.TypeAAAA) for _, rr := range rrs { aaaa, ok := rr.(*dns.AAAA) if !ok { @@ -109,7 +117,7 @@ func (c httpAccessibilityChecker) Check(ctx *scanContext, domain string, method } ips = append(ips, aaaa.AAAA) } - rrs, _ = ctx.Lookup(domain, dns.TypeA) + rrs, _, _ = ctx.Lookup(domain, dns.TypeA) for _, rr := range rrs { a, ok := rr.(*dns.A) if !ok { diff --git a/problem.go b/problem.go index 773cd99..573e2f9 100644 --- a/problem.go +++ b/problem.go @@ -73,3 +73,12 @@ func debugProblem(name, message, detail string) Problem { Severity: SeverityDebug, } } + +func unboundLogs(message, detail string) Problem { + return Problem{ + Name: "UnboundLogs", + Explanation: message, + Detail: detail, + Severity: SeverityDebug, + } +} diff --git a/unbound.go b/unbound.go index 9444e87..688722e 100644 --- a/unbound.go +++ b/unbound.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "io" "io/ioutil" "log" "os" @@ -33,7 +34,7 @@ const ( log-queries: yes num-threads: 1 so-reuseport: yes - verbosity: 2 + verbosity: 9 use-syslog: no log-time-ascii: yes do-ip4: yes @@ -131,14 +132,29 @@ func fileExists(filename string) bool { return true } -func lookup(name string, rrType uint16) ([]dns.RR, error) { +func lookup(name string, rrType uint16) ([]dns.RR, string, error) { + port := <-portChan + + // TODO: pass through a parent context for this? + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer func() { + cancel() + }() + + var b strings.Builder + rr, err := doQuery(ctx, name, rrType, port, &b) + if err != nil { + return nil, b.String(), err + } + return rr, b.String(), err +} + +func doQuery(ctx context.Context, name string, rrType uint16, port int, w io.Writer) ([]dns.RR, error) { if !strings.HasSuffix(name, ".") { name += "." } - port := <-portChan unboundConfFile := filepath.Join(configPath, fmt.Sprintf("unbound%d.conf", port)) - path := os.Getenv("LETSDEBUG_UNBOUND_PATH") if path == "" { path = "unbound" @@ -146,14 +162,11 @@ func lookup(name string, rrType uint16) ([]dns.RR, error) { path = filepath.Join(path, "unbound") } - // TODO: pass through a parent context for this? - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer func() { - cancel() - }() - cmd := exec.CommandContext(ctx, path, "-p", "-d", "-c", unboundConfFile) - errPipe, _ := cmd.StderrPipe() + errPipe, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("unable to setup stderrpipe for unbound: %v", err) + } // start the unbound process debug("[unbound-%d] Starting unbound: %s\n", port, strings.Join(cmd.Args, " ")) @@ -162,7 +175,7 @@ func lookup(name string, rrType uint16) ([]dns.RR, error) { } defer func() { debug("[unbound-%d] unbound closing\n", port) - cmd.Process.Kill() + cmd.Process.Signal(os.Interrupt) cmd.Wait() debug("[unbound-%d] unbound closed\n", port) }() @@ -171,12 +184,15 @@ func lookup(name string, rrType uint16) ([]dns.RR, error) { readyChan := make(chan bool) go func() { scanner := bufio.NewScanner(errPipe) + i := 0 for scanner.Scan() { + i++ + fmt.Fprintln(w, scanner.Text()) if strings.Contains(scanner.Text(), "start of service") { readyChan <- true } - debug("[unbound-%d] output: %s\n", port, scanner.Text()) } + debug("[unbound-%d] %d lines of logs", port, i) }() // wait for unbound From f84f7896168100b1638e6e24ab186eb1f2dc8707 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 8 Oct 2020 18:45:15 +1000 Subject: [PATCH 3/4] Missed newline --- unbound.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unbound.go b/unbound.go index 688722e..c4b17d0 100644 --- a/unbound.go +++ b/unbound.go @@ -192,7 +192,7 @@ func doQuery(ctx context.Context, name string, rrType uint16, port int, w io.Wri readyChan <- true } } - debug("[unbound-%d] %d lines of logs", port, i) + debug("[unbound-%d] %d lines of logs\n", port, i) }() // wait for unbound From c321310df9a06bd0013d95758abafc8f32101daa Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 9 Oct 2020 11:17:27 +1000 Subject: [PATCH 4/4] unbound: store logs in local files for 7 days --- context.go | 26 +++++++------ dns01.go | 4 +- generic.go | 2 +- http01.go | 19 ++++----- unbound.go | 64 +++++++++++++++++++++++++++---- web/templates/layouts/results.tpl | 5 +++ web/web.go | 56 +++++++++++++++++++++++++++ 7 files changed, 144 insertions(+), 32 deletions(-) diff --git a/context.go b/context.go index b270698..3354806 100644 --- a/context.go +++ b/context.go @@ -10,9 +10,10 @@ import ( ) type lookupResult struct { - RRs []dns.RR - Logs string - Error error + RRs []dns.RR + Hash string + LogsFilePath string + Error error } type scanContext struct { @@ -30,7 +31,7 @@ func newScanContext() *scanContext { } } -func (sc *scanContext) Lookup(name string, rrType uint16) ([]dns.RR, string, error) { +func (sc *scanContext) Lookup(name string, rrType uint16, writeLogs bool) ([]dns.RR, string, string, error) { sc.rrsMutex.Lock() rrMap, ok := sc.rrs[name] if !ok { @@ -41,25 +42,26 @@ func (sc *scanContext) Lookup(name string, rrType uint16) ([]dns.RR, string, err sc.rrsMutex.Unlock() if ok { - return result.RRs, result.Logs, result.Error + return result.RRs, result.Hash, result.LogsFilePath, result.Error } - resolved, logs, err := lookup(name, rrType) + resolved, hash, logFilePath, err := lookup(name, rrType, writeLogs) sc.rrsMutex.Lock() rrMap[rrType] = lookupResult{ - RRs: resolved, - Logs: logs, - Error: err, + RRs: resolved, + Hash: hash, + LogsFilePath: logFilePath, + Error: err, } sc.rrsMutex.Unlock() - return resolved, logs, err + return resolved, hash, logFilePath, err } // Only slightly random - it will use AAAA over A if possible. func (sc *scanContext) LookupRandomHTTPRecord(name string) (net.IP, error) { - v6RRs, _, err := sc.Lookup(name, dns.TypeAAAA) + v6RRs, _, _, err := sc.Lookup(name, dns.TypeAAAA, false) if err != nil { return net.IP{}, err } @@ -69,7 +71,7 @@ func (sc *scanContext) LookupRandomHTTPRecord(name string) (net.IP, error) { } } - v4RRs, _, err := sc.Lookup(name, dns.TypeA) + v4RRs, _, _, err := sc.Lookup(name, dns.TypeA, false) if err != nil { return net.IP{}, err } diff --git a/dns01.go b/dns01.go index e03a039..fa148ce 100644 --- a/dns01.go +++ b/dns01.go @@ -47,7 +47,7 @@ func (c txtRecordChecker) Check(ctx *scanContext, domain string, method Validati domain = domain[2:] } - if _, _, err := ctx.Lookup("_acme-challenge."+domain, dns.TypeTXT); err != nil { + if _, _, _, err := ctx.Lookup("_acme-challenge."+domain, dns.TypeTXT, false); err != nil { // report this problem as a fatal problem as that is the purpose of this checker return []Problem{txtRecordError(domain, err)}, nil } @@ -93,7 +93,7 @@ func (c txtDoubledLabelChecker) Check(ctx *scanContext, domain string, method Va doQuery := func(q string) ([]string, string) { found := []string{} combined := []string{} - rrs, _, _ := ctx.Lookup(q, dns.TypeTXT) + rrs, _, _, _ := ctx.Lookup(q, dns.TypeTXT, false) for _, rr := range rrs { txt, ok := rr.(*dns.TXT) if !ok { diff --git a/generic.go b/generic.go index 61c71cf..e80cb62 100644 --- a/generic.go +++ b/generic.go @@ -114,7 +114,7 @@ func (c caaChecker) Check(ctx *scanContext, domain string, method ValidationMeth domain = domain[2:] } - rrs, _, err := ctx.Lookup(domain, dns.TypeCAA) + rrs, _, _, err := ctx.Lookup(domain, dns.TypeCAA, false) if err != nil { probs = append(probs, dnsLookupFailed(domain, "CAA", err)) return probs, nil diff --git a/http01.go b/http01.go index 984659d..8ea27ce 100644 --- a/http01.go +++ b/http01.go @@ -35,7 +35,8 @@ func (c dnsAChecker) Check(ctx *scanContext, domain string, method ValidationMet var probs []Problem var aRRs, aaaaRRs []dns.RR - var aLogs, aaaLogs string + var aHash, aaaaHash string + var aLogPath, aaaaLogPath string var aErr, aaaaErr error var wg sync.WaitGroup @@ -43,12 +44,12 @@ func (c dnsAChecker) Check(ctx *scanContext, domain string, method ValidationMet go func() { defer wg.Done() - aaaaRRs, aaaLogs, aaaaErr = ctx.Lookup(domain, dns.TypeAAAA) + aaaaRRs, aaaaHash, aaaaLogPath, aaaaErr = ctx.Lookup(domain, dns.TypeAAAA, true) }() go func() { defer wg.Done() - aRRs, aLogs, aErr = ctx.Lookup(domain, dns.TypeA) + aRRs, aHash, aLogPath, aErr = ctx.Lookup(domain, dns.TypeA, true) }() wg.Wait() @@ -60,11 +61,11 @@ func (c dnsAChecker) Check(ctx *scanContext, domain string, method ValidationMet probs = append(probs, dnsLookupFailed(domain, "AAAA", aaaaErr)) } - if aLogs != "" { - probs = append(probs, unboundLogs("A record unbound logs for: "+domain, aLogs)) + if aLogPath != "" { + probs = append(probs, unboundLogs("A record unbound logs for: "+domain, aHash+"\n"+aLogPath)) } - if aaaLogs != "" { - probs = append(probs, unboundLogs("AAAA record unbound logs for: "+domain, aaaLogs)) + if aaaaLogPath != "" { + probs = append(probs, unboundLogs("AAAA record unbound logs for: "+domain, aHash+"\n"+aLogPath)) } for _, rr := range aRRs { @@ -109,7 +110,7 @@ func (c httpAccessibilityChecker) Check(ctx *scanContext, domain string, method var ips []net.IP - rrs, _, _ := ctx.Lookup(domain, dns.TypeAAAA) + rrs, _, _, _ := ctx.Lookup(domain, dns.TypeAAAA, false) for _, rr := range rrs { aaaa, ok := rr.(*dns.AAAA) if !ok { @@ -117,7 +118,7 @@ func (c httpAccessibilityChecker) Check(ctx *scanContext, domain string, method } ips = append(ips, aaaa.AAAA) } - rrs, _, _ = ctx.Lookup(domain, dns.TypeA) + rrs, _, _, _ = ctx.Lookup(domain, dns.TypeA, false) for _, rr := range rrs { a, ok := rr.(*dns.A) if !ok { diff --git a/unbound.go b/unbound.go index c4b17d0..c61f09c 100644 --- a/unbound.go +++ b/unbound.go @@ -2,7 +2,9 @@ package letsdebug import ( "bufio" + "bytes" "context" + "crypto/sha256" "fmt" "io" "io/ioutil" @@ -73,6 +75,7 @@ const ( var ( portChan chan int configPath string + logsPath string ) func init() { @@ -98,8 +101,10 @@ func init() { } configPath = filepath.Join(configPath, "letsdebug") + logsPath = filepath.Join(configPath, "logs") _ = os.Mkdir(configPath, 0755) + _ = os.Mkdir(logsPath, 0755) rootKeyFile := filepath.Join(configPath, "root.key") if !fileExists(rootKeyFile) { @@ -122,6 +127,34 @@ func init() { } } } + + go deleteOldLogs() +} + +func deleteOldLogs() { + for { + oldTime := time.Now().AddDate(0, 0, -7) + + matches, _ := filepath.Glob(filepath.Join(logsPath, "*.txt")) + for _, match := range matches { + stat, err := os.Stat(match) + if err != nil { + continue + } + // only easy cross platform time, should be good enough + if stat.ModTime().Before(oldTime) { + err := os.Remove(match) + if err != nil { + debug("[unbound] error removing old log file: %s\n", match) + } else { + debug("[unbound] removed old log file: %s\n", match) + } + } + } + + // run once a day + <-time.After(24 * time.Hour) + } } func fileExists(filename string) bool { @@ -132,21 +165,36 @@ func fileExists(filename string) bool { return true } -func lookup(name string, rrType uint16) ([]dns.RR, string, error) { +func lookup(name string, rrType uint16, writeLogs bool) (rr []dns.RR, hash string, logFilePath string, err error) { port := <-portChan // TODO: pass through a parent context for this? ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer func() { - cancel() - }() + defer cancel() + + if !writeLogs { + rr, err = doQuery(ctx, name, rrType, port, ioutil.Discard) + } else { + var b bytes.Buffer + rr, err = doQuery(ctx, name, rrType, port, &b) + + sum := sha256.Sum256(b.Bytes()) + hash = fmt.Sprintf("%x", sum) + + logFilePath = filepath.Join(logsPath, hash+".txt") + errLogs := ioutil.WriteFile(logFilePath, b.Bytes(), 0644) + if errLogs != nil { + // TODO: pass through this error to front end? + debug("[unbound-%d] error writing logs file %q: %s\n", port, logFilePath, errLogs) + hash = "" + logFilePath = "" + } + } - var b strings.Builder - rr, err := doQuery(ctx, name, rrType, port, &b) if err != nil { - return nil, b.String(), err + return nil, hash, logFilePath, err } - return rr, b.String(), err + return rr, hash, logFilePath, err } func doQuery(ctx context.Context, name string, rrType uint16, port int, w io.Writer) ([]dns.RR, error) { diff --git a/web/templates/layouts/results.tpl b/web/templates/layouts/results.tpl index b4a7af8..f3c4425 100644 --- a/web/templates/layouts/results.tpl +++ b/web/templates/layouts/results.tpl @@ -132,6 +132,7 @@ {{ else }}
+ {{ $test := .Test }} {{ range $index, $problem := .Test.Result.Problems }}
@@ -140,7 +141,11 @@
{{ $problem.Explanation }}
+ {{ if eq $problem.Name "UnboundLogs" }} + Click here to view unbound logs.

Note: These logs are only available for 7 days after a test is submitted. + {{ else }} {{ range $dIndex, $detail := $problem.DetailLines }}{{ $detail }}
{{ end }} + {{ end }}

{{ end }} diff --git a/web/web.go b/web/web.go index 93d3320..294150e 100644 --- a/web/web.go +++ b/web/web.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "html/template" + "io/ioutil" "log" "net" "net/http" @@ -106,6 +107,8 @@ func Serve() error { r.Post("/", s.httpSubmitTest) // - View test results (or test loading page) r.Get("/{domain}/{testID}", s.httpViewTestResult) + // - View unbound logs for a test + r.Get("/{domain}/{testID}/unboundlogs", s.httpViewUnboundLog) // - View all tests for domain r.Get("/{domain}", s.httpViewDomain) // Certwatch query gateway @@ -225,6 +228,59 @@ func (s *server) httpViewDomain(w http.ResponseWriter, r *http.Request) { } } +func (s *server) httpViewUnboundLog(w http.ResponseWriter, r *http.Request) { + domain := chi.URLParam(r, "domain") + testID, err := strconv.Atoi(chi.URLParam(r, "testID")) + + isBrowser := r.Header.Get("accept") != "application/json" + + doError := func(msg string, code int) { + if !isBrowser { + http.Error(w, msg, code) + return + } + s.render(w, code, "results.tpl", map[string]interface{}{ + "Error": msg, + }) + } + + if domain == "" || err != nil { + doError("Invalid request parameters.", http.StatusBadRequest) + return + } + + test, err := s.findTest(domain, testID) + if err != nil { + log.Printf("fetching %s/%d: %v", domain, testID, err) + doError("An internal error occurred fetching that test.", http.StatusInternalServerError) + return + } + + if test == nil { + doError("No such result exists.", http.StatusNotFound) + return + } + + for _, prob := range test.Result.Problems { + if prob.Name == "UnboundLogs" { + lines := prob.DetailLines() + if len(lines) != 2 { + break + } + logFile := lines[1] + b, err := ioutil.ReadFile(logFile) + if err != nil { + break + } + w.Header().Set("Content-Type", "text/plain") + w.Write(b) + return + } + } + + doError("No unbound logs for this test.", http.StatusNotFound) +} + func (s *server) httpViewTestResult(w http.ResponseWriter, r *http.Request) { domain := chi.URLParam(r, "domain") testID, err := strconv.Atoi(chi.URLParam(r, "testID"))