Fix hosts blank line and reading bug, improve hosts reading speed

README.md

@@ -130,6 +130,7 @@ IPv6). Overture will handle both TCP and UDP requests.
 + File: Absolute path like `/path/to/file` is allowed. For Windows users, please use properly escaped path like
   `C:\\path\\to\\file.txt` in the configuration.
 + MinimumTTL: Set the minimum TTL value (in seconds) in order to improve caching efficiency, use `0` to disable.
++ CacheSize: The number of query record to cache, use `0` to disable.
 + RejectQtype: Reject inbound query with specific DNS record types, check [List of DNS record types](https://en.wikipedia.org/wiki/List_of_DNS_record_types) for details.
 
 #### Domain file example (Find domains and suffix match)
@@ -217,7 +218,6 @@ www.qq.com.		43	IN	A	14.17.42.40
 + Dependencies:
     + [dns](https://github.com/miekg/dns): BSD-3-Clause
     + [logrus](https://github.com/Sirupsen/logrus): MIT
-    + [pb](https://github.com/cheggaaa/pb): BSD-3-Clause
 + Code reference:
     + [skydns](https://github.com/skynetservices/skydns): MIT
     + [go-dnsmasq](https://github.com/janeczku/go-dnsmasq):  MIT

core/hosts/hosts.go

@@ -9,12 +9,10 @@ import (
 	"io/ioutil"
 	"net"
 	"strings"
-	"sync"
 )
 
 // Hosts represents a file containing hosts_sample
 type Hosts struct {
-	sync.RWMutex
 	hl       *hostsLineList
 	filePath string
 }
@@ -35,8 +33,6 @@ func New(path string) (*Hosts, error) {
 
 func (h *Hosts) Find(name string) (ipv4List []net.IP, ipv6List []net.IP) {
 	name = strings.TrimSuffix(name, ".")
-	h.RLock()
-	defer h.RUnlock()
 	return h.hl.FindHosts(name)
 }
 

core/hosts/hostsline.go

@@ -5,15 +5,12 @@
 package hosts
 
 import (
-	"fmt"
 	"net"
 	"strings"
-	"sync"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/shawn1m/overture/core/common"
-	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
 type hostsLine struct {
@@ -43,27 +40,11 @@ func newHostsLineList(data []byte) *hostsLineList {
 	ds := string(data)
 	hl := new(hostsLineList)
 
-	isBar := false
-	var bar *pb.ProgressBar
 	defer common.TimeTrack(time.Now(), "Load hosts")
 	lineList := strings.Split(ds, "\n")
-	if len(lineList) >= 10000 {
-		isBar = true
-	}
-	if isBar {
-		log.Info("Prepare to load hosts ...")
-		bar = pb.StartNew(len(lineList))
-		bar.SetRefreshRate(100 * time.Microsecond)
-	}
 
-	wg := new(sync.WaitGroup)
 	for _, l := range lineList {
-		wg.Add(1)
-		go func(l string) {
-			if isBar {
-				defer bar.Increment()
-			}
-			defer wg.Done()
+		func(l string) {
 			if h := parseLine(l); h != nil {
 				err := hl.add(h)
 				if err != nil {
@@ -72,11 +53,6 @@ func newHostsLineList(data []byte) *hostsLineList {
 			}
 		}(l)
 	}
-	wg.Wait()
-
-	if isBar {
-		bar.Finish()
-	}
 
 	return hl
 }
@@ -96,11 +72,12 @@ func (hl *hostsLineList) FindHosts(name string) (ipv4List []net.IP, ipv6List []n
 }
 
 func (hl *hostsLineList) add(h *hostsLine) error {
-	for _, found := range *hl {
-		if found.Equal(h) {
-			return fmt.Errorf("Duplicate hostname entry for %#v", h)
-		}
-	}
+	// Use too much CPU time when hosts file is big
+	//for _, found := range *hl {
+	//	if found.Equal(h) {
+	//		return fmt.Errorf("Duplicate hostname entry for %#v", h)
+	//	}
+	//}
 	*hl = append(*hl, h)
 	return nil
 }
@@ -129,10 +106,13 @@ func parseLine(line string) *hostsLine {
 
 	// Break line into words
 	words := strings.Split(line, " ")
+
+	if len(words) < 2 {
+		return nil
+	}
 	for i, word := range words {
 		words[i] = strings.TrimSpace(word)
 	}
-
 	// Separate the first bit (the ip) from the other bits (the domains)
 	a, h := words[0], words[1]