diff --git a/src/api.go b/src/api.go index 23ea2bb..b8fd360 100644 --- a/src/api.go +++ b/src/api.go @@ -121,6 +121,8 @@ func initAPIs() { authRouter.HandleFunc("/api/analytic/list", AnalyticLoader.HandleSummaryList) authRouter.HandleFunc("/api/analytic/load", AnalyticLoader.HandleLoadTargetDaySummary) authRouter.HandleFunc("/api/analytic/loadRange", AnalyticLoader.HandleLoadTargetRangeSummary) + authRouter.HandleFunc("/api/analytic/exportRange", AnalyticLoader.HandleRangeExport) + authRouter.HandleFunc("/api/analytic/resetRange", AnalyticLoader.HandleRangeReset) //Network utilities authRouter.HandleFunc("/api/tools/ipscan", HandleIpScan) diff --git a/src/go.mod b/src/go.mod index caa94d1..5d33b58 100644 --- a/src/go.mod +++ b/src/go.mod @@ -9,8 +9,8 @@ require ( github.com/gorilla/sessions v1.2.1 github.com/gorilla/websocket v1.4.2 github.com/grandcat/zeroconf v1.0.0 + github.com/microcosm-cc/bluemonday v1.0.24 github.com/oschwald/geoip2-golang v1.8.0 github.com/satori/go.uuid v1.2.0 - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 + golang.org/x/sys v0.8.0 ) diff --git a/src/go.sum b/src/go.sum index 503ee37..c41238b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,3 +1,5 @@ +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -10,6 +12,8 @@ github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3G github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= @@ -18,6 +22,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE= github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs= +github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= +github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs= @@ -52,8 +58,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -69,12 +75,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/src/main.go b/src/main.go index 22f6c2d..28eeb03 100644 --- a/src/main.go +++ b/src/main.go @@ -38,9 +38,9 @@ var ztAuthToken = flag.String("ztauth", "", "ZeroTier authtoken for the local no var ztAPIPort = flag.Int("ztport", 9993, "ZeroTier controller API port") var ( name = "Zoraxy" - version = "2.6.1" + version = "2.6.2" nodeUUID = "generic" - development = true //Set this to false to use embedded web fs + development = false //Set this to false to use embedded web fs bootTime = time.Now().Unix() /* diff --git a/src/mod/statistic/analytic/analytic.go b/src/mod/statistic/analytic/analytic.go index d38ea06..c491788 100644 --- a/src/mod/statistic/analytic/analytic.go +++ b/src/mod/statistic/analytic/analytic.go @@ -1,10 +1,9 @@ package analytic import ( - "encoding/json" + "errors" "net/http" "strings" - "time" "imuslab.com/zoraxy/mod/database" "imuslab.com/zoraxy/mod/statistic" @@ -24,105 +23,49 @@ func NewDataLoader(db *database.Database, sc *statistic.Collector) *DataLoader { } } -func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) { - entries, err := d.Database.ListTable("stats") - if err != nil { - utils.SendErrorResponse(w, "unable to load data from database") - return - } - - entryDates := []string{} - for _, keypairs := range entries { - entryDates = append(entryDates, string(keypairs[0])) - } - - js, _ := json.MarshalIndent(entryDates, "", " ") - utils.SendJSONResponse(w, string(js)) -} - -func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) { - day, err := utils.GetPara(r, "id") +// GetAllStatisticSummaryInRange return all the statisics within the time frame. The second array is the key (dates) of the statistic +func (d *DataLoader) GetAllStatisticSummaryInRange(start, end string) ([]*statistic.DailySummaryExport, []string, error) { + dailySummaries := []*statistic.DailySummaryExport{} + collectedDates := []string{} + //Generate all the dates in between the range + keys, err := generateDateRange(start, end) if err != nil { - utils.SendErrorResponse(w, "id cannot be empty") - return + return dailySummaries, collectedDates, err } - if strings.Contains(day, "-") { - //Must be underscore - day = strings.ReplaceAll(day, "-", "_") - } - - if !statistic.IsBeforeToday(day) { - utils.SendErrorResponse(w, "given date is in the future") - return - } - - var targetDailySummary statistic.DailySummaryExport - - if day == time.Now().Format("2006_01_02") { - targetDailySummary = *d.StatisticCollector.GetExportSummary() - } else { - //Not today data - err = d.Database.Read("stats", day, &targetDailySummary) - if err != nil { - utils.SendErrorResponse(w, "target day data not found") - return + //Load all the data from database + for _, key := range keys { + thisStat := statistic.DailySummaryExport{} + err = d.Database.Read("stats", key, &thisStat) + if err == nil { + dailySummaries = append(dailySummaries, &thisStat) + collectedDates = append(collectedDates, key) } } - js, _ := json.Marshal(targetDailySummary) - utils.SendJSONResponse(w, string(js)) + return dailySummaries, collectedDates, nil + } -func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) { - //Get the start date from POST para +func (d *DataLoader) GetStartAndEndDatesFromRequest(r *http.Request) (string, string, error) { + // Get the start date from POST para start, err := utils.GetPara(r, "start") if err != nil { - utils.SendErrorResponse(w, "start date cannot be empty") - return + return "", "", errors.New("start date cannot be empty") } if strings.Contains(start, "-") { //Must be underscore start = strings.ReplaceAll(start, "-", "_") } - //Get end date from POST para + // Get end date from POST para end, err := utils.GetPara(r, "end") if err != nil { - utils.SendErrorResponse(w, "emd date cannot be empty") - return + return "", "", errors.New("end date cannot be empty") } if strings.Contains(end, "-") { //Must be underscore end = strings.ReplaceAll(end, "-", "_") } - //Generate all the dates in between the range - keys, err := generateDateRange(start, end) - if err != nil { - utils.SendErrorResponse(w, err.Error()) - return - } - - //Load all the data from database - dailySummaries := []*statistic.DailySummaryExport{} - for _, key := range keys { - thisStat := statistic.DailySummaryExport{} - err = d.Database.Read("stats", key, &thisStat) - if err == nil { - dailySummaries = append(dailySummaries, &thisStat) - } - } - - //Merge the summaries into one - mergedSummary := mergeDailySummaryExports(dailySummaries) - - js, _ := json.Marshal(struct { - Summary *statistic.DailySummaryExport - Records []*statistic.DailySummaryExport - }{ - Summary: mergedSummary, - Records: dailySummaries, - }) - - utils.SendJSONResponse(w, string(js)) + return start, end, nil } diff --git a/src/mod/statistic/analytic/handlers.go b/src/mod/statistic/analytic/handlers.go new file mode 100644 index 0000000..283b5dc --- /dev/null +++ b/src/mod/statistic/analytic/handlers.go @@ -0,0 +1,218 @@ +package analytic + +import ( + "encoding/csv" + "encoding/json" + "log" + "net/http" + "strconv" + "strings" + "time" + + "imuslab.com/zoraxy/mod/statistic" + "imuslab.com/zoraxy/mod/utils" +) + +func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) { + entries, err := d.Database.ListTable("stats") + if err != nil { + utils.SendErrorResponse(w, "unable to load data from database") + return + } + + entryDates := []string{} + for _, keypairs := range entries { + entryDates = append(entryDates, string(keypairs[0])) + } + + js, _ := json.MarshalIndent(entryDates, "", " ") + utils.SendJSONResponse(w, string(js)) +} + +func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) { + day, err := utils.GetPara(r, "id") + if err != nil { + utils.SendErrorResponse(w, "id cannot be empty") + return + } + + if strings.Contains(day, "-") { + //Must be underscore + day = strings.ReplaceAll(day, "-", "_") + } + + if !statistic.IsBeforeToday(day) { + utils.SendErrorResponse(w, "given date is in the future") + return + } + + var targetDailySummary statistic.DailySummaryExport + + if day == time.Now().Format("2006_01_02") { + targetDailySummary = *d.StatisticCollector.GetExportSummary() + } else { + //Not today data + err = d.Database.Read("stats", day, &targetDailySummary) + if err != nil { + utils.SendErrorResponse(w, "target day data not found") + return + } + } + + js, _ := json.Marshal(targetDailySummary) + utils.SendJSONResponse(w, string(js)) +} + +func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) { + start, end, err := d.GetStartAndEndDatesFromRequest(r) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + dailySummaries, _, err := d.GetAllStatisticSummaryInRange(start, end) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + //Merge the summaries into one + mergedSummary := mergeDailySummaryExports(dailySummaries) + + js, _ := json.Marshal(struct { + Summary *statistic.DailySummaryExport + Records []*statistic.DailySummaryExport + }{ + Summary: mergedSummary, + Records: dailySummaries, + }) + + utils.SendJSONResponse(w, string(js)) +} + +// Handle exporting of a given range statistics +func (d *DataLoader) HandleRangeExport(w http.ResponseWriter, r *http.Request) { + start, end, err := d.GetStartAndEndDatesFromRequest(r) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + dailySummaries, dates, err := d.GetAllStatisticSummaryInRange(start, end) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + format, err := utils.GetPara(r, "format") + if err != nil { + format = "json" + } + + if format == "csv" { + // Create a buffer to store CSV content + var csvContent strings.Builder + + // Create a CSV writer + writer := csv.NewWriter(&csvContent) + + // Write the header row + header := []string{"Date", "TotalRequest", "ErrorRequest", "ValidRequest", "ForwardTypes", "RequestOrigin", "RequestClientIp", "Referer", "UserAgent", "RequestURL"} + err := writer.Write(header) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Write each data row + for i, item := range dailySummaries { + row := []string{ + dates[i], + strconv.FormatInt(item.TotalRequest, 10), + strconv.FormatInt(item.ErrorRequest, 10), + strconv.FormatInt(item.ValidRequest, 10), + // Convert map values to a comma-separated string + strings.Join(mapToStringSlice(item.ForwardTypes), ","), + strings.Join(mapToStringSlice(item.RequestOrigin), ","), + strings.Join(mapToStringSlice(item.RequestClientIp), ","), + strings.Join(mapToStringSlice(item.Referer), ","), + strings.Join(mapToStringSlice(item.UserAgent), ","), + strings.Join(mapToStringSlice(item.RequestURL), ","), + } + err = writer.Write(row) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + + // Flush the CSV writer + writer.Flush() + + // Check for any errors during writing + if err := writer.Error(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set the response headers + w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Disposition", "attachment; filename=analytics_"+start+"_to_"+end+".csv") + + // Write the CSV content to the response writer + _, err = w.Write([]byte(csvContent.String())) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } else if format == "json" { + type exportData struct { + Stats []*statistic.DailySummaryExport + Dates []string + } + + results := exportData{ + Stats: dailySummaries, + Dates: dates, + } + + js, _ := json.MarshalIndent(results, "", " ") + w.Header().Set("Content-Disposition", "attachment; filename=analytics_"+start+"_to_"+end+".json") + utils.SendJSONResponse(w, string(js)) + } else { + utils.SendErrorResponse(w, "Unsupported export format") + } +} + +// Reset all the keys within the given time period +func (d *DataLoader) HandleRangeReset(w http.ResponseWriter, r *http.Request) { + start, end, err := d.GetStartAndEndDatesFromRequest(r) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + if r.Method != http.MethodDelete { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + keys, err := generateDateRange(start, end) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + for _, key := range keys { + log.Println("DELETING statistics " + key) + d.Database.Delete("stats", key) + + if isTodayDate(key) { + //It is today's date. Also reset statistic collector value + log.Println("RESETING today's in-memory statistics") + d.StatisticCollector.ResetSummaryOfDay() + } + } + + utils.SendOK(w) +} diff --git a/src/mod/statistic/analytic/utils.go b/src/mod/statistic/analytic/utils.go index 239ab82..a373abb 100644 --- a/src/mod/statistic/analytic/utils.go +++ b/src/mod/statistic/analytic/utils.go @@ -70,3 +70,25 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti return mergedExport } + +func mapToStringSlice(m map[string]int) []string { + slice := make([]string, 0, len(m)) + for k := range m { + slice = append(slice, k) + } + return slice +} + +func isTodayDate(dateStr string) bool { + today := time.Now().Local().Format("2006-01-02") + inputDate, err := time.Parse("2006-01-02", dateStr) + if err != nil { + inputDate, err = time.Parse("2006_01_02", dateStr) + if err != nil { + fmt.Println("Invalid date format") + return false + } + } + + return inputDate.Format("2006-01-02") == today +} diff --git a/src/mod/statistic/statistic.go b/src/mod/statistic/statistic.go index 5d24024..06bf11b 100644 --- a/src/mod/statistic/statistic.go +++ b/src/mod/statistic/statistic.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/microcosm-cc/bluemonday" "imuslab.com/zoraxy/mod/database" ) @@ -96,6 +97,11 @@ func (c *Collector) LoadSummaryOfDay(year int, month time.Month, day int) *Daily return &targetSummary } +// Reset today summary, for debug or restoring injections +func (c *Collector) ResetSummaryOfDay() { + c.DailySummary = newDailySummary() +} + // This function gives the current slot in the 288- 5 minutes interval of the day func (c *Collector) GetCurrentRealtimeStatIntervalId() int { now := time.Now() @@ -160,11 +166,15 @@ func (c *Collector) RecordRequest(ri RequestInfo) { } //Record the referer - rf, ok := c.DailySummary.Referer.Load(ri.Referer) + p := bluemonday.StripTagsPolicy() + filteredReferer := p.Sanitize( + ri.Referer, + ) + rf, ok := c.DailySummary.Referer.Load(filteredReferer) if !ok { - c.DailySummary.Referer.Store(ri.Referer, 1) + c.DailySummary.Referer.Store(filteredReferer, 1) } else { - c.DailySummary.Referer.Store(ri.Referer, rf.(int)+1) + c.DailySummary.Referer.Store(filteredReferer, rf.(int)+1) } //Record the UserAgent diff --git a/src/reverseproxy.go b/src/reverseproxy.go index 8081725..06ad50f 100644 --- a/src/reverseproxy.go +++ b/src/reverseproxy.go @@ -604,6 +604,10 @@ func HandleUpdateHttpsRedirect(w http.ResponseWriter, r *http.Request) { js, _ := json.Marshal(currentRedirectToHttps) utils.SendJSONResponse(w, string(js)) } else { + if dynamicProxyRouter.Option.Port == 80 { + utils.SendErrorResponse(w, "This option is not available when listening on port 80") + return + } if useRedirect == "true" { sysdb.Write("settings", "redirect", true) log.Println("Updating force HTTPS redirection to true") diff --git a/src/web/components/stats.html b/src/web/components/stats.html index f8081cb..ba7f43f 100644 --- a/src/web/components/stats.html +++ b/src/web/components/stats.html @@ -15,8 +15,8 @@
Proxy traffic flow on layer 3 via TCP/IP
- +