Merge pull request #13 from openebs/size_stats

Get volume usage stats

backend/file/file.go

@@ -51,6 +51,10 @@ func (f *Wrapper) GetRevisionCounter() (int64, error) {
 	return 1, nil
 }
 
+func (f *Wrapper) GetVolUsage() (types.VolUsage, error) {
+	return types.VolUsage{}, nil
+}
+
 func (f *Wrapper) SetRevisionCounter(counter int64) error {
 	return nil
 }

backend/remote/remote.go

@@ -146,6 +146,33 @@ func (r *Remote) GetRevisionCounter() (int64, error) {
 	return replica.RevisionCounter, nil
 }
 
+func (r *Remote) GetVolUsage() (types.VolUsage, error) {
+	var Details rest.VolUsage
+	var volUsage types.VolUsage
+
+	req, err := http.NewRequest("GET", r.replicaURL+"/volusage", nil)
+	if err != nil {
+		return volUsage, err
+	}
+
+	resp, err := r.httpClient.Do(req)
+	if err != nil {
+		return volUsage, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return volUsage, fmt.Errorf("Bad status: %d %s", resp.StatusCode, resp.Status)
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(&Details)
+	volUsage.UsedLogicalBlocks, _ = strconv.ParseInt(Details.UsedLogicalBlocks, 10, 64)
+	volUsage.UsedBlocks, _ = strconv.ParseInt(Details.UsedBlocks, 10, 64)
+	volUsage.SectorSize, _ = strconv.ParseInt(Details.SectorSize, 10, 64)
+
+	return volUsage, err
+}
+
 func (r *Remote) SetRevisionCounter(counter int64) error {
 	logrus.Infof("Set revision counter of %s to : %v", r.name, counter)
 	return r.doAction("setrevisioncounter", &map[string]int64{"counter": counter})

controller/control.go

@@ -728,15 +728,21 @@ func (c *Controller) shutdownFrontend() error {
 	return nil
 }
 
-func (c *Controller) Stats() types.Stats {
+func (c *Controller) Stats() (types.Stats, error) {
+	var err error
 	// Make sure writing data won't be blocked
 	c.RLock()
 	defer c.RUnlock()
 
 	if c.frontend != nil {
-		return (types.Stats)(c.frontend.Stats())
-	}
-	return types.Stats{}
+		stats := (types.Stats)(c.frontend.Stats())
+		volUsage, err := c.backend.GetVolUsage()
+		stats.UsedLogicalBlocks = volUsage.UsedLogicalBlocks
+		stats.UsedBlocks = volUsage.UsedBlocks
+		stats.SectorSize = volUsage.SectorSize
+		return stats, err
+	}
+	return types.Stats{}, err
 }
 
 func (c *Controller) shutdownBackend() error {

controller/replicator.go

@@ -305,6 +305,23 @@ func (r *replicator) reset(full bool) {
 	}
 }
 
+func (r *replicator) GetVolUsage() (types.VolUsage, error) {
+	var (
+		err      error
+		volUsage types.VolUsage
+	)
+	for _, backend := range r.backends {
+		if backend.mode == types.ERR {
+			continue
+		}
+		if volUsage, err = backend.backend.GetVolUsage(); err != nil {
+			continue
+		}
+		return volUsage, err
+	}
+	return types.VolUsage{}, err
+}
+
 type backendWrapper struct {
 	backend types.Backend
 	mode    types.Mode

controller/rest/model.go

@@ -59,6 +59,10 @@ type VolumeStats struct {
 	WriteIOPS            string `json:"WriteIOPS"`
 	TotalWriteTime       string `json:"TotalWriteTime"`
 	TotalWriteBlockCount string `json:"TotatWriteBlockCount"`
+
+	UsedLogicalBlocks string `json:"UsedLogicalBlocks"`
+	UsedBlocks        string `json:"UsedBlocks"`
+	SectorSize        string `json:"SectorSize"`
 }
 
 type SnapshotInput struct {

controller/rest/volume.go

@@ -35,7 +35,7 @@ func (s *Server) GetVolume(rw http.ResponseWriter, req *http.Request) error {
 
 func (s *Server) GetVolumeStats(rw http.ResponseWriter, req *http.Request) error {
 	apiContext := api.GetApiContext(req)
-	stats := s.c.Stats()
+	stats, _ := s.c.Stats()
 	volumeStats := &VolumeStats{
 		Resource:        client.Resource{Type: "stats"},
 		RevisionCounter: stats.RevisionCounter,
@@ -49,6 +49,10 @@ func (s *Server) GetVolumeStats(rw http.ResponseWriter, req *http.Request) error
 		WriteIOPS:            strconv.FormatInt(stats.WriteIOPS, 10),
 		TotalWriteTime:       strconv.FormatInt(stats.TotalWriteTime, 10),
 		TotalWriteBlockCount: strconv.FormatInt(stats.TotalWriteBlockCount, 10),
+
+		UsedLogicalBlocks: strconv.FormatInt(stats.UsedLogicalBlocks, 10),
+		UsedBlocks:        strconv.FormatInt(stats.UsedBlocks, 10),
+		SectorSize:        strconv.FormatInt(stats.SectorSize, 10),
 	}
 	apiContext.Write(volumeStats)
 	return nil

replica/backup.go

@@ -180,17 +180,16 @@ func preload(d *diffDisk) error {
 		if i == 0 {
 			continue
 		}
-
 		if i == 1 {
 			// Reinitialize to zero so that we can detect holes in the base snapshot
 			for j := 0; j < len(d.location); j++ {
 				d.location[j] = 0
 			}
 		}
-
 		generator := newGenerator(d, f)
 		for offset := range generator.Generate() {
 			d.location[offset] = byte(i)
+			d.UsedBlocks++
 		}
 
 		if generator.Err() != nil {
@@ -200,3 +199,13 @@ func preload(d *diffDisk) error {
 
 	return nil
 }
+
+func PreloadLunMap(d *diffDisk) error {
+	err := preload(d)
+	for _, val := range d.location {
+		if val != 0 {
+			d.UsedLogicalBlocks++
+		}
+	}
+	return err
+}

replica/diff_disk.go

@@ -12,7 +12,9 @@ type diffDisk struct {
 	rmLock sync.Mutex
 	// mapping of sector to index in the files array. a value of 0 is special meaning
 	// we don't know the location yet.
-	location []byte
+	location          []byte
+	UsedLogicalBlocks int64
+	UsedBlocks        int64
 	// list of files in child, parent, grandparent, etc order.
 	// index 0 is nil and index 1 is the active write layer
 	files      []types.DiffDisk
@@ -102,6 +104,12 @@ func (d *diffDisk) fullWriteAt(buf []byte, offset int64) (int, error) {
 
 	// Regardless of err mark bytes as written
 	for i := int64(0); i < sectors; i++ {
+		if val := d.location[startSector+i]; val == 0 {
+			d.UsedLogicalBlocks++
+			d.UsedBlocks++
+		} else if val != target {
+			d.UsedBlocks++
+		}
 		d.location[startSector+i] = target
 	}
 

replica/replica.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"syscall"
 	"time"
+	"unsafe"
 
 	"github.com/Sirupsen/logrus"
 	units "github.com/docker/go-units"
@@ -210,6 +211,7 @@ func construct(readonly bool, size, sectorSize int64, dir, head string, backingF
 	r.insertBackingFile()
 	r.ReplicaType = replicaType
 
+	PreloadLunMap(&r.volume)
 	return r, r.writeVolumeMetaData(true, r.info.Rebuilding)
 }
 
@@ -257,6 +259,14 @@ func (r *Replica) SetRebuilding(rebuilding bool) error {
 	return nil
 }
 
+func (r *Replica) GetUsage() (*types.VolUsage, error) {
+	return &types.VolUsage{
+		UsedLogicalBlocks: r.volume.UsedLogicalBlocks,
+		UsedBlocks:        r.volume.UsedBlocks,
+		SectorSize:        r.volume.sectorSize,
+	}, nil
+}
+
 func (r *Replica) Resize(obj interface{}) error {
 	var sizeInBytes int64
 	chain, err := r.Chain()
@@ -773,6 +783,12 @@ func (r *Replica) createDisk(name string, userCreated bool, created string) erro
 		if err := r.encodeToFile(r.diskData[newSnapName], newSnapName+metadataSuffix); err != nil {
 			return err
 		}
+		size := int64(unsafe.Sizeof(r.diskData[newSnapName]))
+		if size%defaultSectorSize == 0 {
+			r.volume.UsedBlocks += size / defaultSectorSize
+		} else {
+			r.volume.UsedBlocks += (size/defaultSectorSize + 1)
+		}
 
 		r.updateChildDisk(oldHead, newSnapName)
 		r.activeDiskData[len(r.activeDiskData)-1].Name = newSnapName
@@ -846,11 +862,9 @@ func (r *Replica) openLiveChain() error {
 		if err != nil {
 			return err
 		}
-
 		r.volume.files = append(r.volume.files, f)
 		r.activeDiskData = append(r.activeDiskData, r.diskData[parent])
 	}
-
 	return nil
 }
 
@@ -871,12 +885,23 @@ func (r *Replica) readMetadata() (bool, error) {
 				return false, err
 			}
 			r.volume.sectorSize = defaultSectorSize
+			if file.Size()%defaultSectorSize == 0 {
+				r.volume.UsedBlocks += file.Size() / defaultSectorSize
+			} else {
+				r.volume.UsedBlocks += (file.Size()/defaultSectorSize + 1)
+			}
 		} else if strings.HasSuffix(file.Name(), metadataSuffix) {
 			if err := r.readDiskData(file.Name()); err != nil {
 				return false, err
 			}
+			if file.Size()%defaultSectorSize == 0 {
+				r.volume.UsedBlocks += file.Size() / defaultSectorSize
+			} else {
+				r.volume.UsedBlocks += (file.Size()/defaultSectorSize + 1)
+			}
 		}
 	}
+	r.volume.UsedBlocks += 2 // One each for peer.details and revision.counter
 
 	return len(r.diskData) > 0, nil
 }

replica/rest/model.go

@@ -10,18 +10,20 @@ import (
 
 type Replica struct {
 	client.Resource
-	Dirty           bool                        `json:"dirty"`
-	Rebuilding      bool                        `json:"rebuilding"`
-	Head            string                      `json:"head"`
-	Parent          string                      `json:"parent"`
-	Size            string                      `json:"size"`
-	SectorSize      int64                       `json:"sectorSize"`
-	State           string                      `json:"state"`
-	Chain           []string                    `json:"chain"`
-	Disks           map[string]replica.DiskInfo `json:"disks"`
-	RemainSnapshots int                         `json:"remainsnapshots"`
-	RevisionCounter int64                       `json:"revisioncounter"`
-	ReplicaCounter  int64                       `json:"replicacounter"`
+	Dirty             bool                        `json:"dirty"`
+	Rebuilding        bool                        `json:"rebuilding"`
+	Head              string                      `json:"head"`
+	Parent            string                      `json:"parent"`
+	Size              string                      `json:"size"`
+	SectorSize        int64                       `json:"sectorSize"`
+	State             string                      `json:"state"`
+	Chain             []string                    `json:"chain"`
+	Disks             map[string]replica.DiskInfo `json:"disks"`
+	RemainSnapshots   int                         `json:"remainsnapshots"`
+	RevisionCounter   int64                       `json:"revisioncounter"`
+	ReplicaCounter    int64                       `json:"replicacounter"`
+	UsedLogicalBlocks string                      `json:"usedlogicalblocks"`
+	UsedBlocks        string                      `json:"usedblocks"`
 }
 
 type Stats struct {
@@ -58,6 +60,13 @@ type RemoveDiskInput struct {
 	Name string `json:"name"`
 }
 
+type VolUsage struct {
+	client.Resource
+	UsedLogicalBlocks string `json:"usedlogicalblocks"`
+	UsedBlocks        string `json:"usedblocks"`
+	SectorSize        string `json:"sectorSize"`
+}
+
 type ResizeInput struct {
 	client.Resource
 	Name string `json:"name"`

replica/rest/replica.go

@@ -41,6 +41,10 @@ func (s *Server) GetReplicaStats(apiContext *api.ApiContext) *types.Stats {
 	return s.s.Stats()
 }
 
+func (s *Server) GetUsage(apiContext *api.ApiContext) (*types.VolUsage, error) {
+	return s.s.GetUsage()
+}
+
 func (s *Server) GetStats(rw http.ResponseWriter, req *http.Request) error {
 	var stats *types.Stats
 	apiContext := api.GetApiContext(req)
@@ -56,7 +60,25 @@ func (s *Server) GetStats(rw http.ResponseWriter, req *http.Request) error {
 		RevisionCounter: stats.RevisionCounter,
 		ReplicaCounter:  stats.ReplicaCounter,
 	}
+	apiContext.Write(resp)
+	return nil
+}
+
+func (s *Server) GetVolUsage(rw http.ResponseWriter, req *http.Request) error {
+	apiContext := api.GetApiContext(req)
+	usage, _ := s.GetUsage(apiContext)
 
+	resp := &VolUsage{
+		Resource: client.Resource{
+			Type:    "replica",
+			Id:      "1",
+			Actions: map[string]string{},
+			Links:   map[string]string{},
+		},
+		UsedLogicalBlocks: strconv.FormatInt(usage.UsedLogicalBlocks, 10),
+		UsedBlocks:        strconv.FormatInt(usage.UsedBlocks, 10),
+		SectorSize:        strconv.FormatInt(usage.SectorSize, 10),
+	}
 	apiContext.Write(resp)
 	return nil
 }

replica/rest/router.go

@@ -50,6 +50,7 @@ func NewRouter(s *Server) *mux.Router {
 	// Replicas
 	router.Methods("GET").Path("/v1/replicas").Handler(f(schemas, s.ListReplicas))
 	router.Methods("GET").Path("/v1/replicas/{id}").Handler(f(schemas, s.GetReplica))
+	router.Methods("GET").Path("/v1/replicas/{id}/volusage").Handler(f(schemas, s.GetVolUsage))
 	router.Methods("DELETE").Path("/v1/replicas/{id}").Handler(f(schemas, s.DeleteReplica))
 
 	// Actions

replica/server.go

@@ -158,8 +158,12 @@ func (s *Server) Stats() *types.Stats {
 		RevisionCounter: r.revisionCache,
 		ReplicaCounter:  r.peerCache.ReplicaCount,
 	}
+}
 
+func (s *Server) GetUsage() (*types.VolUsage, error) {
+	return s.r.GetUsage()
 }
+
 func (s *Server) SetRebuilding(rebuilding bool) error {
 	s.Lock()
 	defer s.Unlock()

types/types.go

@@ -36,6 +36,7 @@ type Backend interface {
 	SectorSize() (int64, error)
 	RemainSnapshots() (int, error)
 	GetRevisionCounter() (int64, error)
+	GetVolUsage() (VolUsage, error)
 	SetRevisionCounter(counter int64) error
 	UpdatePeerDetails(replicaCount int64, quorumReplicaCount int64) error
 	SetRebuilding(rebuilding bool) error
@@ -48,6 +49,12 @@ type BackendFactory interface {
 	SignalToAdd(string, string) error
 }
 
+type VolUsage struct {
+	UsedLogicalBlocks int64
+	UsedBlocks        int64
+	SectorSize        int64
+}
+
 type Controller interface {
 	AddReplica(address string) error
 	RemoveReplica(address string) error
@@ -98,6 +105,10 @@ type Stats struct {
 	WriteIOPS            int64
 	TotalWriteTime       int64
 	TotalWriteBlockCount int64
+
+	UsedLogicalBlocks int64
+	UsedBlocks        int64
+	SectorSize        int64
 }
 
 type Interface interface{}