fix(core): Add a few metrics and fix a bug (#9298)

Author: Harshil Goel

edgraph/server.go

@@ -1282,11 +1282,10 @@ func (s *Server) doQuery(ctx context.Context, req *Request) (resp *api.Response,
 	l := &query.Latency{}
 	l.Start = time.Now()
 
-	// TODO: Following trace messages have been commented out as stringified trace messages allocate
-	// too much memory. These trace messages need to be generated if tracing is enabled.
-	// if bool(glog.V(3)) || worker.LogDQLRequestEnabled() {
-	// 	glog.Infof("Got a query, DQL form: %+v at %+v", req.req, l.Start.Format(time.RFC3339))
-	// }
+	if bool(glog.V(3)) || worker.LogDQLRequestEnabled() {
+		glog.Infof("Got a query, DQL form: %+v %+v at %+v",
+			req.req.Query, req.req.Mutations, l.Start.Format(time.RFC3339))
+	}
 
 	isMutation := len(req.req.Mutations) > 0
 	methodRequest := methodQuery

posting/mvcc.go

@@ -347,9 +347,9 @@ func RemoveCacheFor(key []byte) {
 type Cache struct {
 	data *ristretto.Cache[[]byte, *CachePL]
 
-	numCacheRead      int64
-	numCacheReadFails int64
-	numCacheSave      int64
+	numCacheRead      atomic.Int64
+	numCacheReadFails atomic.Int64
+	numCacheSave      atomic.Int64
 }
 
 func (c *Cache) wait() {
@@ -365,23 +365,22 @@ func (c *Cache) get(key []byte) (*CachePL, bool) {
 	}
 	val, ok := c.data.Get(key)
 	if !ok {
-		atomic.AddInt64(&c.numCacheReadFails, 1)
+		c.numCacheReadFails.Add(1)
 		return val, ok
 	}
 	if val.list == nil {
-		atomic.AddInt64(&c.numCacheReadFails, 1)
+		c.numCacheReadFails.Add(1)
 		return nil, false
 	}
-	atomic.AddInt64(&c.numCacheRead, 1)
-	c.numCacheRead += 1
+	c.numCacheRead.Add(1)
 	return val, true
 }
 
 func (c *Cache) set(key []byte, i *CachePL) {
 	if c == nil {
 		return
 	}
-	c.numCacheSave += 1
+	c.numCacheSave.Add(1)
 	c.data.Set(key, i, 1)
 }
 
@@ -407,8 +406,7 @@ type MemoryLayer struct {
 	cache *Cache
 
 	// metrics
-	statsHolder  *StatsHolder
-	numDisksRead int64
+	statsHolder *StatsHolder
 }
 
 func (ml *MemoryLayer) clear() {
@@ -448,6 +446,15 @@ func initMemoryLayer(cacheSize int64, deleteOnUpdates bool) *MemoryLayer {
 			for range ticker.C {
 				// Record the posting list cache hit ratio
 				ostats.Record(context.Background(), x.PLCacheHitRatio.M(m.Ratio()))
+
+				x.NumPostingListCacheSave.M(ml.cache.numCacheRead.Load())
+				ml.cache.numCacheSave.Store(0)
+
+				x.NumPostingListCacheRead.M(ml.cache.numCacheRead.Load())
+				ml.cache.numCacheRead.Store(0)
+
+				x.NumPostingListCacheReadFail.M(ml.cache.numCacheReadFails.Load())
+				ml.cache.numCacheReadFails.Store(0)
 			}
 		}()
 
@@ -657,7 +664,6 @@ func (ml *MemoryLayer) readFromCache(key []byte, readTs uint64) *List {
 }
 
 func (ml *MemoryLayer) readFromDisk(key []byte, pstore *badger.DB, readTs uint64) (*List, error) {
-	atomic.AddInt64(&ml.numDisksRead, 1)
 	txn := pstore.NewTransactionAt(readTs, false)
 	defer txn.Discard()
 

systest/backup/minio/docker-compose.yml

@@ -23,7 +23,7 @@ services:
         read_only: true
     command:
       /gobin/dgraph  ${COVERAGE_OUTPUT} alpha --telemetry "reports=false; sentry=false;"
-      --my=alpha1:7080 --zero=zero1:5080 --logtostderr -v=2 --security
+      --my=alpha1:7080 --zero=zero1:5080 --logtostderr --cache "size-mb=500;" -v=2 --security
       "whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16;" --tls "ca-cert=/dgraph-tls/ca.crt;
       server-cert=/dgraph-tls/node.crt; server-key=/dgraph-tls/node.key; internal-port=true;
       client-cert=/dgraph-tls/client.alpha1.crt; client-key=/dgraph-tls/client.alpha1.key;"
@@ -48,7 +48,7 @@ services:
         read_only: true
     command:
       /gobin/dgraph  ${COVERAGE_OUTPUT} alpha --telemetry "reports=false; sentry=false;"
-      --my=alpha2:7080 --zero=zero1:5080 --logtostderr -v=2 --security
+      --my=alpha2:7080 --zero=zero1:5080 --logtostderr --cache "size-mb=500;" -v=2 --security
       "whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16;" --tls "ca-cert=/dgraph-tls/ca.crt;
       server-cert=/dgraph-tls/node.crt; server-key=/dgraph-tls/node.key; internal-port=true;
       client-cert=/dgraph-tls/client.alpha2.crt; client-key=/dgraph-tls/client.alpha2.key;"
@@ -73,7 +73,7 @@ services:
         read_only: true
     command:
       /gobin/dgraph  ${COVERAGE_OUTPUT} alpha --telemetry "reports=false; sentry=false;"
-      --my=alpha3:7080 --zero=zero1:5080 --logtostderr -v=2 --security
+      --my=alpha3:7080 --zero=zero1:5080 --logtostderr --cache "size-mb=500;" -v=2 --security
       "whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16;" --tls "ca-cert=/dgraph-tls/ca.crt;
       server-cert=/dgraph-tls/node.crt; server-key=/dgraph-tls/node.key; internal-port=true;
       client-cert=/dgraph-tls/client.alpha3.crt; client-key=/dgraph-tls/client.alpha3.key;"

worker/task.go

@@ -838,10 +838,8 @@ func (qs *queryState) handleUidPostings(
 			}
 
 			if srcFn.fnType == compareAttrFn {
-				pl.RLock()
 				posting.GetStatsHolder().InsertRecord(
 					q.Attr, []byte(srcFn.tokens[i]), uint64(pl.ApproxLen()))
-				pl.RUnlock()
 			}
 
 			switch {

x/metrics.go

@@ -147,6 +147,12 @@ var (
 	// RaftLeaderChanges records the total number of leader changes seen.
 	RaftLeaderChanges = ostats.Int64("raft_leader_changes_total",
 		"Total number of leader changes seen", ostats.UnitDimensionless)
+	NumPostingListCacheRead = ostats.Int64("num_posting_list_cache_reads",
+		"Number of times cache was read", ostats.UnitDimensionless)
+	NumPostingListCacheReadFail = ostats.Int64("num_posting_list_cache_reads_fail",
+		"Number of times cache was read", ostats.UnitDimensionless)
+	NumPostingListCacheSave = ostats.Int64("num_posting_list_cache_saves",
+		"Number of times item was saved in cache", ostats.UnitDimensionless)
 
 	// Conf holds the metrics config.
 	// TODO: Request statistics, latencies, 500, timeouts
@@ -202,6 +208,27 @@ var (
 			Aggregation: view.Count(),
 			TagKeys:     allTagKeys,
 		},
+		{
+			Name:        NumPostingListCacheRead.Name(),
+			Measure:     NumPostingListCacheRead,
+			Description: NumPostingListCacheRead.Description(),
+			Aggregation: view.Count(),
+			TagKeys:     allTagKeys,
+		},
+		{
+			Name:        NumPostingListCacheReadFail.Name(),
+			Measure:     NumPostingListCacheReadFail,
+			Description: NumPostingListCacheReadFail.Description(),
+			Aggregation: view.Count(),
+			TagKeys:     allTagKeys,
+		},
+		{
+			Name:        NumPostingListCacheSave.Name(),
+			Measure:     NumPostingListCacheSave,
+			Description: NumPostingListCacheSave.Description(),
+			Aggregation: view.Count(),
+			TagKeys:     allTagKeys,
+		},
 		{
 			Name:        NumEdges.Name(),
 			Measure:     NumEdges,