fix(core): put new metrics behind a feature flag (#9337) (#9348)

Cherry pick

Author: Harshil Goel

dgraph/cmd/alpha/run.go

@@ -268,6 +268,7 @@ they form a Raft group and provide synchronous replication.
 		Flag("normalize-compatibility-mode", "configure @normalize response formatting."+
 			" 'v20': returns values with repeated key for fields with same alias (same as v20.11)."+
 			" For more details, see https://github.com/hypermodeinc/dgraph/pull/7639").
+		Flag("enable-detailed-metrics", "Enable metrics about disk reads and cache per predicate").
 		String())
 }
 
@@ -753,6 +754,7 @@ func run() {
 	featureFlagsConf := z.NewSuperFlag(Alpha.Conf.GetString("feature-flags")).MergeAndCheckDefault(
 		worker.FeatureFlagsDefaults)
 	x.Config.NormalizeCompatibilityMode = featureFlagsConf.GetString("normalize-compatibility-mode")
+	enableDetailedMetrics := featureFlagsConf.GetBool("enable-detailed-metrics")
 
 	x.PrintVersion()
 	glog.Infof("x.Config: %+v", x.Config)
@@ -777,6 +779,7 @@ func run() {
 	// schema before calling posting.Init().
 	schema.Init(worker.State.Pstore)
 	posting.Init(worker.State.Pstore, postingListCacheSize, removeOnUpdate)
+	posting.SetEnabledDetailedMetrics(enableDetailedMetrics)
 	defer posting.Cleanup()
 	worker.Init(worker.State.Pstore)
 

posting/lists.go

@@ -28,8 +28,9 @@ const (
 )
 
 var (
-	pstore *badger.DB
-	closer *z.Closer
+	pstore                *badger.DB
+	closer                *z.Closer
+	EnableDetailedMetrics bool
 )
 
 // Init initializes the posting lists package, the in memory and dirty list hash.
@@ -41,6 +42,10 @@ func Init(ps *badger.DB, cacheSize int64, removeOnUpdate bool) {
 	memoryLayer = initMemoryLayer(cacheSize, removeOnUpdate)
 }
 
+func SetEnabledDetailedMetrics(enableMetrics bool) {
+	EnableDetailedMetrics = enableMetrics
+}
+
 func UpdateMaxCost(maxCost int64) {
 }
 
@@ -297,15 +302,17 @@ func (lc *LocalCache) getInternal(key []byte, readFromDisk bool) (*List, error)
 }
 
 func (lc *LocalCache) readPostingListAt(key []byte) (*pb.PostingList, error) {
-	start := time.Now()
-	defer func() {
-		pk, _ := x.Parse(key)
-		ms := x.SinceMs(start)
-		var tags []tag.Mutator
-		tags = append(tags, tag.Upsert(x.KeyMethod, "get"))
-		tags = append(tags, tag.Upsert(x.KeyStatus, pk.Attr))
-		_ = ostats.RecordWithTags(context.Background(), tags, x.BadgerReadLatencyMs.M(ms))
-	}()
+	if EnableDetailedMetrics {
+		start := time.Now()
+		defer func() {
+			ms := x.SinceMs(start)
+			pk, _ := x.Parse(key)
+			var tags []tag.Mutator
+			tags = append(tags, tag.Upsert(x.KeyMethod, "get"))
+			tags = append(tags, tag.Upsert(x.KeyStatus, pk.Attr))
+			_ = ostats.RecordWithTags(context.Background(), tags, x.BadgerReadLatencyMs.M(ms))
+		}()
+	}
 
 	pl := &pb.PostingList{}
 	txn := pstore.NewTransactionAt(lc.startTs, false)

posting/mvcc.go

@@ -437,13 +437,14 @@ func initMemoryLayer(cacheSize int64, removeOnUpdate bool) *MemoryLayer {
 				// Record the posting list cache hit ratio
 				ostats.Record(context.Background(), x.PLCacheHitRatio.M(m.Ratio()))
 
-				x.NumPostingListCacheSave.M(ml.cache.numCacheRead.Load())
-				ml.cache.numCacheSave.Store(0)
+				if EnableDetailedMetrics {
+					x.NumPostingListCacheSave.M(ml.cache.numCacheRead.Load())
+					x.NumPostingListCacheRead.M(ml.cache.numCacheRead.Load())
+					x.NumPostingListCacheReadFail.M(ml.cache.numCacheReadFails.Load())
+				}
 
-				x.NumPostingListCacheRead.M(ml.cache.numCacheRead.Load())
+				ml.cache.numCacheSave.Store(0)
 				ml.cache.numCacheRead.Store(0)
-
-				x.NumPostingListCacheReadFail.M(ml.cache.numCacheReadFails.Load())
 				ml.cache.numCacheReadFails.Store(0)
 			}
 		}()
@@ -547,10 +548,12 @@ func ReadPostingList(key []byte, it *badger.Iterator) (*List, error) {
 	start := time.Now()
 	defer func() {
 		ms := x.SinceMs(start)
-		var tags []tag.Mutator
-		tags = append(tags, tag.Upsert(x.KeyMethod, "iterate"))
-		tags = append(tags, tag.Upsert(x.KeyStatus, pk.Attr))
-		_ = ostats.RecordWithTags(context.Background(), tags, x.BadgerReadLatencyMs.M(ms))
+		if EnableDetailedMetrics {
+			var tags []tag.Mutator
+			tags = append(tags, tag.Upsert(x.KeyMethod, "iterate"))
+			tags = append(tags, tag.Upsert(x.KeyStatus, pk.Attr))
+			_ = ostats.RecordWithTags(context.Background(), tags, x.BadgerReadLatencyMs.M(ms))
+		}
 	}()
 
 	if pk.HasStartUid {

worker/server_state.go

@@ -42,7 +42,7 @@ const (
 	GraphQLDefaults    = `introspection=true; debug=false; extensions=true; poll-interval=1s; ` +
 		`lambda-url=;`
 	CacheDefaults        = `size-mb=1024; percentage=40,40,20; remove-on-update=false`
-	FeatureFlagsDefaults = `normalize-compatibility-mode=`
+	FeatureFlagsDefaults = `normalize-compatibility-mode=; enable-detailed-metrics=false`
 )
 
 // ServerState holds the state of the Dgraph server.