fix(bm25): store TF/doclen in facets and fix query pipeline integration

Three critical bugs fixed:

1. REF postings lose Value during rollup: The posting list encode/rollup
   cycle strips the Value field from REF postings without facets (list.go:1630).
   BM25 term frequencies and doc lengths were stored in Value and lost.
   Fix: Store TF and doclen as facets on REF postings, which are preserved.

2. Missing function validation: query/query.go has a separate isValidFuncName
   check from dql/parser.go. "bm25" was only added to the parser, causing
   "Invalid function name: bm25" at query time.

3. Unsorted UIDs break query pipeline: BM25 returned UIDs sorted by score,
   but the query pipeline (algo.MergeSorted, child predicate fetching) requires
   UID-ascending order. Fix: Sort UIDs ascending in UidMatrix, apply
   first/offset pagination on score-sorted results before UID sorting.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

Author: shaunpatterson

posting/index.go

@@ -28,6 +28,7 @@ import (
 	"github.com/dgraph-io/badger/v4"
 	"github.com/dgraph-io/badger/v4/options"
 	bpb "github.com/dgraph-io/badger/v4/pb"
+	"github.com/dgraph-io/dgo/v250/protos/api"
 	"github.com/dgraph-io/dgraph/v25/protos/pb"
 	"github.com/dgraph-io/dgraph/v25/schema"
 	"github.com/dgraph-io/dgraph/v25/tok"
@@ -304,15 +305,15 @@ func (txn *Txn) addBM25IndexMutations(ctx context.Context, info *indexMutationIn
 		if err != nil {
 			return err
 		}
-		// Store uid in the posting list. The TF is encoded in the Value field.
+		// Store uid in the posting list. TF is stored as a facet so it survives
+		// the rollup cycle (REF postings without facets lose their Value field).
 		tfBuf := make([]byte, 4)
 		binary.BigEndian.PutUint32(tfBuf, tf)
 		edge := &pb.DirectedEdge{
-			ValueId:   uid,
-			Attr:      attr,
-			Value:     tfBuf,
-			ValueType: pb.Posting_INT,
-			Op:        pb.DirectedEdge_SET,
+			ValueId: uid,
+			Attr:    attr,
+			Op:      pb.DirectedEdge_SET,
+			Facets:  []*api.Facet{{Key: "tf", Value: tfBuf, ValType: api.Facet_INT}},
 		}
 		if err := plist.addMutation(ctx, txn, edge); err != nil {
 			return err
@@ -328,11 +329,10 @@ func (txn *Txn) addBM25IndexMutations(ctx context.Context, info *indexMutationIn
 	dlBuf := make([]byte, 4)
 	binary.BigEndian.PutUint32(dlBuf, docLen)
 	dlEdge := &pb.DirectedEdge{
-		ValueId:   uid,
-		Attr:      attr,
-		Value:     dlBuf,
-		ValueType: pb.Posting_INT,
-		Op:        pb.DirectedEdge_SET,
+		ValueId: uid,
+		Attr:    attr,
+		Op:      pb.DirectedEdge_SET,
+		Facets:  []*api.Facet{{Key: "dl", Value: dlBuf, ValType: api.Facet_INT}},
 	}
 	if err := dlPlist.addMutation(ctx, txn, dlEdge); err != nil {
 		return err

query/query.go

@@ -2751,7 +2751,7 @@ func isValidArg(a string) bool {
 func isValidFuncName(f string) bool {
 	switch f {
 	case "anyofterms", "allofterms", "val", "regexp", "anyoftext", "alloftext", "ngram",
-		"has", "uid", "uid_in", "anyof", "allof", "type", "match", "similar_to":
+		"has", "uid", "uid_in", "anyof", "allof", "type", "match", "similar_to", "bm25":
 		return true
 	}
 	return isInequalityFn(f) || types.IsGeoFunc(f)

query/query_bm25_test.go

@@ -10,7 +10,6 @@ package query
 
 import (
 	"context"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -32,6 +31,8 @@ func TestBM25Basic(t *testing.T) {
 }
 
 func TestBM25Ordering(t *testing.T) {
+	// BM25 returns all matching documents. Use first:1 to verify the highest-scored
+	// document is "fox fox fox" (tf=3, short doc).
 	query := `
 	{
 		me(func: bm25(description_bm25, "fox")) {
@@ -41,14 +42,22 @@ func TestBM25Ordering(t *testing.T) {
 	}
 	`
 	js := processQueryNoErr(t, query)
-	// Document 503 has "fox fox fox" (tf=3, short doc) so should rank highest.
-	// Verify it appears before other fox-containing documents in the output.
-	foxFoxFoxIdx := strings.Index(js, "fox fox fox")
-	quickBrownIdx := strings.Index(js, "quick brown fox jumps")
-	require.Greater(t, foxFoxFoxIdx, -1, "should contain 'fox fox fox'")
-	require.Greater(t, quickBrownIdx, -1, "should contain 'quick brown fox jumps'")
-	require.Less(t, foxFoxFoxIdx, quickBrownIdx,
-		"'fox fox fox' (higher tf, shorter doc) should rank before 'quick brown fox jumps'")
+	// Should contain all fox-mentioning documents.
+	require.Contains(t, js, "fox fox fox")
+	require.Contains(t, js, "quick brown fox jumps")
+
+	// first:1 should return the top-ranked document.
+	topQuery := `
+	{
+		me(func: bm25(description_bm25, "fox"), first: 1) {
+			uid
+			description_bm25
+		}
+	}
+	`
+	topJs := processQueryNoErr(t, topQuery)
+	require.Contains(t, topJs, "fox fox fox",
+		"top-1 BM25 result for 'fox' should be 'fox fox fox' (highest tf, shortest doc)")
 }
 
 func TestBM25WithParams(t *testing.T) {

worker/task.go

@@ -1337,8 +1337,11 @@ func (qs *queryState) handleBM25Search(ctx context.Context, args funcArgs) error
 				}
 			}
 			tf := uint32(1)
-			if len(p.Value) >= 4 {
-				tf = binary.BigEndian.Uint32(p.Value[:4])
+			for _, f := range p.Facets {
+				if f.Key == "tf" && len(f.Value) >= 4 {
+					tf = binary.BigEndian.Uint32(f.Value[:4])
+					break
+				}
 			}
 			ti.uidTFs[p.Uid] = tf
 			return nil
@@ -1369,8 +1372,11 @@ func (qs *queryState) handleBM25Search(ctx context.Context, args funcArgs) error
 			}
 			if _, needed := allUids[p.Uid]; needed {
 				dl := uint32(1)
-				if len(p.Value) >= 4 {
-					dl = binary.BigEndian.Uint32(p.Value[:4])
+				for _, f := range p.Facets {
+					if f.Key == "dl" && len(f.Value) >= 4 {
+						dl = binary.BigEndian.Uint32(f.Value[:4])
+						break
+					}
 				}
 				docLens[p.Uid] = dl
 				remaining--
@@ -1402,18 +1408,32 @@ func (qs *queryState) handleBM25Search(ctx context.Context, args funcArgs) error
 	for uid, score := range scores {
 		results = append(results, uidScore{uid: uid, score: score})
 	}
+	// Sort by score descending for ordering, then collect UIDs.
 	sort.Slice(results, func(i, j int) bool {
 		if results[i].score != results[j].score {
 			return results[i].score > results[j].score
 		}
 		return results[i].uid < results[j].uid
 	})
 
-	// Build output UIDs.
+	// Apply first/offset pagination on score-sorted results before returning UIDs.
+	if q.First > 0 || q.Offset > 0 {
+		offset := int(q.Offset)
+		if offset > len(results) {
+			offset = len(results)
+		}
+		results = results[offset:]
+		if q.First > 0 && int(q.First) < len(results) {
+			results = results[:int(q.First)]
+		}
+	}
+
+	// Build output UIDs sorted by UID (ascending) as required by the query pipeline.
 	uids := make([]uint64, len(results))
 	for i, r := range results {
 		uids[i] = r.uid
 	}
+	sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] })
 
 	args.out.UidMatrix = append(args.out.UidMatrix, &pb.List{Uids: uids})
 	return nil