diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index 801e42b..c38b7ed 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -996,9 +996,44 @@ func neighbourPages(nav *NavResult, pages []NavPage, leafTitle map[string]string return out } +// sectionRunnerUps are the leaves to fill a section-tree answer's open +// slots with, best first: the leaves of the pages read just below the +// evidence, then every other leaf in navigation's own ranking. The pages +// read all belong to the one or two leaves navigation chose, so on their +// own they add nothing new; on QASPER that left 15 of 50 questions with +// 1-3 units, and those found the gold paragraph 7% of the time against 88% +// for the questions with all 5 (HAL-3388). seen holds the leaves already +// selected and is updated. +func sectionRunnerUps(nav *NavResult, seen map[string]bool) []tree.SectionID { + var out []tree.SectionID + add := func(id string) { + if id == "" || seen[id] || len(out) >= maxSectionRunnerUps { + return + } + seen[id] = true + out = append(out, tree.SectionID(id)) + } + for _, ps := range runnerUps(nav) { + add(ps.Page.LeafID) + } + if nav != nil { + for _, ls := range nav.Leaves { // every leaf, best first + add(ls.Leaf.ID) + } + } + return out +} + +// maxSectionRunnerUps caps the leaves offered to fill open slots; the +// response still takes only as many as max_sections leaves room for. +const maxSectionRunnerUps = 16 + // runnerUps are the pages navigation read in full that did not make the // evidence, best first, at most maxRunnerUps of them. func runnerUps(nav *NavResult) []PageScore { + if nav == nil { + return nil + } type key struct { leaf string page int @@ -1327,13 +1362,7 @@ func (s *JudgeWalkStrategy) selectOnSectionTree(ctx context.Context, t *tree.Tre if len(nav.Evidence) > 0 { best = nav.Evidence[0].P } - var runnerUp []tree.SectionID - for _, ps := range runnerUps(nav) { - if id := tree.SectionID(ps.Page.LeafID); !seen[ps.Page.LeafID] { - seen[ps.Page.LeafID] = true - runnerUp = append(runnerUp, id) - } - } + runnerUp := sectionRunnerUps(nav, seen) if ordinal { ranges = nil // the units were numbered by position; there are no pages to cite } diff --git a/pkg/retrieval/units_test.go b/pkg/retrieval/units_test.go index aeab388..626514e 100644 --- a/pkg/retrieval/units_test.go +++ b/pkg/retrieval/units_test.go @@ -267,3 +267,25 @@ func TestWholeDocumentThreshold(t *testing.T) { t.Error("negative turns it off") } } + +// On a section tree, open slots fill from navigation's own ranking of every +// leaf once the pages read are used up (HAL-3388). +func TestSectionRunnerUpsFallBackToTheLeafRanking(t *testing.T) { + leaf := func(id string) NavLeaf { return NavLeaf{ID: id} } + nav := &NavResult{ + Leaves: []LeafScore{{Leaf: leaf("a"), P: 0.9}, {Leaf: leaf("b"), P: 0.6}, {Leaf: leaf("c"), P: 0.4}, {Leaf: leaf("d"), P: 0.1}}, + Pages: []PageScore{{Page: NavPage{Number: 1, LeafID: "a"}, P: 0.9}, {Page: NavPage{Number: 2, LeafID: "a"}, P: 0.5}}, + Evidence: []PageScore{{Page: NavPage{Number: 1, LeafID: "a"}, P: 0.9}}, + } + seen := map[string]bool{"a": true} + got := sectionRunnerUps(nav, seen) + if fmt.Sprint(got) != "[b c d]" { + t.Fatalf("runner-ups = %v, want the other leaves best first", got) + } + if !seen["b"] || !seen["d"] { + t.Error("seen not updated") + } + if sectionRunnerUps(nil, map[string]bool{}) != nil { + t.Error("nil navigation, no runner-ups") + } +}