lib: generic iterator based on stripes iterator (#45017)

In a few places we have different patterns around pagination:
- Passing a closure in
- Returning a cursor for the client to use
- Just collect everything into a big slice

Other go libraries often will setup pagination patterns / use
interfaces. The API I quite like is the stripe-go one which is minimal
and reminds me a lot of Scanner from the stdlib. You just do something
like

  for it.Next() {
    doSomething(it.Current())
  }
  if it.Err() != nil {
    handle
  }

I imagine if we had this interface a few useful methods like

  Map[A,B](func(A() B, Iterator[A]) Iterator[B]
  Collect(Iterator[T]) ([]T, error)

etc. Basically could implement the same functions as present in
https://pkg.go.dev/golang.org/x/exp/slices

lib/iterator/functions.go

@@ -0,0 +1,25 @@
+package iterator
+
+// From is a convenience function to create an iterator from the slice s.
+//
+// Note: this function keeps a reference to s, so do not mutate it.
+func From[T any](s []T) *Iterator[T] {
+	done := false
+	return New(func() ([]T, error) {
+		if done {
+			return nil, nil
+		}
+		done = true
+		return s, nil
+	})
+}
+
+// Collect transforms the iterator it into a slice. It returns the slice and
+// the value of Err.
+func Collect[T any](it *Iterator[T]) ([]T, error) {
+	var s []T
+	for it.Next() {
+		s = append(s, it.Current())
+	}
+	return s, it.Err()
+}

lib/iterator/functions_test.go

@@ -0,0 +1,17 @@
+package iterator_test
+
+import (
+	"fmt"
+
+	"github.com/sourcegraph/sourcegraph/lib/iterator"
+)
+
+func ExampleCollect() {
+	it := iterator.From([]string{"Hello", "world"})
+	v, err := iterator.Collect(it)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(v)
+	// Output: [Hello world]
+}

lib/iterator/iterator.go

@@ -0,0 +1,67 @@
+package iterator
+
+import "fmt"
+
+// New returns an Iterator for next.
+//
+// next is a function which is repeatedly called until no items are returned
+// or there is a non-nil error. These items are returned one by one via Next
+// and Current.
+func New[T any](next func() ([]T, error)) *Iterator[T] {
+	return &Iterator[T]{next: next}
+}
+
+// Iterator provides a convenient interface for iterating over items which are
+// fetched in batches and can error. In particular this is designed for
+// pagination.
+//
+// Iterating stops as soon as the underlying next function returns an error or
+// no items. If an error is returned, Err will return a non-nil error.
+type Iterator[T any] struct {
+	items []T
+	err   error
+	done  bool
+
+	next func() ([]T, error)
+}
+
+// Next advances the iterator to the next item, which will then be available
+// from Current. It returns false when the iterator stops, either due to the
+// end of the input or an error occurred. After Next returns false Err() will
+// return the error occurred or nil if none.
+func (it *Iterator[T]) Next() bool {
+	if it.done {
+		return false
+	}
+
+	if len(it.items) > 1 {
+		it.items = it.items[1:]
+		return true
+	}
+
+	it.items, it.err = it.next()
+	if len(it.items) == 0 || it.err != nil {
+		it.items = nil // clear out so Current fails with err.
+		it.done = true
+	}
+
+	return !it.done
+}
+
+// Current returns the latest item advanced by Next. Note: this will panic if
+// Next returned false or if Next was never called.
+func (it *Iterator[T]) Current() T {
+	if len(it.items) == 0 {
+		if it.done {
+			panic(fmt.Sprintf("%T.Current() called after Next() returned false", it))
+		} else {
+			panic(fmt.Sprintf("%T.Current() called before first call to Next()", it))
+		}
+	}
+	return it.items[0]
+}
+
+// Err returns the first non-nil error encountered by Next.
+func (it *Iterator[T]) Err() error {
+	return it.err
+}

lib/iterator/iterator_test.go

@@ -0,0 +1,80 @@
+package iterator_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"github.com/sourcegraph/sourcegraph/lib/iterator"
+	"github.com/stretchr/testify/assert"
+)
+
+func ExampleIterator() {
+	x := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	it := iterator.New(func() ([]int, error) {
+		if len(x) == 0 {
+			return nil, nil
+		}
+		y := x[:2]
+		x = x[2:]
+		return y, nil
+	})
+
+	for it.Next() {
+		fmt.Printf("%d ", it.Current())
+	}
+
+	if it.Err() != nil {
+		fmt.Println(it.Err())
+	}
+
+	// Output: 1 2 3 4 5 6 7 8 9 10
+}
+
+func TestIterator_Err(t *testing.T) {
+	assert := assert.New(t)
+
+	sendErr := false
+	it := iterator.New(func() ([]int, error) {
+		var err error
+		if sendErr {
+			err = errors.New("boom")
+		}
+		sendErr = true
+		// We always return items, to test that we stop collecting after err.
+		return []int{1, 2, 3}, err
+	})
+
+	got, err := iterator.Collect(it)
+	assert.Equal([]int{1, 2, 3}, got)
+	assert.ErrorContains(err, "boom")
+
+	// Double check it is safe to call Next and Err again.
+	assert.Falsef(it.Next(), "expected collected Next to return false")
+	assert.Errorf(it.Err(), "expected collected Err to be non-nil")
+
+	// Ensure we panic on calling Current.
+	assert.Panics(func() { it.Current() })
+}
+
+func TestIterator_Current(t *testing.T) {
+	assert := assert.New(t)
+
+	it := iterator.From([]int{1})
+	assert.PanicsWithValue(
+		"*iterator.Iterator[int].Current() called before first call to Next()",
+		func() { it.Current() },
+		"Current before Next should panic",
+	)
+
+	assert.True(it.Next())
+	assert.Equal(1, it.Current())
+	assert.Equal(1, it.Current(), "Current should be idempotent")
+
+	assert.False(it.Next())
+	assert.PanicsWithValue(
+		"*iterator.Iterator[int].Current() called after Next() returned false",
+		func() { it.Current() },
+		"Current after Next is false should panic",
+	)
+}