artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 18:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

github: add ability to exclude repositories based on size & stars (#58377)

Browse Source
This commit is contained in:
Thorsten Ball 2023-11-16 19:03:24 +01:00 committed by GitHub
parent b714d56338
commit 8f0c04090c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 563 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
internal/bytesize/BUILD.bazel Normal file
View File

@ -0,0 +1,16 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//dev:go_defs.bzl", "go_test")
go_library(
name = "bytesize",
srcs = ["bytesize.go"],
importpath = "github.com/sourcegraph/sourcegraph/internal/bytesize",
visibility = ["//:__subpackages__"],
deps = ["//lib/errors"],
)
go_test(
name = "bytesize_test",
srcs = ["bytesize_test.go"],
embed = [":bytesize"],
)

90
internal/bytesize/bytesize.go Normal file
View File

@ -0,0 +1,90 @@
// package bytesize provides utilities to work with bytes in human-readable
// form.
package bytesize
import (
"strconv"
"strings"
"unicode"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
// Bytes represents an amount of bytes.
type Bytes int64
const (
maxSize Bytes = 1<<63 - 1
)
const (
B Bytes = 1
KB = 1_000 * B
KiB = 1_024 * B
MB = 1_000 * KB
MiB = 1_024 * KiB
GB = 1_000 * MB
GiB = 1_024 * MiB
)
// Parse parses string that represents an amount of bytes and returns the
// amount in Bytes.
//
// Only positive amounts are supported.
//
// Bytes are represented as int64. If the value overflows, an error is
// returned.
//
// Example inputs: "3 MB", "4 GiB", "172 KiB". See the tests for more examples.
func Parse(str string) (Bytes, error) {
str = strings.TrimSpace(str)
num, unitIndex := readNumber(str)
if unitIndex == 0 {
return 0, errors.Newf("missing number at start of string: %s", str)
}
unit, err := parseUnit(str[unitIndex:])
if err != nil {
return 0, err
}
result := Bytes(num) * unit
if result < 0 {
return 0, errors.Newf("value overflows max size of %d bytes", maxSize)
}
return result, nil
}
func readNumber(str string) (int, int) {
for i, c := range str {
if unicode.IsDigit(c) {
continue
}
number, _ := strconv.Atoi(str[0:i])
return number, i
}
return 0, 0
}
func parseUnit(unit string) (Bytes, error) {
switch strings.TrimSpace(unit) {
case "B", "b":
return B, nil
case "kB", "KB":
return KB, nil
case "kiB", "KiB":
return KiB, nil
case "MiB":
return MiB, nil
case "MB":
return MB, nil
case "GiB":
return GiB, nil
case "GB":
return GB, nil
default:
return 0, errors.Newf("unknown unit: %s", unit)
}
}

121
internal/bytesize/bytesize_test.go Normal file
View File

@ -0,0 +1,121 @@
package bytesize
import (
"fmt"
"testing"
)
func TestParse(t *testing.T) {
tests := []struct {
input string
want Bytes
}{
// Happy paths
{"1 B", 1},
{"1 KB", 1000},
{"1 KiB", 1024},
{"1 MB", 1_000_000},
{"1 MiB", 1024 * 1024},
{"1 GB", 1_000_000_000},
{"1 GiB", 1024 * 1024 * 1024},
// Whitespace
{" 1 B", 1},
{" 100 KB", 100_000},
// Various
{"100 B", 100},
{"1024 B", 1024},
{"12 KB", 12_000},
{"72 KiB", 73_728},
{"3 MB", 3000000},
{"3 MiB", 3145728},
{"7 GB", 7_000_000_000},
{"7 GiB", 7_516_192_768},
}
for _, tt := range tests {
got, err := Parse(tt.input)
if err != nil {
t.Errorf("FromString(%q), got error: %s", tt.input, err)
}
if got != tt.want {
t.Errorf("FromString(%q) = %d, want %d", tt.input, got, tt.want)
}
}
invalid := []string{
" ",
"aaa",
"1",
"1 count",
"1 megabyte",
"10 horses",
"1324 k b",
}
for _, tt := range invalid {
_, err := Parse(tt)
if err == nil {
t.Errorf("Parse(%q) expected error but got nil", tt)
}
}
}
func TestParseOverflow(t *testing.T) {
input := fmt.Sprintf("%d KB", (maxSize / 2))
_, err := Parse(input)
if err == nil {
t.Errorf("Parse(%q) expected error, but got none", input)
}
}
func TestReadNumber(t *testing.T) {
tests := []struct {
input string
want int
len int
}{
{"1234 foobar", 1234, 4},
{"1234foobar", 1234, 4},
{"12 34", 12, 2},
{"1f", 1, 1},
{"foobar", 0, 0},
}
for _, tt := range tests {
num, numLen := readNumber(tt.input)
if num != tt.want {
t.Errorf("readNumber(%q) = %d, want %d", tt.input, num, tt.want)
}
if numLen != tt.len {
t.Errorf("readNumber(%q) length = %d, want %d", tt.input, numLen, tt.len)
}
}
}
func TestMultiplication(t *testing.T) {
tests := []struct {
inputNumber int
inputUnit Bytes
want Bytes
}{
{10, B, 10},
{25, KB, 25_000},
{25_000, KB, 25_000_000},
}
for _, tt := range tests {
got := Bytes(tt.inputNumber) * tt.inputUnit
if got != tt.want {
t.Errorf("FromUnit(%d, %v) = %d, want %d", tt.inputNumber, tt.inputUnit, got, tt.want)
}
}
}

1
internal/extsvc/github/BUILD.bazel
View File

@ -13,6 +13,7 @@ go_library(
visibility = ["//:__subpackages__"],
deps = [
"//internal/api",
"//internal/bytesize",
"//internal/conf",
"//internal/encryption",
"//internal/env",

5
internal/extsvc/github/common.go
View File

@ -22,6 +22,7 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/bytesize"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/encryption"
"github.com/sourcegraph/sourcegraph/internal/env"
@ -1813,6 +1814,10 @@ type Repository struct {
DiskUsageKibibytes int `json:"DiskUsage,omitempty"`
}
func (r *Repository) SizeBytes() bytesize.Bytes {
return bytesize.Bytes(r.DiskUsageKibibytes) * bytesize.KiB
}
// ParentRepository is the parent of a GitHub repository.
type ParentRepository struct {
NameWithOwner string

2
internal/repos/BUILD.bazel
View File

@ -47,6 +47,7 @@ go_library(
"//cmd/frontend/envvar",
"//internal/actor",
"//internal/api",
"//internal/bytesize",
"//internal/codeintel/dependencies",
"//internal/conf",
"//internal/conf/conftypes",
@ -128,6 +129,7 @@ go_test(
"azuredevops_test.go",
"bitbucketcloud_test.go",
"bitbucketserver_test.go",
"exclude_test.go",
"gerrit_test.go",
"github_test.go",
"gitlab_test.go",

122
internal/repos/exclude.go
View File

@ -1,9 +1,15 @@
package repos
import (
"strconv"
"strings"
"github.com/grafana/regexp"
"github.com/sourcegraph/sourcegraph/internal/bytesize"
"github.com/sourcegraph/sourcegraph/internal/extsvc/github"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
// excludeFunc takes either a generic object and returns true if the repo should be excluded. In
@ -77,3 +83,119 @@ func (e *excludeBuilder) Build() (excludeFunc, error) {
return false
}, e.err
}
func buildGitHubExcludeRule(rule *schema.ExcludedGitHubRepo) (excludeFunc, error) {
var fns []gitHubExcludeFunc
if rule.Stars != "" {
fn, err := buildStarsConstraintsExcludeFn(rule.Stars)
if err != nil {
return nil, err
}
fns = append(fns, fn)
}
if rule.Size != "" {
fn, err := buildSizeConstraintsExcludeFn(rule.Size)
if err != nil {
return nil, err
}
fns = append(fns, fn)
}
return func(repo any) bool {
githubRepo, ok := repo.(github.Repository)
if !ok {
return false
}
// We're AND'ing the functions together. If one of them does NOT exclude
// the repository, then we don't exclude it.
for _, fn := range fns {
excluded := fn(githubRepo)
if !excluded {
return false
}
}
return true
}, nil
}
type gitHubExcludeFunc func(github.Repository) bool
var starsConstraintRegex = regexp.MustCompile(`([<>=]{1,2})\s*(\d+)`)
var sizeConstraintRegex = regexp.MustCompile(`([<>=]{1,2})\s*(\d+\s*\w+)`)
func buildStarsConstraintsExcludeFn(constraint string) (gitHubExcludeFunc, error) {
matches := starsConstraintRegex.FindStringSubmatch(constraint)
if matches == nil {
return nil, errors.Newf("invalid stars constraint format: %q", constraint)
}
operator, err := newOperator(matches[1])
if err != nil {
return nil, errors.Wrapf(err, "failed to evaluate stars constraint")
}
count, err := strconv.Atoi(matches[2])
if err != nil {
return nil, err
}
return func(r github.Repository) bool {
return operator.Eval(r.StargazerCount, count)
}, nil
}
func buildSizeConstraintsExcludeFn(constraint string) (gitHubExcludeFunc, error) {
sizeMatch := sizeConstraintRegex.FindStringSubmatch(constraint)
if sizeMatch == nil {
return nil, errors.Newf("invalid size constraint format: %q", constraint)
}
operator, err := newOperator(sizeMatch[1])
if err != nil {
return nil, errors.Wrapf(err, "failed to evaluate size constraint")
}
size, err := bytesize.Parse(sizeMatch[2])
if err != nil {
return nil, err
}
return func(r github.Repository) bool {
return operator.Eval(int(r.SizeBytes()), int(size))
}, nil
}
type operator string
const (
opLess operator = "<"
opLessOrEqual operator = "<="
opGreater operator = ">"
opGreaterOrEqual operator = ">="
)
func newOperator(input string) (operator, error) {
if input != "<" && input != "<=" && input != ">" && input != ">=" {
return "", errors.Newf("invalid operator %q", input)
}
return operator(input), nil
}
func (o operator) Eval(left, right int) bool {
switch o {
case "<":
return left < right
case ">":
return left > right
case "<=":
return left <= right
case ">=":
return left >= right
default:
// I wish Go had enums
panic(errors.Newf("unknown operator: %q", o))
}
}

119
internal/repos/exclude_test.go Normal file
View File

@ -0,0 +1,119 @@
package repos
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/sourcegraph/internal/extsvc/github"
"github.com/sourcegraph/sourcegraph/schema"
)
func TestBuildGitHubExcludeRule(t *testing.T) {
assertExcluded := func(t *testing.T, rule *schema.ExcludedGitHubRepo, repo github.Repository, wantExcluded bool) {
t.Helper()
fn, err := buildGitHubExcludeRule(rule)
assert.Nil(t, err)
assert.Equal(
t,
wantExcluded,
fn(repo),
"rule.Stars=%q, rule.Size=%q, repo.StarGazerCount=%d, repo.DiskUsageKibibytes=%d",
rule.Stars,
rule.Size,
repo.StargazerCount,
repo.DiskUsageKibibytes,
)
}
t.Run("stars", func(t *testing.T) {
tests := []struct {
rule string
stars int
wantExcluded bool
}{
{"< 100", 99, true},
{"< 100", 100, false},
{"<= 100", 100, true},
{"<= 100", 99, true},
{"<= 100", 101, false},
{"> 100", 101, true},
{"> 100", 100, false},
{">= 100", 100, true},
{">= 100", 101, true},
{">= 100", 99, false},
}
for _, tt := range tests {
excludeRule := &schema.ExcludedGitHubRepo{Stars: tt.rule}
repo := github.Repository{StargazerCount: tt.stars}
assertExcluded(t, excludeRule, repo, tt.wantExcluded)
}
})
t.Run("size", func(t *testing.T) {
tests := []struct {
rule string
sizeKibibytes int
wantExcluded bool
}{
{"< 100 KiB", 99, true},
{"< 100 KiB", 100, false},
{"<= 100 KiB", 100, true},
{"<= 100 KiB", 99, true},
{"<= 100 KiB", 101, false},
{"> 100 KiB", 101, true},
{"> 100 KiB", 100, false},
{">= 100 KiB", 100, true},
{">= 100 KiB", 101, true},
{">= 100 KiB", 99, false},
{"< 1025 B", 1, true},
{"< 1024 B", 1, false},
{"< 1025 B", 2, false},
{"< 100 KiB", 99, true},
{"< 100 KiB", 100, false},
{"< 100 KB", 99, false},
{"< 102 KB", 99, true},
{"< 100 MiB", 99, true},
{"< 100 MiB", 102400, false},
{"< 101 MiB", 102400, true},
{"< 100 MB", 102400, false},
{"< 105 MB", 102400, true},
{"< 1 GiB", 1024*1024 - 1, true},
{"< 1 GiB", 1024 * 1024, false},
{"< 1 GB", 1024*1024 - 1, false},
{"< 2 GB", 1024 * 1024, true},
}
for _, tt := range tests {
excludeRule := &schema.ExcludedGitHubRepo{Size: tt.rule}
repo := github.Repository{DiskUsageKibibytes: tt.sizeKibibytes}
assertExcluded(t, excludeRule, repo, tt.wantExcluded)
}
})
t.Run("stars and size", func(t *testing.T) {
rule := &schema.ExcludedGitHubRepo{Stars: "< 100", Size: ">= 1GB"}
// Less than 100 stars, equal or greater than 1GB in size
assertExcluded(t, rule, github.Repository{StargazerCount: 99, DiskUsageKibibytes: 976563}, true)
assertExcluded(t, rule, github.Repository{StargazerCount: 99, DiskUsageKibibytes: 976563 + 1}, true)
// Equal or greater than 100 stars, greater than 1GB in size
assertExcluded(t, rule, github.Repository{StargazerCount: 100, DiskUsageKibibytes: 976563 + 1}, false)
assertExcluded(t, rule, github.Repository{StargazerCount: 101, DiskUsageKibibytes: 976563 + 1}, false)
// Greater than 100 stars, less than 1 GB
assertExcluded(t, rule, github.Repository{StargazerCount: 101, DiskUsageKibibytes: 500}, false)
})
}

13
internal/repos/github.go
View File

@ -134,7 +134,20 @@ func newGitHubSource(
}
return false
}
for _, r := range c.Exclude {
// TODO: Size/Stars are special-case'd here and are AND'ed if both set.
// This condition here should be replace with something that builds an
// exclude-function for all possible values a schema.ExcludedGitHubRepo
// could have.
if r.Size != "" || r.Stars != "" {
fn, err := buildGitHubExcludeRule(r)
if err != nil {
return nil, err
}
eb.Generic(fn)
}
if r.Archived {
eb.Generic(excludeArchived)
}

1
internal/repos/github_test.go
View File

@ -775,6 +775,7 @@ func TestGithubSource_ListRepos(t *testing.T) {
// uses rcache, a caching layer that uses Redis.
// We need to clear the cache before we run the tests
rcache.SetupForTest(t)
ratelimit.SetupForTest(t)
var (
cf *httpcli.Factory

10
internal/repos/sources_test.go
View File

@ -182,13 +182,21 @@ func TestSources_ListRepos_Excluded(t *testing.T) {
"sourcegraph/sourcegraph",
"keegancsmith/sqlf",
"tsenart/VEGETA",
"tsenart/go-tsz", // fork
"tsenart/go-tsz", // fork
"sourcegraph/about", // has >500MB and < 200 stars
"facebook/react", // has 215k stars as of now
"torvalds/linux", // has ~4GB
"avelino/awesome-go", // has < 20 MB and > 100k stars
},
Exclude: []*schema.ExcludedGitHubRepo{
{Name: "tsenart/Vegeta"},
{Id: "MDEwOlJlcG9zaXRvcnkxNTM2NTcyNDU="}, // tsenart/patrol ID
{Pattern: "^keegancsmith/.*"},
{Forks: true},
{Stars: "> 215000"}, // exclude facebook/react
{Size: "> 3GB"}, // exclude torvalds/linux
{Size: ">= 500MB", Stars: "< 200"}, // exclude about repo
{Size: "<= 20MB", Stars: "> 100000"}, // exclude awesome-go
},
})),
},

80
internal/repos/testdata/sources/GITHUB/excluded-repos-are-never-yielded.yaml vendored
View File

@ -2,43 +2,56 @@
version: 1
interactions:
- request:
body: '{"query":"\nfragment RepositoryFields on Repository {\n\tid\n\tdatabaseId\n\tnameWithOwner\n\tdescription\n\turl\n\tisPrivate\n\tisFork\n\tisArchived\n\tisLocked\n\tisDisabled\n\tviewerPermission\n\tstargazerCount\n\tforkCount\n}\n\tquery
{\nrepo0: repository(owner: \"sourcegraph\", name: \"Sourcegraph\") { ... on
Repository { ...RepositoryFields } }\nrepo1: repository(owner: \"keegancsmith\",
name: \"sqlf\") { ... on Repository { ...RepositoryFields } }\nrepo2: repository(owner:
\"tsenart\", name: \"VEGETA\") { ... on Repository { ...RepositoryFields } }\nrepo3:
repository(owner: \"tsenart\", name: \"go-tsz\") { ... on Repository { ...RepositoryFields
} }\n}","variables":{}}'
body: '{"query":"\nfragment RepositoryFields on Repository {\n\tid\n\tdatabaseId\n\tnameWithOwner\n\tdescription\n\turl\n\tisPrivate\n\tisFork\n\tisArchived\n\tisLocked\n\tisDisabled\n\tviewerPermission\n\tstargazerCount\n\tforkCount\n\tdiskUsage\n\trepositoryTopics(first:100)
{\n\t\tnodes {\n\t\t\ttopic {\n\t\t\t\tname\n\t\t\t}\n\t\t}\n\t}\n}\n\tquery
{\nrepo0: repository(owner: \"sourcegraph\", name: \"sourcegraph\") { ... on
Repository { ...RepositoryFields parent { nameWithOwner, isFork } } }\nrepo1:
repository(owner: \"keegancsmith\", name: \"sqlf\") { ... on Repository { ...RepositoryFields
parent { nameWithOwner, isFork } } }\nrepo2: repository(owner: \"tsenart\",
name: \"VEGETA\") { ... on Repository { ...RepositoryFields parent { nameWithOwner,
isFork } } }\nrepo3: repository(owner: \"tsenart\", name: \"go-tsz\") { ...
on Repository { ...RepositoryFields parent { nameWithOwner, isFork } } }\nrepo4:
repository(owner: \"sourcegraph\", name: \"about\") { ... on Repository { ...RepositoryFields
parent { nameWithOwner, isFork } } }\nrepo5: repository(owner: \"facebook\",
name: \"react\") { ... on Repository { ...RepositoryFields parent { nameWithOwner,
isFork } } }\nrepo6: repository(owner: \"torvalds\", name: \"linux\") { ...
on Repository { ...RepositoryFields parent { nameWithOwner, isFork } } }\nrepo7:
repository(owner: \"avelino\", name: \"awesome-go\") { ... on Repository { ...RepositoryFields
parent { nameWithOwner, isFork } } }\n}","variables":{}}'
form: {}
headers:
Accept:
- application/vnd.github.antiope-preview+json
Cache-Control:
- max-age=0
Content-Type:
- application/json; charset=utf-8
url: https://api.github.com/graphql
method: POST
response:
body: '{"data":{"repo0":{"id":"MDEwOlJlcG9zaXRvcnk0MTI4ODcwOA==","databaseId":41288708,"nameWithOwner":"sourcegraph/sourcegraph","description":"Universal
code search (self-hosted)","url":"https://github.com/sourcegraph/sourcegraph","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"ADMIN","stargazerCount":4888,"forkCount":579},"repo1":{"id":"MDEwOlJlcG9zaXRvcnk1ODk1ODk0Mg==","databaseId":58958942,"nameWithOwner":"keegancsmith/sqlf","description":"sqlf
generates parameterized SQL statements in Go, sprintf style","url":"https://github.com/keegancsmith/sqlf","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":54,"forkCount":3},"repo2":{"id":"MDEwOlJlcG9zaXRvcnkxMjA4MDU1MQ==","databaseId":12080551,"nameWithOwner":"tsenart/vegeta","description":"HTTP
load testing tool and library. It''s over 9000!","url":"https://github.com/tsenart/vegeta","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"ADMIN","stargazerCount":17729,"forkCount":1101},"repo3":{"id":"MDEwOlJlcG9zaXRvcnkxNDE3OTgwNzU=","databaseId":141798075,"nameWithOwner":"tsenart/go-tsz","description":"Time
series compression algorithm from Facebook''s Gorilla paper","url":"https://github.com/tsenart/go-tsz","isPrivate":false,"isFork":true,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"ADMIN","stargazerCount":4,"forkCount":5}}}'
body: '{"data":{"repo0":{"id":"MDEwOlJlcG9zaXRvcnk0MTI4ODcwOA==","databaseId":41288708,"nameWithOwner":"sourcegraph/sourcegraph","description":"Code
AI platform with Code Search & Cody","url":"https://github.com/sourcegraph/sourcegraph","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"ADMIN","stargazerCount":9145,"forkCount":1192,"diskUsage":1162487,"repositoryTopics":{"nodes":[{"topic":{"name":"sourcegraph"}},{"topic":{"name":"open-source"}},{"topic":{"name":"code-search"}},{"topic":{"name":"code-intelligence"}},{"topic":{"name":"repo-type-main"}},{"topic":{"name":"lsif-enabled"}}]},"parent":null},"repo1":{"id":"MDEwOlJlcG9zaXRvcnk1ODk1ODk0Mg==","databaseId":58958942,"nameWithOwner":"keegancsmith/sqlf","description":"sqlf
generates parameterized SQL statements in Go, sprintf style","url":"https://github.com/keegancsmith/sqlf","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":123,"forkCount":8,"diskUsage":17,"repositoryTopics":{"nodes":[{"topic":{"name":"golang"}},{"topic":{"name":"go"}},{"topic":{"name":"sql"}},{"topic":{"name":"sprintf-style"}}]},"parent":null},"repo2":{"id":"MDEwOlJlcG9zaXRvcnkxMjA4MDU1MQ==","databaseId":12080551,"nameWithOwner":"tsenart/vegeta","description":"HTTP
load testing tool and library. It''s over 9000!","url":"https://github.com/tsenart/vegeta","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":22112,"forkCount":1364,"diskUsage":2553,"repositoryTopics":{"nodes":[{"topic":{"name":"load-testing"}},{"topic":{"name":"go"}},{"topic":{"name":"benchmarking"}},{"topic":{"name":"http"}}]},"parent":null},"repo3":{"id":"MDEwOlJlcG9zaXRvcnkxNDE3OTgwNzU=","databaseId":141798075,"nameWithOwner":"tsenart/go-tsz","description":"Time
series compression algorithm from Facebook''s Gorilla paper","url":"https://github.com/tsenart/go-tsz","isPrivate":false,"isFork":true,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":5,"forkCount":4,"diskUsage":326,"repositoryTopics":{"nodes":[]},"parent":{"nameWithOwner":"dgryski/go-tsz","isFork":false}},"repo4":{"id":"MDEwOlJlcG9zaXRvcnkxNDk1MTg3ODM=","databaseId":149518783,"nameWithOwner":"sourcegraph/about","description":"Sourcegraph
blog, feature announcements, and website (about.sourcegraph.com)","url":"https://github.com/sourcegraph/about","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"ADMIN","stargazerCount":89,"forkCount":168,"diskUsage":524322,"repositoryTopics":{"nodes":[{"topic":{"name":"team"}}]},"parent":null},"repo5":{"id":"MDEwOlJlcG9zaXRvcnkxMDI3MDI1MA==","databaseId":10270250,"nameWithOwner":"facebook/react","description":"The
library for web and native user interfaces.","url":"https://github.com/facebook/react","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":215295,"forkCount":45372,"diskUsage":364386,"repositoryTopics":{"nodes":[{"topic":{"name":"javascript"}},{"topic":{"name":"react"}},{"topic":{"name":"frontend"}},{"topic":{"name":"declarative"}},{"topic":{"name":"ui"}},{"topic":{"name":"library"}}]},"parent":null},"repo6":{"id":"MDEwOlJlcG9zaXRvcnkyMzI1Mjk4","databaseId":2325298,"nameWithOwner":"torvalds/linux","description":"Linux
kernel source tree","url":"https://github.com/torvalds/linux","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":161083,"forkCount":51069,"diskUsage":4786194,"repositoryTopics":{"nodes":[]},"parent":null},"repo7":{"id":"MDEwOlJlcG9zaXRvcnkyMTU0MDc1OQ==","databaseId":21540759,"nameWithOwner":"avelino/awesome-go","description":"A
curated list of awesome Go frameworks, libraries and software","url":"https://github.com/avelino/awesome-go","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":111633,"forkCount":11434,"diskUsage":10576,"repositoryTopics":{"nodes":[{"topic":{"name":"golang"}},{"topic":{"name":"golang-library"}},{"topic":{"name":"go"}},{"topic":{"name":"awesome"}},{"topic":{"name":"awesome-list"}},{"topic":{"name":"hacktoberfest"}}]},"parent":null}}}'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, Deprecation,
Sunset
Cache-Control:
- no-cache
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
X-GitHub-Request-Id, Deprecation, Sunset
Content-Security-Policy:
- default-src 'none'
Content-Type:
- application/json; charset=utf-8
Date:
- Tue, 20 Jul 2021 18:51:56 GMT
- Thu, 16 Nov 2023 14:56:51 GMT
Referrer-Policy:
- origin-when-cross-origin, strict-origin-when-cross-origin
Server:
@ -47,8 +60,6 @@ interactions:
- max-age=31536000; includeSubdomains; preload
Vary:
- Accept-Encoding, Accept, X-Requested-With
X-Accepted-Oauth-Scopes:
- repo
X-Content-Type-Options:
- nosniff
X-Frame-Options:
@ -56,28 +67,29 @@ interactions:
X-Github-Media-Type:
- github.v4; param=antiope-preview; format=json
X-Github-Request-Id:
- B8E2:9885:4A73C8:4C0935:60F71B4C
X-Oauth-Scopes:
- ""
- F7AF:56FE:3B280F2C:3BD93ED2:65562DB3
X-Ratelimit-Resource:
- graphql
X-Ratelimit-Used:
- "1"
- "13"
X-Xss-Protection:
- "0"
status: 200 OK
code: 200
duration: ""
- request:
body: '{"query":"\nfragment RepositoryFields on Repository {\n\tid\n\tdatabaseId\n\tnameWithOwner\n\tdescription\n\turl\n\tisPrivate\n\tisFork\n\tisArchived\n\tisLocked\n\tisDisabled\n\tviewerPermission\n\tstargazerCount\n\tforkCount\n}\n\t\nquery($query:
body: '{"query":"\nfragment RepositoryFields on Repository {\n\tid\n\tdatabaseId\n\tnameWithOwner\n\tdescription\n\turl\n\tisPrivate\n\tisFork\n\tisArchived\n\tisLocked\n\tisDisabled\n\tviewerPermission\n\tstargazerCount\n\tforkCount\n\tdiskUsage\n\trepositoryTopics(first:100)
{\n\t\tnodes {\n\t\t\ttopic {\n\t\t\t\tname\n\t\t\t}\n\t\t}\n\t}\n}\n\t\nquery($query:
String!, $type: SearchType!, $after: String, $first: Int!) {\n\tsearch(query:
$query, type: $type, after: $after, first: $first) {\n\t\trepositoryCount\n\t\tpageInfo
{ hasNextPage, endCursor }\n\t\tnodes { ... on Repository { ...RepositoryFields
} }\n\t}\n}","variables":{"first":100,"query":"user:tsenart in:name patrol","type":"REPOSITORY"}}'
} }\n\t}\n}","variables":{"first":100,"query":"user:tsenart in:name patrol created:2007-06-01T00:00:00+00:00..2023-11-16T14:56:51+00:00","type":"REPOSITORY"}}'
form: {}
headers:
Accept:
- application/vnd.github.antiope-preview+json
Cache-Control:
- max-age=0
Content-Type:
- application/json; charset=utf-8
url: https://api.github.com/graphql
@ -85,23 +97,21 @@ interactions:
response:
body: '{"data":{"search":{"repositoryCount":1,"pageInfo":{"hasNextPage":false,"endCursor":"Y3Vyc29yOjE="},"nodes":[{"id":"MDEwOlJlcG9zaXRvcnkxNTM2NTcyNDU=","databaseId":153657245,"nameWithOwner":"tsenart/patrol","description":"Patrol
is an operator friendly distributed rate limiting HTTP API with strong eventually
consistent CvRDT based replication.","url":"https://github.com/tsenart/patrol","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"ADMIN","stargazerCount":33,"forkCount":3}]}}}'
consistent CvRDT based replication.","url":"https://github.com/tsenart/patrol","isPrivate":false,"isFork":false,"isArchived":false,"isLocked":false,"isDisabled":false,"viewerPermission":"READ","stargazerCount":35,"forkCount":3,"diskUsage":95,"repositoryTopics":{"nodes":[]}}]}}}'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, Deprecation,
Sunset
Cache-Control:
- no-cache
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
X-GitHub-Request-Id, Deprecation, Sunset
Content-Security-Policy:
- default-src 'none'
Content-Type:
- application/json; charset=utf-8
Date:
- Tue, 20 Jul 2021 18:51:56 GMT
- Thu, 16 Nov 2023 14:56:52 GMT
Referrer-Policy:
- origin-when-cross-origin, strict-origin-when-cross-origin
Server:
@ -110,8 +120,6 @@ interactions:
- max-age=31536000; includeSubdomains; preload
Vary:
- Accept-Encoding, Accept, X-Requested-With
X-Accepted-Oauth-Scopes:
- repo
X-Content-Type-Options:
- nosniff
X-Frame-Options:
@ -119,13 +127,11 @@ interactions:
X-Github-Media-Type:
- github.v4; param=antiope-preview; format=json
X-Github-Request-Id:
- B8E2:9885:4A740B:4C0981:60F71B4C
X-Oauth-Scopes:
- ""
- F7AF:56FE:3B2812A4:3BD94230:65562DB3
X-Ratelimit-Resource:
- graphql
X-Ratelimit-Used:
- "2"
- "14"
X-Xss-Protection:
- "0"
status: 200 OK

19
schema/github.schema.json
View File

@ -107,7 +107,9 @@
{ "required": ["id"] },
{ "required": ["pattern"] },
{ "required": ["forks"] },
{ "required": ["archived"] }
{ "required": ["archived"] },
{ "required": ["stars"] },
{ "required": ["size"] }
],
"properties": {
"archived": {
@ -132,13 +134,26 @@
"description": "Regular expression which matches against the name of a GitHub repository (\"owner/name\").",
"type": "string",
"format": "regex"
},
"size": {
"description": "If set, repositories with a size above the specified one will be excluded. Specify in kb.",
"type": "string",
"minLength": 2,
"pattern": "^[<>]{1}[=]{0,1}\\s*\\d+\\s*\\w+$"
},
"stars": {
"description": "If set, repositories stars less than the specified number will be.",
"type": "string",
"minLength": 2,
"pattern": "^[<>]{1}[=]{0,1}\\s*\\d+$"
}
}
},
"examples": [
[{ "forks": true }],
[{ "name": "owner/name" }, { "id": "MDEwOlJlcG9zaXRvcnkxMTczMDM0Mg==" }],
[{ "name": "vuejs/vue" }, { "name": "php/php-src" }, { "pattern": "^topsecretorg/.*" }]
[{ "name": "vuejs/vue" }, { "name": "php/php-src" }, { "pattern": "^topsecretorg/.*" }],
[{ "size": ">= 1GB", "stars": "< 100" }]
]
},
"repositoryQuery": {

4
schema/schema.go
View File

@ -787,6 +787,10 @@ type ExcludedGitHubRepo struct {
Name string `json:"name,omitempty"`
// Pattern description: Regular expression which matches against the name of a GitHub repository ("owner/name").
Pattern string `json:"pattern,omitempty"`
// Size description: If set, repositories with a size above the specified one will be excluded. Specify in kb.
Size string `json:"size,omitempty"`
// Stars description: If set, repositories stars less than the specified number will be.
Stars string `json:"stars,omitempty"`
}
type ExcludedGitLabProject struct {
// EmptyRepos description: Whether to exclude empty repositories.