extsvc: Change default rate limits of npm and Go external services (#34042)

This commit changes the default rate limit configs of npm and Go
external services to be unlimited by default, and for a higher default
of 16 req/s to be filled in when auto-completing in the Monaco editor of
the UI.

Unlike the GitHub or GitLab APIs, neither the public npm registry nor
the public Go modules proxy (i.e proxy.golang.org) document an enforced
req/s rate limit. For private registries / proxies, site-admins can
manually configure their desired rate limits.

The request pattern for npm and Go sources is also different than GitHub
and GitLab sources in that they execute a lot more individual requests
since there are no batch endpoints used (except for multiple versions of
a single package). This means we need a far higher rate limit in order
for these external services to sync in a reasonable amount of time,
as well as leave some quota for lazy-syncing individual packages via
dependencies search.

internal/extsvc/types.go

@@ -436,14 +436,20 @@ func GetLimitFromConfig(kind string, config interface{}) (rate.Limit, error) {
 			limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
 		}
 	case *schema.NpmPackagesConnection:
-		// 3000 per hour is the same default we use in our schema
-		limit = rate.Limit(3000.0 / 3600.0)
+		// Unlike the GitHub or GitLab APIs, the public npm registry (i.e. https://registry.npmjs.org)
+		// doesn't document an enforced req/s rate limit AND we do a lot more individual
+		// requests in comparison since they don't offer enough batch APIs. For private registries,
+		// site-admins can manually configure their desired rate limits if needed.
+		limit = rate.Inf
 		if c != nil && c.RateLimit != nil {
 			limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
 		}
 	case *schema.GoModulesConnection:
-		// 3000 per hour is the same default we use in our schema
-		limit = rate.Limit(3000.0 / 3600.0)
+		// Unlike the GitHub or GitLab APIs, the public npm registry (i.e. https://proxy.golang.org)
+		// doesn't document an enforced req/s rate limit AND we do a lot more individual
+		// requests in comparison since they don't offer enough batch APIs. For private proxies,
+		// site-admins can manually configure their desired rate limits if needed.
+		limit = rate.Inf
 		if c != nil && c.RateLimit != nil {
 			limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
 		}

internal/extsvc/types_test.go

@@ -112,7 +112,7 @@ func TestExtractRateLimitConfig(t *testing.T) {
 			name:   "NPM default",
 			config: `{"registry": "https://registry.npmjs.org"}`,
 			kind:   KindNpmPackages,
-			want:   3000.0 / 3600.0,
+			want:   rate.Inf,
 		},
 		{
 			name:   "NPM non-default",
@@ -124,7 +124,7 @@ func TestExtractRateLimitConfig(t *testing.T) {
 			name:   "Go mod default",
 			config: `{"urls": ["https://example.com"]}`,
 			kind:   KindGoModules,
-			want:   3000.0 / 3600.0,
+			want:   rate.Inf,
 		},
 		{
 			name:   "Go mod non-default",

schema/go-modules.schema.json

@@ -31,13 +31,13 @@
         "requestsPerHour": {
           "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.",
           "type": "number",
-          "default": 3000,
+          "default": 57600,
           "minimum": 0
         }
       },
       "default": {
         "enabled": true,
-        "requestsPerHour": 3000
+        "requestsPerHour": 57600
       }
     },
     "dependencies": {

schema/npm-packages.schema.json

@@ -34,13 +34,13 @@
         "requestsPerHour": {
           "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.",
           "type": "number",
-          "default": 3000,
+          "default": 57600,
           "minimum": 0
         }
       },
       "default": {
         "enabled": true,
-        "requestsPerHour": 3000
+        "requestsPerHour": 57600
       }
     },
     "dependencies": {