docs: embeddings list automation (#52879)

This PR will merge and update the list of embedded repos we have on
sourcegraph.com. It uses the sourcegraph graphQL to pull a list of all
embedded repos, filters & sorts the list, and then updates the markdown
`embedded-repos.md`.


The update workflow will run everyday @ `0:00 UTC` and the automatic
merge follows @ `0:05 UTC`.


## Test plan
`npm test`

run locally:
1. get access from
[here](https://start.1password.com/open/i?a=HEDEDSLHPBFGRBTKAKJWE23XX4&v=dnrhbauihkhjs5ag6vszsme45a&i=za6swt25wax766z6pe7wpczxxe&h=team-sourcegraph.1password.com)
2. set access token `set SOURCEGRAPH_DOCS_ACCESS_TOKEN=<access_token>`
or hard code
`const access_token = <access_token>`
3. run `ts-node src/index.ts`

output should be embedded-repos.md
<!-- All pull requests REQUIRE a test plan:
https://docs.sourcegraph.com/dev/background-information/testing_principles
-->

---------

Co-authored-by: Beatrix <68532117+abeatrix@users.noreply.github.com>
This commit is contained in:
Kalan 2023-06-05 15:30:34 -07:00 committed by GitHub
parent 5706e85663
commit c0ccde2ccf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 489 additions and 0 deletions

29
.github/workflows/merge-pr.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: Merge embeddings list updates
on:
schedule:
# everyday at 0:10 UTC
- cron: 10 0 * * *
workflow_dispatch: {}
permissions: write-all
jobs:
run:
runs-on: ubuntu-latest
env:
GITHUB_TOKEN: ${{ secrets.MERGE_PR_TOKEN_DEVX_BOT }}
steps:
- uses: actions/checkout@v3
- run: |
PR_NUMBER="$(gh pr list --label automerge --label embeddings-list-update --json number | jq -r '.[0].number')"
if [ "$PR_NUMBER" = "null" ]
then
echo "no oustanding PR...skipping"
exit 0
else
echo "approving PR"
gh pr review $PR_NUMBER --approve
echo "squashing PR: $PR_NUMBER"
gh pr merge $PR_NUMBER --auto --squash
fi

View File

@ -0,0 +1,44 @@
name: Update list of Embedded Repos
on:
schedule:
# everyday at 0:00 UTC
- cron: 0 0 * * *
workflow_dispatch: {}
jobs:
build:
runs-on: ubuntu-latest
env:
SOURCEGRAPH_DOCS_ACCESS_TOKEN: ${{ secrets.SOURCEGRAPH_DOCS_ACCESS_TOKEN }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 18
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H:%M:%S')"
- name: Install dependencies
run: npm install
- name: Compile Javascript
run: tsc src/index.ts --outDir build
- name: check output
run: ls
- name: run script
run: node build/index.js
- name: move into doc folder
run: mv embedded-repos.md doc/cody/embedded-repos.md
- name: check list in doc folder
run: head -n 10 doc/cody/embedded-repos.md
- name: Create Pull Request
uses: peter-evans/create-pull-request@v4
with:
title: "Cody: Update list of embedded repos"
commit-message: update list - ${{ steps.date.outputs.date }}
author: github-actions <github-actions[bot]@users.noreply.github.com>
body: |
- This is an automated PR to update the list of [embedded repos](https://docs.sourcegraph.com/cody/embedded-repos).
labels: |
embeddings-list-update
automerge

2
dev/update-embeddings-list/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
node_modules/
build/

View File

@ -0,0 +1,3 @@
module.exports = {
presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
}

View File

@ -0,0 +1,24 @@
{
"name": "list-embeddings",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start:dev": "npx nodemon",
"build": "ts-node ./src/index.ts",
"test": "jest ./src/index.test.ts --forceExit"
},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@types/node": "^18.16.1",
"ts-node": "^10.9.1",
"typescript": "^5.0.4"
},
"dependencies": {
"node-fetch": "^2.6.5",
"typescript": "^5.0.4",
"@types/node": "^18.16.1"
}
}

View File

@ -0,0 +1,141 @@
import { filter, sort, embeddedReposToMarkdown } from './index'
const fetch = require('node-fetch')
interface Repo {
name: string
url: string
}
interface Embedding {
id: string
state: string
repo: Repo
}
describe('filter', () => {
test('filters and sorts repo embedding jobs', done => {
const input: Embedding[] = [
{
id: '1',
state: 'COMPLETED',
repo: {
name: 'b',
url: 'https://github.com/b',
},
},
{
id: '2',
state: 'COMPLETED',
repo: {
name: 'a',
url: 'https://github.com/a',
},
},
{
id: '3',
state: 'PROCESSING',
repo: {
name: 'c',
url: 'https://github.com/c',
},
},
]
const expected = [
{
id: '1',
state: 'COMPLETED',
repo: {
name: 'b',
url: 'https://github.com/b',
},
},
{
id: '2',
state: 'COMPLETED',
repo: {
name: 'a',
url: 'https://github.com/a',
},
},
]
expect(filter(input)).toEqual(expected)
done()
})
})
describe('sort', () => {
test('sorts repos alphabetically', () => {
const input: Repo[] = [
{
name: 'c/repo1',
url: 'https://github.com/c/repo1',
},
{
name: 'b/repo2',
url: 'https://github.com/b/repo2',
},
{
name: 'a/repo3',
url: 'https://github.com/a/repo3',
},
]
const expected = [
{
name: 'a/repo3',
url: 'https://github.com/a/repo3',
},
{
name: 'b/repo2',
url: 'https://github.com/b/repo2',
},
{
name: 'c/repo1',
url: 'https://github.com/c/repo1',
},
]
expect(sort(input)).toEqual(expected)
})
})
describe('embeddedReposToMarkdown', () => {
test('generates markdown for embedded repos', () => {
const input: Embedding[] = [
{
id: '1',
state: 'COMPLETED',
repo: {
name: 'sourcegraph/sourcegraph',
url: 'https://github.com/sourcegraph/sourcegraph',
},
},
{
id: '2',
state: 'COMPLETED',
repo: {
name: 'golang/go',
url: 'https://github.com/golang/go',
},
},
]
const expected = `# Embeddings for repositories with 5+ stars
Last updated: ${new Date().toLocaleString('en-US', {
month: '2-digit',
day: '2-digit',
year: 'numeric',
hour: '2-digit',
minute: '2-digit',
timeZoneName: 'short',
})}
1. [golang/go](https://github.com/golang/go)
1. [sourcegraph/sourcegraph](https://github.com/sourcegraph/sourcegraph)
`
expect(embeddedReposToMarkdown(input)).toEqual(expected)
})
test('throws error if no repos', () => {
expect(() => embeddedReposToMarkdown(undefined)).toThrowError('no embedded repos found!')
})
})

View File

@ -0,0 +1,133 @@
import * as fs from 'fs'
const fetch = require('node-fetch')
interface Repo {
name: string
url: string
}
interface Embedding {
id: string
state: string
repo: Repo
}
const access_token = process.env.SOURCEGRAPH_DOCS_ACCESS_TOKEN
const endpoint = 'https://sourcegraph.com/.api/graphql'
async function start() {
try {
let embeddedRepos = await gqlRequest(endpoint)
embeddedRepos = filter(embeddedRepos)
let markdown = embeddedReposToMarkdown(embeddedRepos)
fs.writeFileSync('embedded-repos.md', markdown)
} catch (err) {
console.error(err)
}
}
async function gqlRequest(endpoint: string): Promise<Embedding[]> {
const embeddedRepos: Embedding[] = []
try {
let pagination = true
let endCursor = ''
while (pagination) {
let query = `
query RepoEmbeddingJobs {
repoEmbeddingJobs(first: 100, after: ${endCursor ? '"' + endCursor + '"' : null}) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
id
state
repo {
name
url
}
}
}
}`
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `token ${access_token}`,
},
body: JSON.stringify({ query }),
})
const { data } = await response.json()
embeddedRepos.push(...data.repoEmbeddingJobs.nodes)
pagination = data.repoEmbeddingJobs.pageInfo.hasNextPage
endCursor = data.repoEmbeddingJobs.pageInfo.endCursor
}
} catch (err) {
console.error(err)
}
return embeddedRepos
}
export function filter(repos: Embedding[]): Embedding[] {
const filtered = repos.filter(item => item.state === 'COMPLETED')
const result = Array.from(new Set(filtered.map(x => x.repo?.name))).map(name =>
filtered.find(x => x.repo?.name === name)
) as Embedding[]
return result
}
export function sort(repos: Repo[]): Repo[] {
// sort alphabetically
repos.sort((a: Repo, b: Repo) => (a.name > b.name ? 1 : b.name > a.name ? -1 : 0))
return repos
}
export function embeddedReposToMarkdown(repos: Embedding[] | undefined): string {
const listOfRepos: Repo[] = []
const today = new Date()
let markdown = `# Embeddings for repositories with 5+ stars\n\n`
markdown += `Last updated: ${today.toLocaleString('en-US', {
month: '2-digit',
day: '2-digit',
year: 'numeric',
hour: '2-digit',
minute: '2-digit',
timeZoneName: 'short',
})} \n\n`
repos?.forEach(repo => {
let repoName: string | undefined = repo.repo?.name
let repoUrl: string | undefined = repo.repo?.url
if (repoName === undefined || repoUrl === undefined) {
return
}
const r: Repo = {
name: repo.repo?.name.replace('github.com/', ''),
url: repoUrl.replace(/^\//, 'https://'),
}
listOfRepos.push(r)
})
if (listOfRepos.length === 0) {
throw new Error('no embedded repos found!')
}
const sorted = sort(listOfRepos)
sorted?.forEach(repo => {
markdown += `1. [${repo?.name}](${repo?.url})\n`
})
return markdown
}
start()

View File

@ -0,0 +1,113 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
"lib": [
"es6",
] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
"module": "commonjs" /* Specify what module code is generated. */,
"rootDir": "src" /* Specify the root folder within your source files. */,
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
// "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
// "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
"resolveJsonModule": true /* Enable importing .json files. */,
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
"allowJs": true /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */,
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "build" /* Specify an output folder for all emitted files. */,
// "removeComments": true, /* Disable emitting comments. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": true /* Enable all strict type-checking options. */,
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied 'any' type. */,
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */,
},
"include": ["src/**/*"],
"exclude": ["doc", "**/*.test.ts"],
}