diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index f3ca556..7488369 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest - digest: sha256:ddb19a6df6c1fa081bc99fb29658f306dd64668bc26f75d1353b28296f3a78e6 -# created: 2022-06-07T21:18:30.024751809Z + digest: sha256:bb493bf01d28519e82ab61c490c20122c85a7119c03a978ad0c34b4239fbad15 +# created: 2022-08-23T18:40:55.597313991Z diff --git a/.kokoro/publish.sh b/.kokoro/publish.sh index 77a5def..949e3e1 100755 --- a/.kokoro/publish.sh +++ b/.kokoro/publish.sh @@ -19,7 +19,6 @@ set -eo pipefail export NPM_CONFIG_PREFIX=${HOME}/.npm-global # Start the releasetool reporter -python3 -m pip install gcp-releasetool python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script cd $(dirname $0)/.. diff --git a/.kokoro/release/docs.sh b/.kokoro/release/docs.sh index 4c866c8..1d8f3f4 100755 --- a/.kokoro/release/docs.sh +++ b/.kokoro/release/docs.sh @@ -29,7 +29,6 @@ npm run docs # create docs.metadata, based on package.json and .repo-metadata.json. npm i json@9.0.6 -g -python3 -m pip install --user gcp-docuploader python3 -m docuploader create-metadata \ --name=$(cat .repo-metadata.json | json name) \ --version=$(cat package.json | json version) \ diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a4ae1..6af1d86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://www.npmjs.com/package/@google/repo?activeTab=versions +## [6.1.0](https://github.com/googleapis/github-repo-automation/compare/v6.0.0...v6.1.0) (2022-08-29) + + +### Features + +* use cache when listing PRs and issues ([#616](https://github.com/googleapis/github-repo-automation/issues/616)) ([6572a87](https://github.com/googleapis/github-repo-automation/commit/6572a87c73be076d6b685a73a2e0a1b81fa5ae55)) + + +### Bug Fixes + +* remove pip install statements ([#1546](https://github.com/googleapis/github-repo-automation/issues/1546)) ([#613](https://github.com/googleapis/github-repo-automation/issues/613)) ([a046b29](https://github.com/googleapis/github-repo-automation/commit/a046b295487d5da45597d26110effad0c15982ad)) + ## [6.0.0](https://github.com/googleapis/github-repo-automation/compare/v5.0.0...v6.0.0) (2022-07-27) diff --git a/package.json b/package.json index c8fc414..ec77be5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@google/repo", - "version": "6.0.0", + "version": "6.1.0", "repository": "googleapis/github-repo-automation", "description": "A tool for automating multiple GitHub repositories.", "engines": { @@ -18,6 +18,7 @@ "dependencies": { "@types/command-line-usage": "^5.0.2", "@types/tmp": "^0.2.3", + "async-mutex": "^0.3.2", "chalk": "^5.0.1", "command-line-usage": "^6.1.3", "extend": "^3.0.2", @@ -47,7 +48,7 @@ "nock": "^13.2.9", "proxyquire": "^2.1.3", "sinon": "^14.0.0", - "typescript": "^4.7.4" + "typescript": "~4.7.4" }, "scripts": { "lint": "gts check", diff --git a/samples/package.json b/samples/package.json index 6f8acbe..bfc2a7d 100644 --- a/samples/package.json +++ b/samples/package.json @@ -14,7 +14,7 @@ "test": "mocha system-test" }, "dependencies": { - "@google/repo": "^6.0.0" + "@google/repo": "^6.1.0" }, "devDependencies": { "mocha": "^8.0.0" diff --git a/src/cli.ts b/src/cli.ts index 5f7ae3e..9f5da7c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -97,6 +97,7 @@ const cli = meow( concurrency: {type: 'string'}, author: {type: 'string'}, yespleasedoit: {type: 'boolean'}, + nocache: {type: 'boolean'}, }, } ); diff --git a/src/lib/asyncItemIterator.ts b/src/lib/asyncItemIterator.ts index 959a539..3fbae55 100644 --- a/src/lib/asyncItemIterator.ts +++ b/src/lib/asyncItemIterator.ts @@ -20,6 +20,7 @@ const debug = debuglog('repo'); import * as configLib from './config.js'; import {GitHub, GitHubRepository, PullRequest, Issue} from './github.js'; +import {CacheType, readFromCache, saveToCache} from './cache.js'; /** * Retry the promise returned by a function if the promise throws @@ -127,7 +128,7 @@ export interface IteratorOptions { async function process( cli: meow.Result, options: PRIteratorOptions | IssueIteratorOptions, - processIssues = false + type: CacheType ) { if ( !cli.flags.title && @@ -175,27 +176,40 @@ async function process( const orb1 = ora( `[${scanned}/${repos.length}] Scanning repos for ${ - processIssues ? 'issues' : 'PR' + type === 'issues' ? 'issues' : 'PR' }s` ).start(); // Concurrently find all PRs or issues in all relevant repositories + let cached = 0; const q = new Q({concurrency}); q.addAll( repos.map(repo => { return async () => { try { let localItems; - if (processIssues) { - localItems = await retryException(async () => { - if (delay) await delayMs(nextDelay(delay)); - return await repo.listIssues(); - }, retryStrategy); - } else { - localItems = await retryException(async () => { - if (delay) await delayMs(nextDelay(delay)); - return await repo.listPullRequests(); - }, retryStrategy); + if (!cli.flags.nocache) { + const cachedData = await readFromCache(repo, type); + if (cachedData !== null) { + localItems = + (type === 'issues' ? cachedData.issues : cachedData.prs) ?? []; + ++cached; + } + } + if (!localItems) { + if (type === 'issues') { + localItems = await retryException(async () => { + if (delay) await delayMs(nextDelay(delay)); + return await repo.listIssues(); + }, retryStrategy); + await saveToCache(repo, type, {prs: [], issues: localItems}); + } else { + localItems = await retryException(async () => { + if (delay) await delayMs(nextDelay(delay)); + return await repo.listPullRequests(); + }, retryStrategy); + await saveToCache(repo, type, {prs: localItems, issues: []}); + } } items.push( ...localItems.map(item => { @@ -203,9 +217,11 @@ async function process( }) ); scanned++; - orb1.text = `[${scanned}/${repos.length}] Scanning repos for PRs`; + orb1.text = `[${scanned}/${repos.length}] Scanning repos for ${ + type === 'issues' ? 'issue' : 'PR' + }`; } catch (err) { - error = `cannot list open ${processIssues ? 'issue' : 'PR'}s: ${( + error = `cannot list open ${type === 'issues' ? 'issue' : 'PR'}s: ${( err as Error ).toString()}`; } @@ -213,6 +229,11 @@ async function process( }) ); await q.onIdle(); + if (cached > 0) { + console.log( + `\nData for ${cached} repositories was taken from cache. Use --nocache to override.` + ); + } // Filter the list of PRs or Issues to ones who match the PR title and/or the branch name items = items.filter(itemSet => itemSet.item.title.match(regex)); @@ -246,7 +267,7 @@ async function process( orb1.succeed( `[${scanned}/${repos.length}] repositories scanned, ${ items.length - } matching ${processIssues ? 'issue' : 'PR'}s found` + } matching ${type === 'issues' ? 'issue' : 'PR'}s found` ); // Concurrently process each relevant PR or Issue @@ -260,11 +281,11 @@ async function process( if (title.match(regex)) { orb2.text = `[${processed}/${items.length}] ${ options.commandActive - } ${processIssues ? 'issue' : 'PR'}s`; + } ${type === 'issues' ? 'issue' : 'PR'}s`; let result; // By setting the process issues flag, the iterator can be made to // process a list of issues rather than PR: - if (processIssues) { + if (type === 'issues') { const opts = options as IssueIteratorOptions; result = await retryBoolean(async () => { if (delay) await delayMs(nextDelay(delay)); @@ -293,7 +314,7 @@ async function process( processed++; orb2.text = `[${processed}/${items.length}] ${ options.commandActive - } ${processIssues ? 'issue' : 'PR'}s`; + } ${type === 'issues' ? 'issue' : 'PR'}s`; } }; }) @@ -301,7 +322,7 @@ async function process( await q.onIdle(); orb2.succeed( - `[${processed}/${items.length}] ${processIssues ? 'issue' : 'PR'}s ${ + `[${processed}/${items.length}] ${type === 'issues' ? 'issue' : 'PR'}s ${ options.commandNamePastTense }` ); @@ -317,7 +338,7 @@ async function process( console.log( `Successfully processed: ${successful.length} ${ - processIssues ? 'issue' : 'PR' + type === 'issues' ? 'issue' : 'PR' }s` ); for (const item of successful) { @@ -326,7 +347,9 @@ async function process( if (failed.length > 0) { console.log( - `Unable to process: ${failed.length} ${processIssues ? 'issue' : 'PR'}(s)` + `Unable to process: ${failed.length} ${ + type === 'issues' ? 'issue' : 'PR' + }(s)` ); for (const item of failed) { console.log(` ${item.html_url.padEnd(maxUrlLength, ' ')} ${item.title}`); @@ -335,7 +358,7 @@ async function process( if (error) { console.log( - `Error when processing ${processIssues ? 'issue' : 'PR'}s: ${error}` + `Error when processing ${type === 'issues' ? 'issue' : 'PR'}s: ${error}` ); } } @@ -345,7 +368,7 @@ export async function processPRs( cli: meow.Result, options: PRIteratorOptions | IssueIteratorOptions ) { - return process(cli, options, false); + return process(cli, options, 'prs'); } // Shorthand for processing list of issues: @@ -353,5 +376,5 @@ export async function processIssues( cli: meow.Result, options: PRIteratorOptions | IssueIteratorOptions ) { - return process(cli, options, true); + return process(cli, options, 'issues'); } diff --git a/src/lib/cache.ts b/src/lib/cache.ts new file mode 100644 index 0000000..75acb2b --- /dev/null +++ b/src/lib/cache.ts @@ -0,0 +1,102 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {existsSync} from 'fs'; +import {mkdir, readFile, stat, unlink, writeFile} from 'fs/promises'; +import {tmpdir} from 'os'; +import {join} from 'path'; +import {Mutex} from 'async-mutex'; +import {GitHubRepository, Issue, PullRequest} from './github'; + +const cacheDirectory = join(tmpdir(), 'google-repo-cache'); +const cacheMaxAge = 60 * 60 * 1000; // 1 hour + +export type CachedData = {issues?: Issue[]; prs?: PullRequest[]}; +export type CacheType = 'prs' | 'issues'; + +const mutex = new Mutex(); + +async function initCache() { + if (!existsSync(cacheDirectory)) { + await mkdir(cacheDirectory); + } +} + +function cacheFilename(repo: GitHubRepository, type: CacheType) { + const owner = repo.repository.owner.login; + const name = repo.repository.name; + return join( + cacheDirectory, + `${owner}-${name}`.replace(/\W/g, '-') + `-${type}` + ); +} + +export async function readFromCache(repo: GitHubRepository, type: CacheType) { + const release = await mutex.acquire(); + try { + await initCache(); + const cacheFile = cacheFilename(repo, type); + if (!existsSync(cacheFile)) { + return null; + } + const cacheStat = await stat(cacheFile); + const mtime = cacheStat.mtimeMs ?? cacheStat.ctimeMs; + const now = Date.now(); + if (now - mtime >= cacheMaxAge) { + await unlink(cacheFile); + return null; + } + + const content = await readFile(cacheFile); + const json = JSON.parse(content.toString()) as CachedData; + return json; + } finally { + release(); + } +} + +export async function saveToCache( + repo: GitHubRepository, + type: CacheType, + data: CachedData +) { + const release = await mutex.acquire(); + try { + await initCache(); + const cacheFile = cacheFilename(repo, type); + if (!data.issues) { + data.issues = []; + } + if (!data.prs) { + data.prs = []; + } + const content = JSON.stringify(data, null, ' '); + await writeFile(cacheFile, content); + } finally { + release(); + } +} + +export async function deleteCache(repo: GitHubRepository, type: CacheType) { + const release = await mutex.acquire(); + try { + await initCache(); + const cacheFile = cacheFilename(repo, type); + if (existsSync(cacheFile)) { + await unlink(cacheFile); + } + } finally { + release(); + } +} diff --git a/test/async-iterator.ts b/test/async-iterator.ts index 1d7e69a..bd11f3f 100644 --- a/test/async-iterator.ts +++ b/test/async-iterator.ts @@ -25,6 +25,7 @@ import * as sinon from 'sinon'; import {GitHubRepository, PullRequest} from '../src/lib/github.js'; import * as config from '../src/lib/config.js'; import {processPRs} from '../src/lib/asyncItemIterator.js'; +import {deleteCache} from '../src/lib/cache.js'; nock.disableNetConnect(); @@ -44,6 +45,7 @@ describe('asyncItemIterator', () => { flags: { title: '.*', retry: true, + nocache: true, }, } as unknown as ReturnType; const githubRequests = nock('https://api.github.com') @@ -76,6 +78,97 @@ describe('asyncItemIterator', () => { }); githubRequests.done(); }); + + it('should use cache', async () => { + await deleteCache( + { + repository: { + owner: { + login: 'googleapis', + }, + name: 'foo', + }, + } as GitHubRepository, + 'prs' + ); + sinon.stub(config.GetConfig, 'getConfig').resolves({ + githubToken: 'abc123', + clonePath: '/foo/bar', + repoSearch: + 'org:googleapis language:typescript language:javascript is:public archived:false', + }); + const cli = { + flags: { + title: '.*', + retry: true, + nocache: false, + }, + } as unknown as ReturnType; + const githubRequests = nock('https://api.github.com') + // for the first invocation + .get( + '/search/repositories?per_page=100&page=1&q=org%3Agoogleapis%20language%3Atypescript%20language%3Ajavascript%20is%3Apublic%20archived%3Afalse' + ) + .reply(200, { + items: [ + { + full_name: 'googleapis/foo', + default_branch: 'main', + }, + ], + }) + .get('/repos/googleapis/foo/pulls?state=open&page=1') + .reply(200, [ + { + title: 'feat: foo pull request', + html_url: 'http://example.com/pr/2', + }, + ]) + .get('/repos/googleapis/foo/pulls?state=open&page=2') + .reply(200) + // just the repositories for the second invocation + .get( + '/search/repositories?per_page=100&page=1&q=org%3Agoogleapis%20language%3Atypescript%20language%3Ajavascript%20is%3Apublic%20archived%3Afalse' + ) + .reply(200, { + items: [ + { + full_name: 'googleapis/foo', + default_branch: 'main', + }, + ], + }); + + // First invocation: fill the cache + await processPRs(cli, { + commandName: 'update', + commandActive: 'updating', + commandNamePastTense: 'updated', + commandDesc: + 'Iterates over all PRs matching the regex, and updates them, to the latest on the base branch.', + processMethod: async () => { + return true; + }, + }); + + // Second invocation: must use the cache + const titles: string[] = []; + await processPRs(cli, { + commandName: 'update', + commandActive: 'updating', + commandNamePastTense: 'updated', + commandDesc: + 'Iterates over all PRs matching the regex, and updates them, to the latest on the base branch.', + processMethod: async (_repo: unknown, pr: {title: string}) => { + titles.push(pr.title); + return true; + }, + }); + githubRequests.done(); + assert.strictEqual(titles.length, 1); + assert.strictEqual(titles[0], 'feat: foo pull request'); + }); + it('should retry process method if it returns false', async () => { sinon.stub(config.GetConfig, 'getConfig').resolves({ githubToken: 'abc123', @@ -88,6 +181,7 @@ describe('asyncItemIterator', () => { flags: { title: '.*', retry: true, + nocache: true, }, } as unknown as ReturnType; const githubRequests = nock('https://api.github.com')