diff --git a/CHANGELOG.md b/CHANGELOG.md index 70f5563..ef2bddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://www.npmjs.com/package/@google/repo?activeTab=versions +## [4.8.0](https://github.com/googleapis/github-repo-automation/compare/v4.7.0...v4.8.0) (2022-01-17) + + +### Features + +* delay by default ([#568](https://github.com/googleapis/github-repo-automation/issues/568)) ([2e522e4](https://github.com/googleapis/github-repo-automation/commit/2e522e4ef4ff8b9aad6ce204b5736abe47bab0f2)) +* detailed debugging now configured with NODE_DEBUG=repo ([#567](https://github.com/googleapis/github-repo-automation/issues/567)) ([5306b21](https://github.com/googleapis/github-repo-automation/commit/5306b2155387e5d99f199762390aff0baa12ffe8)) + ## [4.7.0](https://github.com/googleapis/github-repo-automation/compare/v4.6.1...v4.7.0) (2022-01-14) diff --git a/README.md b/README.md index 8f59607..fe6257f 100644 --- a/README.md +++ b/README.md @@ -311,5 +311,13 @@ There are settings you can configure to make this less likely: When running against a large number of repos, try the following as a starting point: ```bash -repo [command] --delay=2500 --concurrency=4 --retry --title='.*some title.*' +repo [command] --delay=1000 --concurrency=2 --retry --title='.*some title.*' ``` + +If you are continuing to run into problems, run with: + +``` +NODE_DEBUG=repo repo [command] --delay=1000 --concurrency=2 --retry --title='.*some title.*' +``` + +And share the debug output in an issue, along with the command you are running. diff --git a/package.json b/package.json index c72ddd9..63e838e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@google/repo", - "version": "4.7.0", + "version": "4.8.0", "repository": "googleapis/github-repo-automation", "description": "A tool for automating multiple GitHub repositories.", "engines": { diff --git a/src/lib/asyncItemIterator.ts b/src/lib/asyncItemIterator.ts index c6a641b..c217cc9 100644 --- a/src/lib/asyncItemIterator.ts +++ b/src/lib/asyncItemIterator.ts @@ -16,6 +16,8 @@ import * as meow from 'meow'; import {meowFlags} from '../cli'; import Q from 'p-queue'; import ora = require('ora'); +import {debuglog} from 'util'; +const debug = debuglog('repo'); import * as configLib from './config'; import {GitHub, GitHubRepository, PullRequest, Issue} from './github'; @@ -37,9 +39,9 @@ async function retryException( return result; } catch (err) { if (i < retryStrategy.length) { - console.error(`\noperation failed: ${err.toString()}`); + debug(`operation failed: ${err.toString()}`); const delay = nextDelay(retryStrategy[i]); - console.info(`\nretrying in ${delay}ms`); + debug(`retrying in ${delay}ms`); await delayMs(delay); continue; } @@ -63,7 +65,7 @@ async function retryBoolean( const result = await eventual(); if (!result && i < retryStrategy.length) { const delay = nextDelay(retryStrategy[i]); - console.info(`\nretrying in ${delay}ms`); + debug(`retrying in ${delay}ms`); await delayMs(delay); continue; } else { @@ -153,7 +155,7 @@ async function process( : 15; // Introduce a delay between requests, this may be necessary if // processing many repos in a row to avoid rate limits: - const delay: number = cli.flags.delay ? Number(cli.flags.delay) : 0; + const delay: number = cli.flags.delay ? Number(cli.flags.delay) : 500; const retry: boolean = cli.flags.retry ? Boolean(cli.flags.retry) : false; const config = await configLib.getConfig(); const retryStrategy = retry @@ -187,12 +189,12 @@ async function process( let localItems; if (processIssues) { localItems = await retryException(async () => { - if (delay) delayMs(delay); + if (delay) await delayMs(nextDelay(delay)); return await repo.listIssues(); }, retryStrategy); } else { localItems = await retryException(async () => { - if (delay) delayMs(delay); + if (delay) await delayMs(nextDelay(delay)); return await repo.listPullRequests(); }, retryStrategy); } @@ -266,7 +268,7 @@ async function process( if (processIssues) { const opts = options as IssueIteratorOptions; result = await retryBoolean(async () => { - if (delay) await delayMs(delay); + if (delay) await delayMs(nextDelay(delay)); return await opts.processMethod( itemSet.repo, itemSet.item as Issue, @@ -276,7 +278,7 @@ async function process( } else { const opts = options as PRIteratorOptions; result = await retryBoolean(async () => { - if (delay) await delayMs(delay); + if (delay) await delayMs(nextDelay(delay)); return await opts.processMethod( itemSet.repo, itemSet.item as PullRequest, diff --git a/src/lib/github.ts b/src/lib/github.ts index c4ed5f1..581da4c 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -16,14 +16,36 @@ * @fileoverview Wraps some octokit GitHub API calls. */ -import {Gaxios} from 'gaxios'; +import {Gaxios, GaxiosPromise, GaxiosOptions} from 'gaxios'; import {Config} from './config'; +import {debuglog} from 'util'; +const debug = debuglog('repo'); export function getClient(config: Config) { - return new Gaxios({ + const client = new Gaxios({ baseURL: 'https://api.github.com', headers: {Authorization: `token ${config.githubToken}`}, }); + // Report rate limit information if NODE_DEBUG=repo set. + let counter = 0; + const request = client.request.bind(client); + client.request = async (opts: GaxiosOptions): GaxiosPromise => { + const resp = await request(opts); + const rateLimit = resp.headers['x-ratelimit-limit'] + ? Number(resp.headers['x-ratelimit-limit']) + : 0; + const rateLimitRemaining = resp.headers['x-ratelimit-remaining'] + ? Number(resp.headers['x-ratelimit-remaining']) + : 0; + const reset = resp.headers['x-ratelimit-reset']; + if (counter++ % 10 === 0) { + debug( + `GitHub rate limit: limit = ${rateLimit} remaining = ${rateLimitRemaining} reset epoch = ${reset}` + ); + } + return resp; + }; + return client; } interface SearchReposResponse {