diff --git a/lib/get-client.js b/lib/get-client.js index f0725a65..66584aaf 100644 --- a/lib/get-client.js +++ b/lib/get-client.js @@ -1,35 +1,17 @@ -const {memoize, get} = require('lodash'); const {Octokit} = require('@octokit/rest'); -const pRetry = require('p-retry'); -const Bottleneck = require('bottleneck'); +const {throttling} = require('@octokit/plugin-throttling'); +const {retry} = require('@octokit/plugin-retry'); const urljoin = require('url-join'); const HttpProxyAgent = require('http-proxy-agent'); const HttpsProxyAgent = require('https-proxy-agent'); -const {RETRY_CONF, RATE_LIMITS, GLOBAL_RATE_LIMIT} = require('./definitions/rate-limit'); +const SemanticReleaseOctokit = Octokit.plugin(throttling, retry); -/** - * Http error status for which to not retry. - */ -const SKIP_RETRY_CODES = new Set([400, 401, 403]); - -/** - * Create or retrieve the throttler function for a given rate limit group. - * - * @param {Array} rate The rate limit group. - * @param {String} limit The rate limits per API endpoints. - * @param {Bottleneck} globalThrottler The global throttler. - * - * @return {Bottleneck} The throller function for the given rate limit group. - */ -const getThrottler = memoize((rate, globalThrottler) => - new Bottleneck({minTime: get(RATE_LIMITS, rate)}).chain(globalThrottler) -); +const {RETRY_CONF} = require('./definitions/rate-limit'); module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => { const baseUrl = githubUrl && urljoin(githubUrl, githubApiPathPrefix); - const globalThrottler = new Bottleneck({minTime: GLOBAL_RATE_LIMIT}); - const github = new Octokit({ + const github = new SemanticReleaseOctokit({ auth: `token ${githubToken}`, baseUrl, request: { @@ -39,24 +21,21 @@ module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => { : new HttpsProxyAgent(proxy) : undefined, }, - }); + throttle: { + onRateLimit: (retryAfter, options) => { + github.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`); - github.hook.wrap('request', (request, options) => { - const access = options.method === 'GET' ? 'read' : 'write'; - const rateCategory = options.url.startsWith('/search') ? 'search' : 'core'; - const limitKey = [rateCategory, RATE_LIMITS[rateCategory][access] && access].filter(Boolean).join('.'); - - return pRetry(async () => { - try { - return await getThrottler(limitKey, globalThrottler).wrap(request)(options); - } catch (error) { - if (SKIP_RETRY_CODES.has(error.status)) { - throw new pRetry.AbortError(error); + if (options.request.retryCount <= RETRY_CONF.retries) { + github.log.debug(`Will retry after ${retryAfter}.`); + return true; } - throw error; - } - }, RETRY_CONF); + return false; + }, + onAbuseLimit: (retryAfter, options) => { + github.log.warn(`Abuse detected for request ${options.method} ${options.url}`); + }, + }, }); return github; diff --git a/package-lock.json b/package-lock.json index ed66058c..320a85e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -528,6 +528,24 @@ "deprecation": "^2.3.1" } }, + "@octokit/plugin-retry": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.7.tgz", + "integrity": "sha512-n08BPfVeKj5wnyH7IaOWnuKbx+e9rSJkhDHMJWXLPv61625uWjsN8G7sAW3zWm9n9vnS4friE7LL/XLcyGeG8Q==", + "requires": { + "@octokit/types": "^6.0.3", + "bottleneck": "^2.15.3" + } + }, + "@octokit/plugin-throttling": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz", + "integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==", + "requires": { + "@octokit/types": "^6.0.1", + "bottleneck": "^2.15.3" + } + }, "@octokit/request": { "version": "5.4.15", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.15.tgz", @@ -782,7 +800,8 @@ "@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true }, "@typescript-eslint/eslint-plugin": { "version": "4.23.0", @@ -7769,6 +7788,7 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.5.0.tgz", "integrity": "sha512-5Hwh4aVQSu6BEP+w2zKlVXtFAaYQe1qWuVADSgoeVlLjwe/Q/AMSoRR4MDeaAfu8llT+YNbEijWu/YF3m6avkg==", + "dev": true, "requires": { "@types/retry": "^0.12.0", "retry": "^0.12.0" @@ -8475,7 +8495,8 @@ "retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", + "dev": true }, "reusify": { "version": "1.0.4", diff --git a/package.json b/package.json index a8cf5657..6b49d1a7 100644 --- a/package.json +++ b/package.json @@ -16,10 +16,11 @@ "Gregor Martynus (https://twitter.com/gr2m)" ], "dependencies": { + "@octokit/plugin-retry": "^3.0.0", + "@octokit/plugin-throttling": "^3.4.0", "@octokit/rest": "^18.0.0", "@semantic-release/error": "^2.2.0", "aggregate-error": "^3.0.0", - "bottleneck": "^2.18.1", "debug": "^4.0.0", "dir-glob": "^3.0.0", "fs-extra": "^10.0.0", @@ -30,7 +31,6 @@ "lodash": "^4.17.4", "mime": "^2.4.3", "p-filter": "^2.0.0", - "p-retry": "^4.0.0", "url-join": "^4.0.0" }, "devDependencies": { diff --git a/test/get-client.test.js b/test/get-client.test.js index ee005b2a..d9bceedf 100644 --- a/test/get-client.test.js +++ b/test/get-client.test.js @@ -4,12 +4,10 @@ const https = require('https'); const {promisify} = require('util'); const {readFile} = require('fs-extra'); const test = require('ava'); -const {inRange} = require('lodash'); -const {stub, spy} = require('sinon'); +const {spy} = require('sinon'); const proxyquire = require('proxyquire'); const Proxy = require('proxy'); const serverDestroy = require('server-destroy'); -const {Octokit} = require('@octokit/rest'); const rateLimit = require('./helpers/rate-limit'); const getClient = proxyquire('../lib/get-client', {'./definitions/rate-limit': rateLimit}); @@ -114,124 +112,3 @@ test.serial('Do not use a proxy if set to false', async (t) => { await promisify(server.destroy).bind(server)(); }); - -test('Use the global throttler for all endpoints', async (t) => { - const rate = 150; - - const octokit = new Octokit(); - octokit.hook.wrap('request', () => Date.now()); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': {RATE_LIMITS: {search: 1, core: 1}, GLOBAL_RATE_LIMIT: rate}, - })({githubToken: 'token'}); - - /* eslint-disable unicorn/prevent-abbreviations */ - - const a = await github.repos.createRelease(); - const b = await github.issues.createComment(); - const c = await github.repos.createRelease(); - const d = await github.issues.createComment(); - const e = await github.search.issuesAndPullRequests(); - const f = await github.search.issuesAndPullRequests(); - - // `issues.createComment` should be called `rate` ms after `repos.createRelease` - t.true(inRange(b - a, rate - 50, rate + 50)); - // `repos.createRelease` should be called `rate` ms after `issues.createComment` - t.true(inRange(c - b, rate - 50, rate + 50)); - // `issues.createComment` should be called `rate` ms after `repos.createRelease` - t.true(inRange(d - c, rate - 50, rate + 50)); - // `search.issuesAndPullRequests` should be called `rate` ms after `issues.createComment` - t.true(inRange(e - d, rate - 50, rate + 50)); - // `search.issuesAndPullRequests` should be called `rate` ms after `search.issuesAndPullRequests` - t.true(inRange(f - e, rate - 50, rate + 50)); - - /* eslint-enable unicorn/prevent-abbreviations */ -}); - -test('Use the same throttler for endpoints in the same rate limit group', async (t) => { - const searchRate = 300; - const coreRate = 150; - - const octokit = new Octokit(); - octokit.hook.wrap('request', () => Date.now()); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': {RATE_LIMITS: {search: searchRate, core: coreRate}, GLOBAL_RATE_LIMIT: 1}, - })({githubToken: 'token'}); - - /* eslint-disable unicorn/prevent-abbreviations */ - - const a = await github.repos.createRelease(); - const b = await github.issues.createComment(); - const c = await github.repos.createRelease(); - const d = await github.issues.createComment(); - const e = await github.search.issuesAndPullRequests(); - const f = await github.search.issuesAndPullRequests(); - - // `issues.createComment` should be called `coreRate` ms after `repos.createRelease` - t.true(inRange(b - a, coreRate - 50, coreRate + 50)); - // `repos.createRelease` should be called `coreRate` ms after `issues.createComment` - t.true(inRange(c - b, coreRate - 50, coreRate + 50)); - // `issues.createComment` should be called `coreRate` ms after `repos.createRelease` - t.true(inRange(d - c, coreRate - 50, coreRate + 50)); - - // The first search should be called immediately as it uses a different throttler - t.true(inRange(e - d, -50, 50)); - // The second search should be called only after `searchRate` ms - t.true(inRange(f - e, searchRate - 50, searchRate + 50)); - - /* eslint-enable unicorn/prevent-abbreviations */ -}); - -test('Use different throttler for read and write endpoints', async (t) => { - const writeRate = 300; - const readRate = 150; - - const octokit = new Octokit(); - octokit.hook.wrap('request', () => Date.now()); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': {RATE_LIMITS: {core: {write: writeRate, read: readRate}}, GLOBAL_RATE_LIMIT: 1}, - })({githubToken: 'token'}); - - const a = await github.repos.get(); - const b = await github.repos.get(); - const c = await github.repos.createRelease(); - const d = await github.repos.createRelease(); - - // `repos.get` should be called `readRate` ms after `repos.get` - t.true(inRange(b - a, readRate - 50, readRate + 50)); - // `repos.createRelease` should be called `coreRate` ms after `repos.createRelease` - t.true(inRange(d - c, writeRate - 50, writeRate + 50)); -}); - -test('Use the same throttler when retrying', async (t) => { - const coreRate = 200; - const request = stub().callsFake(async () => { - const err = new Error(); - err.time = Date.now(); - err.status = 404; - throw err; - }); - const octokit = new Octokit(); - octokit.hook.wrap('request', request); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': { - RETRY_CONF: {retries: 3, factor: 1, minTimeout: 1}, - RATE_LIMITS: {core: coreRate}, - GLOBAL_RATE_LIMIT: 1, - }, - })({githubToken: 'token'}); - - await t.throwsAsync(github.repos.createRelease()); - const {time: a} = await t.throwsAsync(request.getCall(0).returnValue); - const {time: b} = await t.throwsAsync(request.getCall(1).returnValue); - const {time: c} = await t.throwsAsync(request.getCall(2).returnValue); - const {time: d} = await t.throwsAsync(request.getCall(3).returnValue); - - // Each retry should be done after `coreRate` ms - t.true(inRange(b - a, coreRate - 50, coreRate + 50)); - t.true(inRange(c - b, coreRate - 50, coreRate + 50)); - t.true(inRange(d - c, coreRate - 50, coreRate + 50)); -}); diff --git a/test/verify.test.js b/test/verify.test.js index f428e741..be8e1678 100644 --- a/test/verify.test.js +++ b/test/verify.test.js @@ -65,7 +65,11 @@ test.serial( await t.notThrowsAsync( verify( {proxy, assets, successComment, failTitle, failComment, labels}, - {env, options: {repositoryUrl: `git+https://othertesturl.com/${owner}/${repo}.git`}, logger: t.context.logger} + { + env, + options: {repositoryUrl: `git+https://othertesturl.com/${owner}/${repo}.git`}, + logger: t.context.logger, + } ) ); t.true(github.isDone()); @@ -440,7 +444,11 @@ test('Throw SemanticReleaseError for missing github token', async (t) => { const [error, ...errors] = await t.throwsAsync( verify( {}, - {env: {}, options: {repositoryUrl: 'https://github.com/semantic-release/github.git'}, logger: t.context.logger} + { + env: {}, + options: {repositoryUrl: 'https://github.com/semantic-release/github.git'}, + logger: t.context.logger, + } ) ); @@ -526,7 +534,7 @@ test.serial("Throw SemanticReleaseError if the repository doesn't exist", async const owner = 'test_user'; const repo = 'test_repo'; const env = {GH_TOKEN: 'github_token'}; - const github = authenticate(env).get(`/repos/${owner}/${repo}`).times(4).reply(404); + const github = authenticate(env).get(`/repos/${owner}/${repo}`).reply(404); const [error, ...errors] = await t.throwsAsync( verify({}, {env, options: {repositoryUrl: `https://github.com/${owner}/${repo}.git`}, logger: t.context.logger})