Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 23f6154

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Add file system cache for git commands (#6509)
1 parent f4952ef commit 23f6154

File tree

6 files changed

+528
-152
lines changed

6 files changed

+528
-152
lines changed

‎integration-tests/ci-visibility/git-cache.spec.js‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
'use strict'
2+
3+
const { expect } = require('chai')
4+
const fs = require('fs')
5+
const path = require('path')
6+
const os = require('os')
7+
const rimraf = require('rimraf')
8+
const { execSync } = require('child_process')
9+
10+
const { createSandbox } = require('../helpers')
11+
12+
const FIXED_COMMIT_MESSAGE = 'Test commit message for caching'
13+
const GET_COMMIT_MESSAGE_COMMAND_ARGS = ['log', '-1', '--pretty=format:%s']
14+
15+
function removeGitFromPath () {
16+
process.env.PATH = process.env.PATH
17+
.split(path.delimiter)
18+
.filter(dir => {
19+
return !dir.includes('git') &&
20+
!dir.includes('Git') &&
21+
!dir.includes('usr/bin') &&
22+
!dir.includes('bin')
23+
})
24+
.join(path.delimiter)
25+
}
26+
27+
describe('git-cache integration tests', () => {
28+
let sandbox, cwd, gitCache, testRepoPath
29+
let originalPath, originalCwd
30+
let cacheDir
31+
let originalCacheEnabled, originalCacheDir
32+
33+
before(async () => {
34+
sandbox = await createSandbox([], true)
35+
cwd = sandbox.folder
36+
testRepoPath = cwd
37+
38+
cacheDir = path.join(os.tmpdir(), 'dd-trace-git-cache-integration-test')
39+
originalCacheEnabled = process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED
40+
originalCacheDir = process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR
41+
42+
execSync(`git commit --allow-empty -m '${FIXED_COMMIT_MESSAGE}'`, { cwd: testRepoPath })
43+
})
44+
45+
after(async () => {
46+
await sandbox.remove()
47+
})
48+
49+
beforeEach(() => {
50+
process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED = 'true'
51+
process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR = cacheDir
52+
// We need this, otherwise the file is already loaded and the cache is disabled
53+
delete require.cache[require.resolve('../../packages/dd-trace/src/plugins/util/git-cache')]
54+
gitCache = require('../../packages/dd-trace/src/plugins/util/git-cache')
55+
56+
originalPath = process.env.PATH
57+
originalCwd = process.cwd()
58+
process.chdir(testRepoPath)
59+
})
60+
61+
afterEach(() => {
62+
if (cacheDir && fs.existsSync(cacheDir)) {
63+
rimraf.sync(cacheDir)
64+
}
65+
process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED = originalCacheEnabled
66+
process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR = originalCacheDir
67+
process.env.PATH = originalPath
68+
process.chdir(originalCwd)
69+
})
70+
71+
it('should cache git commands', function () {
72+
const firstResult = gitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
73+
74+
const firstResultStr = firstResult.toString().trim()
75+
expect(firstResultStr).to.equal(FIXED_COMMIT_MESSAGE)
76+
77+
const cacheKey = gitCache.getCacheKey('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
78+
const cacheFilePath = gitCache.getCacheFilePath(cacheKey)
79+
expect(fs.existsSync(cacheFilePath)).to.be.true
80+
81+
const cachedContent = fs.readFileSync(cacheFilePath, 'utf8')
82+
expect(cachedContent).to.equal(firstResultStr)
83+
})
84+
85+
it('should return cached results when git is unavailable', function () {
86+
const firstResult = gitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS).toString().trim()
87+
88+
expect(firstResult).to.equal(FIXED_COMMIT_MESSAGE)
89+
90+
const cacheKey = gitCache.getCacheKey('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
91+
const cacheFilePath = gitCache.getCacheFilePath(cacheKey)
92+
expect(fs.existsSync(cacheFilePath)).to.be.true
93+
94+
removeGitFromPath()
95+
96+
const secondResult = gitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS).toString().trim()
97+
expect(secondResult).to.equal(firstResult)
98+
99+
let secondError
100+
try {
101+
gitCache.cachedExec('git', ['rev-parse', 'HEAD'])
102+
} catch (error) {
103+
secondError = error
104+
}
105+
expect(secondError).to.be.an('error')
106+
expect(secondError.code).to.equal('ENOENT')
107+
expect(secondError.message).to.include('git')
108+
})
109+
110+
it('should cache git command failures and throw the same error on subsequent calls', function () {
111+
const gitArgs = ['nonexistent-command']
112+
113+
let firstError
114+
try {
115+
firstError = gitCache.cachedExec('git', gitArgs)
116+
} catch (error) {
117+
firstError = error
118+
}
119+
120+
expect(firstError).to.be.an('error')
121+
122+
const cacheKey = gitCache.getCacheKey('git', gitArgs)
123+
const cacheFilePath = gitCache.getCacheFilePath(cacheKey)
124+
expect(fs.existsSync(cacheFilePath)).to.be.true
125+
126+
const cachedData = fs.readFileSync(cacheFilePath, 'utf8')
127+
expect(cachedData).to.include('__GIT_COMMAND_FAILED__')
128+
129+
removeGitFromPath()
130+
131+
// Second call: should throw the same error from cache
132+
let secondError
133+
try {
134+
gitCache.cachedExec('git', gitArgs)
135+
} catch (error) {
136+
secondError = error
137+
}
138+
139+
expect(secondError).to.be.an('error')
140+
expect(secondError.message).to.equal(firstError.message)
141+
expect(secondError.code).to.equal(firstError.code)
142+
expect(secondError.status).to.equal(firstError.status)
143+
expect(secondError.errno).to.equal(firstError.errno)
144+
})
145+
146+
it('should not cache when DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED is not set to true', function () {
147+
delete process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED
148+
149+
delete require.cache[require.resolve('../../packages/dd-trace/src/plugins/util/git-cache')]
150+
const disabledGitCache = require('../../packages/dd-trace/src/plugins/util/git-cache')
151+
152+
const firstResult = disabledGitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
153+
const firstResultStr = firstResult.toString().trim()
154+
expect(firstResultStr).to.equal(FIXED_COMMIT_MESSAGE)
155+
156+
const cacheKey = disabledGitCache.getCacheKey('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
157+
const cacheFilePath = disabledGitCache.getCacheFilePath(cacheKey)
158+
expect(fs.existsSync(cacheFilePath)).to.be.false
159+
160+
removeGitFromPath()
161+
162+
let secondError
163+
try {
164+
disabledGitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
165+
} catch (error) {
166+
secondError = error
167+
}
168+
169+
expect(secondError).to.be.an('error')
170+
expect(secondError.code).to.equal('ENOENT')
171+
expect(secondError.message).to.include('git')
172+
})
173+
174+
context('invalid DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR', () => {
175+
function runInvalidCacheTest (invalidCacheDir) {
176+
process.env.DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR = invalidCacheDir
177+
178+
delete require.cache[require.resolve('../../packages/dd-trace/src/plugins/util/git-cache')]
179+
const invalidDirGitCache = require('../../packages/dd-trace/src/plugins/util/git-cache')
180+
181+
const firstResult = invalidDirGitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
182+
const firstResultStr = firstResult.toString().trim()
183+
expect(firstResultStr).to.equal(FIXED_COMMIT_MESSAGE)
184+
185+
const cacheKey = invalidDirGitCache.getCacheKey('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
186+
const cacheFilePath = invalidDirGitCache.getCacheFilePath(cacheKey)
187+
expect(fs.existsSync(cacheFilePath)).to.be.false
188+
189+
removeGitFromPath()
190+
191+
let secondError
192+
try {
193+
invalidDirGitCache.cachedExec('git', GET_COMMIT_MESSAGE_COMMAND_ARGS)
194+
} catch (error) {
195+
secondError = error
196+
}
197+
198+
expect(secondError).to.be.an('error')
199+
expect(secondError.code).to.equal('ENOENT')
200+
expect(secondError.message).to.include('git')
201+
}
202+
203+
it('set to a file', () => {
204+
const filePath = path.join(os.tmpdir(), 'invalid-cache-folder.txt')
205+
206+
fs.writeFileSync(filePath, 'this is a file, not a directory')
207+
runInvalidCacheTest(filePath)
208+
if (fs.existsSync(filePath)) {
209+
fs.unlinkSync(filePath)
210+
}
211+
})
212+
213+
it('set to a directory without write permissions', () => {
214+
const tempDir = path.join(os.tmpdir(), 'dd-trace-git-cache-permission-test')
215+
216+
try {
217+
fs.mkdirSync(tempDir, { recursive: true })
218+
219+
// Remove write permissions for the current user (chmod 444 = read-only)
220+
fs.chmodSync(tempDir, 0o444)
221+
222+
runInvalidCacheTest(tempDir)
223+
} finally {
224+
try {
225+
fs.chmodSync(tempDir, 0o755)
226+
if (fs.existsSync(tempDir)) {
227+
rimraf.sync(tempDir)
228+
}
229+
} catch (cleanupError) {
230+
try {
231+
rimraf.sync(tempDir)
232+
} catch {
233+
// Ignore cleanup errors
234+
}
235+
}
236+
}
237+
})
238+
})
239+
})

‎packages/dd-trace/src/plugins/util/git-cache.js‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
'use strict'
2+
3+
const os = require('os')
4+
const path = require('path')
5+
const fs = require('fs')
6+
const crypto = require('crypto')
7+
const cp = require('child_process')
8+
9+
const log = require('../../log')
10+
const { getEnvironmentVariable } = require('../../config-helper')
11+
const { isTrue } = require('../../util')
12+
13+
let isGitEnabled = isTrue(getEnvironmentVariable('DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED'))
14+
const GIT_CACHE_DIR = getEnvironmentVariable('DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR') ||
15+
path.join(os.tmpdir(), 'dd-trace-git-cache')
16+
17+
function ensureCacheDir () {
18+
if (!isGitEnabled) return false
19+
20+
try {
21+
if (fs.existsSync(GIT_CACHE_DIR)) {
22+
const stats = fs.statSync(GIT_CACHE_DIR)
23+
if (!stats.isDirectory()) {
24+
throw new Error(`Cache directory path exists but is not a directory: ${GIT_CACHE_DIR}`)
25+
}
26+
} else {
27+
fs.mkdirSync(GIT_CACHE_DIR, { recursive: true })
28+
}
29+
return true
30+
} catch (err) {
31+
log.error('Failed to create git cache directory, disabling cache', err)
32+
isGitEnabled = false
33+
return false
34+
}
35+
}
36+
37+
// Initialize cache directory at module load time
38+
ensureCacheDir()
39+
40+
function getCacheKey (cmd, flags) {
41+
// Create a hash of the command and flags to use as cache key
42+
const commandString = `${cmd} ${flags.join(' ')}`
43+
return crypto.createHash('sha256').update(commandString).digest('hex')
44+
}
45+
46+
function getCacheFilePath (cacheKey) {
47+
return path.join(GIT_CACHE_DIR, `${cacheKey}.cache`)
48+
}
49+
50+
function getCache (cacheKey) {
51+
if (!isGitEnabled) return null
52+
53+
try {
54+
const cacheFilePath = getCacheFilePath(cacheKey)
55+
if (!fs.existsSync(cacheFilePath)) {
56+
return null
57+
}
58+
59+
const content = fs.readFileSync(cacheFilePath, 'utf8')
60+
return content
61+
} catch (err) {
62+
log.error('Failed to read git cache', err)
63+
return null
64+
}
65+
}
66+
67+
function setCache (cacheKey, result) {
68+
if (!isGitEnabled) return
69+
70+
// Ensure cache directory exists
71+
if (!ensureCacheDir()) return
72+
73+
try {
74+
const cacheFilePath = getCacheFilePath(cacheKey)
75+
fs.writeFileSync(cacheFilePath, result, 'utf8')
76+
} catch (err) {
77+
log.error('Failed to write git cache', err)
78+
}
79+
}
80+
81+
function cachedExec (cmd, flags, options) {
82+
if (options === undefined) {
83+
options = { stdio: 'pipe' }
84+
}
85+
if (!isGitEnabled) {
86+
return cp.execFileSync(cmd, flags, options)
87+
}
88+
const cacheKey = getCacheKey(cmd, flags)
89+
const cachedResult = getCache(cacheKey)
90+
if (cachedResult !== null) {
91+
if (cachedResult.startsWith('__GIT_COMMAND_FAILED__')) {
92+
let error
93+
try {
94+
const errorData = cachedResult.replace('__GIT_COMMAND_FAILED__', '')
95+
const { message, code, status, errno } = JSON.parse(errorData)
96+
error = new Error(message)
97+
error.code = code
98+
error.status = status
99+
error.errno = errno
100+
} catch {
101+
// we couldn't parse the error data, so we'll throw a generic error
102+
throw new Error('Git command failed')
103+
}
104+
throw error
105+
}
106+
return cachedResult
107+
}
108+
try {
109+
const result = cp.execFileSync(cmd, flags, options)
110+
setCache(cacheKey, result)
111+
return result
112+
} catch (err) {
113+
const cacheValue = '__GIT_COMMAND_FAILED__' +
114+
JSON.stringify({
115+
code: err.code,
116+
status: err.status,
117+
errno: err.errno,
118+
message: err.message
119+
})
120+
setCache(cacheKey, cacheValue)
121+
throw err
122+
}
123+
}
124+
125+
module.exports = {
126+
getCacheKey,
127+
getCacheFilePath,
128+
cachedExec
129+
}

0 commit comments

Comments
 (0)