Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ac3111f

Browse files
fabpotfabpot
committed
bug #17377 Fix performance (PHP5) and memory (PHP7) issues when using token_get_all (nicolas-grekas, peteward)
This PR was merged into the 2.3 branch. Discussion ---------- Fix performance (PHP5) and memory (PHP7) issues when using token_get_all | Q | A | ------------- | --- | Bug fix? | yes | New feature? | no | BC breaks? | no | Deprecations? | no | Tests pass? | yes | Fixed tickets | #16868 | License | MIT | Doc PR | - Although it's not the case anymore on PHP 7, on PHP 5, `is_array` checks are much slower than `isset` checks. Also from @peteward in #17384: > New PHP7 memory manager will not release small buckets to OS automatically in cases exposed by `token_get_all()`. This function call addition specifically for PHP7 will reclaim this memory to keep the footprint down of long processe > See above ticket and suggested actions by PHP internals team for long-running tasks (https://bugs.php.net/70098) - I think `cache:clear/warmup` on a heavy app justifies this. > We're running on cloud-based hosting platforms under memory limitations (Platform.sh). When memory is exceeded we're into swap and the cache clearing process goes from seconds to minutes for the initial deployment, which really slows our development workflow and also causes holding page delays. Commits ------- e555aad Add gc_mem_caches() call for PHP7 after itoken_get_all() as new memory manager will not release small buckets to OS automatically d1f72d8 Fix perf and mem issue when using token_get_all
2 parents 5444231 + e555aad commit ac3111f

File tree

6 files changed

+64
-33
lines changed

6 files changed

+64
-33
lines changed

‎src/Symfony/Bundle/FrameworkBundle/Translation/PhpExtractor.php

+8-3
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ public function extract($directory, MessageCatalogue $catalog)
6161
$files = $finder->files()->name('*.php')->in($directory);
6262
foreach ($files as $file) {
6363
$this->parseTokens(token_get_all(file_get_contents($file)), $catalog);
64+
65+
if (PHP_VERSION_ID >= 70000) {
66+
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
67+
gc_mem_caches();
68+
}
6469
}
6570
}
6671

@@ -81,7 +86,7 @@ public function setPrefix($prefix)
8186
*/
8287
protected function normalizeToken($token)
8388
{
84-
if (is_array($token)) {
89+
if (isset($token[1]) && 'b"' !== $token) {
8590
return $token[1];
8691
}
8792

@@ -95,7 +100,7 @@ private function seekToNextRelevantToken(\Iterator $tokenIterator)
95100
{
96101
for (; $tokenIterator->valid(); $tokenIterator->next()) {
97102
$t = $tokenIterator->current();
98-
if (!is_array($t) || ($t[0] !== T_WHITESPACE)) {
103+
if (T_WHITESPACE !== $t[0]) {
99104
break;
100105
}
101106
}
@@ -112,7 +117,7 @@ private function getMessage(\Iterator $tokenIterator)
112117

113118
for (; $tokenIterator->valid(); $tokenIterator->next()) {
114119
$t = $tokenIterator->current();
115-
if (!is_array($t)) {
120+
if (!isset($t[1])) {
116121
break;
117122
}
118123

‎src/Symfony/Component/ClassLoader/ClassCollectionLoader.php

+18-9
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,9 @@ public static function fixNamespaceDeclarations($source)
149149
$inNamespace = false;
150150
$tokens = token_get_all($source);
151151

152-
for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
153-
if (is_string($token)) {
152+
for ($i = 0; isset($tokens[$i]); ++$i) {
153+
$token = $tokens[$i];
154+
if (!isset($token[1]) || 'b"' === $token) {
154155
$rawChunk .= $token;
155156
} elseif (in_array($token[0], array(T_COMMENT, T_DOC_COMMENT))) {
156157
// strip comments
@@ -162,21 +163,21 @@ public static function fixNamespaceDeclarations($source)
162163
$rawChunk .= $token[1];
163164

164165
// namespace name and whitespaces
165-
while (($t = next($tokens)) && is_array($t) && in_array($t[0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
166-
$rawChunk .= $t[1];
166+
while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
167+
$rawChunk .= $tokens[$i][1];
167168
}
168-
if ('{' === $t) {
169+
if ('{' === $tokens[$i]) {
169170
$inNamespace = false;
170-
prev($tokens);
171+
--$i;
171172
} else {
172173
$rawChunk = rtrim($rawChunk)."\n{";
173174
$inNamespace = true;
174175
}
175176
} elseif (T_START_HEREDOC === $token[0]) {
176177
$output .= self::compressCode($rawChunk).$token[1];
177178
do {
178-
$token = next($tokens);
179-
$output .= is_string($token) ? $token : $token[1];
179+
$token = $tokens[++$i];
180+
$output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
180181
} while ($token[0] !== T_END_HEREDOC);
181182
$output .= "\n";
182183
$rawChunk = '';
@@ -192,7 +193,15 @@ public static function fixNamespaceDeclarations($source)
192193
$rawChunk .= "}\n";
193194
}
194195

195-
return $output.self::compressCode($rawChunk);
196+
$output .= self::compressCode($rawChunk);
197+
198+
if (PHP_VERSION_ID >= 70000) {
199+
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
200+
unset($tokens, $rawChunk);
201+
gc_mem_caches();
202+
}
203+
204+
return $output;
196205
}
197206

198207
/**

‎src/Symfony/Component/ClassLoader/ClassMapGenerator.php

+14-8
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public static function createMap($dir)
7272

7373
$classes = self::findClasses($path);
7474

75+
if (PHP_VERSION_ID >= 70000) {
76+
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
77+
gc_mem_caches();
78+
}
79+
7580
foreach ($classes as $class) {
7681
$map[$class] = $path;
7782
}
@@ -95,10 +100,10 @@ private static function findClasses($path)
95100
$classes = array();
96101

97102
$namespace = '';
98-
for ($i = 0, $max = count($tokens); $i < $max; ++$i) {
103+
for ($i = 0; isset($tokens[$i]); ++$i) {
99104
$token = $tokens[$i];
100105

101-
if (is_string($token)) {
106+
if (!isset($token[1])) {
102107
continue;
103108
}
104109

@@ -108,9 +113,9 @@ private static function findClasses($path)
108113
case T_NAMESPACE:
109114
$namespace = '';
110115
// If there is a namespace, extract it
111-
while (($t = $tokens[++$i]) && is_array($t)) {
112-
if (in_array($t[0], array(T_STRING, T_NS_SEPARATOR))) {
113-
$namespace .= $t[1];
116+
while (isset($tokens[++$i][1])) {
117+
if (in_array($tokens[$i][0], array(T_STRING, T_NS_SEPARATOR))) {
118+
$namespace .= $tokens[$i][1];
114119
}
115120
}
116121
$namespace .= '\\';
@@ -121,7 +126,7 @@ private static function findClasses($path)
121126
// Skip usage of ::class constant
122127
$isClassConstant = false;
123128
for ($j = $i - 1; $j > 0; --$j) {
124-
if (is_string($tokens[$j])) {
129+
if (!isset($tokens[$j][1])) {
125130
break;
126131
}
127132

@@ -138,10 +143,11 @@ private static function findClasses($path)
138143
}
139144

140145
// Find the classname
141-
while (($t = $tokens[++$i]) && is_array($t)) {
146+
while (isset($tokens[++$i][1])) {
147+
$t = $tokens[$i];
142148
if (T_STRING === $t[0]) {
143149
$class .= $t[1];
144-
} elseif ($class !== '' && T_WHITESPACE == $t[0]) {
150+
} elseif ('' !== $class && T_WHITESPACE === $t[0]) {
145151
break;
146152
}
147153
}

‎src/Symfony/Component/HttpKernel/Kernel.php

+11-4
Original file line numberDiff line numberDiff line change
@@ -699,14 +699,15 @@ public static function stripComments($source)
699699
$output = '';
700700
$tokens = token_get_all($source);
701701
$ignoreSpace = false;
702-
for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
703-
if (is_string($token)) {
702+
for ($i = 0; isset($tokens[$i]); ++$i) {
703+
$token = $tokens[$i];
704+
if (!isset($token[1]) || 'b"' === $token) {
704705
$rawChunk .= $token;
705706
} elseif (T_START_HEREDOC === $token[0]) {
706707
$output .= $rawChunk.$token[1];
707708
do {
708-
$token = next($tokens);
709-
$output .= $token[1];
709+
$token = $tokens[++$i];
710+
$output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
710711
} while ($token[0] !== T_END_HEREDOC);
711712
$rawChunk = '';
712713
} elseif (T_WHITESPACE === $token[0]) {
@@ -732,6 +733,12 @@ public static function stripComments($source)
732733

733734
$output .= $rawChunk;
734735

736+
if (PHP_VERSION_ID >= 70000) {
737+
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
738+
unset($tokens, $rawChunk);
739+
gc_mem_caches();
740+
}
741+
735742
return $output;
736743
}
737744

‎src/Symfony/Component/HttpKernel/Tests/KernelTest.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ public function testStripComments()
307307
$heredoc = <<<HD
308308
309309
310-
Heredoc should not be modified
310+
Heredoc should not be modified {$a[1+$b]}
311311
312312
313313
HD;
@@ -343,7 +343,7 @@ public function doStuff()
343343
$heredoc = <<<HD
344344
345345
346-
Heredoc should not be modified
346+
Heredoc should not be modified {$a[1+$b]}
347347
348348
349349
HD;

‎src/Symfony/Component/Routing/Loader/AnnotationFileLoader.php

+11-7
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ public function load($file, $type = null)
6464
$collection->addResource(new FileResource($path));
6565
$collection->addCollection($this->loader->load($class, $type));
6666
}
67+
if (PHP_VERSION_ID >= 70000) {
68+
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
69+
gc_mem_caches();
70+
}
6771

6872
return $collection;
6973
}
@@ -88,10 +92,10 @@ protected function findClass($file)
8892
$class = false;
8993
$namespace = false;
9094
$tokens = token_get_all(file_get_contents($file));
91-
for ($i = 0, $count = count($tokens); $i < $count; ++$i) {
95+
for ($i = 0; isset($tokens[$i]); ++$i) {
9296
$token = $tokens[$i];
9397

94-
if (!is_array($token)) {
98+
if (!isset($token[1])) {
9599
continue;
96100
}
97101

@@ -100,11 +104,11 @@ protected function findClass($file)
100104
}
101105

102106
if (true === $namespace && T_STRING === $token[0]) {
103-
$namespace = '';
104-
do {
105-
$namespace .= $token[1];
106-
$token = $tokens[++$i];
107-
} while ($i < $count && is_array($token) && in_array($token[0], array(T_NS_SEPARATOR, T_STRING)));
107+
$namespace = $token[1];
108+
while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_NS_SEPARATOR, T_STRING))) {
109+
$namespace .= $tokens[$i][1];
110+
}
111+
$token = $tokens[$i];
108112
}
109113

110114
if (T_CLASS === $token[0]) {

0 commit comments

Comments
 (0)