Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[Intl] [Emoji] Move emoji data in a new component #53096

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions .github/workflows/intl-data-tests.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
name: Intl data
name: Intl/Emoji data

on:
push:
paths:
- 'src/Symfony/Component/Emoji/*.php'
- 'src/Symfony/Component/Emoji/Resources/data/**'
- 'src/Symfony/Component/Emoji/Tests/*Test.php'
- 'src/Symfony/Component/Intl/*.php'
- 'src/Symfony/Component/Intl/Util/GitRepository.php'
- 'src/Symfony/Component/Intl/Resources/data/**'
- 'src/Symfony/Component/Intl/Tests/*Test.php'
- 'src/Symfony/Component/Intl/Tests/Util/GitRepositoryTest.php'
pull_request:
paths:
- 'src/Symfony/Component/Emoji/*.php'
- 'src/Symfony/Component/Emoji/Resources/data/**'
- 'src/Symfony/Component/Emoji/Tests/*Test.php'
- 'src/Symfony/Component/Intl/*.php'
- 'src/Symfony/Component/Intl/Util/GitRepository.php'
- 'src/Symfony/Component/Intl/Resources/data/**'
Expand All @@ -29,7 +35,7 @@ permissions:

jobs:
tests:
name: Intl data
name: Intl/Emoji data
runs-on: Ubuntu-20.04

steps:
Expand Down Expand Up @@ -80,15 +86,23 @@ jobs:
- name: Run intl-data tests
run: ./phpunit --group intl-data -v

- name: Test with compressed data
- name: Test intl-data with compressed data
run: |
[ -f src/Symfony/Component/Intl/Resources/data/locales/en.php ]
[ ! -f src/Symfony/Component/Intl/Resources/data/locales/en.php.gz ]
[ -f src/Symfony/Component/Intl/Resources/data/transliterator/emoji/emoji-en.php ]
[ ! -f src/Symfony/Component/Intl/Resources/data/transliterator/emoji/emoji-en.php.gz ]
src/Symfony/Component/Intl/Resources/bin/compress
[ ! -f src/Symfony/Component/Intl/Resources/data/locales/en.php ]
[ -f src/Symfony/Component/Intl/Resources/data/locales/en.php.gz ]
[ ! -f src/Symfony/Component/Intl/Resources/data/transliterator/emoji/emoji-en.php ]
[ -f src/Symfony/Component/Intl/Resources/data/transliterator/emoji/emoji-en.php.gz ]
./phpunit src/Symfony/Component/Intl

- name: Run Emoji tests
run: ./phpunit src/Symfony/Component/Emoji -v

- name: Test Emoji with compressed data
run: |
[ -f src/Symfony/Component/Emoji/Resources/data/emoji-en.php ]
[ ! -f src/Symfony/Component/Emoji/Resources/data/emoji-en.php.gz ]
src/Symfony/Component/Emoji/Resources/bin/compress
[ ! -f src/Symfony/Component/Emoji/Resources/data/emoji-en.php ]
[ -f src/Symfony/Component/Emoji/Resources/data/emoji-en.php.gz ]
./phpunit src/Symfony/Component/Emoji
2 changes: 1 addition & 1 deletion .github/workflows/package-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:

- name: Find packages
id: find-packages
run: echo "packages=$(php .github/get-modified-packages.php $(find src/Symfony -mindepth 2 -type f -name composer.json -printf '%h\n' | grep -v src/Symfony/Component/Intl/Resources/emoji |jq -R -s -c 'split("\n")[:-1]') $(git diff --name-only origin/${{ github.base_ref }} HEAD | grep src/ | jq -R -s -c 'split("\n")[:-1]'))" >> $GITHUB_OUTPUT
run: echo "packages=$(php .github/get-modified-packages.php $(find src/Symfony -mindepth 2 -type f -name composer.json -printf '%h\n' | grep -v src/Symfony/Component/Emoji/Resources/bin |jq -R -s -c 'split("\n")[:-1]') $(git diff --name-only origin/${{ github.base_ref }} HEAD | grep src/ | jq -R -s -c 'split("\n")[:-1]'))" >> $GITHUB_OUTPUT

- name: Verify meta files are correct
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
echo SYMFONY_DEPRECATIONS_HELPER=weak >> $GITHUB_ENV
cp composer.json composer.json.orig
echo -e '{\n"require":{'"$(grep phpunit-bridge composer.json)"'"php":"*"},"minimum-stability":"dev"}' > composer.json
php .github/build-packages.php HEAD^ $SYMFONY_VERSION $(find src/Symfony -mindepth 2 -type f -name composer.json -printf '%h\n' | grep -v src/Symfony/Component/Intl/Resources/emoji)
php .github/build-packages.php HEAD^ $SYMFONY_VERSION $(find src/Symfony -mindepth 2 -type f -name composer.json -printf '%h\n' | grep -v src/Symfony/Component/Emoji/Resources/bin)
mv composer.json composer.json.phpunit
mv composer.json.orig composer.json
fi
Expand Down
1 change: 1 addition & 0 deletions .php-cs-fixer.dist.php
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
'Symfony/Bundle/FrameworkBundle/Resources/views/Form',
// explicit trigger_error tests
'Symfony/Bridge/PhpUnit/Tests/DeprecationErrorHandler/',
'Symfony/Component/Emoji/Resources/',
'Symfony/Component/Intl/Resources/data/',
])
// explicit tests for ommited @param type, against `no_superfluous_phpdoc_tags`
Expand Down
1 change: 1 addition & 0 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
"symfony/doctrine-bridge": "self.version",
"symfony/dom-crawler": "self.version",
"symfony/dotenv": "self.version",
"symfony/emoji": "self.version",
"symfony/error-handler": "self.version",
"symfony/event-dispatcher": "self.version",
"symfony/expression-language": "self.version",
Expand Down
3 changes: 2 additions & 1 deletion psalm.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
<directory name="src/Symfony/*/*/Tests" />
<directory name="src/Symfony/*/*/*/Tests" />
<directory name="src/Symfony/*/*/*/*/Tests" />
<directory name="src/Symfony/Component/Intl/Resources/emoji/" />
<directory name="src/Symfony/Component/Intl/Resources/data/" />
<directory name="src/Symfony/Component/Emoji/Resources/data/" />
<directory name="vendor" />
</ignoreFiles>
</projectFiles>
Expand Down
8 changes: 8 additions & 0 deletions src/Symfony/Component/Emoji/.gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/Resources/bin/build.php export-ignore
/Resources/bin/composer.json export-ignore
/Resources/bin/Makefile export-ignore
/Resources/bin/README.md export-ignore
/Tests export-ignore
/phpunit.xml.dist export-ignore
/.gitattributes export-ignore
/.gitignore export-ignore
3 changes: 3 additions & 0 deletions src/Symfony/Component/Emoji/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
vendor/
composer.lock
phpunit.xml
7 changes: 7 additions & 0 deletions src/Symfony/Component/Emoji/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CHANGELOG
=========

7.1
---

* Add the component
141 changes: 141 additions & 0 deletions src/Symfony/Component/Emoji/EmojiTransliterator.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
<?php

/*
* This file is part of the Symfony package.
*
* (c) Fabien Potencier <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace Symfony\Component\Emoji;

use Symfony\Component\Emoji\Util\GzipStreamWrapper;

if (!class_exists(\Transliterator::class)) {
throw new \LogicException(sprintf('You cannot use the "%s\EmojiTransliterator" class as the "intl" extension is not installed. See https://php.net/intl.', __NAMESPACE__));
}

final class EmojiTransliterator extends \Transliterator
{
private const QUICK_CHECK = "\xA9\xAE\xE2\xE3\xF0";
private const REVERSEABLE_IDS = [
'emoji-github' => 'github-emoji',
'emoji-slack' => 'slack-emoji',
'github-emoji' => 'emoji-github',
'slack-emoji' => 'emoji-slack',
];

public readonly string $id;

private array $map;

private string $quickCheck;

private \Transliterator $transliterator;

public static function create(string $id, int $direction = self::FORWARD): self
{
$id = strtolower($id);

if (!isset(self::REVERSEABLE_IDS[$id]) && !str_starts_with($id, 'emoji-')) {
$id = 'emoji-'.$id;
}

if (self::REVERSE === $direction) {
if (!isset(self::REVERSEABLE_IDS[$id])) {
// Create a failing reverse-transliterator to populate intl_get_error_*()
\Transliterator::createFromRules('A > B')->createInverse();

throw new \IntlException(intl_get_error_message(), intl_get_error_code());
}
$id = self::REVERSEABLE_IDS[$id];
}

$file = __DIR__."/Resources/data/{$id}.php";
if (!preg_match('/^[a-z0-9@_\\.\\-]*$/', $id) || !is_file($file) && !is_file($file .= '.gz')) {
\Transliterator::create($id); // Populate intl_get_error_*()

throw new \IntlException(intl_get_error_message(), intl_get_error_code());
}

/**
* @var array<string, array> $maps
*/
static $maps;

// Create an instance of \Transliterator with a custom id; that's the only way
static $newInstance;
$instance = ($newInstance ??= (new \ReflectionClass(self::class))->newInstanceWithoutConstructor(...))();
$instance->id = $id;
$instance->map = $maps[$id] ??= str_ends_with($file, '.gz') ? GzipStreamWrapper::require($file) : require $file;

return $instance;
}

public function createInverse(): self
{
return self::create($this->id, \Transliterator::REVERSE);
}

public function getErrorCode(): int|false
{
return $this->transliterator?->getErrorCode() ?? 0;
}

public function getErrorMessage(): string|false
{
return $this->transliterator?->getErrorMessage() ?? false;
}

public static function listIDs(): array
{
static $ids = [];

if ($ids) {
return $ids;
}

foreach (scandir(__DIR__.'/Resources/data/') as $file) {
if (str_ends_with($file, '.php.gz')) {
$ids[] = substr($file, 0, -7);
} elseif (str_ends_with($file, '.php')) {
$ids[] = substr($file, 0, -4);
}
}

return $ids;
}

public function transliterate(string $string, int $start = 0, int $end = -1): string|false
{
$this->quickCheck ??= str_starts_with(array_key_first($this->map), ':') ? ':' : self::QUICK_CHECK;

if (0 === $start && -1 === $end && preg_match('//u', $string)) {
return \strlen($string) === strcspn($string, $this->quickCheck) ? $string : strtr($string, $this->map);
}

// Here we rely on intl to validate the $string, $start and $end arguments
// and to slice the string. Slicing is done by replacing the part if $string
// between $start and $end by a unique cookie that can be reliably used to
// identify which part of $string should be transliterated.

static $cookie;
static $transliterator;

$cookie ??= hash('xxh128', random_bytes(8));
$this->transliterator ??= clone $transliterator ??= \Transliterator::createFromRules('[:any:]* > '.$cookie);

if (false === $result = $this->transliterator->transliterate($string, $start, $end)) {
return false;
}

$parts = explode($cookie, $result);
$start = \strlen($parts[0]);
$length = -\strlen($parts[1]) ?: null;
$string = substr($string, $start, $length);

return $parts[0].(\strlen($string) === strcspn($string, $this->quickCheck) ? $string : strtr($string, $this->map)).$parts[1];
}
}
19 changes: 19 additions & 0 deletions src/Symfony/Component/Emoji/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
Copyright (c) 2024-present Fabien Potencier

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
18 changes: 18 additions & 0 deletions src/Symfony/Component/Emoji/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
Emoji Component
===============

The Emoji component provides access to emoji characters and sequences from
the [Unicode CLDR](https://cldr.unicode.org/index).

If you have the zlib extension enabled, you can compress the data by running:

php vendor/symfony/emoji/Resources/bin/compress

Resources
---------

* [Documentation](https://symfony.com/doc/current/components/emoji.html)
* [Contributing](https://symfony.com/doc/current/contributing/index.html)
* [Report issues](https://github.com/symfony/symfony/issues) and
[send Pull Requests](https://github.com/symfony/symfony/pulls)
in the [main Symfony repository](https://github.com/symfony/symfony)
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

final class Builder
{
private const TARGET_DIR = __DIR__.'/../data/transliterator/emoji/';
private const TARGET_DIR = __DIR__.'/../data/';

public static function getEmojisCodePoints(): array
{
Expand Down Expand Up @@ -233,7 +233,7 @@ public static function saveRules(iterable $rulesByLocale): void
sort($firstChars);

$quickCheck = '"'.str_replace('%', '\\x', rawurlencode(implode('', $firstChars))).'"';
$file = dirname(__DIR__, 2).'/Transliterator/EmojiTransliterator.php';
$file = dirname(__DIR__, 2).'/EmojiTransliterator.php';
file_put_contents($file, preg_replace('/QUICK_CHECK = .*;/m', "QUICK_CHECK = {$quickCheck};", file_get_contents($file)));
}

Expand Down
36 changes: 36 additions & 0 deletions src/Symfony/Component/Emoji/Resources/bin/compress
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env php
<?php

/*
* This file is part of the Symfony package.
*
* (c) Fabien Potencier <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

if ('cli' !== PHP_SAPI) {
throw new Exception('This script must be run from the command line.');
}
if (!extension_loaded('zlib')) {
throw new Exception('This script requires the zlib extension.');
}

$iterator = new RecursiveIteratorIterator(
new RecursiveDirectoryIterator(
dirname(__DIR__).'/data',
FilesystemIterator::CURRENT_AS_FILEINFO | FilesystemIterator::SKIP_DOTS
)
);

foreach ($iterator as $file) {
if ('php' !== $file->getExtension()) {
continue;
}

$data = file_get_contents($file);
file_put_contents('compress.zlib://'.$file.'.gz', $data);

unlink($file.(filesize($file.'.gz') >= strlen($data) ? '.gz' : ''));
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
* file that was distributed with this source code.
*/

namespace Symfony\Component\Intl\Tests\Transliterator;
namespace Symfony\Component\Emoji\Tests;

use PHPUnit\Framework\TestCase;
use Symfony\Component\Emoji\EmojiTransliterator;
use Symfony\Component\Finder\Finder;
use Symfony\Component\Intl\Transliterator\EmojiTransliterator;

/**
* @requires extension intl
Expand Down Expand Up @@ -95,7 +95,7 @@ public function testAllTransliterator(string $locale)
public static function provideLocaleTest(): iterable
{
$file = (new Finder())
->in(__DIR__.'/../../Resources/data/transliterator/emoji')
->in(__DIR__.'/../Resources/data')
->name('*.php')
->notName('emoji-strip.php')
->files()
Expand Down
Loading