diff --git a/README.md b/README.md index f1d77f0..cbeaee2 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,13 @@ This module makes job entities searchable with Solr. In addition it enables face Build status: [![Build Status](https://travis-ci.org/yawik/Solr.svg?branch=master)](https://travis-ci.org/yawik/Solr) -[![Coverage Status](https://coveralls.io/repos/github/yawik/Solr/badge.svg?branch=develop)](https://coveralls.io/github/yawik/Solr?branch=develop) +[![Coverage Status](https://coveralls.io/repos/github/yawik/Solr/badge.svg?branch=develop)](https://coveralls.io/github/yawik/Solr?branch=master) Requirements ------------ * running [YAWIK](https://github.com/cross-solution/YAWIK) -* running Solr Server < 7 - -This module currently requires a Solr 6 +* running Solr Server >= 8 Installation @@ -40,4 +38,4 @@ http://jobs.yawik.org Licence ------- -MIT \ No newline at end of file +MITf \ No newline at end of file diff --git a/composer.lock b/composer.lock index 9acb1ac..bb02932 100644 --- a/composer.lock +++ b/composer.lock @@ -2205,16 +2205,16 @@ }, { "name": "laminas/laminas-form", - "version": "2.14.5", + "version": "2.14.6", "source": { "type": "git", "url": "https://github.com/laminas/laminas-form.git", - "reference": "3e22e09751cf6ae031be87a44e092e7925ce5b7b" + "reference": "c19b62ed8394bcf2038ab3f51a49b7d0ef4e1700" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laminas/laminas-form/zipball/3e22e09751cf6ae031be87a44e092e7925ce5b7b", - "reference": "3e22e09751cf6ae031be87a44e092e7925ce5b7b", + "url": "https://api.github.com/repos/laminas/laminas-form/zipball/c19b62ed8394bcf2038ab3f51a49b7d0ef4e1700", + "reference": "c19b62ed8394bcf2038ab3f51a49b7d0ef4e1700", "shasum": "" }, "require": { @@ -2283,20 +2283,20 @@ "form", "laminas" ], - "time": "2020-03-29T12:46:16+00:00" + "time": "2020-06-22T20:17:02+00:00" }, { "name": "laminas/laminas-http", - "version": "2.11.2", + "version": "2.12.0", "source": { "type": "git", "url": "https://github.com/laminas/laminas-http.git", - "reference": "8c66963b933c80da59433da56a44dfa979f3ec88" + "reference": "48bd06ffa3a6875e2b77d6852405eb7b1589d575" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laminas/laminas-http/zipball/8c66963b933c80da59433da56a44dfa979f3ec88", - "reference": "8c66963b933c80da59433da56a44dfa979f3ec88", + "url": "https://api.github.com/repos/laminas/laminas-http/zipball/48bd06ffa3a6875e2b77d6852405eb7b1589d575", + "reference": "48bd06ffa3a6875e2b77d6852405eb7b1589d575", "shasum": "" }, "require": { @@ -2308,7 +2308,7 @@ "php": "^5.6 || ^7.0" }, "replace": { - "zendframework/zend-http": "self.version" + "zendframework/zend-http": "^2.11.2" }, "require-dev": { "laminas/laminas-coding-standard": "~1.0.0", @@ -2321,8 +2321,8 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.11.x-dev", - "dev-develop": "2.12.x-dev" + "dev-master": "2.12.x-dev", + "dev-develop": "2.13.x-dev" } }, "autoload": { @@ -2341,7 +2341,7 @@ "http client", "laminas" ], - "time": "2019-12-31T17:02:36+00:00" + "time": "2020-06-23T15:14:37+00:00" }, { "name": "laminas/laminas-hydrator", @@ -3582,6 +3582,75 @@ ], "time": "2019-12-31T17:35:22+00:00" }, + { + "name": "laminas/laminas-paginator", + "version": "2.8.2", + "source": { + "type": "git", + "url": "https://github.com/laminas/laminas-paginator.git", + "reference": "5e53d927776b2d20e420bc2b289fa0c364a6b0bd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laminas/laminas-paginator/zipball/5e53d927776b2d20e420bc2b289fa0c364a6b0bd", + "reference": "5e53d927776b2d20e420bc2b289fa0c364a6b0bd", + "shasum": "" + }, + "require": { + "laminas/laminas-stdlib": "^2.7 || ^3.0", + "laminas/laminas-zendframework-bridge": "^1.0", + "php": "^7.0 || ^5.6" + }, + "replace": { + "zendframework/zend-paginator": "self.version" + }, + "require-dev": { + "laminas/laminas-cache": "^2.6.1", + "laminas/laminas-coding-standard": "~1.0.0", + "laminas/laminas-config": "^2.6.0", + "laminas/laminas-db": "^2.9.2", + "laminas/laminas-filter": "^2.6.1", + "laminas/laminas-json": "^2.6.1", + "laminas/laminas-servicemanager": "^2.7.5 || ^3.0.3", + "laminas/laminas-view": "^2.6.3", + "phpunit/phpunit": "^6.2.1 || ^5.7.15" + }, + "suggest": { + "laminas/laminas-cache": "Laminas\\Cache component to support cache features", + "laminas/laminas-db": "Laminas\\Db component", + "laminas/laminas-filter": "Laminas\\Filter component", + "laminas/laminas-json": "Laminas\\Json component", + "laminas/laminas-servicemanager": "Laminas\\ServiceManager component", + "laminas/laminas-view": "Laminas\\View component" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.8.x-dev", + "dev-develop": "2.9.x-dev" + }, + "laminas": { + "component": "Laminas\\Paginator", + "config-provider": "Laminas\\Paginator\\ConfigProvider" + } + }, + "autoload": { + "psr-4": { + "Laminas\\Paginator\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "description": "Paginate collections of data from arbitrary sources", + "homepage": "https://laminas.dev", + "keywords": [ + "laminas", + "paginator" + ], + "time": "2019-12-31T17:36:22+00:00" + }, { "name": "laminas/laminas-permissions-acl", "version": "2.7.1", @@ -5214,16 +5283,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.17.0", + "version": "v1.17.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "e94c8b1bbe2bc77507a1056cdb06451c75b427f9" + "reference": "2edd75b8b35d62fd3eeabba73b26b8f1f60ce13d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/e94c8b1bbe2bc77507a1056cdb06451c75b427f9", - "reference": "e94c8b1bbe2bc77507a1056cdb06451c75b427f9", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/2edd75b8b35d62fd3eeabba73b26b8f1f60ce13d", + "reference": "2edd75b8b35d62fd3eeabba73b26b8f1f60ce13d", "shasum": "" }, "require": { @@ -5236,6 +5305,10 @@ "extra": { "branch-alias": { "dev-master": "1.17-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -5268,20 +5341,20 @@ "polyfill", "portable" ], - "time": "2020-05-12T16:14:59+00:00" + "time": "2020-06-06T08:46:27+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.17.0", + "version": "v1.17.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "fa79b11539418b02fc5e1897267673ba2c19419c" + "reference": "7110338d81ce1cbc3e273136e4574663627037a7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fa79b11539418b02fc5e1897267673ba2c19419c", - "reference": "fa79b11539418b02fc5e1897267673ba2c19419c", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/7110338d81ce1cbc3e273136e4574663627037a7", + "reference": "7110338d81ce1cbc3e273136e4574663627037a7", "shasum": "" }, "require": { @@ -5294,6 +5367,10 @@ "extra": { "branch-alias": { "dev-master": "1.17-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -5327,20 +5404,20 @@ "portable", "shim" ], - "time": "2020-05-12T16:47:27+00:00" + "time": "2020-06-06T08:46:27+00:00" }, { "name": "symfony/polyfill-php73", - "version": "v1.17.0", + "version": "v1.17.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php73.git", - "reference": "a760d8964ff79ab9bf057613a5808284ec852ccc" + "reference": "fa0837fe02d617d31fbb25f990655861bb27bd1a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php73/zipball/a760d8964ff79ab9bf057613a5808284ec852ccc", - "reference": "a760d8964ff79ab9bf057613a5808284ec852ccc", + "url": "https://api.github.com/repos/symfony/polyfill-php73/zipball/fa0837fe02d617d31fbb25f990655861bb27bd1a", + "reference": "fa0837fe02d617d31fbb25f990655861bb27bd1a", "shasum": "" }, "require": { @@ -5350,6 +5427,10 @@ "extra": { "branch-alias": { "dev-master": "1.17-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -5385,20 +5466,20 @@ "portable", "shim" ], - "time": "2020-05-12T16:47:27+00:00" + "time": "2020-06-06T08:46:27+00:00" }, { "name": "symfony/polyfill-php80", - "version": "v1.17.0", + "version": "v1.17.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "5e30b2799bc1ad68f7feb62b60a73743589438dd" + "reference": "4a5b6bba3259902e386eb80dd1956181ee90b5b2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/5e30b2799bc1ad68f7feb62b60a73743589438dd", - "reference": "5e30b2799bc1ad68f7feb62b60a73743589438dd", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/4a5b6bba3259902e386eb80dd1956181ee90b5b2", + "reference": "4a5b6bba3259902e386eb80dd1956181ee90b5b2", "shasum": "" }, "require": { @@ -5408,6 +5489,10 @@ "extra": { "branch-alias": { "dev-master": "1.17-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -5447,7 +5532,7 @@ "portable", "shim" ], - "time": "2020-05-12T16:47:27+00:00" + "time": "2020-06-06T08:46:27+00:00" }, { "name": "symfony/process", @@ -6250,72 +6335,6 @@ "zf3" ], "time": "2020-06-12T12:10:33+00:00" - }, - { - "name": "zendframework/zend-paginator", - "version": "2.8.2", - "source": { - "type": "git", - "url": "https://github.com/zendframework/zend-paginator.git", - "reference": "2b4d07d9475ed581278a28d065b238a0941402e2" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/zendframework/zend-paginator/zipball/2b4d07d9475ed581278a28d065b238a0941402e2", - "reference": "2b4d07d9475ed581278a28d065b238a0941402e2", - "shasum": "" - }, - "require": { - "php": "^7.0 || ^5.6", - "zendframework/zend-stdlib": "^2.7 || ^3.0" - }, - "require-dev": { - "phpunit/phpunit": "^6.2.1 || ^5.7.15", - "zendframework/zend-cache": "^2.6.1", - "zendframework/zend-coding-standard": "~1.0.0", - "zendframework/zend-config": "^2.6.0", - "zendframework/zend-db": "^2.9.2", - "zendframework/zend-filter": "^2.6.1", - "zendframework/zend-json": "^2.6.1", - "zendframework/zend-servicemanager": "^2.7.5 || ^3.0.3", - "zendframework/zend-view": "^2.6.3" - }, - "suggest": { - "zendframework/zend-cache": "Zend\\Cache component to support cache features", - "zendframework/zend-db": "Zend\\Db component", - "zendframework/zend-filter": "Zend\\Filter component", - "zendframework/zend-json": "Zend\\Json component", - "zendframework/zend-servicemanager": "Zend\\ServiceManager component", - "zendframework/zend-view": "Zend\\View component" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.8.x-dev", - "dev-develop": "2.9.x-dev" - }, - "zf": { - "component": "Zend\\Paginator", - "config-provider": "Zend\\Paginator\\ConfigProvider" - } - }, - "autoload": { - "psr-4": { - "Zend\\Paginator\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "BSD-3-Clause" - ], - "description": "Paginate collections of data from arbitrary sources", - "keywords": [ - "ZendFramework", - "paginator", - "zf" - ], - "abandoned": "laminas/laminas-paginator", - "time": "2019-08-21T13:31:03+00:00" } ], "packages-dev": [ @@ -7248,16 +7267,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.1.0", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "7462d5f123dfc080dfdf26897032a6513644fc95" + "reference": "30441f2752e493c639526b215ed81d54f369d693" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/7462d5f123dfc080dfdf26897032a6513644fc95", - "reference": "7462d5f123dfc080dfdf26897032a6513644fc95", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30441f2752e493c639526b215ed81d54f369d693", + "reference": "30441f2752e493c639526b215ed81d54f369d693", "shasum": "" }, "require": { @@ -7271,7 +7290,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "1.x-dev" + "dev-1.x": "1.x-dev" } }, "autoload": { @@ -7290,7 +7309,7 @@ } ], "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", - "time": "2020-02-18T18:59:58+00:00" + "time": "2020-06-19T20:22:09+00:00" }, { "name": "phpspec/prophecy", @@ -7609,16 +7628,16 @@ }, { "name": "phpunit/phpunit", - "version": "8.5.5", + "version": "8.5.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "63dda3b212a0025d380a745f91bdb4d8c985adb7" + "reference": "34c18baa6a44f1d1fbf0338907139e9dce95b997" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/63dda3b212a0025d380a745f91bdb4d8c985adb7", - "reference": "63dda3b212a0025d380a745f91bdb4d8c985adb7", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/34c18baa6a44f1d1fbf0338907139e9dce95b997", + "reference": "34c18baa6a44f1d1fbf0338907139e9dce95b997", "shasum": "" }, "require": { @@ -7688,7 +7707,7 @@ "testing", "xunit" ], - "time": "2020-05-22T13:51:52+00:00" + "time": "2020-06-22T07:06:58+00:00" }, { "name": "se/selenium-server-standalone", @@ -8426,7 +8445,7 @@ }, { "name": "symfony/css-selector", - "version": "v5.1.1", + "version": "v5.1.2", "source": { "type": "git", "url": "https://github.com/symfony/css-selector.git", @@ -8920,7 +8939,7 @@ }, { "name": "symfony/yaml", - "version": "v5.1.1", + "version": "v5.1.2", "source": { "type": "git", "url": "https://github.com/symfony/yaml.git", @@ -9023,16 +9042,16 @@ }, { "name": "webmozart/assert", - "version": "1.8.0", + "version": "1.9.0", "source": { "type": "git", "url": "https://github.com/webmozart/assert.git", - "reference": "ab2cb0b3b559010b75981b1bdce728da3ee90ad6" + "reference": "9dc4f203e36f2b486149058bade43c851dd97451" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/webmozart/assert/zipball/ab2cb0b3b559010b75981b1bdce728da3ee90ad6", - "reference": "ab2cb0b3b559010b75981b1bdce728da3ee90ad6", + "url": "https://api.github.com/repos/webmozart/assert/zipball/9dc4f203e36f2b486149058bade43c851dd97451", + "reference": "9dc4f203e36f2b486149058bade43c851dd97451", "shasum": "" }, "require": { @@ -9040,6 +9059,7 @@ "symfony/polyfill-ctype": "^1.8" }, "conflict": { + "phpstan/phpstan": "<0.12.20", "vimeo/psalm": "<3.9.1" }, "require-dev": { @@ -9067,7 +9087,7 @@ "check", "validate" ], - "time": "2020-04-18T12:12:48+00:00" + "time": "2020-06-16T10:16:42+00:00" }, { "name": "yawik/behat", diff --git a/config/console.config.php b/config/console.config.php index b66dab8..490007a 100644 --- a/config/console.config.php +++ b/config/console.config.php @@ -13,7 +13,7 @@ 'routes' => [ 'solr-index-job' => [ 'options' => [ - 'route' => 'solr index job', + 'route' => 'solr index job [--batch=]', 'defaults' => [ 'controller' => 'Solr/Console', 'action' => 'activeJobIndex', diff --git a/contrib/conf/managed-schema b/contrib/conf/managed-schema index 7b367f0..0e2dc22 100644 --- a/contrib/conf/managed-schema +++ b/contrib/conf/managed-schema @@ -42,7 +42,6 @@ - @@ -450,9 +449,9 @@ + - @@ -461,10 +460,8 @@ - - @@ -489,12 +486,14 @@ + + diff --git a/contrib/conf/schema.xml b/contrib/conf/schema.xml index db9cf56..bfed7b6 100644 --- a/contrib/conf/schema.xml +++ b/contrib/conf/schema.xml @@ -1,500 +1,493 @@ - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Controller/ConsoleController.php b/src/Controller/ConsoleController.php index 1ac4a0f..7ea286c 100644 --- a/src/Controller/ConsoleController.php +++ b/src/Controller/ConsoleController.php @@ -10,6 +10,7 @@ namespace Solr\Controller; use Core\Console\ProgressBar; +use Jobs\Entity\StatusInterface; use Jobs\Repository\Job as JobRepository; use SolrClient; use Laminas\Mvc\Controller\AbstractActionController; @@ -32,17 +33,17 @@ class ConsoleController extends AbstractActionController * @var SolrClient */ protected $solrClient; - + /** * @var JobRepository */ protected $jobRepository; - + /** * @var callable */ protected $progressBarFactory; - + /** * @param SolrClient $solrClient * @param JobRepository $jobRepository @@ -59,33 +60,77 @@ public function __construct(SolrClient $solrClient, JobRepository $jobRepository public function activeJobIndexAction() { - $jobs = $this->jobRepository->findActiveJob(); - $count = $jobs->count(); - + + $limit = $this->params('batch', false); + if ($limit) { + $file = getcwd() . '/var/cache/solr-index.dat'; + echo $file; exit; + $skip = file_exists($file) ? file_get_contents($file) : 0; + file_put_contents($file, ($skip + $limit)); + } + + $qb = $this->jobRepository->createQueryBuilder() + ->hydrate(true) + ->field('status.name')->in([StatusInterface::ACTIVE]) + ->field('isDraft')->equals(false) + ->readOnly() + ; + if ($limit) { + $qb->limit($limit)->skip($skip); + } + $q = $qb->getQuery(); + $jobs = $q->execute(); + + $count = $jobs->count(true); + // check if there is any active job if (0 === $count) { + if ($limit) { + unlink($file); + } return 'There is no active job' . PHP_EOL; } - + + if ($count > 2500 && !$limit) { + return 'There are to many active jobs, please use --batch'; + } + + if ($limit) { + $upper = ($skip + $limit); + $total = $jobs->count(); + $upper = $upper > $total ? $total : $upper; + echo "Processing jobs $skip - $upper of $total", PHP_EOL; + } + $i = 1; $progressBarFactory = $this->progressBarFactory; $progressBar = $progressBarFactory($count); $entityToDocument = new \Solr\Filter\EntityToDocument\JobEntityToSolrDocument($this->options); - + // add jobs in the Solr index foreach ($jobs as $job) { $document = $entityToDocument->filter($job); $this->solrClient->addDocument($document); $progressBar->update($i, 'Job ' . $i . ' / ' . $count); $i++; + if ($i % 1000 === 0) { + $this->solrClient->commit(true, false); + $this->solrClient->optimize(1, true, false); + } } - + $this->solrClient->commit(true, false); $this->solrClient->optimize(1, true, false); + $this->jobRepository->getDocumentManager()->clear(); + if ($limit && $count < $limit) { + echo "No jobs left."; + unlink($file); + exit(1); + } return PHP_EOL; } - + /** * @return callable */ diff --git a/src/Filter/EntityToDocument/JobEntityToSolrDocument.php b/src/Filter/EntityToDocument/JobEntityToSolrDocument.php index 6985846..dc6190f 100644 --- a/src/Filter/EntityToDocument/JobEntityToSolrDocument.php +++ b/src/Filter/EntityToDocument/JobEntityToSolrDocument.php @@ -40,7 +40,7 @@ public function filter($job) if (!$job instanceof JobEntity) { throw new InvalidArgumentException(sprintf('$job must be instance of "%s"', JobEntity::class)); } - + $document = new SolrInputDocument(); $document->addField('id', $job->getId()); $document->addField('applyId', $job->getApplyId()); @@ -73,9 +73,9 @@ public function filter($job) } elseif (!is_null($job->getOrganization())) { $this->processOrganization($job, $document); } - + $plainText = $job->getMetaData('plainText'); - + if ($plainText) { $html = $plainText; } else { @@ -84,13 +84,13 @@ public function filter($job) $stripTags = new StripTags(); $stripTags->setAttributesAllowed([])->setTagsAllowed([]); $description = $stripTags->filter($description); - + $qualification = $stripTags($templateValues->getQualifications()); $requirements = $stripTags($templateValues->getRequirements()); $benefits = $stripTags($templateValues->getBenefits()); $html = "$description " . $job->getTitle() ." $requirements $qualification $benefits"; } - + $document->addField('html', $html); foreach ($job->getClassifications()->getProfessions()->getItems() as $profession) { /* @var $profession \Jobs\Entity\Category */ @@ -106,7 +106,7 @@ public function filter($job) return $document; } - + /** * @param JobEntity $job * @return array @@ -114,17 +114,17 @@ public function filter($job) public function getDocumentIds(JobEntity $job) { $ids = [$job->getId()]; - + /* @var $location \Jobs\Entity\Location */ foreach ($job->getLocations() as $location) { if (is_object($location->getCoordinates())) { $ids[] = $this->getLocationDocumentId($job, Util::convertLocationCoordinates($location)); } } - + return $ids; } - + /** * @param JobEntity $job * @param SolrInputDocument $document @@ -156,8 +156,6 @@ public function processLocation(JobEntity $job, SolrInputDocument $document) $region = $location->getRegion(); $city = $location->getCity(); $loc->addField('point', $coordinate); - $loc->addField('latLon', $coordinate); - $document->addField('locations', $coordinate); $document->addField('points', $coordinate); $loc->addField('id', $this->getLocationDocumentId($job, $coordinate)); $loc->addField('city', $city); @@ -169,10 +167,10 @@ public function processLocation(JobEntity $job, SolrInputDocument $document) $document->addChildDocument($loc); } } - + $document->addField('location', $job->getLocation()); } - + /** * @param JobEntity $job * @param string $coordinate diff --git a/src/Module.php b/src/Module.php index 3921b55..2a0b983 100644 --- a/src/Module.php +++ b/src/Module.php @@ -22,7 +22,7 @@ * @since 0.26 * @package Solr */ -class Module implements ConsoleUsageProviderInterface,VersionProviderInterface +class Module implements ConsoleUsageProviderInterface, VersionProviderInterface { use VersionProviderTrait; @@ -32,6 +32,11 @@ public function getConsoleUsage(AdapterInterface $console) { return [ 'solr index job' => 'Indexing active jobs', + ['--batch=', 'Indexing jobs in batches of jobs.'], + ['', 'Each invokation will continue the indexing with the next batch.'], + ['', 'When the last batch is indexed, it exists with a non-zero exit code.'], + ['', 'So you can do something like:'], + ['', 'while true; do [yawik] solr index job --batch 2500 || break; done'], ]; } diff --git a/test/SolrTest/Controller/ConsoleControllerTest.php b/test/SolrTest/Controller/ConsoleControllerTest.php index ffd56fc..a71f63b 100644 --- a/test/SolrTest/Controller/ConsoleControllerTest.php +++ b/test/SolrTest/Controller/ConsoleControllerTest.php @@ -14,14 +14,15 @@ use Core\Console\ProgressBar; use Doctrine\MongoDB\CursorInterface; -use Solr\Filter\EntityToDocument\JobEntityToSolrDocument; +use Doctrine\ODM\MongoDB\DocumentManager; use Solr\Options\ModuleOptions; use SolrClient; use Jobs\Entity\Job; use Jobs\Repository\Job as JobRepository; use Solr\Controller\ConsoleController; use stdClass; -use Laminas\Log\Filter\Mock; +use Laminas\Mvc\Controller\PluginManager; +use Laminas\ServiceManager\ServiceManager; /** * Class ConsoleControllerTest @@ -34,27 +35,27 @@ */ class ConsoleControllerTest extends TestCase { - + /** * @var ConsoleController */ protected $controller; - + /** * @var SolrClient|MockObject */ protected $client; - + /** * @var CursorInterface|MockObject */ protected $cursor; - + /** * @var ProgressBar|MockObject */ protected $progressBar; - + /** * @var MockObject */ @@ -64,7 +65,11 @@ class ConsoleControllerTest extends TestCase * @var \Solr\Options\ModuleOptions */ protected $options; - + + protected $params; + + protected $qb; + /** * {@inheritDoc} */ @@ -73,17 +78,40 @@ protected function setUp():void $this->client = $this->getMockBuilder(SolrClient::class) ->disableOriginalConstructor() ->getMock(); - + $this->cursor = $this->getMockBuilder(CursorInterface::class) ->getMock(); - + $jobRepo = $this->getMockBuilder(JobRepository::class) ->disableOriginalConstructor() ->getMock(); + + $dm = $this->getMockBuilder(DocumentManager::class)->disableOriginalConstructor()->getMock(); + $dm->expects($this->any())->method('clear'); + + $jobRepo->expects($this->any())->method('getDocumentManager')->willReturn($dm); + + $query = $this->getMockBuilder(\Doctrine\MongoDb\Query\Query::class) + ->disableOriginalConstructor() + ->setMethods(['execute']) + ->getMock(); + $query->expects($this->any())->method('execute')->willReturn($this->cursor); + + $qb = $this->getMockBuilder(\Doctrine\MongoDB\Query\Builder::class) + ->disableOriginalConstructor() + ->setMethods(['hydrate', 'field', 'in', 'equals', 'readOnly', 'limit', 'skip', 'getQuery']) + ->getMock(); + $qb->expects($this->any())->method('hydrate')->will($this->returnSelf()); + $qb->expects($this->any())->method('field')->will($this->returnSelf()); + $qb->expects($this->any())->method('in')->will($this->returnSelf()); + $qb->expects($this->any())->method('equals')->will($this->returnSelf()); + $qb->expects($this->any())->method('readOnly')->will($this->returnSelf()); + $qb->expects($this->any())->method('getQuery')->willReturn($query); $jobRepo->expects($this->any()) - ->method('findActiveJob') - ->willReturn($this->cursor); - + ->method('createQueryBuilder') + ->willReturn($qb); + + $this->qb = $qb; $this->progressBar = $this->getMockBuilder(ProgressBar::class) ->disableOriginalConstructor() ->getMock(); @@ -100,36 +128,48 @@ protected function setUp():void ->willReturn($this->progressBar); $this->controller = new ConsoleController($this->client, $jobRepo, $this->progressBarFactory, $this->options); + + $this->params = $this->getMockBuilder(\Laminas\Mvc\Controller\Plugin\Params::class) + ->disableOriginalConstructor() + ->setMethods(['__invoke']) + ->getMock(); + $plugins = new PluginManager(new ServiceManager()); + $plugins->setService('params', $this->params); + + $this->controller->setPluginManager($plugins); } - + /** * @covers ::__construct() * @covers ::activeJobIndexAction() */ public function testActiveJobIndexActionWithoutJobs() { + $this->qb->expects($this->never())->method('limit'); + $this->qb->expects($this->never())->method('skip'); + $this->params->expects($this->once())->method('__invoke')->willReturn(null); $this->cursor->expects($this->once()) ->method('count') ->willReturn(0); $this->cursor->expects($this->never()) ->method('rewind'); - + $this->progressBarFactory->expects($this->never()) ->method('__invoke'); - + $this->progressBar->expects($this->never()) ->method('update'); - + $this->client->expects($this->never()) ->method('addDocument'); $this->client->expects($this->never()) ->method('commit'); $this->client->expects($this->never()) ->method('optimize'); - + $this->assertStringContainsString('no active job', $this->controller->activeJobIndexAction()); } - + /** * @covers ::__construct() * @covers ::activeJobIndexAction() @@ -138,7 +178,7 @@ public function testActiveJobIndexActionWithJobs() { $job = new Job(); $count = 2; - + $this->cursor->expects($this->once()) ->method('count') ->willReturn($count); @@ -148,21 +188,21 @@ public function testActiveJobIndexActionWithJobs() $this->cursor->expects($this->exactly($count)) ->method('current') ->willReturn($job); - + $this->progressBar->expects($this->exactly($count)) ->method('update') ->withConsecutive([1, 'Job 1 / 2'], [2, 'Job 2 / 2']); - + $this->client->expects($this->exactly($count)) ->method('addDocument'); $this->client->expects($this->once()) ->method('commit'); $this->client->expects($this->once()) ->method('optimize'); - + $this->assertEmpty(trim($this->controller->activeJobIndexAction())); } - + /** * @covers ::getProgressBarFactory() */ diff --git a/test/SolrTest/Filter/EntityToDocument/JobEntityToSolrDocumentTest.php b/test/SolrTest/Filter/EntityToDocument/JobEntityToSolrDocumentTest.php index c1beb29..5690103 100644 --- a/test/SolrTest/Filter/EntityToDocument/JobEntityToSolrDocumentTest.php +++ b/test/SolrTest/Filter/EntityToDocument/JobEntityToSolrDocumentTest.php @@ -73,7 +73,7 @@ public function testFilterWithValidJob() ->setApplyId('some-external-id'); $document = $this->jobFilter->filter($job); - + $this->assertInstanceOf(SolrInputDocument::class, $document); $this->assertSame($job->getId(), $this->getDocumentValue($document, 'id')); $this->assertSame($job->getApplyId(), $this->getDocumentValue($document, 'applyId')); @@ -88,7 +88,7 @@ public function testFilterWithValidJob() $this->assertFalse((bool)$this->getDocumentValue($document, 'isActive')); $this->assertSame($job->getLanguage(), $this->getDocumentValue($document, 'lang')); } - + /** * @covers ::filter() */ @@ -96,10 +96,10 @@ public function testFilterWithJobWithOrganization() { $organization = $this->getMockBuilder(Organization::class) ->getMock(); - + $job = new Job(); $job->setOrganization($organization); - + $jobFilter = $this->getMockBuilder(JobFilter::class) ->disableOriginalConstructor() ->setMethods(['processOrganization']) @@ -107,10 +107,10 @@ public function testFilterWithJobWithOrganization() $jobFilter->expects($this->once()) ->method('processOrganization') ->with($this->identicalTo($job), $this->isInstanceOf(SolrInputDocument::class)); - + $jobFilter->filter($job); } - + /** * @covers ::getDocumentIds() * @covers ::getLocationDocumentId() @@ -120,9 +120,9 @@ public function testGetDocumentIds() $id = 'some-id'; $job = new Job(); $job->setId($id); - + $this->assertSame([$id], $this->jobFilter->getDocumentIds($job)); - + $coordinate1 = 1.2; $coordinate2 = 2.1; $coordinatesWrapper = $this->getMockBuilder(stdClass::class) @@ -136,7 +136,7 @@ public function testGetDocumentIds() ->willReturn($coordinatesWrapper); $locations = new ArrayCollection([$location]); $job->setLocations($locations); - + $this->assertSame([$id, "{$id}-{$coordinate2},{$coordinate1}"], $this->jobFilter->getDocumentIds($job)); } @@ -148,19 +148,19 @@ public function testProcessOrganization() $companyLogo = 'some-uri'; $organizationName = 'some-name'; $organizationId = 'some-id'; - + $org = $this->getMockBuilder(Organization::class) ->getMock(); $org->expects($this->once()) ->method('getId') ->willReturn($organizationId); - + $orgName = $this->getMockBuilder(OrganizationName::class) ->getMock(); $orgName->expects($this->once()) ->method('getName') ->willReturn($organizationName); - + $orgImage = $this->getMockBuilder(OrganizationImage::class) ->getMock(); $orgImage->expects($this->once()) @@ -177,7 +177,7 @@ public function testProcessOrganization() ->willReturn($orgImage); $document = new SolrInputDocument(); - + $this->jobFilter->processOrganization($job, $document); $this->assertSame($companyLogo, $this->getDocumentValue($document, 'companyLogo')); $this->assertSame($organizationName, $this->getDocumentValue($document, 'organizationName')); @@ -203,7 +203,7 @@ public function testProcessLocation() $location = $this->getMockBuilder(Location::class)->getMock(); $coordinates = $this->getMockBuilder(CoordinatesInterface::class)->getMock(); $locations = [$location]; - + $job->setId('job-id'); $job->setLocation($locationText); $job->expects($this->once()) @@ -226,9 +226,9 @@ public function testProcessLocation() $coordinates->expects($this->once()) ->method('getCoordinates') ->willReturn($coordinatesArray); - + $document = new SolrInputDocument(); - + $this->jobFilter->processLocation($job, $document); $this->assertSame($job->getLocation(), $this->getDocumentValue($document, 'location')); $this->assertSame($region, $this->getDocumentValue($document, 'region_MultiString')); @@ -238,15 +238,14 @@ public function testProcessLocation() $childDocument = reset($childDocuments); $this->assertSame('location', $this->getDocumentValue($childDocument, 'entityName')); $this->assertSame($coordinatesConverted, $this->getDocumentValue($childDocument, 'point')); - $this->assertSame($coordinatesConverted, $this->getDocumentValue($childDocument, 'latLon')); $this->assertSame("{$job->getId()}-$coordinatesConverted", $this->getDocumentValue($childDocument, 'id')); $this->assertSame($city, $this->getDocumentValue($childDocument, 'city')); $this->assertSame($postalCode, $this->getDocumentValue($childDocument, 'postalCode')); $this->assertSame($country, $this->getDocumentValue($childDocument, 'country')); $this->assertSame($region, $this->getDocumentValue($childDocument, 'region')); - + } - + /** * @param SolrInputDocument $document * @param string $fieldName diff --git a/test/sandbox/config/autoload/solr.moduleoptions.local.php~ b/test/sandbox/config/autoload/solr.moduleoptions.local.php~ deleted file mode 100644 index 8518116..0000000 --- a/test/sandbox/config/autoload/solr.moduleoptions.local.php~ +++ /dev/null @@ -1,83 +0,0 @@ - false, - - // The hostname for the Solr server - 'hostname' => 'solr.hq.cross', - - // The port number - 'port' => 8983, - - // The username used for HTTP Authentication, if any - 'username' => null, #'yawik', - - // The HTTP Authentication password - 'password' => null, #'3qaS2uQU86dGbMXjDds2', - - // A path for solr jobs index - 'jobsPath' => '/solr/Gastro24', - - // List of Facet Fields. Fieldnames must exist in the solr index. Each facet field must have a 'name'. - // 'label' is optional. It can be used as a headline of the facet result. - 'facetFields' => [ - [ - 'name' => 'region_MultiString', - 'label' => 'Region' - ], - [ - 'name' => 'city_MultiString', - 'label' => 'Stadt' - ], - [ - 'name' => 'organizationTag', - 'label' => 'Firma' - ], - [ - 'name' => 'profession_MultiString', - 'label' => 'Berufsfeld' - ], - [ - 'name' => 'industry_MultiString', - 'label' => 'Branche' - ], - [ - 'name' => 'employmentType_MultiString', - 'label' => 'Art der Anstellung' - ] - - ], - // - 'parameterNames' => [ - 'q' => [ - 'name' => 'q' - ], - 'l' => [ - 'name' => 'l' - ], - 'd' => [ - 'name' => 'd' - ] - ], - 'sorts' => [ - 'random' => \SolrQuery::ORDER_ASC, - ], - 'filterQueries' => [ - // 'text:(vertrieb OR sales OR Verkäufer OR "Key Account") AND NOT (oracle OR softwareentwicklung OR Netzwerktechniker OR SAP or JAVA OR sachbearbeiter OR Maschinenbau OR P - ], - 'boostQueries' => [ - 'organizationName:(hays orizon)^-1', - ], - - - -]; - -/* - * Do not change below this line - */ -return [ 'options' => [ 'Solr/Options/Module' => [ 'options' => $options ] ] ];