jikan-me · irfan-dahir · Jul 27, 2022 · Jul 27, 2022 · Jul 27, 2022 · Jul 27, 2022
@@ -4,7 +4,6 @@ Homestead.json
 Homestead.yaml
 .env
 composer.phar
-composer.lock
 .php_cs.cache
 .phpunit.result.cache
 .env.v4

@@ -48,42 +48,6 @@ public function handle()
 
         echo "Note: If an entry already exists, it will be updated instead.\n\n";
 
-        /**
-         * Producers
-         */
-        echo "Indexing Producers...\n";
-        $results = \json_decode(
-            app('SerializerV4')->serialize(
-                app('JikanParser')
-                    ->getProducers(new ProducersRequest()),
-                'json'
-            ),
-            true
-        )['producers'];
-
-        if (HttpHelper::hasError($results)) {
-            echo "FAILED: {$results->original['error']}\n";
-            return;
-        }
-
-        $itemCount = count($results);
-        echo "Parsed {$itemCount} producers\n";
-        foreach ($results as $i => $item) {
-            $result = DB::table('producers')
-                ->updateOrInsert(
-                    [
-                        'mal_id' => $item['mal_id']
-                    ],
-                    [
-                        'mal_id' => $item['mal_id'],
-                        'name' => $item['name'],
-                        'url' => $item['url'],
-                        'count' => $item['count']
-                    ]
-                );
-            echo "Indexing {$i}/{$itemCount} \r";
-        }
-
         /**
          * Magazines
          */

@@ -0,0 +1,178 @@
+<?php
+
+namespace App\Console\Commands\Indexer;
+
+use App\Exceptions\Console\FileNotFoundException;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Storage;
+use Jikan\Request\Producer\ProducersRequest;
+
+
+/**
+ * Class ProducersIndexer
+ * @package App\Console\Commands\Indexer
+ */
+class ProducersIndexer extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *`
+     * @var string
+     */
+    protected $signature = 'indexer:producers
+                            {--failed : Run only entries that failed to index last time}
+                            {--resume : Resume from the last position}
+                            {--reverse : Start from the end of the array}
+                            {--index=0 : Start from a specific index}
+                            {--delay=3 : Set a delay between requests}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Index all producers';
+
+    /**
+     * @var array
+     */
+    private array $ids;
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return void
+     * @throws FileNotFoundException
+     */
+    public function handle()
+    {
+
+        $failed = $this->option('failed') ?? false;
+        $resume = $this->option('resume') ?? false;
+        $reverse = $this->option('reverse') ?? false;
+        $delay = $this->option('delay') ?? 3;
+        $index = $this->option('index') ?? 0;
+
+        $index = (int)$index;
+        $delay = (int)$delay;
+
+        $this->info("Info: ProducersIndexer scrapes available MAL IDs and updates/indexes them\n\n");
+
+        if ($failed && Storage::exists('indexer/indexer_producers.save')) {
+            $this->ids = $this->loadFailedMalIds();
+        }
+
+        if (!$failed) {
+            $this->ids = $this->fetchMalIds();
+        }
+
+        // start from the end
+        if ($reverse) {
+            $this->ids = array_reverse($this->ids);
+        }
+
+        // Resume
+        if ($resume && Storage::exists('indexer/indexer_producers.save')) {
+            $index = (int)Storage::get('indexer/indexer_producers.save');
+
+            $this->info("Resuming from index: {$index}");
+        }
+
+        // check if index even exists
+        if ($index > 0 && !isset($this->ids[$index])) {
+            $index = 0;
+            $this->warn('Invalid index; set back to 0');
+        }
+
+        // initialize and index
+        Storage::put('indexer/indexer_producers.save', 0);
+
+        echo "Loading MAL IDs\n";
+        $count = count($this->ids);
+        $failedIds = [];
+        $success = [];
+
+        echo "{$count} entries available\n";
+        for ($i = $index; $i <= ($count - 1); $i++) {
+            $id = $this->ids[$i];
+
+            $url = env('APP_URL') . "/v4/producers/{$id}";
+
+            echo "Indexing/Updating " . ($i + 1) . "/{$count} {$url} [MAL ID: {$id}] \n";
+
+            try {
+                $response = json_decode(file_get_contents($url), true);
+
+                if (isset($response['error']) && $response['status'] != 404) {
+                    echo "[SKIPPED] Failed to fetch {$url} - {$response['error']}\n";
+                    $failedIds[] = $id;
+                    Storage::put('indexer/indexer_producers.failed', json_encode($failedIds));
+                }
+
+                sleep($delay);
+            } catch (\Exception $e) {
+                echo "[SKIPPED] Failed to fetch {$url}\n";
+                $failedIds[] = $id;
+                Storage::put('indexer/indexer_producers.failed', json_encode($failedIds));
+            }
+
+            $success[] = $id;
+            Storage::put('indexer/indexer_producers.save', $i);
+        }
+
+        Storage::delete('indexer/indexer_producers.save');
+
+        echo "---------\nIndexing complete\n";
+        echo count($success) . " entries indexed or updated\n";
+        echo count($failedIds) . " entries failed to index or update. Re-run with --failed to requeue failed entries only\n";
+    }
+
+    /**
+     * @return array
+     */
+    private function fetchMalIds() : array
+    {
+        $this->info("Scraping Producer MAL IDs from https://myanimelist.net/anime/producer...\n");
+
+        $producers = \json_decode(
+            app('SerializerV4')->serialize(
+                app('JikanParser')
+                    ->getProducers(new ProducersRequest()),
+                'json'
+            ),
+            true
+        )['producers'];
+
+        foreach ($producers as $producer) {
+            $this->ids[] = $producer->mal_id;
+        }
+
+        Storage::put('indexer/producers_mal_id.json', json_encode($this->ids));
+
+        return json_decode(Storage::get('indexer/producers_mal_id.json'));
+    }
+
+    /**
+     * @return array
+     * @throws FileNotFoundException
+     */
+    private function loadFailedMalIds() : array
+    {
+        if (!Storage::exists('indexer/indexer_producers.failed')) {
+            throw new FileNotFoundException('"indexer/indexer_producers.failed" does not exist');
+        }
+
+        return json_decode(Storage::get('indexer/indexer_producers.failed'));
+    }
+
+}
@@ -10,6 +10,7 @@
 use App\Console\Commands\Indexer\CurrentSeasonIndexer;
 use App\Console\Commands\Indexer\GenreIndexer;
 use App\Console\Commands\Indexer\MangaIndexer;
+use App\Console\Commands\Indexer\ProducersIndexer;
 use App\Console\Commands\ManageMicrocaching;
 use App\Console\Commands\ModifyCacheDriver;
 use App\Console\Commands\ModifyCacheMethod;
@@ -34,7 +35,8 @@ class Kernel extends ConsoleKernel
         ManageMicrocaching::class,
         AnimeIndexer::class,
         MangaIndexer::class,
-        GenreIndexer::class
+        GenreIndexer::class,
+        ProducersIndexer::class
     ];
 
     /**
@@ -60,5 +62,8 @@ protected function schedule(Schedule $schedule)
 
         $schedule->command('indexer:genres')
             ->daily();
+
+        $schedule->command('indexer:producers')
+            ->daily();
     }
 }
@@ -44,6 +44,7 @@
 use Jikan\Request\Anime\AnimeRequest;
 use Jikan\Request\Anime\AnimeReviewsRequest;
 use Jikan\Request\Anime\AnimeStatsRequest;
+use Jikan\Request\Anime\AnimeVideosEpisodesRequest;
 use Jikan\Request\Anime\AnimeVideosRequest;
 use Laravel\Lumen\Http\ResponseFactory;
 use MongoDB\BSON\UTCDateTime;
@@ -713,6 +714,105 @@ public function videos(Request $request, int $id)
         );
     }
 
+    /**
+     *  @OA\Get(
+     *     path="/anime/{id}/videos/episodes",
+     *     operationId="getAnimeVideosEpisodes",
+     *     tags={"anime"},
+     *
+     *     @OA\Parameter(
+     *       name="id",
+     *       in="path",
+     *       required=true,
+     *       @OA\Schema(type="integer")
+     *     ),
+     *
+     *     @OA\Parameter(ref="#/components/parameters/page"),
+     *
+     *     @OA\Response(
+     *         response="200",
+     *         description="Returns episode videos related to the entry",
+     *         @OA\JsonContent(
+     *              ref="#/components/schemas/anime_videos_episodes"
+     *         )
+     *     ),
+     *     @OA\Response(
+     *         response="400",
+     *         description="Error: Bad request. When required parameters were not supplied.",
+     *     ),
+     *  ),
+     *
+     *
+     *  @OA\Schema(
+     *      schema="anime_videos_episodes",
+     *      description="Anime Videos Episodes Resource",
+     *
+     *      allOf={
+     *          @OA\Schema(ref="#/components/schemas/pagination"),
+     *          @OA\Schema(
+     *               @OA\Property(
+     *                    property="data",
+     *                    type="array",
+     *                    @OA\Items(
+     *                        type="object",
+     *                        @OA\Property(
+     *                            property="mal_id",
+     *                            type="integer",
+     *                            description="MyAnimeList ID or Episode Number"
+     *                        ),
+     *                        @OA\Property(
+     *                            property="title",
+     *                            type="string",
+     *                            description="Episode Title"
+     *                        ),
+     *                        @OA\Property(
+     *                            property="episode",
+     *                            type="string",
+     *                            description="Episode Subtitle"
+     *                        ),
+     *                        @OA\Property(
+     *                            property="url",
+     *                            type="string",
+     *                            description="Episode Page URL",
+     *                        ),
+     *                        @OA\Property(
+     *                            property="images",
+     *                            ref="#/components/schemas/common_images"
+     *                        ),
+     *                    ),
+     *               ),
+     *          ),
+     *      }
+     *  )
+     */
+    public function videosEpisodes(Request $request, int $id)
+    {
+        $results = DB::table($this->getRouteTable($request))
+            ->where('request_hash', $this->fingerprint)
+            ->get();
+
+        if (
+            $results->isEmpty()
+            || $this->isExpired($request, $results)
+        ) {
+            $page = $request->get('page') ?? 1;
+            $anime = $this->jikan->getAnimeVideosEpisodes(new AnimeVideosEpisodesRequest($id, $page));
+            $response = \json_decode($this->serializer->serialize($anime, 'json'), true);
+
+            $results = $this->updateCache($request, $results, $response);
+        }
+
+        $response = (new AnimeEpisodesResource(
+            $results->first()
+        ))->response();
+
+        return $this->prepareResponse(
+            $response,
+            $results,
+            $request
+        );
+    }
+
     /**
      *  @OA\Get(
      *     path="/anime/{id}/pictures",