javinizer · jvlflame · Jan 25, 2025 · Jan 25, 2025 · Jan 25, 2025 · Jan 25, 2025
diff --git a/.github/CHANGELOG.md b/.github/CHANGELOG.md
@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.6.0]
+
+### Added
+
+- Added setting `sleep` to add a delay between scraper threads
+
+### Changed
+
+- Reduced throttlelimit from 10 -> 3
+- Added custom UserAgent identifier to r18.dev scraper requests
+- Set JAVLibrary scrapers to disabled by default
+- Set throttlelimit to 1 by default
+
+### Removed
+
+- Removed all cloudflare validation from JAVLibrary scraper
+
+## [2.5.18]
+
+### Fixed
+
+- Fixed age verification error on DmmJa
+
 ## [2.5.17]
 
 ### Fixed

diff --git a/src/Javinizer/Javinizer.psd1 b/src/Javinizer/Javinizer.psd1
@@ -13,7 +13,7 @@
 
     # Version number of this module.
 
-    ModuleVersion     = '2.5.18'
+    ModuleVersion     = '2.6.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @('Core')

diff --git a/src/Javinizer/Public/Get-R18DevData.ps1 b/src/Javinizer/Public/Get-R18DevData.ps1
@@ -1,3 +1,5 @@
+$UserAgent = 'Javinizer (+https://github.com/javinizer/Javinizer)'
+
 function Get-R18DevData {
     [CmdletBinding()]
     param (
@@ -31,7 +33,7 @@ function Get-R18DevData {
 
         try {
             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$apiUrl]"
-            $webRequest = (Invoke-WebRequest -Uri $apiUrl -Method Get -Verbose:$false).Content | ConvertFrom-Json
+            $webRequest = (Invoke-WebRequest -Uri $apiUrl -UserAgent $UserAgent -Method Get -Verbose:$false).Content | ConvertFrom-Json
         } catch {
             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Error [GET] on URL [$Url]: $PSItem" -Action 'Continue'
         }

diff --git a/src/Javinizer/Public/Get-R18DevUrl.ps1 b/src/Javinizer/Public/Get-R18DevUrl.ps1
@@ -1,3 +1,5 @@
+$UserAgent = 'Javinizer (+https://github.com/javinizer/Javinizer)'
+
 function Get-R18DevUrl {
     [CmdletBinding()]
     param (
@@ -34,7 +36,7 @@ function Get-R18DevUrl {
         # Try matching the video with Video ID
         try {
             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$searchUrl]"
-            $webRequest = Invoke-WebRequest -Uri $searchUrl -Method Get -Verbose:$false | ConvertFrom-Json
+            $webRequest = Invoke-WebRequest -Uri $searchUrl -UserAgent $UserAgent -Method Get -Verbose:$false | ConvertFrom-Json
         } catch {
             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Error occured on [GET] on URL [$searchUrl]: $PSItem" -Action 'Continue'
         }
@@ -44,7 +46,7 @@ function Get-R18DevUrl {
 
             try {
                 Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on Uri [$testUrl]"
-                $webRequest = Invoke-WebRequest -Uri $testUrl -Method Get -Verbose:$false | ConvertFrom-Json
+                $webRequest = Invoke-WebRequest -Uri $testUrl -UserAgent $UserAgent -Method Get -Verbose:$false | ConvertFrom-Json
             } catch {
                 $webRequest = $null
             }

diff --git a/src/Javinizer/Public/Javinizer.ps1 b/src/Javinizer/Public/Javinizer.ps1
@@ -393,11 +393,6 @@ function Javinizer {
         [Parameter(ParameterSetName = 'Info')]
         [Switch]$AllResults,
 
-        [Parameter(ParameterSetName = 'Info')]
-        [Parameter(ParameterSetName = 'Path')]
-        [Parameter(ParameterSetName = 'Javlibrary')]
-        [PSObject]$CfSession,
-
         [Parameter(ParameterSetName = 'Emby')]
         [Switch]$SetEmbyThumbs,
 
@@ -701,10 +696,6 @@ function Javinizer {
             }
 
             'Info' {
-                if (($Javlibrary -or $JavlibraryZh -or $JavlibraryJa) -or ($Find -like '*javlibrary*' -or $Find -like '*g46e*' -or $Find -like '*m45e*')) {
-                    $CfSession = Test-JavlibraryCf -Settings $Settings -CfSession $CfSession
-                }
-
                 if ($Find -match 'https?:\/\/') {
                     $urlObject = Get-JVUrlLocation -Url $Find -Settings $Settings
                     $data = foreach ($item in $urlObject) {
@@ -756,7 +747,7 @@ function Javinizer {
                     $data = Get-JVData -Id $Find -Javlibrary:$Javlibrary -JavlibraryJa:$JavlibraryJa -JavlibraryZh:$JavlibraryZh -Dmm:$Dmm `
                         -DmmJa:$DmmJa -R18Dev:$R18Dev -Javbus:$Javbus -JavbusJa:$JavbusJa -JavbusZh:$JavbusZh -Jav321Ja:$Jav321Ja -JavlibraryBaseUrl $Settings.'javlibrary.baseurl' `
                         -MgstageJa:$MgstageJa -Aventertainment:$Aventertainment -AventertainmentJa:$AventertainmentJa -Tokyohot:$Tokyohot -TokyohotJa:$TokyohotJa -TokyohotZh:$TokyohotZh -UncensorCsvPath $uncensorCsvPath -Strict:$Strict `
-                        -Javdb:$Javdb -JavdbZh:$JavdbZh -Session:$CfSession -JavdbSession:$Settings.'javdb.cookie.session' -AllResults:$AllResults
+                        -Javdb:$Javdb -JavdbZh:$JavdbZh -JavdbSession:$Settings.'javdb.cookie.session' -AllResults:$AllResults
                 }
 
                 if ($Aggregated) {
@@ -885,14 +876,13 @@ function Javinizer {
             }
 
             'Javlibrary' {
-                $CfSession = Test-JavlibraryCf -Settings $Settings -CfSession $CfSession
                 try {
                     if (!($Path)) {
                         $Path = $null -ne $Settings.'location.input' -and $Settings.'location.input' -ne '' ? $Settings.'location.input' : (Get-Location).Path
                     }
 
                     $javlibraryBaseUrl = $Settings.'javlibrary.baseurl'
-                    $request = Invoke-WebRequest -Uri "$javlibraryBaseUrl/en/mv_owned.php" -WebSession $CfSession -UserAgent $CfSession.UserAgent -Verbose:$false -Headers @{
+                    $request = Invoke-WebRequest -Uri "$javlibraryBaseUrl/en/mv_owned.php" -Verbose:$false -Headers @{
                         "method"                    = "GET"
                         "authority"                 = "$javlibraryBaseUrl"
                         "scheme"                    = "https"
@@ -940,7 +930,7 @@ function Javinizer {
                     $index = 1
                     foreach ($movieId in $unowned) {
                         Write-Progress -Id 1 -Activity "Javinizer" -Status "Remaining Jobs: $($unowned.Count-$index)" -PercentComplete ($index / $unowned.Count * 100) -CurrentOperation "Setting owned: $movieId"
-                        Set-JavlibraryOwned -Id $movieId -UserId $Settings.'javlibrary.cookie.userid' -LoginSession $Settings.'javlibrary.cookie.session' -Session:$CfSession -BaseUrl $javlibraryBaseUrl
+                        Set-JavlibraryOwned -Id $movieId -UserId $Settings.'javlibrary.cookie.userid' -LoginSession $Settings.'javlibrary.cookie.session' -BaseUrl $javlibraryBaseUrl
                         $index++
                     }
                 } else {
@@ -957,10 +947,6 @@ function Javinizer {
             }
 
             'Path' {
-                if (!$IsThread -and ($Settings.'scraper.movie.javlibrary' -or $Settings.'scraper.movie.javlibraryja' -or $Settings.'scraper.movie.javlibraryzh')) {
-                    $CfSession = Test-JavlibraryCf -Settings $Settings -CfSession $CfSession
-                }
-
                 if (!($Path)) {
                     try {
                         # Default path to location.input in settings if not specified
@@ -1008,7 +994,7 @@ function Javinizer {
                         $mediaInfo = Get-JVMediaInfo -Path $movie.FullName
                     }
 
-                    $javData = Get-JVData -Url $Url -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Session:$CfSession -JavdbSession:$Settings.'javdb.cookie.session'
+                    $javData = Get-JVData -Url $Url -Settings $Settings -UncensorCsvPath $uncensorCsvPath -JavdbSession:$Settings.'javdb.cookie.session'
                     if ($null -ne $javData) {
                         $javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings -FileName $javMovies.BaseName -MediaInfo $mediaInfo | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
                         if ($javAggregatedData.NullFields -eq '') {
@@ -1040,8 +1026,8 @@ function Javinizer {
                         }
                     }
                 } else {
-                    if ($Settings.'throttlelimit' -lt 1 -or $Settings.'throttlelimit' -gt 10) {
-                        Write-JVLog -Write $script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Setting 'scraper.throttlelimit' must be within accepted values (1-10)"
+                    if ($Settings.'throttlelimit' -lt 1 -or $Settings.'throttlelimit' -gt 3) {
+                        Write-JVLog -Write $script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Setting 'scraper.throttlelimit' must be within accepted values (1-3)"
                     }
 
                     if (!($PSboundParameters.ContainsKey('IsThread'))) {
@@ -1052,15 +1038,15 @@ function Javinizer {
                                     Import-Module $using:jvModulePath
                                     $jvMovie = $_
                                     $Settings = $using:Settings
-                                    $jvData = Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -WebTempPath:$using:WebTempPath -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
+                                    $jvData = Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -WebTempPath:$using:WebTempPath -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
                                     Write-Output $jvData
                                 }
                             } else {
                                 $javMovies | Invoke-JVParallel -IsWeb -IsWebType 'sort' -MaxQueue $Settings.'throttlelimit' -Throttle $Settings.'throttlelimit' -Quiet:$true -ScriptBlock {
                                     Import-Module $using:jvModulePath
                                     $jvMovie = $_
                                     $Settings = $using:Settings
-                                    Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
+                                    Javinizer -IsThread -IsWeb -IsWebType $using:IsWebType -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
                                 }
                             }
 
@@ -1069,20 +1055,31 @@ function Javinizer {
                                 Import-Module $using:jvModulePath
                                 $jvMovie = $_
                                 $Settings = $using:Settings
-                                $jvData = Javinizer -IsThread -Search -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
+                                $jvData = Javinizer -IsThread -Search -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
                                 Write-Output $jvData
                             }
                         } else {
                             $javMovies | Invoke-JVParallel -MaxQueue $Settings.'throttlelimit' -Throttle $Settings.'throttlelimit' -Quiet:$HideProgress -ScriptBlock {
                                 Import-Module $using:jvModulePath
                                 $jvMovie = $_
                                 $Settings = $using:Settings
-                                Javinizer -IsThread -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
+                                Javinizer -IsThread -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
                             }
                         }
                     }
 
                     if ($PSboundParameters.ContainsKey('IsThread')) {
+                        $sleepSeconds = $Settings.'sleep'
+
+                        if (!($sleepSeconds)) {
+                            # Default sleep to 5 seconds if not specified
+                            $sleepSeconds = 5
+                        }
+
+                        if ($sleepSeconds -gt 0) {
+                            Start-Sleep -Seconds $sleepSeconds
+                        }
+
                         foreach ($movie in $javMovies) {
                             if (!$MoveToFolder -and ($DestinationPath -ne $movie.Directory)) {
                                 $DestinationPath = $movie.Directory
@@ -1092,7 +1089,7 @@ function Javinizer {
                                 $mediaInfo = Get-JVMediaInfo -Path $movie.FullName
                             }
 
-                            $javData = Get-JVData -Id $movie.Id -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Strict:$Strict -Session:$CfSession -JavdbSession:$Settings.'javdb.cookie.session'
+                            $javData = Get-JVData -Id $movie.Id -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Strict:$Strict -JavdbSession:$Settings.'javdb.cookie.session'
                             $javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings -FileName $movie.BaseName -MediaInfo $mediaInfo | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
 
                             if ($PSBoundParameters.ContainsKey('IsWeb') -or $PSBoundParameters.ContainsKey('Search')) {

diff --git a/src/Javinizer/Universal/Repository/javinizergui.ps1 b/src/Javinizer/Universal/Repository/javinizergui.ps1
@@ -1,4 +1,4 @@
-$cache:guiVersion = '2.5.17-1'
+$cache:guiVersion = '2.6.0-1'
 
 # Define Javinizer module file paths
 $cache:modulePath = (Get-InstalledModule -Name Javinizer).InstalledLocation
@@ -679,7 +679,6 @@ function Show-JVCfModal {
                 $cf_chl_prog = (Get-UDElement -Id 'textbox-javlibrary-cfchlprog').value
                 $cf_clearance = (Get-UDElement -Id 'textbox-javlibrary-cfclearance').value
                 $UserAgent = (Get-UDElement -Id 'textbox-javlibrary-useragent').value
-                $cache:cfSession = Get-CfSession -cf_chl_2 $cf_chl_2 -cf_chl_prog $cf_chl_prog -cf_clearance $cf_clearance -UserAgent $useragent -BaseUrl $cache:settings.'javlibrary.baseurl'
             } catch {
                 Show-JVToast -Type Error -Message "$PSItem"
                 Hide-UDModal
@@ -792,41 +791,6 @@ function Invoke-JavinizerWeb {
 
     process {
         if (!($cache:inProgress)) {
-            # Check if javlibrary cloudflare protection is enabled
-            if ($cache:settings.'scraper.movie.javlibrary' -or $cache:settings.'scraper.movie.javlibraryja' -or $cache:settings.'scraper.movie.javlibraryzh') {
-                if (-not ($cache:cfSession)) {
-                    try {
-                        Invoke-WebRequest -Uri $cache:settings.'javlibrary.baseurl' -Verbose:$false | Out-Null
-                    } catch {
-                        try {
-                            # Test with persisted settings
-                            if ($cache:settings.'javlibrary.cookie.cf_chl_2' -and $cache:settings.'javlibrary.cookie.cf_chl_prog' -and $cache:settings.'javlibrary.cookie.cf_clearance' -and $cache:settings.'javlibrary.browser.useragent') {
-                                $cache:cfSession = Get-CfSession -cf_chl_2:$cache:settings.'javlibrary.cookie.cf_chl_2' -cf_chl_prog:$cache:settings.'javlibrary.cookie.cf_chl_prog' -cf_clearance:$cache:settings.'javlibrary.cookie.cf_clearance' `
-                                    -UserAgent:$cache:settings.'javlibrary.browser.useragent' -BaseUrl $cache:settings.'javlibrary.baseurl'
-
-                                # Testing with the newly created session sometimes fails if there is no wait time
-                                Start-Sleep -Seconds 1
-
-                                Invoke-WebRequest -Uri $cache:settings.'javlibrary.baseurl' -WebSession $cache:cfSession -UserAgent $cache:cfSession.UserAgent -Verbose:$false | Out-Null
-                            } else {
-                                Show-JVCfModal
-                                return
-                            }
-                        } catch {
-                            Show-JVCfModal
-                            return
-                        }
-                    }
-                } else {
-                    try {
-                        Invoke-WebRequest -Uri $cache:settings.'javlibrary.baseurl' -WebSession $cache:cfSession -UserAgent $cache:cfSession.UserAgent -Verbose:$false | Out-Null
-                    } catch {
-                        Show-JVCfModal
-                        return
-                    }
-                }
-            }
-
             Show-JVProgressModal -Sort
 
             if ($interactive) {