@@ -372,6 +372,11 @@ func testFakePath(urlChan chan string) {
372
372
atomic .AddInt32 (& currentFakeCount , 1 )
373
373
}
374
374
}
375
+ func addToCompleteList (newUrl string ) {
376
+ completedPathList = append (completedPathList ,newUrl )
377
+ //fmt.Println("add "+newUrl+" "+strconv.Itoa(len(completedPathList)))
378
+ }
379
+
375
380
func checkURL (urlChan chan string ) {
376
381
var tmpResultList3 []string
377
382
for v := range urlChan {
@@ -708,6 +713,9 @@ func getUrlWorker(urlChan chan string) {
708
713
} else if strings .Contains (err .Error (),"server gave HTTP response to HTTPS client" ) {
709
714
fmt .Printf ("%s [%s] [%d of %d]\n " ,newUrl , color .RedString ("302" ),currentListCount ,totalListCount )
710
715
log .Printf ("%s [%s] [%d of %d]\n " ,newUrl , color .RedString ("302" ),currentListCount ,totalListCount )
716
+ } else if strings .Contains (err .Error (),"network is unreachablet" ) {
717
+ fmt .Printf ("%s [%s] [%d of %d]\n " ,newUrl , color .RedString ("Unreachable" ),currentListCount ,totalListCount )
718
+ log .Printf ("%s [%s] [%d of %d]\n " ,newUrl , color .RedString ("Unreachable" ),currentListCount ,totalListCount )
711
719
} else {
712
720
fmt .Printf ("%s [%s] [%d of %d]\n " ,newUrl , color .RedString (err .Error ()))
713
721
log .Printf ("%s [%s] [%d of %d]\n " ,newUrl , color .RedString (err .Error ()))
@@ -731,10 +739,6 @@ func getUrlWorker(urlChan chan string) {
731
739
finalURL := resp .Request .URL .String ()
732
740
var tmpTitle = ""
733
741
if finalURL == newUrl {
734
- //tmpTitle1,tmpStatus:=GetHtmlTitle(string(body))
735
- //if tmpStatus==true {
736
- // tmpTitle=tmpTitle1
737
- //}
738
742
tmpTitle = getPageTitle (newUrl )
739
743
}
740
744
if intelligentMode == true && CMSmode == false {
@@ -772,19 +776,25 @@ func getUrlWorker(urlChan chan string) {
772
776
}
773
777
}
774
778
}
775
- if tmpStatusCode == "200" {
776
- fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .BlueString (initialStatusCode ), lenBody , tmpTitle ,currentListCount ,totalListCount )
777
- log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .BlueString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
778
- } else if tmpStatusCode == "401" {
779
- fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .GreenString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
780
- log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .GreenString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
781
- } else {
782
- if initialStatusCode == "0" {
783
- fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString ("" ), lenBody , tmpTitle , currentListCount ,totalListCount )
784
- log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString ("" ), lenBody , tmpTitle , currentListCount ,totalListCount )
779
+ if tmpFound == true {
780
+ if tmpStatusCode == "200" {
781
+ fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .BlueString (initialStatusCode ), lenBody , tmpTitle ,currentListCount ,totalListCount )
782
+ log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .BlueString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
783
+ //addToCompleteList(newUrl)
784
+ } else if tmpStatusCode == "401" {
785
+ fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .GreenString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
786
+ log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .GreenString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
787
+ //addToCompleteList(newUrl)
785
788
} else {
786
- fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
787
- log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
789
+ if initialStatusCode == "0" {
790
+ fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString ("" ), lenBody , tmpTitle , currentListCount ,totalListCount )
791
+ log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString ("" ), lenBody , tmpTitle , currentListCount ,totalListCount )
792
+ //addToCompleteList(newUrl)
793
+ } else {
794
+ fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
795
+ log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
796
+ //addToCompleteList(newUrl)
797
+ }
788
798
}
789
799
}
790
800
}
@@ -801,21 +811,26 @@ func getUrlWorker(urlChan chan string) {
801
811
log .Printf ("%s [%s] [%d of %d]\n " ,newURL2 , color .RedString (initialStatusCode ), currentListCount ,totalListCount )
802
812
var a = [][]string {{newURL2 , initialStatusCode , "" ,"" }}
803
813
tmpResultList = append (tmpResultList ,a ... )
814
+ //addToCompleteList(newUrl)
804
815
} else if (resp .StatusCode != 401 && initialStatusCode == "401" ) {
805
816
fmt .Printf ("%s [%s] [%d of %d]\n " ,newURL2 , color .RedString (initialStatusCode ), currentListCount ,totalListCount )
806
817
log .Printf ("%s [%s] [%d of %d]\n " ,newURL2 , color .RedString (initialStatusCode ), currentListCount ,totalListCount )
807
818
var a = [][]string {{newURL2 , initialStatusCode , "" ,"" }}
808
819
tmpResultList = append (tmpResultList ,a ... )
820
+ //addToCompleteList(newUrl)
809
821
} else {
810
822
if tmpStatusCode == "200" {
811
823
fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .BlueString (initialStatusCode ), lenBody , tmpTitle ,currentListCount ,totalListCount )
812
824
log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .BlueString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
825
+ //addToCompleteList(newUrl)
813
826
} else if tmpStatusCode == "401" {
814
827
fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .GreenString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
815
828
log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .GreenString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
829
+ //addToCompleteList(newUrl)
816
830
} else {
817
831
fmt .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
818
832
log .Printf ("%s [%s] [%d] [%s] [%d of %d]\n " ,newUrl , color .RedString (initialStatusCode ), lenBody , tmpTitle , currentListCount ,totalListCount )
833
+ //addToCompleteList(newUrl)
819
834
}
820
835
}
821
836
}
@@ -889,32 +904,28 @@ func getUrlWorker(urlChan chan string) {
889
904
resp .Body .Close ()
890
905
_ = resp
891
906
_ = tmpTitle
892
- }
907
+ }
908
+
893
909
}
894
910
895
911
if currentListCount == totalListCount {
896
- if ! stringInSlice (newUrl ,completedPathList ) {
897
- mu .Lock ()
898
- completedPathList = append (completedPathList ,newUrl )
899
- mu .Unlock ()
900
- }
912
+ //if !stringInSlice(newUrl,completedPathList) {
913
+ //fmt.Println("a1 "+newUrl+" "+strconv.Itoa(len(completedPathList)))
914
+ addToCompleteList (newUrl )
915
+ //}
901
916
reachedTheEnd = true
902
917
} else {
903
- if ! stringInSlice (newUrl ,completedPathList ) {
904
- mu .Lock ()
905
- completedPathList = append (completedPathList ,newUrl )
906
- mu .Unlock ()
907
- currentListCount += 1
908
- }
909
- }
918
+ addToCompleteList (newUrl )
919
+ }
920
+
910
921
_ = err
911
922
} else {
912
- if ! stringInSlice (newUrl ,completedPathList ) {
913
- mu . Lock ( )
914
- completedPathList = append ( completedPathList , newUrl )
915
- mu . Unlock ()
916
- currentListCount += 1
917
- }
923
+ // if !stringInSlice(newUrl,completedPathList) {
924
+ //fmt.Println("a3 "+newUrl+" "+strconv.Itoa(len(completedPathList)) )
925
+ addToCompleteList ( newUrl )
926
+ currentListCount += 1
927
+ //}
928
+ //mu.Unlock()
918
929
}
919
930
}
920
931
}
@@ -1779,9 +1790,7 @@ func main() {
1779
1790
} else {
1780
1791
for _ , v := range tmpResultList {
1781
1792
if ! stringInSlice (v [0 ],tmpResultList1 ) {
1782
- //fmt.Println("xxx ",v[0])
1783
1793
tmpResultList1 = append (tmpResultList1 , v [0 ])
1784
- //tmpResultList1 = append(tmpResultList1, v[0])
1785
1794
}
1786
1795
}
1787
1796
@@ -1800,7 +1809,9 @@ func main() {
1800
1809
//Wait until all processes have been completed
1801
1810
for {
1802
1811
if len (completedPathList )!= len (finalList ) {
1803
- time .Sleep (5 * time .Millisecond )
1812
+ //fmt.Println("a "+strconv.Itoa(len(completedPathList))+" "+strconv.Itoa(len(finalList)))
1813
+ //time.Sleep(5 * time.Millisecond)
1814
+ time .Sleep (5 * time .Second )
1804
1815
} else {
1805
1816
break
1806
1817
}
0 commit comments