Add files via upload

c/checkrobots.h

@@ -72,12 +72,12 @@ int checkrobots(char *rURLprefix, char *rDomain, char *rURLpath)
 		if(fp = fopen(outfilename,"wb")){
 			//set curl options
 			curl_easy_setopt(curl, CURLOPT_URL, robotsurl);// set URL to get here 
-			curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; WebCrawler; SearchEngine)"); 
+			curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Wibybot; https://wiby.me/)"); 
 			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_checkrobots);// send all data to this function  // 
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);// write the page body to this file handle  
 			curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);//allow redirects
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L);
-			curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 55L);
+			curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15L);
 			curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);//max num of redirects
 			curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, 1000000L);//don't download if over 1MB
 			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);//0 or 1 to verify ssl

c/cr.c

@@ -511,7 +511,7 @@ int main(int argc, char **argv)
 					curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);// write the page body to this file handle  
 					curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);//allow redirects
 					curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L);
-					curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 55L);					
+					curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);					
 					curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);//max num of redirects
 					curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, 5000000L);//don't download if over 5MB
 					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);//0 or 1 to verify ssl

c/urlparse.h

@@ -142,7 +142,7 @@ void urlparse(char* url){
 	
 		//check for file extension like html/htm/txt if no prefix in url
 		if(noprebutisdomain==1 && urlsize>4){
-			if(checkDomain(urlnopathnoprefix_fromlist,".html",".HTML",urlnopathnoprefix_len,5)==1 || checkDomain(urlnopathnoprefix_fromlist,".htm",".HTM",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".txt",".txt",urlnopathnoprefix_len,4)==1){
+			if(checkDomain(urlnopathnoprefix_fromlist,".html",".HTML",urlnopathnoprefix_len,5)==1 || checkDomain(urlnopathnoprefix_fromlist,".htm",".HTM",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".txt",".txt",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".php",".PHP",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".shtml",".SHTML",urlnopathnoprefix_len,6)==1 || checkDomain(urlnopathnoprefix_fromlist,".xhtml",".XHTML",urlnopathnoprefix_len,6)==1 || checkDomain(urlnopathnoprefix_fromlist,".cgi",".CGI",urlnopathnoprefix_len,4)==1){
 				memset(domain,0,1000);
 				memset(urlnoprefix_fromlist,0,1000);
 				memset(urlnopathnoprefix_fromlist,0,1000);