beenull/wiby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add files via upload

Browse source
This commit is contained in:
wibyweb 2023-08-10 00:06:45 -04:00 committed by GitHub
parent 904f821399
commit 19e3c3702c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
c/checkrobots.h
View file

@ -72,12 +72,12 @@ int checkrobots(char *rURLprefix, char *rDomain, char *rURLpath)
if(fp = fopen(outfilename,"wb")){
//set curl options
curl_easy_setopt(curl, CURLOPT_URL, robotsurl);// set URL to get here
curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; WebCrawler; SearchEngine)");
curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Wibybot; https://wiby.me/)");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_checkrobots);// send all data to this function //
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);// write the page body to this file handle
curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);//allow redirects
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 55L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15L);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);//max num of redirects
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, 1000000L);//don't download if over 1MB
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);//0 or 1 to verify ssl

2
c/cr.c
View file

@ -511,7 +511,7 @@ int main(int argc, char **argv)
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);// write the page body to this file handle
curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);//allow redirects
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 55L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);//max num of redirects
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, 5000000L);//don't download if over 5MB
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);//0 or 1 to verify ssl

2
c/urlparse.h
View file

@ -142,7 +142,7 @@ void urlparse(char* url){
//check for file extension like html/htm/txt if no prefix in url
if(noprebutisdomain==1 && urlsize>4){
if(checkDomain(urlnopathnoprefix_fromlist,".html",".HTML",urlnopathnoprefix_len,5)==1 || checkDomain(urlnopathnoprefix_fromlist,".htm",".HTM",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".txt",".txt",urlnopathnoprefix_len,4)==1){
if(checkDomain(urlnopathnoprefix_fromlist,".html",".HTML",urlnopathnoprefix_len,5)==1 || checkDomain(urlnopathnoprefix_fromlist,".htm",".HTM",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".txt",".txt",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".php",".PHP",urlnopathnoprefix_len,4)==1 || checkDomain(urlnopathnoprefix_fromlist,".shtml",".SHTML",urlnopathnoprefix_len,6)==1 || checkDomain(urlnopathnoprefix_fromlist,".xhtml",".XHTML",urlnopathnoprefix_len,6)==1 || checkDomain(urlnopathnoprefix_fromlist,".cgi",".CGI",urlnopathnoprefix_len,4)==1){
memset(domain,0,1000);
memset(urlnoprefix_fromlist,0,1000);
memset(urlnopathnoprefix_fromlist,0,1000);