0ct0pu5/wiby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add files via upload

Browse source
This commit is contained in:
wibyweb 2023-06-20 23:28:33 -04:00 • committed by GitHub
parent 8d73643940
commit 9b1605592d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
c/cr.c
View file

@ -296,7 +296,7 @@ int main(int argc, char **argv)
memset(checkurl,0,checkurlsize);
if(task == 0 || task[0] == '2'){//index request did not come from refresh scheduler, or is an autocrawl url
//strcpy(checkurl,"SELECT id,updatable,title,enable,fault,url FROM windex WHERE url = 'http://"); //replace this with a simple check for url_noprefix column match
strcpy(checkurl,"SELECT id,updatable,title,enable,fault,url,url_noprefix,shard FROM windex WHERE url_noprefix = '");
strcpy(checkurl,"SELECT id,updatable,title,enable,fault,url,shard FROM windex WHERE url_noprefix = '");
if(slashfound==0)
{
strcat(checkurl,urlnoprefix);
@ -346,7 +346,6 @@ int main(int argc, char **argv)
char *dbtitle;
char *fault;
char *dburl;
char *dburl_noprefix;
char *shard;
//Catalog the previous crawl attempts (to see if they are all for the same page - which would be a bad sign)
@ -369,8 +368,7 @@ int main(int argc, char **argv)
enableOldDBval = row[3];
fault = row[4];
dburl=row[5];
dburl_noprefix=row[6];
shard=row[7];
shard=row[6];
if(task != 0 && task[0]=='2')
alreadydone=1;
}
@ -618,12 +616,12 @@ int main(int argc, char **argv)
}
}
int finalURL_prefixsize = httpswww+httpwww+https+http;
prefixsize = httpswww+httpwww+https+http;
urlcount=urlnoprefixcount=0;
//store the url without prefix to urlnoprefix
while(finalURL[urlcount] != 0){
if(urlcount>finalURL_prefixsize-1)
if(urlcount>prefixsize-1)
{
finalURLnoprefix[urlnoprefixcount]=finalURL[urlcount];
urlnoprefixcount++;
@ -637,7 +635,7 @@ int main(int argc, char **argv)
mysql_free_result(resulturlcheck);
char doublecheckurl[finalURLsize+100];
memset(doublecheckurl,0,finalURLsize+100);
strcpy(doublecheckurl,"SELECT id,updatable,title,enable,fault,url,url_noprefix,shard FROM windex WHERE url = '");
strcpy(doublecheckurl,"SELECT id,updatable,title,enable,fault,url,shard FROM windex WHERE url = '");
strcat(doublecheckurl,finalURL);
strcat(doublecheckurl,"';");
if (mysql_query(con, doublecheckurl))
@ -661,9 +659,8 @@ int main(int argc, char **argv)
enableOldDBval = row[3];
fault = row[4];
dburl=row[5];
dburl_noprefix=row[6];
shard=row[7];
if((task != 0 && task[0]=='2') || updatableOldDBval[0] == '0')
shard=row[6];
if(task != 0 && task[0]=='2')
alreadydone=1;
foundindoublecheck=1;
}
@ -911,13 +908,27 @@ int main(int argc, char **argv)
}
//check if original dburl is now getting redirected from finalurl (should be sent to review)
int finalURLnoprefix_size = strlen(finalURLnoprefix), dburl_noprefix_size = strlen(dburl_noprefix);
if(finalURLnoprefix_size != dburl_noprefix_size){
int finalUrlsize_noprefix, dburlsize_noprefix = 0, finalURL_prefixsize = 0, dburl_prefixsize = 0,dburlsize=strlen(dburl);
if(finalURL[4] == ':'){//if its just a switch from http to https, ignore
finalUrlsize_noprefix = finalURLsize - 7;
finalURL_prefixsize = 7;
}else{
finalUrlsize_noprefix = finalURLsize - 8;
finalURL_prefixsize = 8;
}
if(dburl[4] == ':'){
dburlsize_noprefix = dburlsize - 7;
dburl_prefixsize = 7;
}else{
dburlsize_noprefix = dburlsize - 8;
dburl_prefixsize = 8;
}
if(finalURLsize-finalURL_prefixsize != dburlsize-dburl_prefixsize){
redirected = 1;
printf("\nIndexed page is being redirected.");
}else{
for(int i=0;i<finalURLnoprefix_size;i++){
if(dburl_noprefix[i] != finalURLnoprefix[i]){
for(int i=0;i<finalUrlsize_noprefix;i++){
if(dburl[i+dburl_prefixsize] != finalURL[i+finalURL_prefixsize]){
redirected = 1;
printf("\nIndexed page is being redirected.");
break;