From 8ac5d6e1147b7f5401e137cb57d269e561ae50a7 Mon Sep 17 00:00:00 2001 From: wibyweb <49052850+wibyweb@users.noreply.github.com> Date: Thu, 10 Aug 2023 00:08:15 -0400 Subject: [PATCH] Update checkrobots.h --- c/checkrobots.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/checkrobots.h b/c/checkrobots.h index 3e721a4..2dff449 100755 --- a/c/checkrobots.h +++ b/c/checkrobots.h @@ -72,7 +72,7 @@ int checkrobots(char *rURLprefix, char *rDomain, char *rURLpath) if(fp = fopen(outfilename,"wb")){ //set curl options curl_easy_setopt(curl, CURLOPT_URL, robotsurl);// set URL to get here - curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Wibybot; https://wiby.me/)"); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; WebCrawler; SearchEngine)"); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_checkrobots);// send all data to this function // curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);// write the page body to this file handle curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);//allow redirects