Updated system, spring remix with broken link checker
This commit is contained in:
markseu
parent
3c0a3e09b8
commit
7b52807337
2 changed files with 243 additions and 40 deletions
|
|
@ -1,13 +1,14 @@
|
|||
<?php
|
||||
// Command plugin, https://github.com/datenstrom/yellow-plugins/tree/master/command
|
||||
// Copyright (c) 2013-2017 Datenstrom, https://datenstrom.se
|
||||
// Copyright (c) 2013-2018 Datenstrom, https://datenstrom.se
|
||||
// This file may be used and distributed under the terms of the public license.
|
||||
|
||||
class YellowCommand
|
||||
{
|
||||
const VERSION = "0.7.3";
|
||||
const VERSION = "0.7.4";
|
||||
var $yellow; //access to API
|
||||
var $files; //number of files
|
||||
var $links; //number of links
|
||||
var $errors; //number of errors
|
||||
var $locationsArgs; //locations with location arguments detected
|
||||
var $locationsArgsPagination; //locations with pagination arguments detected
|
||||
|
|
@ -26,6 +27,7 @@ class YellowCommand
|
|||
{
|
||||
case "": $statusCode = $this->helpCommand(); break;
|
||||
case "build": $statusCode = $this->buildCommand($args); break;
|
||||
case "check": $statusCode = $this->checkCommand($args); break;
|
||||
case "clean": $statusCode = $this->cleanCommand($args); break;
|
||||
case "version": $statusCode = $this->versionCommand($args); break;
|
||||
default: $statusCode = 0;
|
||||
|
|
@ -37,6 +39,7 @@ class YellowCommand
|
|||
function onCommandHelp()
|
||||
{
|
||||
$help .= "build [DIRECTORY LOCATION]\n";
|
||||
$help .= "check [DIRECTORY LOCATION]\n";
|
||||
$help .= "clean [DIRECTORY LOCATION]\n";
|
||||
$help .= "version\n";
|
||||
return $help;
|
||||
|
|
@ -51,7 +54,7 @@ class YellowCommand
|
|||
return 200;
|
||||
}
|
||||
|
||||
// Build static files
|
||||
// Build static website
|
||||
function buildCommand($args)
|
||||
{
|
||||
$statusCode = 0;
|
||||
|
|
@ -60,7 +63,7 @@ class YellowCommand
|
|||
{
|
||||
if($this->checkStaticConfig())
|
||||
{
|
||||
$statusCode = $this->buildStatic($path, $location);
|
||||
$statusCode = $this->buildStaticFiles($path, $location);
|
||||
} else {
|
||||
$statusCode = 500;
|
||||
$this->files = 0; $this->errors = 1;
|
||||
|
|
@ -68,8 +71,7 @@ class YellowCommand
|
|||
echo "ERROR building files: Please configure StaticUrl in file '$fileName'!\n";
|
||||
}
|
||||
echo "Yellow $command: $this->files file".($this->files!=1 ? 's' : '');
|
||||
echo ", $this->errors error".($this->errors!=1 ? 's' : '');
|
||||
echo ", status $statusCode\n";
|
||||
echo ", $this->errors error".($this->errors!=1 ? 's' : '')."\n";
|
||||
} else {
|
||||
$statusCode = 400;
|
||||
echo "Yellow $command: Invalid arguments\n";
|
||||
|
|
@ -77,36 +79,41 @@ class YellowCommand
|
|||
return $statusCode;
|
||||
}
|
||||
|
||||
// Build static files and additional locations
|
||||
function buildStatic($path, $location)
|
||||
// Build static files
|
||||
function buildStaticFiles($path, $locationFilter)
|
||||
{
|
||||
$path = rtrim(empty($path) ? $this->yellow->config->get("staticDir") : $path, '/');
|
||||
$this->files = $this->errors = $statusCode = 0;
|
||||
$this->files = $this->errors = 0;
|
||||
$this->locationsArgs = $this->locationsArgsPagination = array();
|
||||
if(empty($location))
|
||||
$statusCode = empty($locationFilter) ? $this->cleanStaticFiles($path, $locationFilter) : 200;
|
||||
$staticUrl = $this->yellow->config->get("staticUrl");
|
||||
list($scheme, $address, $base) = $this->yellow->lookup->getUrlInformation($staticUrl);
|
||||
foreach($this->getContentLocations() as $location)
|
||||
{
|
||||
$statusCode = $this->cleanStatic($path, $location);
|
||||
foreach($this->getContentLocations() as $location)
|
||||
if(!preg_match("#^$base$locationFilter#", $location)) continue;
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
|
||||
}
|
||||
foreach($this->locationsArgs as $location)
|
||||
{
|
||||
if(!preg_match("#^$base$locationFilter#", $location)) continue;
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
|
||||
}
|
||||
foreach($this->locationsArgsPagination as $location)
|
||||
{
|
||||
if(!preg_match("#^$base$locationFilter#", $location)) continue;
|
||||
if(substru($location, -1)!=$this->yellow->toolbox->getLocationArgsSeparator())
|
||||
{
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location, false, true));
|
||||
}
|
||||
foreach($this->locationsArgs as $location)
|
||||
for($pageNumber=2; $pageNumber<=999; ++$pageNumber)
|
||||
{
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
|
||||
}
|
||||
foreach($this->locationsArgsPagination as $location)
|
||||
{
|
||||
if(substru($location, -1)!=$this->yellow->toolbox->getLocationArgsSeparator())
|
||||
{
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location, false, true));
|
||||
}
|
||||
for($pageNumber=2; $pageNumber<=999; ++$pageNumber)
|
||||
{
|
||||
$statusCodeLocation = $this->buildStaticFile($path, $location.$pageNumber, false, true);
|
||||
$statusCode = max($statusCode, $statusCodeLocation);
|
||||
if($statusCodeLocation==100) break;
|
||||
}
|
||||
$statusCodeLocation = $this->buildStaticFile($path, $location.$pageNumber, false, true);
|
||||
$statusCode = max($statusCode, $statusCodeLocation);
|
||||
if($statusCodeLocation==100) break;
|
||||
}
|
||||
}
|
||||
if(empty($locationFilter))
|
||||
{
|
||||
foreach($this->getMediaLocations() as $location)
|
||||
{
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location));
|
||||
|
|
@ -116,8 +123,6 @@ class YellowCommand
|
|||
$statusCode = max($statusCode, $this->buildStaticFile($path, $location));
|
||||
}
|
||||
$statusCode = max($statusCode, $this->buildStaticFile($path, "/error/", false, false, true));
|
||||
} else {
|
||||
$statusCode = $this->buildStaticFile($path, $location);
|
||||
}
|
||||
return $statusCode;
|
||||
}
|
||||
|
|
@ -232,6 +237,161 @@ class YellowCommand
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check static files for broken links
|
||||
function checkCommand($args)
|
||||
{
|
||||
$statusCode = 0;
|
||||
list($command, $path, $location) = $args;
|
||||
if(empty($location) || $location[0]=='/')
|
||||
{
|
||||
if($this->checkStaticConfig())
|
||||
{
|
||||
$statusCode = $this->checkStaticFiles($path, $location);
|
||||
} else {
|
||||
$statusCode = 500;
|
||||
$this->files = $this->links = 0;
|
||||
$fileName = $this->yellow->config->get("configDir").$this->yellow->config->get("configFile");
|
||||
echo "ERROR checking files: Please configure StaticUrl in file '$fileName'!\n";
|
||||
}
|
||||
echo "Yellow $command: $this->files file".($this->files!=1 ? 's' : '');
|
||||
echo ", $this->links link".($this->links!=1 ? 's' : '')."\n";
|
||||
} else {
|
||||
$statusCode = 400;
|
||||
echo "Yellow $command: Invalid arguments\n";
|
||||
}
|
||||
return $statusCode;
|
||||
}
|
||||
|
||||
// Check static files
|
||||
function checkStaticFiles($path, $locationFilter)
|
||||
{
|
||||
$path = rtrim(empty($path) ? $this->yellow->config->get("staticDir") : $path, '/');
|
||||
$this->files = $this->links = 0;
|
||||
$regex = "/^[^.]+$|".$this->yellow->config->get("staticDefaultFile")."$/";
|
||||
$fileNames = $this->yellow->toolbox->getDirectoryEntriesRecursive($path, $regex, false, false);
|
||||
list($statusCodeFiles, $links) = $this->analyseStaticFiles($path, $locationFilter, $fileNames);
|
||||
list($statusCodeLinks, $broken, $redirect) = $this->analyseLinks($path, $links);
|
||||
if($statusCodeLinks!=200)
|
||||
{
|
||||
$this->showLinks($broken, "Broken links");
|
||||
$this->showLinks($redirect, "Redirect links");
|
||||
}
|
||||
return max($statusCodeFiles, $statusCodeLinks);
|
||||
}
|
||||
|
||||
// Analyse static files, detect links
|
||||
function analyseStaticFiles($path, $locationFilter, $fileNames)
|
||||
{
|
||||
$statusCode = 200;
|
||||
$links = array();
|
||||
if(!empty($fileNames))
|
||||
{
|
||||
$staticUrl = $this->yellow->config->get("staticUrl");
|
||||
list($scheme, $address, $base) = $this->yellow->lookup->getUrlInformation($staticUrl);
|
||||
foreach($fileNames as $fileName)
|
||||
{
|
||||
if(is_readable($fileName))
|
||||
{
|
||||
$locationSource = $this->getStaticLocation($path, $fileName);
|
||||
if(!preg_match("#^$base$locationFilter#", $locationSource)) continue;
|
||||
$fileData = $this->yellow->toolbox->readFile($fileName);
|
||||
preg_match_all("/<(.*?)href=\"([^\"]+)\"(.*?)>/i", $fileData, $matches);
|
||||
foreach($matches[2] as $match)
|
||||
{
|
||||
$location = rawurldecode($match);
|
||||
if(preg_match("/^(.*?)#(.*)$/", $location, $tokens)) $location = $tokens[1];
|
||||
if(preg_match("/^(\w+):\/\/([^\/]+)(.*)$/", $location, $matches))
|
||||
{
|
||||
$url = $location.(empty($matches[3]) ? "/" : "");
|
||||
if(!is_null($links[$url])) $links[$url] .= ",";
|
||||
$links[$url] .= $locationSource;
|
||||
if(defined("DEBUG") && DEBUG>=2) echo "YellowCommand::analyseStaticFiles detected url:$url<br/>\n";
|
||||
} else if($location[0]=='/') {
|
||||
$url = "$scheme://$address$location";
|
||||
if(!is_null($links[$url])) $links[$url] .= ",";
|
||||
$links[$url] .= $locationSource;
|
||||
if(defined("DEBUG") && DEBUG>=2) echo "YellowCommand::analyseStaticFiles detected url:$url<br/>\n";
|
||||
}
|
||||
}
|
||||
++$this->files;
|
||||
} else {
|
||||
$statusCode = 500;
|
||||
echo "ERROR reading files: Can't read file '$fileName'!\n";
|
||||
}
|
||||
}
|
||||
$this->links = count($links);
|
||||
} else {
|
||||
$statusCode = 500;
|
||||
echo "ERROR reading files: Can't find files in directory '$path'!\n";
|
||||
}
|
||||
return array($statusCode, $links);
|
||||
}
|
||||
|
||||
// Analyse links, detect status
|
||||
function analyseLinks($path, $links)
|
||||
{
|
||||
$statusCode = 200;
|
||||
$broken = $redirect = $data = array();
|
||||
if(extension_loaded("curl"))
|
||||
{
|
||||
$staticUrl = $this->yellow->config->get("staticUrl");
|
||||
list($scheme, $address, $base) = $this->yellow->lookup->getUrlInformation($staticUrl);
|
||||
$staticLocations = $this->getContentLocations(true);
|
||||
uksort($links, "strnatcasecmp");
|
||||
foreach($links as $url=>$value)
|
||||
{
|
||||
if(defined("DEBUG") && DEBUG>=1) echo "YellowCommand::analyseLinks url:$url\n";
|
||||
if(preg_match("#^$staticUrl#", $url))
|
||||
{
|
||||
$location = substru($url, 32);
|
||||
$fileName = $path.substru($url, 32);
|
||||
if(is_readable($fileName)) continue;
|
||||
if(in_array($location, $staticLocations)) continue;
|
||||
}
|
||||
if(preg_match("/^(http|https):/", $url))
|
||||
{
|
||||
$referer = "$scheme://$address".(($pos = strposu($value, ',')) ? substru($value, 0, $pos) : $value);
|
||||
$statusCodeUrl = $this->getLinkStatus($url, $referer);
|
||||
if($statusCodeUrl!=200)
|
||||
{
|
||||
$statusCode = max($statusCode, $statusCodeUrl);
|
||||
$data[$url] = "$statusCodeUrl,$value";
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach($data as $url=>$value)
|
||||
{
|
||||
$locations = preg_split("/\s*,\s*/", $value);
|
||||
$statusCodeUrl = array_shift($locations);
|
||||
foreach($locations as $location)
|
||||
{
|
||||
if($statusCodeUrl>=300 && $statusCodeUrl<=399) {
|
||||
$redirect["$scheme://$address$location -> $url - ".$this->getStatusFormatted($statusCodeUrl)] = $statusCodeUrl;
|
||||
} else {
|
||||
$broken["$scheme://$address$location -> $url - ".$this->getStatusFormatted($statusCodeUrl)] = $statusCodeUrl;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$statusCode = 500;
|
||||
echo "ERROR checking links: Plugin 'command' requires cURL library!\n";
|
||||
}
|
||||
return array($statusCode, $broken, $redirect);
|
||||
}
|
||||
|
||||
// Show links
|
||||
function showLinks($data, $text)
|
||||
{
|
||||
if(!empty($data))
|
||||
{
|
||||
echo "$text\n\n";
|
||||
uksort($data, "strnatcasecmp");
|
||||
$data = array_slice($data, 0, 99);
|
||||
foreach($data as $key=>$value) echo "- $key\n";
|
||||
echo "\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Clean static files
|
||||
function cleanCommand($args)
|
||||
|
|
@ -240,7 +400,7 @@ class YellowCommand
|
|||
list($command, $path, $location) = $args;
|
||||
if(empty($location) || $location[0]=='/')
|
||||
{
|
||||
$statusCode = $this->cleanStatic($path, $location);
|
||||
$statusCode = $this->cleanStaticFiles($path, $location);
|
||||
echo "Yellow $command: Static file".(empty($location) ? "s" : "")." ".($statusCode!=200 ? "not " : "")."cleaned\n";
|
||||
} else {
|
||||
$statusCode = 400;
|
||||
|
|
@ -250,7 +410,7 @@ class YellowCommand
|
|||
}
|
||||
|
||||
// Clean static files and directories
|
||||
function cleanStatic($path, $location)
|
||||
function cleanStaticFiles($path, $location)
|
||||
{
|
||||
$statusCode = 200;
|
||||
$path = rtrim(empty($path) ? $this->yellow->config->get("staticDir") : $path, '/');
|
||||
|
|
@ -259,7 +419,13 @@ class YellowCommand
|
|||
$statusCode = max($statusCode, $this->commandBroadcast("clean", "all"));
|
||||
$statusCode = max($statusCode, $this->cleanStaticDirectory($path));
|
||||
} else {
|
||||
$statusCode = $this->cleanStaticFile($path, $location);
|
||||
if($this->yellow->lookup->isFileLocation($location))
|
||||
{
|
||||
$fileName = $this->getStaticFile($path, $location, $statusCode);
|
||||
$statusCode = $this->cleanStaticFile($fileName);
|
||||
} else {
|
||||
$statusCode = $this->cleanStaticDirectory($path.$location);
|
||||
}
|
||||
}
|
||||
return $statusCode;
|
||||
}
|
||||
|
|
@ -280,10 +446,9 @@ class YellowCommand
|
|||
}
|
||||
|
||||
// Clean static file
|
||||
function cleanStaticFile($path, $location)
|
||||
function cleanStaticFile($fileName)
|
||||
{
|
||||
$statusCode = 200;
|
||||
$fileName = $this->getStaticFile($path, $location, $statusCode);
|
||||
if(is_file($fileName))
|
||||
{
|
||||
if(!$this->yellow->toolbox->deleteFile($fileName))
|
||||
|
|
@ -365,6 +530,18 @@ class YellowCommand
|
|||
return $fileName;
|
||||
}
|
||||
|
||||
// Return static location
|
||||
function getStaticLocation($path, $fileName)
|
||||
{
|
||||
$location = substru($fileName, strlenu($path));
|
||||
if(basename($location)==$this->yellow->config->get("staticDefaultFile"))
|
||||
{
|
||||
$defaultFileLength = strlenu($this->yellow->config->get("staticDefaultFile"));
|
||||
$location = substru($location, 0, -$defaultFileLength);
|
||||
}
|
||||
return $location;
|
||||
}
|
||||
|
||||
// Return static redirect
|
||||
function getStaticRedirect($location)
|
||||
{
|
||||
|
|
@ -374,9 +551,15 @@ class YellowCommand
|
|||
$output .= "</head>\n</html>";
|
||||
return $output;
|
||||
}
|
||||
|
||||
// Return human readable status
|
||||
function getStatusFormatted($statusCode)
|
||||
{
|
||||
return $this->yellow->toolbox->getHttpStatusFormatted($statusCode, true);
|
||||
}
|
||||
|
||||
// Return content locations
|
||||
function getContentLocations()
|
||||
function getContentLocations($includeAll = false)
|
||||
{
|
||||
$locations = array();
|
||||
$staticUrl = $this->yellow->config->get("staticUrl");
|
||||
|
|
@ -384,7 +567,7 @@ class YellowCommand
|
|||
$this->yellow->page->setRequestInformation($scheme, $address, $base, "", "");
|
||||
foreach($this->yellow->pages->index(true, true) as $page)
|
||||
{
|
||||
if($page->get("status")!="ignore" && $page->get("status")!="draft")
|
||||
if(($page->get("status")!="ignore" && $page->get("status")!="draft") || $includeAll)
|
||||
{
|
||||
array_push($locations, $page->location);
|
||||
}
|
||||
|
|
@ -458,6 +641,27 @@ class YellowCommand
|
|||
}
|
||||
return array($statusCode, $data);
|
||||
}
|
||||
|
||||
// Return link status
|
||||
function getLinkStatus($url, $referer)
|
||||
{
|
||||
if(extension_loaded("curl"))
|
||||
{
|
||||
$curlHandle = curl_init();
|
||||
curl_setopt($curlHandle, CURLOPT_URL, $url);
|
||||
curl_setopt($curlHandle, CURLOPT_REFERER, $referer);
|
||||
curl_setopt($curlHandle, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; DatenstromYellow/".YellowCore::VERSION."; LinkChecker)");
|
||||
curl_setopt($curlHandle, CURLOPT_NOBODY, 1);
|
||||
curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_exec($curlHandle);
|
||||
$statusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE);
|
||||
curl_close($curlHandle);
|
||||
if(defined("DEBUG") && DEBUG>=2) echo "YellowCommand::getLinkStatus status:$statusCode url:$url<br/>\n";
|
||||
} else {
|
||||
$statusCode = 500;
|
||||
}
|
||||
return $statusCode;
|
||||
}
|
||||
}
|
||||
|
||||
$yellow->plugins->register("command", "YellowCommand", YellowCommand::VERSION);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
class YellowUpdate
|
||||
{
|
||||
const VERSION = "0.7.9";
|
||||
const VERSION = "0.7.10";
|
||||
var $yellow; //access to API
|
||||
var $updates; //number of updates
|
||||
|
||||
|
|
@ -18,7 +18,6 @@ class YellowUpdate
|
|||
$this->yellow->config->setDefault("updateInformationFile", "update.ini");
|
||||
$this->yellow->config->setDefault("updateVersionFile", "version.ini");
|
||||
$this->yellow->config->setDefault("updateResourceFile", "resource.ini");
|
||||
$this->yellow->config->setDefault("newFile", "page-new-(.*).txt"); //TODO: remove with next release
|
||||
}
|
||||
|
||||
// Handle startup
|
||||
|
|
@ -826,7 +825,7 @@ class YellowUpdate
|
|||
}
|
||||
$curlHandle = curl_init();
|
||||
curl_setopt($curlHandle, CURLOPT_URL, $urlRequest);
|
||||
curl_setopt($curlHandle, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; YellowCore/".YellowCore::VERSION).")";
|
||||
curl_setopt($curlHandle, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; DatenstromYellow/".YellowCore::VERSION."; SoftwareUpdater)");
|
||||
curl_setopt($curlHandle, CURLOPT_RETURNTRANSFER, 1);
|
||||
curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
$rawData = curl_exec($curlHandle);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue