badmanners/shimmie2
badmanners/shimmie2
Archived
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added lock file usage to cron uploader to prevent concurrent runs.

Browse source 
Changed extension manager to allow author to be a comma-separated list.
This commit is contained in:
Matthew Barbour 2019-06-15 11:01:13 -05:00 committed by matthew
parent e940d87c22
commit 0202597f88
3 changed files with 162 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
ext/cron_uploader/main.php
View file

@ -1,12 +1,14 @@
<?php <?php
/* /*
* Name: Cron Uploader * Name: Cron Uploader
* Author: YaoiFox <admin@yaoifox.com> * Authors: YaoiFox <admin@yaoifox.com>, Matthew Barbour <matthew@darkholme.net>
* Link: http://www.yaoifox.com/ * Link: http://www.yaoifox.com/
* License: GPLv2 * License: GPLv2
* Description: Uploads images automatically using Cron Jobs * Description: Uploads images automatically using Cron Jobs
* Documentation: Installation guide: activate this extension and navigate to www.yoursite.com/cron_upload * Documentation: Installation guide: activate this extension and navigate to www.yoursite.com/cron_upload
*/ */
class CronUploader extends Extension class CronUploader extends Extension
{ {
// TODO: Checkbox option to only allow localhost + a list of additional IP adresses that can be set in /cron_upload // TODO: Checkbox option to only allow localhost + a list of additional IP adresses that can be set in /cron_upload
@ -51,7 +53,18 @@ class CronUploader extends Extension
// If the key is in the url, upload // If the key is in the url, upload
if ($this->upload_key != "" && $event->get_arg(0) == $this->upload_key) { if ($this->upload_key != "" && $event->get_arg(0) == $this->upload_key) {
// log in as admin // log in as admin
$this->process_upload(); // Start upload $this->set_dir();
$lockfile = fopen($this->root_dir . "/.lock", "w");
try {
if (!flock($lockfile, LOCK_EX | LOCK_NB)) {
throw new Exception("Cron upload process is already running");
}
$this->process_upload(); // Start upload
} finally {
flock($lockfile, LOCK_UN);
fclose($lockfile);
}
} elseif ($user->is_admin()) { } elseif ($user->is_admin()) {
$this->set_dir(); $this->set_dir();
$this->display_documentation(); $this->display_documentation();
@ -131,6 +144,8 @@ class CronUploader extends Extension
<br />This link can be found under 'Cron Command' in the board config, just remove the 'wget ' part and only the url remains. <br />This link can be found under 'Cron Command' in the board config, just remove the 'wget ' part and only the url remains.
<br />(<b>$cron_url</b>)"; <br />(<b>$cron_url</b>)";
$page->set_title("Cron Uploader");
$page->set_heading("Cron Uploader");
$block = new Block("Cron Uploader", $info_html, "main", 10); $block = new Block("Cron Uploader", $info_html, "main", 10);
$block_install = new Block("Installation Guide", $install_html, "main", 20); $block_install = new Block("Installation Guide", $install_html, "main", 20);
@ -143,7 +158,7 @@ class CronUploader extends Extension
global $config; global $config;
// Set default values // Set default values
$this->upload_key = $config->get_string("cron_uploader_key", ""); $this->upload_key = $config->get_string("cron_uploader_key", "");
if (strlen($this->upload_key)<=0) { if (strlen($this->upload_key) <= 0) {
$this->upload_key = $this->generate_key(); $this->upload_key = $this->generate_key();
$config->set_default_int('cron_uploader_count', 1); $config->set_default_int('cron_uploader_count', 1);
@ -181,7 +196,7 @@ class CronUploader extends Extension
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; $characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$randomString = ''; $randomString = '';
for ($i = 0; $i < $length; $i ++) { for ($i = 0; $i < $length; $i++) {
$randomString .= $characters [rand(0, strlen($characters) - 1)]; $randomString .= $characters [rand(0, strlen($characters) - 1)];
} }
@ -224,11 +239,11 @@ class CronUploader extends Extension
*/ */
public function scan_dir(string $path): array public function scan_dir(string $path): array
{ {
$bytestotal=0; $bytestotal = 0;
$nbfiles=0; $nbfiles = 0;
$ite=new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS); $ite = new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $filename=>$cur) { foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) {
$filesize = $cur->getSize(); $filesize = $cur->getSize();
$bytestotal += $filesize; $bytestotal += $filesize;
$nbfiles++; $nbfiles++;
@ -236,7 +251,7 @@ class CronUploader extends Extension
$size_mb = $bytestotal / 1048576; // to mb $size_mb = $bytestotal / 1048576; // to mb
$size_mb = number_format($size_mb, 2, '.', ''); $size_mb = number_format($size_mb, 2, '.', '');
return ['total_files'=>$nbfiles,'total_mb'=>$size_mb]; return ['total_files' => $nbfiles, 'total_mb' => $size_mb];
} }
/** /**
@ -246,10 +261,10 @@ class CronUploader extends Extension
{ {
global $config, $database; global $config, $database;
set_time_limit(0); //set_time_limit(0);
$output_subdir = date('Ymd-His', time())."/";
$this->set_dir(); $output_subdir = date('Ymd-His', time()) . "/";
$this->generate_image_queue(); $this->generate_image_queue();
// Gets amount of imgs to upload // Gets amount of imgs to upload
@ -274,7 +289,7 @@ class CronUploader extends Extension
$failedItems = []; $failedItems = [];
// Upload the file(s) // Upload the file(s)
for ($i = 0; $i < $upload_count && sizeof($this->image_queue)>0; $i++) { for ($i = 0; $i < $upload_count && sizeof($this->image_queue) > 0; $i++) {
$img = array_pop($this->image_queue); $img = array_pop($this->image_queue);
try { try {
@ -282,7 +297,7 @@ class CronUploader extends Extension
$result = $this->add_image($img[0], $img[1], $img[2]); $result = $this->add_image($img[0], $img[1], $img[2]);
$database->commit(); $database->commit();
$this->move_uploaded($img[0], $img[1], $output_subdir, false); $this->move_uploaded($img[0], $img[1], $output_subdir, false);
if ($result==null) { if ($result == null) {
$merged++; $merged++;
} else { } else {
$added++; $added++;
@ -290,7 +305,7 @@ class CronUploader extends Extension
} catch (Exception $e) { } catch (Exception $e) {
$failed++; $failed++;
$this->move_uploaded($img[0], $img[1], $output_subdir, true); $this->move_uploaded($img[0], $img[1], $output_subdir, true);
$msgNumber = $this->add_upload_info("(".gettype($e).") ".$e->getMessage()); $msgNumber = $this->add_upload_info("(" . gettype($e) . ") " . $e->getMessage());
$msgNumber = $this->add_upload_info($e->getTraceAsString()); $msgNumber = $this->add_upload_info($e->getTraceAsString());
if (strpos($e->getMessage(), 'SQLSTATE') !== false) { if (strpos($e->getMessage(), 'SQLSTATE') !== false) {
// Postgres invalidates the transaction if there is an SQL error, // Postgres invalidates the transaction if there is an SQL error,
@ -310,11 +325,11 @@ class CronUploader extends Extension
$msgNumber = $this->add_upload_info("Items failed: $failed"); $msgNumber = $this->add_upload_info("Items failed: $failed");
// Display & save upload log // Display & save upload log
$this->handle_log(); $this->handle_log();
return true; return true;
} }
private function move_uploaded($path, $filename, $output_subdir, $corrupt = false) private function move_uploaded($path, $filename, $output_subdir, $corrupt = false)
@ -329,20 +344,20 @@ class CronUploader extends Extension
// Determine which dir to move to // Determine which dir to move to
if ($corrupt) { if ($corrupt) {
// Move to corrupt dir // Move to corrupt dir
$newDir .= "/failed_to_upload/".$output_subdir.$relativeDir; $newDir .= "/failed_to_upload/" . $output_subdir . $relativeDir;
$info = "ERROR: Image was not uploaded."; $info = "ERROR: Image was not uploaded.";
} else { } else {
$newDir .= "/uploaded/".$output_subdir.$relativeDir; $newDir .= "/uploaded/" . $output_subdir . $relativeDir;
$info = "Image successfully uploaded. "; $info = "Image successfully uploaded. ";
} }
$newDir = str_replace("//", "/", $newDir."/"); $newDir = str_replace("//", "/", $newDir . "/");
if (!is_dir($newDir)) { if (!is_dir($newDir)) {
mkdir($newDir, 0775, true); mkdir($newDir, 0775, true);
} }
// move file to correct dir // move file to correct dir
rename($path, $newDir.$filename); rename($path, $newDir . $filename);
$this->add_upload_info($info . "Image \"$filename\" moved from queue to \"$newDir\"."); $this->add_upload_info($info . "Image \"$filename\" moved from queue to \"$newDir\".");
} }
@ -382,13 +397,13 @@ class CronUploader extends Extension
{ {
$base = $this->root_dir . "/queue"; $base = $this->root_dir . "/queue";
if (! is_dir($base)) { if (!is_dir($base)) {
$this->add_upload_info("Image Queue Directory could not be found at \"$base\"."); $this->add_upload_info("Image Queue Directory could not be found at \"$base\".");
return; return;
} }
$ite=new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS); $ite = new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $fullpath=>$cur) { foreach (new RecursiveIteratorIterator($ite) as $fullpath => $cur) {
if (!is_link($fullpath) && !is_dir($fullpath)) { if (!is_link($fullpath) && !is_dir($fullpath)) {
$pathinfo = pathinfo($fullpath); $pathinfo = pathinfo($fullpath);
@ -396,9 +411,9 @@ class CronUploader extends Extension
$tags = path_to_tags($relativePath); $tags = path_to_tags($relativePath);
$img = [ $img = [
0 => $fullpath, 0 => $fullpath,
1 => $pathinfo ["basename"], 1 => $pathinfo ["basename"],
2 => $tags 2 => $tags
]; ];
array_push($this->image_queue, $img); array_push($this->image_queue, $img);
} }
@ -411,14 +426,14 @@ class CronUploader extends Extension
private function add_upload_info(string $text, int $addon = 0): int private function add_upload_info(string $text, int $addon = 0): int
{ {
$info = $this->upload_info; $info = $this->upload_info;
$time = "[" .date('Y-m-d H:i:s'). "]"; $time = "[" . date('Y-m-d H:i:s') . "]";
// If addon function is not used // If addon function is not used
if ($addon == 0) { if ($addon == 0) {
$this->upload_info .= "$time $text\r\n"; $this->upload_info .= "$time $text\r\n";
// Returns the number of the current line // Returns the number of the current line
$currentLine = substr_count($this->upload_info, "\n") -1; $currentLine = substr_count($this->upload_info, "\n") - 1;
return $currentLine; return $currentLine;
} }
@ -451,7 +466,7 @@ class CronUploader extends Extension
$prev_content = ""; $prev_content = "";
} }
$content = $prev_content ."\r\n".$this->upload_info; $content = $prev_content . "\r\n" . $this->upload_info;
file_put_contents($log_path, $content); file_put_contents($log_path, $content);
} }
} }

54
ext/ext_manager/main.php
View file

@ -22,8 +22,7 @@ class ExtensionInfo
public $ext_name; public $ext_name;
public $name; public $name;
public $link; public $link;
public $author; public $authors;
public $email;
public $description; public $description;
public $documentation; public $documentation;
public $version; public $version;
@ -39,8 +38,9 @@ class ExtensionInfo
$this->ext_name = $matches[1]; $this->ext_name = $matches[1];
$this->name = $this->ext_name; $this->name = $this->ext_name;
$this->enabled = $this->is_enabled($this->ext_name); $this->enabled = $this->is_enabled($this->ext_name);
$this->authors = [];
for ($i=0; $i<$number_of_lines; $i++) { for ($i = 0; $i < $number_of_lines; $i++) {
$line = $lines[$i]; $line = $lines[$i];
if (preg_match("/Name: (.*)/", $line, $matches)) { if (preg_match("/Name: (.*)/", $line, $matches)) {
$this->name = $matches[1]; $this->name = $matches[1];
@ -53,25 +53,31 @@ class ExtensionInfo
} }
} elseif (preg_match("/Version: (.*)/", $line, $matches)) { } elseif (preg_match("/Version: (.*)/", $line, $matches)) {
$this->version = $matches[1]; $this->version = $matches[1];
} elseif (preg_match("/Author: (.*) [<\(](.*@.*)[>\)]/", $line, $matches)) { } elseif (preg_match("/Authors?: (.*)/", $line, $matches)) {
$this->author = $matches[1]; $author_list = explode(',', $matches[1]);
$this->email = $matches[2]; foreach ($author_list as $author) {
} elseif (preg_match("/Author: (.*)/", $line, $matches)) { if (preg_match("/(.*) [<\(](.*@.*)[>\)]/", $author, $matches)) {
$this->author = $matches[1]; $this->authors[] = new ExtensionAuthor($matches[1], $matches[2]);
} else {
$this->authors[] = new ExtensionAuthor($author, null);
}
}
} elseif (preg_match("/(.*)Description: ?(.*)/", $line, $matches)) { } elseif (preg_match("/(.*)Description: ?(.*)/", $line, $matches)) {
$this->description = $matches[2]; $this->description = $matches[2];
$start = $matches[1]." "; $start = $matches[1] . " ";
$start_len = strlen($start); $start_len = strlen($start);
while (substr($lines[$i+1], 0, $start_len) == $start) { while (substr($lines[$i + 1], 0, $start_len) == $start) {
$this->description .= " ".substr($lines[$i+1], $start_len); $this->description .= " " . substr($lines[$i + 1], $start_len);
$i++; $i++;
} }
} elseif (preg_match("/(.*)Documentation: ?(.*)/", $line, $matches)) { } elseif (preg_match("/(.*)Documentation: ?(.*)/", $line, $matches)) {
$this->documentation = $matches[2]; $this->documentation = $matches[2];
$start = $matches[1]." "; $start = $matches[1] . " ";
$start_len = strlen($start); $start_len = strlen($start);
while (substr($lines[$i+1], 0, $start_len) == $start) { while (substr($lines[$i + 1], 0, $start_len) == $start) {
$this->documentation .= " ".substr($lines[$i+1], $start_len); $this->documentation .= " " . substr($lines[$i + 1], $start_len);
$i++; $i++;
} }
$this->documentation = str_replace('$site', make_http(get_base_href()), $this->documentation); $this->documentation = str_replace('$site', make_http(get_base_href()), $this->documentation);
@ -96,6 +102,18 @@ class ExtensionInfo
} }
} }
class ExtensionAuthor
{
public $name;
public $email;
public function __construct(string $name, string $email)
{
$this->name = $name;
$this->email = $email;
}
}
class ExtManager extends Extension class ExtManager extends Extension
{ {
public function onPageRequest(PageRequestEvent $event) public function onPageRequest(PageRequestEvent $event)
@ -166,7 +184,7 @@ class ExtManager extends Extension
if ($all) { if ($all) {
$exts = zglob("ext/*/main.php"); $exts = zglob("ext/*/main.php");
} else { } else {
$exts = zglob("ext/{".ENABLED_EXTS."}/main.php"); $exts = zglob("ext/{" . ENABLED_EXTS . "}/main.php");
} }
foreach ($exts as $main) { foreach ($exts as $main) {
$extensions[] = new ExtensionInfo($main); $extensions[] = new ExtensionInfo($main);
@ -200,9 +218,9 @@ class ExtManager extends Extension
{ {
file_put_contents( file_put_contents(
"data/config/extensions.conf.php", "data/config/extensions.conf.php",
'<'.'?php'."\n". '<' . '?php' . "\n" .
'define("EXTRA_EXTS", "'.implode(",", $extras).'");'."\n". 'define("EXTRA_EXTS", "' . implode(",", $extras) . '");' . "\n" .
'?'.">" '?' . ">"
); );
// when the list of active extensions changes, we can be // when the list of active extensions changes, we can be

41
ext/ext_manager/theme.php
View file

@ -9,7 +9,7 @@ class ExtManagerTheme extends Themelet
{ {
$h_en = $editable ? "<th>Enabled</th>" : ""; $h_en = $editable ? "<th>Enabled</th>" : "";
$html = " $html = "
".make_form(make_link("ext_manager/set"))." " . make_form(make_link("ext_manager/set")) . "
<table id='extensions' class='zebra sortable'> <table id='extensions' class='zebra sortable'>
<thead> <thead>
<tr> <tr>
@ -26,17 +26,17 @@ class ExtManagerTheme extends Themelet
continue; continue;
} }
$h_name = html_escape(empty($extension->name) ? $extension->ext_name : $extension->name); $h_name = html_escape(empty($extension->name) ? $extension->ext_name : $extension->name);
$h_description = html_escape($extension->description); $h_description = html_escape($extension->description);
$h_link = make_link("ext_doc/".url_escape($extension->ext_name)); $h_link = make_link("ext_doc/" . url_escape($extension->ext_name));
$h_enabled = ($extension->enabled === true ? " checked='checked'" : ($extension->enabled === false ? "" : " disabled checked='checked'")); $h_enabled = ($extension->enabled === true ? " checked='checked'" : ($extension->enabled === false ? "" : " disabled checked='checked'"));
$h_enabled_box = $editable ? "<td><input type='checkbox' name='ext_".html_escape($extension->ext_name)."' id='ext_".html_escape($extension->ext_name)."'$h_enabled></td>" : ""; $h_enabled_box = $editable ? "<td><input type='checkbox' name='ext_" . html_escape($extension->ext_name) . "' id='ext_" . html_escape($extension->ext_name) . "'$h_enabled></td>" : "";
$h_docs = ($extension->documentation ? "<a href='$h_link'>■</a>" : ""); //TODO: A proper "docs" symbol would be preferred here. $h_docs = ($extension->documentation ? "<a href='$h_link'>■</a>" : ""); //TODO: A proper "docs" symbol would be preferred here.
$html .= " $html .= "
<tr data-ext='{$extension->ext_name}'> <tr data-ext='{$extension->ext_name}'>
{$h_enabled_box} {$h_enabled_box}
<td><label for='ext_".html_escape($extension->ext_name)."'>{$h_name}</label></td> <td><label for='ext_" . html_escape($extension->ext_name) . "'>{$h_name}</label></td>
<td>{$h_docs}</td> <td>{$h_docs}</td>
<td style='text-align: left;'>{$h_description}</td> <td style='text-align: left;'>{$h_description}</td>
</tr>"; </tr>";
@ -116,15 +116,24 @@ class ExtManagerTheme extends Themelet
public function display_doc(Page $page, ExtensionInfo $info) public function display_doc(Page $page, ExtensionInfo $info)
{ {
$author = ""; $author = "";
if ($info->author) { if (count($info->authors) > 0) {
if ($info->email) { $author = "<br /><b>Author";
$author = "<br><b>Author:</b> <a href=\"mailto:".html_escape($info->email)."\">".html_escape($info->author)."</a>"; if (count($info->authors) > 1) {
} else { $author .= "s";
$author = "<br><b>Author:</b> ".html_escape($info->author);
} }
$author .= ":</b>";
foreach ($info->authors as $auth) {
if (!empty($auth->email)) {
$author .= "<a href=\"mailto:" . html_escape($auth->email) . "\">" . html_escape($auth->name) . "</a>";
} else {
$author .= html_escape($auth->name);
}
}
} }
$version = ($info->version) ? "<br><b>Version:</b> ".html_escape($info->version) : "";
$link = ($info->link) ? "<br><b>Home Page:</b> <a href=\"".html_escape($info->link)."\">Link</a>" : ""; $version = ($info->version) ? "<br><b>Version:</b> " . html_escape($info->version) : "";
$link = ($info->link) ? "<br><b>Home Page:</b> <a href=\"" . html_escape($info->link) . "\">Link</a>" : "";
$doc = $info->documentation; $doc = $info->documentation;
$html = " $html = "
<div style='margin: auto; text-align: left; width: 512px;'> <div style='margin: auto; text-align: left; width: 512px;'>
@ -133,10 +142,10 @@ class ExtManagerTheme extends Themelet
$link $link
<p>$doc <p>$doc
<hr> <hr>
<p><a href='".make_link("ext_manager")."'>Back to the list</a> <p><a href='" . make_link("ext_manager") . "'>Back to the list</a>
</div>"; </div>";
$page->set_title("Documentation for ".html_escape($info->name)); $page->set_title("Documentation for " . html_escape($info->name));
$page->set_heading(html_escape($info->name)); $page->set_heading(html_escape($info->name));
$page->add_block(new NavBlock()); $page->add_block(new NavBlock());
$page->add_block(new Block("Documentation", $html)); $page->add_block(new Block("Documentation", $html));