badmanners/shimmie2
badmanners/shimmie2
Archived
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #686 from sanmadjack/cron_upload

Browse source 
Cron upload
This commit is contained in:
Shish 2019-10-18 16:20:10 +01:00 committed by GitHub
commit de39f3e5d7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 610 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/permissions.php
View file

@ -80,4 +80,5 @@ abstract class Permissions
public const NOTES_ADMIN = "notes_admin"; public const NOTES_ADMIN = "notes_admin";
public const POOLS_ADMIN = "pools_admin"; public const POOLS_ADMIN = "pools_admin";
public const TIPS_ADMIN = "tips_admin"; public const TIPS_ADMIN = "tips_admin";
public const CRON_ADMIN = "cron_admin";
} }

2
core/userclass.php
View file

@ -150,6 +150,7 @@ new UserClass("base", null, [
Permissions::NOTES_ADMIN => false, Permissions::NOTES_ADMIN => false,
Permissions::POOLS_ADMIN => false, Permissions::POOLS_ADMIN => false,
Permissions::TIPS_ADMIN => false, Permissions::TIPS_ADMIN => false,
Permissions::CRON_ADMIN => false,
]); ]);
new UserClass("anonymous", "base", [ new UserClass("anonymous", "base", [
@ -226,6 +227,7 @@ new UserClass("admin", "base", [
Permissions::NOTES_ADMIN => true, Permissions::NOTES_ADMIN => true,
Permissions::POOLS_ADMIN => true, Permissions::POOLS_ADMIN => true,
Permissions::TIPS_ADMIN => true, Permissions::TIPS_ADMIN => true,
Permissions::CRON_ADMIN => true,
]); ]);
new UserClass("hellbanned", "user", [ new UserClass("hellbanned", "user", [

45
core/util.php
View file

@ -350,6 +350,51 @@ function join_url(string $base, string ...$paths)
return $output; return $output;
} }
function get_dir_contents(string $dir): array
{
if(empty($dir)) {
throw new Exception("dir required");
}
if(!is_dir($dir)) {
return [];
}
$results = array_diff(
scandir(
$dir),
['..', '.']);
return $results;
}
/**
* Returns amount of files & total size of dir.
*/
function scan_dir(string $path): array
{
$bytestotal = 0;
$nbfiles = 0;
$ite = new RecursiveDirectoryIterator(
$path,
FilesystemIterator::KEY_AS_PATHNAME |
FilesystemIterator::CURRENT_AS_FILEINFO |
FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) {
try {
$filesize = $cur->getSize();
$bytestotal += $filesize;
$nbfiles++;
} catch (RuntimeException $e) {
// This usually just means that the file got eaten by the import
continue;
}
}
$size_mb = $bytestotal / 1048576; // to mb
$size_mb = number_format($size_mb, 2, '.', '');
return ['path' => $path, 'total_files' => $nbfiles, 'total_mb' => $size_mb];
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Debugging functions * * Debugging functions *

98
ext/cron_uploader/config.php Normal file
View file

@ -0,0 +1,98 @@
<?php
abstract class CronUploaderConfig
{
const DEFAULT_PATH = "cron_uploader";
const KEY = "cron_uploader_key";
const COUNT = "cron_uploader_count";
const DIR = "cron_uploader_dir";
const USER = "cron_uploader_user";
public static function set_defaults(): void
{
global $config;
$config->set_default_int(self::COUNT, 1);
$config->set_default_string(self::DIR, data_path(self::DEFAULT_PATH));
$upload_key = $config->get_string(self::KEY, "");
if (empty($upload_key)) {
$upload_key = self::generate_key();
$config->set_string(self::KEY, $upload_key);
}
}
public static function get_user(): int
{
global $config;
return $config->get_int(self::USER);
}
public static function set_user(int $value): void
{
global $config;
$config->set_int(self::USER, $value);
}
public static function get_key(): string
{
global $config;
return $config->get_string(self::KEY);
}
public static function set_key(string $value): void
{
global $config;
$config->set_string(self::KEY, $value);
}
public static function get_count(): int
{
global $config;
return $config->get_int(self::COUNT);
}
public static function set_count(int $value): int
{
global $config;
$config->get_int(self::COUNT, $value);
}
public static function get_dir(): string
{
global $config;
$value = $config->get_string(self::DIR);
if (empty($value)) {
$value = data_path("cron_uploader");
self::set_dir($value);
}
return $value;
}
public static function set_dir(string $value): void
{
global $config;
$config->set_string(self::DIR, $value);
}
/*
* Generates a unique key for the website to prevent unauthorized access.
*/
private static function generate_key()
{
$length = 20;
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$randomString = '';
for ($i = 0; $i < $length; $i++) {
$randomString .= $characters [rand(0, strlen($characters) - 1)];
}
return $randomString;
}
}

581
ext/cron_uploader/main.php
View file

@ -1,43 +1,31 @@
<?php <?php
require_once "config.php";
class CronUploader extends Extension class CronUploader extends Extension
{ {
// TODO: Checkbox option to only allow localhost + a list of additional IP adresses that can be set in /cron_upload public const NAME = "cron_uploader";
// TODO: Change logging to MySQL + display log at /cron_upload
// TODO: Move stuff to theme.php // TODO: Checkbox option to only allow localhost + a list of additional IP addresses that can be set in /cron_upload
const QUEUE_DIR = "queue"; const QUEUE_DIR = "queue";
const UPLOADED_DIR = "uploaded"; const UPLOADED_DIR = "uploaded";
const FAILED_DIR = "failed_to_upload"; const FAILED_DIR = "failed_to_upload";
const CONFIG_KEY = "cron_uploader_key"; public $output_buffer = [];
const CONFIG_COUNT = "cron_uploader_count";
const CONFIG_DIR = "cron_uploader_dir";
/** public function onInitExt(InitExtEvent $event)
* Lists all log events this session {
* @var string // Set default values
*/ CronUploaderConfig::set_defaults();
private $upload_info = ""; }
/** public function onPageSubNavBuilding(PageSubNavBuildingEvent $event)
* Lists all files & info required to upload. {
* @var array if($event->parent=="system") {
*/ $event->add_nav_link("cron_docs", new Link('cron_upload'), "Cron Upload");
private $image_queue = []; }
}
/**
* Cron Uploader root directory
* @var string
*/
private $root_dir = "";
/**
* Key used to identify uploader
* @var string
*/
private $upload_key = "";
/** /**
* Checks if the cron upload page has been accessed * Checks if the cron upload page has been accessed
@ -45,237 +33,266 @@ class CronUploader extends Extension
*/ */
public function onPageRequest(PageRequestEvent $event) public function onPageRequest(PageRequestEvent $event)
{ {
global $config, $user; global $user;
if ($event->page_matches("cron_upload")) { if ($event->page_matches("cron_upload")) {
$this->upload_key = $config->get_string(self::CONFIG_KEY, ""); $key = $event->get_arg(0);
if (!empty($key)) {
// If the key is in the url, upload $this->process_upload($key); // Start upload
if ($this->upload_key != "" && $event->get_arg(0) == $this->upload_key) { } elseif ($user->can(Permissions::CRON_ADMIN)) {
// log in as admin
$this->set_dir();
$lockfile = fopen($this->root_dir . "/.lock", "w");
if (!flock($lockfile, LOCK_EX | LOCK_NB)) {
throw new Exception("Cron upload process is already running");
}
try {
$this->process_upload(); // Start upload
} finally {
flock($lockfile, LOCK_UN);
fclose($lockfile);
}
} elseif ($user->can(Permissions::BULK_ADD)) {
$this->set_dir();
$this->display_documentation(); $this->display_documentation();
} }
} }
} }
private function display_documentation()
{
global $page;
$this->set_dir(); // Determines path to cron_uploader_dir
$queue_dir = $this->root_dir . "/" . self::QUEUE_DIR;
$uploaded_dir = $this->root_dir . "/" . self::UPLOADED_DIR;
$failed_dir = $this->root_dir . "/" . self::FAILED_DIR;
$queue_dirinfo = $this->scan_dir($queue_dir);
$uploaded_dirinfo = $this->scan_dir($uploaded_dir);
$failed_dirinfo = $this->scan_dir($failed_dir);
$cron_url = make_http(make_link("/cron_upload/" . $this->upload_key));
$cron_cmd = "curl --silent $cron_url";
$log_path = $this->root_dir . "/uploads.log";
$info_html = "<b>Information</b>
<br>
<table style='width:470px;'>
<tr>
<td style='width:90px;'><b>Directory</b></td>
<td style='width:90px;'><b>Files</b></td>
<td style='width:90px;'><b>Size (MB)</b></td>
<td style='width:200px;'><b>Directory Path</b></td>
</tr><tr>
<td>Queue</td>
<td>{$queue_dirinfo['total_files']}</td>
<td>{$queue_dirinfo['total_mb']}</td>
<td><input type='text' style='width:150px;' value='$queue_dir'></td>
</tr><tr>
<td>Uploaded</td>
<td>{$uploaded_dirinfo['total_files']}</td>
<td>{$uploaded_dirinfo['total_mb']}</td>
<td><input type='text' style='width:150px;' value='$uploaded_dir'></td>
</tr><tr>
<td>Failed</td>
<td>{$failed_dirinfo['total_files']}</td>
<td>{$failed_dirinfo['total_mb']}</td>
<td><input type='text' style='width:150px;' value='$failed_dir'></td>
</tr></table>
<br>Cron Command: <input type='text' size='60' value='$cron_cmd'><br>
Create a cron job with the command above.<br/>
Read the documentation if you're not sure what to do.<br>";
$install_html = "
This cron uploader is fairly easy to use but has to be configured first.
<br />1. Install & activate this plugin.
<br />
<br />2. Upload your images you want to be uploaded to the queue directory using your FTP client.
<br />(<b>$queue_dir</b>)
<br />This also supports directory names to be used as tags.
<br />
<br />3. Go to the Board Config to the Cron Uploader menu and copy the Cron Command.
<br />(<b>$cron_cmd</b>)
<br />
<br />4. Create a cron job or something else that can open a url on specified times.
<br />If you're not sure how to do this, you can give the command to your web host and you can ask them to create the cron job for you.
<br />When you create the cron job, you choose when to upload new images.
<br />
<br />5. When the cron command is set up, your image queue will upload x file(s) at the specified times.
<br />You can see any uploads or failed uploads in the log file. (<b>$log_path</b>)
<br />Your uploaded images will be moved to the 'uploaded' directory, it's recommended that you remove everything out of this directory from time to time.
<br />(<b>$uploaded_dir</b>)
<br />
<br />Whenever the url in that cron job command is opened, a new file will upload from the queue.
<br />So when you want to manually upload an image, all you have to do is open the link once.
<br />This link can be found under 'Cron Command' in the board config, just remove the 'wget ' part and only the url remains.
<br />(<b>$cron_url</b>)";
$page->set_title("Cron Uploader");
$page->set_heading("Cron Uploader");
$block = new Block("Cron Uploader", $info_html, "main", 10);
$block_install = new Block("Installation Guide", $install_html, "main", 20);
$page->add_block($block);
$page->add_block($block_install);
}
public function onInitExt(InitExtEvent $event)
{
global $config;
// Set default values
$config->set_default_int(self::CONFIG_COUNT, 1);
$this->set_dir();
$this->upload_key = $config->get_string(self::CONFIG_KEY, "");
if (empty($this->upload_key)) {
$this->upload_key = $this->generate_key();
$config->set_string(self::CONFIG_KEY, $this->upload_key);
}
}
public function onSetupBuilding(SetupBuildingEvent $event) public function onSetupBuilding(SetupBuildingEvent $event)
{ {
$this->set_dir(); global $database;
$cron_url = make_http(make_link("/cron_upload/" . $this->upload_key));
$cron_cmd = "curl --silent $cron_url";
$documentation_link = make_http(make_link("cron_upload")); $documentation_link = make_http(make_link("cron_upload"));
$sb = new SetupBlock("Cron Uploader"); $users = $database->get_pairs("SELECT name, id FROM users UNION ALL SELECT '', null order by name");
$sb->add_label("<b>Settings</b><br>");
$sb->add_int_option(self::CONFIG_COUNT, "How many to upload each time");
$sb->add_text_option(self::CONFIG_DIR, "<br>Set Cron Uploader root directory<br>");
$sb->add_label("<br>Cron Command: <input type='text' size='60' readonly='readonly' value='" . html_escape($cron_cmd) . "'><br> $sb = new SetupBlock("Cron Uploader");
Create a cron job with the command above.<br/> $sb->start_table();
<a href='$documentation_link'>Read the documentation</a> if you're not sure what to do."); $sb->add_int_option(CronUploaderConfig::COUNT, "Upload per run", true);
$sb->add_text_option(CronUploaderConfig::DIR, "Root dir", true);
$sb->add_text_option(CronUploaderConfig::KEY, "Key", true);
$sb->add_choice_option(CronUploaderConfig::USER, $users,"User", true);
$sb->end_table();
$sb->add_label("<a href='$documentation_link'>Read the documentation</a> for cron setup instructions.");
$event->panel->add_block($sb); $event->panel->add_block($sb);
} }
/* public function onAdminBuilding(AdminBuildingEvent $event)
* Generates a unique key for the website to prevent unauthorized access.
*/
private function generate_key()
{ {
$length = 20; $failed_dir = $this->get_failed_dir();
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; $results = get_dir_contents($failed_dir);
$randomString = '';
for ($i = 0; $i < $length; $i++) { $failed_dirs = [];
$randomString .= $characters [rand(0, strlen($characters) - 1)]; foreach ($results as $result) {
$path = join_path($failed_dir, $result);
if (is_dir($path)) {
$failed_dirs[] = $result;
}
} }
return $randomString; $this->theme->display_form($failed_dirs);
} }
/* public function onAdminAction(AdminActionEvent $event)
* Set the directory for the image queue. If no directory was given, set it to the default directory. {
*/ $action = $event->action;
private function set_dir() switch ($action) {
case "cron_uploader_clear_queue":
$event->redirect = true;
$this->clear_folder(self::QUEUE_DIR);
break;
case "cron_uploader_clear_uploaded":
$event->redirect = true;
$this->clear_folder(self::UPLOADED_DIR);
break;
case "cron_uploader_clear_failed":
$event->redirect = true;
$this->clear_folder(self::FAILED_DIR);
break;
case "cron_uploader_restage":
$event->redirect = true;
if (array_key_exists("failed_dir", $_POST) && !empty($_POST["failed_dir"])) {
$this->restage_folder($_POST["failed_dir"]);
}
break;
}
}
private function restage_folder(string $folder)
{
if (empty($folder)) {
throw new Exception("folder empty");
}
$queue_dir = $this->get_queue_dir();
$stage_dir = join_path($this->get_failed_dir(), $folder);
if (!is_dir($stage_dir)) {
throw new Exception("Could not find $stage_dir");
}
$this->prep_root_dir();
$results = get_dir_contents($queue_dir);
if (count($results) > 0) {
flash_message("Queue folder must be empty to re-stage", "error");
return;
}
$results = get_dir_contents($stage_dir);
if (count($results) == 0) {
if(rmdir($stage_dir)===false) {
flash_message("Nothing to stage from $folder, cannot remove folder");
} else {
flash_message("Nothing to stage from $folder, removing folder");
}
return;
}
foreach ($results as $result) {
$original_path = join_path($stage_dir, $result);
$new_path = join_path($queue_dir, $result);
rename($original_path, $new_path);
}
flash_message("Re-staged $folder to queue");
rmdir($stage_dir);
}
private function clear_folder($folder)
{
$path = join_path(CronUploaderConfig::get_dir(), $folder);
deltree($path);
flash_message("Cleared $path");
}
private function get_cron_url()
{
return make_http(make_link("/cron_upload/" . CronUploaderConfig::get_key()));
}
private function get_cron_cmd()
{
return "curl --silent " . $this->get_cron_url();
}
private function display_documentation()
{
global $database;
$this->prep_root_dir();
$queue_dir = $this->get_queue_dir();
$uploaded_dir = $this->get_uploaded_dir();
$failed_dir = $this->get_failed_dir();
$queue_dirinfo = scan_dir($queue_dir);
$uploaded_dirinfo = scan_dir($uploaded_dir);
$failed_dirinfo = scan_dir($failed_dir);
$running = false;
$lockfile = fopen($this->get_lock_file(), "w");
try {
if (!flock($lockfile, LOCK_EX | LOCK_NB)) {
$running = true;
} else {
flock($lockfile, LOCK_UN);
}
} finally {
fclose($lockfile);
}
$logs = [];
if (Extension::is_enabled(LogDatabaseInfo::KEY)) {
$logs = $database->get_all(
"SELECT * FROM score_log WHERE section = :section ORDER BY date_sent DESC LIMIT 100",
["section" => self::NAME]
);
}
$this->theme->display_documentation(
$running, $queue_dirinfo, $uploaded_dirinfo, $failed_dirinfo,
$this->get_cron_cmd(), $this->get_cron_url(), $logs
);
}
function get_queue_dir()
{
$dir = CronUploaderConfig::get_dir();
return join_path($dir, self::QUEUE_DIR);
}
function get_uploaded_dir()
{
$dir = CronUploaderConfig::get_dir();
return join_path($dir, self::UPLOADED_DIR);
}
function get_failed_dir()
{
$dir = CronUploaderConfig::get_dir();
return join_path($dir, self::FAILED_DIR);
}
private function prep_root_dir(): string
{ {
global $config;
// Determine directory (none = default) // Determine directory (none = default)
$dir = CronUploaderConfig::get_dir();
$dir = $config->get_string(self::CONFIG_DIR, "");
// Sets new default dir if not in config yet/anymore
if ($dir == "") {
$dir = data_path("cron_uploader");
$config->set_string(self::CONFIG_DIR, $dir);
}
// Make the directory if it doesn't exist yet // Make the directory if it doesn't exist yet
if (!is_dir($dir . "/" . self::QUEUE_DIR . "/")) { if (!is_dir($this->get_queue_dir())) {
mkdir($dir . "/" . self::QUEUE_DIR . "/", 0775, true); mkdir($this->get_queue_dir(), 0775, true);
} }
if (!is_dir($dir . "/" . self::UPLOADED_DIR . "/")) { if (!is_dir($this->get_uploaded_dir())) {
mkdir($dir . "/" . self::UPLOADED_DIR . "/", 0775, true); mkdir($this->get_uploaded_dir(), 0775, true);
} }
if (!is_dir($dir . "/" . self::FAILED_DIR . "/")) { if (!is_dir($this->get_failed_dir())) {
mkdir($dir . "/" . self::FAILED_DIR . "/", 0775, true); mkdir($this->get_failed_dir(), 0775, true);
} }
$this->root_dir = $dir;
return $dir; return $dir;
} }
/** private function get_lock_file(): string
* Returns amount of files & total size of dir.
*/
public function scan_dir(string $path): array
{ {
$bytestotal = 0; $root_dir = CronUploaderConfig::get_dir();
$nbfiles = 0; return join_path($root_dir, ".lock");
$ite = new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) {
$filesize = $cur->getSize();
$bytestotal += $filesize;
$nbfiles++;
}
$size_mb = $bytestotal / 1048576; // to mb
$size_mb = number_format($size_mb, 2, '.', '');
return ['total_files' => $nbfiles, 'total_mb' => $size_mb];
} }
/** /**
* Uploads the image & handles everything * Uploads the image & handles everything
*/ */
public function process_upload(int $upload_count = 0): bool public function process_upload(string $key, ?int $upload_count = null): bool
{ {
global $config, $database; global $database;
//set_time_limit(0); if ($key!=CronUploaderConfig::get_key()) {
throw new SCoreException("Cron upload key incorrect");
}
$output_subdir = date('Ymd-His', time()) . "/"; $user_id = CronUploaderConfig::get_user();
$this->generate_image_queue(); if(empty($user_id)) {
throw new SCoreException("Cron upload user not set");
// Gets amount of imgs to upload }
if ($upload_count == 0) { $user = User::by_id($user_id);
$upload_count = $config->get_int(self::CONFIG_COUNT, 1); if ($user == null) {
throw new SCoreException("No user found for cron upload user $user_id");
} }
send_event(new UserLoginEvent($user));
$this->log_message(SCORE_LOG_INFO, "Logged in as user {$user->name}");
$lockfile = fopen($this->get_lock_file(), "w");
if (!flock($lockfile, LOCK_EX | LOCK_NB)) {
throw new SCoreException("Cron upload process is already running");
}
try {
//set_time_limit(0);
// Gets amount of imgs to upload
if ($upload_count == null) {
$upload_count = CronUploaderConfig::get_count();
}
$output_subdir = date('Ymd-His', time());
$image_queue = $this->generate_image_queue($upload_count);
// Throw exception if there's nothing in the queue // Throw exception if there's nothing in the queue
if (count($this->image_queue) == 0) { if (count($image_queue) == 0) {
$this->add_upload_info("Your queue is empty so nothing could be uploaded."); $this->log_message(SCORE_LOG_WARNING, "Your queue is empty so nothing could be uploaded.");
$this->handle_log(); $this->handle_log();
return false; return false;
} }
@ -288,12 +305,12 @@ class CronUploader extends Extension
$failed = 0; $failed = 0;
// Upload the file(s) // Upload the file(s)
for ($i = 0; $i < $upload_count && sizeof($this->image_queue) > 0; $i++) { for ($i = 0; $i < $upload_count && sizeof($image_queue) > 0; $i++) {
$img = array_pop($this->image_queue); $img = array_pop($image_queue);
$database->beginTransaction();
try { try {
$this->add_upload_info("Adding file: {$img[1]} - tags: {$img[2]}"); $database->beginTransaction();
$this->log_message(SCORE_LOG_INFO, "Adding file: {$img[0]} - tags: {$img[2]}");
$result = $this->add_image($img[0], $img[1], $img[2]); $result = $this->add_image($img[0], $img[1], $img[2]);
$database->commit(); $database->commit();
$this->move_uploaded($img[0], $img[1], $output_subdir, false); $this->move_uploaded($img[0], $img[1], $output_subdir, false);
@ -303,50 +320,65 @@ class CronUploader extends Extension
$added++; $added++;
} }
} catch (Exception $e) { } catch (Exception $e) {
try {
$database->rollback(); $database->rollback();
} catch (Exception $e) {
}
$failed++; $failed++;
$this->move_uploaded($img[0], $img[1], $output_subdir, true); $this->move_uploaded($img[0], $img[1], $output_subdir, true);
$msgNumber = $this->add_upload_info("(" . gettype($e) . ") " . $e->getMessage()); $this->log_message(SCORE_LOG_ERROR, "(" . gettype($e) . ") " . $e->getMessage());
$msgNumber = $this->add_upload_info($e->getTraceAsString()); $this->log_message(SCORE_LOG_ERROR, $e->getTraceAsString());
} }
} }
$msgNumber = $this->add_upload_info("Items added: $added");
$msgNumber = $this->add_upload_info("Items merged: $merged");
$msgNumber = $this->add_upload_info("Items failed: $failed");
// Display & save upload log $this->log_message(SCORE_LOG_INFO, "Items added: $added");
$this->log_message(SCORE_LOG_INFO, "Items merged: $merged");
$this->log_message(SCORE_LOG_INFO, "Items failed: $failed");
// Display upload log
$this->handle_log(); $this->handle_log();
return true; return true;
} finally {
flock($lockfile, LOCK_UN);
fclose($lockfile);
} }
private function move_uploaded($path, $filename, $output_subdir, $corrupt = false) }
{
// Create
$newDir = $this->root_dir;
$relativeDir = dirname(substr($path, strlen($this->root_dir) + 7)); private function move_uploaded(string $path, string $filename, string $output_subdir, bool $corrupt = false)
{
$relativeDir = dirname(substr($path, strlen(CronUploaderConfig::get_dir()) + 7));
if($relativeDir==".") {
$relativeDir = "";
}
// Determine which dir to move to // Determine which dir to move to
if ($corrupt) { if ($corrupt) {
// Move to corrupt dir // Move to corrupt dir
$newDir .= "/" . self::FAILED_DIR . "/" . $output_subdir . $relativeDir; $newDir = join_path($this->get_failed_dir(), $output_subdir, $relativeDir);
$info = "ERROR: Image was not uploaded."; $info = "ERROR: Image was not uploaded. ";
} else { } else {
$newDir .= "/" . self::UPLOADED_DIR . "/" . $output_subdir . $relativeDir; $newDir = join_path($this->get_uploaded_dir(), $output_subdir, $relativeDir);
$info = "Image successfully uploaded. "; $info = "Image successfully uploaded. ";
} }
$newDir = str_replace("//", "/", $newDir . "/"); $newDir = str_replace(DIRECTORY_SEPARATOR . DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR, $newDir);
if (!is_dir($newDir)) { if (!is_dir($newDir)) {
mkdir($newDir, 0775, true); mkdir($newDir, 0775, true);
} }
$newFile = join_path($newDir, $filename);
// move file to correct dir // move file to correct dir
rename($path, $newDir . $filename); rename($path, $newFile);
$this->add_upload_info($info . "Image \"$filename\" moved from queue to \"$newDir\"."); $this->log_message(SCORE_LOG_INFO, $info . "Image \"$filename\" moved from queue to \"$newDir\".");
} }
/** /**
@ -357,7 +389,7 @@ class CronUploader extends Extension
assert(file_exists($tmpname)); assert(file_exists($tmpname));
$tagArray = Tag::explode($tags); $tagArray = Tag::explode($tags);
if (count($tagArray)==0) { if (count($tagArray) == 0) {
$tagArray[] = "tagme"; $tagArray[] = "tagme";
} }
@ -377,11 +409,11 @@ class CronUploader extends Extension
if ($event->image_id == -1) { if ($event->image_id == -1) {
throw new Exception("File type not recognised. Filename: {$filename}"); throw new Exception("File type not recognised. Filename: {$filename}");
} elseif ($event->merged === true) { } elseif ($event->merged === true) {
$infomsg = "Image merged. ID: {$event->image_id} Filename: {$filename}"; $infomsg = "Image merged. ID: {$event->image_id} - Filename: {$filename}";
} else { } else {
$infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename}"; $infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename}";
} }
$msgNumber = $this->add_upload_info($infomsg); $this->log_message(SCORE_LOG_INFO, $infomsg);
// Set tags // Set tags
$img = Image::by_id($event->image_id); $img = Image::by_id($event->image_id);
@ -390,18 +422,31 @@ class CronUploader extends Extension
return $event; return $event;
} }
private function generate_image_queue(): void private const PARTIAL_DOWNLOAD_EXTENSIONS = ['crdownload','part'];
private function is_skippable_file(string $path) {
$info = pathinfo($path);
if(in_array(strtolower($info['extension']),self::PARTIAL_DOWNLOAD_EXTENSIONS)) {
return true;
}
return false;
}
private function generate_image_queue(string $root_dir, ?int $limit = null): array
{ {
$base = $this->root_dir . "/" . self::QUEUE_DIR; $base = $this->get_queue_dir();
$output = [];
if (!is_dir($base)) { if (!is_dir($base)) {
$this->add_upload_info("Image Queue Directory could not be found at \"$base\"."); $this->log_message(SCORE_LOG_WARNING, "Image Queue Directory could not be found at \"$base\".");
return; return [];
} }
$ite = new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS); $ite = new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $fullpath => $cur) { foreach (new RecursiveIteratorIterator($ite) as $fullpath => $cur) {
if (!is_link($fullpath) && !is_dir($fullpath)) { if (!is_link($fullpath) && !is_dir($fullpath) && !$this->is_skippable_file($fullpath)) {
$pathinfo = pathinfo($fullpath); $pathinfo = pathinfo($fullpath);
$relativePath = substr($fullpath, strlen($base)); $relativePath = substr($fullpath, strlen($base));
@ -412,34 +457,33 @@ class CronUploader extends Extension
1 => $pathinfo ["basename"], 1 => $pathinfo ["basename"],
2 => $tags 2 => $tags
]; ];
array_push($this->image_queue, $img); $output[] = $img;
if (!empty($limit) && count($output) >= $limit) {
break;
} }
} }
} }
return $output;
}
/**
* Adds a message to the info being published at the end private function log_message(int $severity, string $message): void
*/
private function add_upload_info(string $text, int $addon = 0): int
{ {
$info = $this->upload_info; global $database;
log_msg(self::NAME, $severity, $message);
$time = "[" . date('Y-m-d H:i:s') . "]"; $time = "[" . date('Y-m-d H:i:s') . "]";
$this->output_buffer[] = $time . " " . $message;
// If addon function is not used $log_path = $this->get_log_file();
if ($addon == 0) {
$this->upload_info .= "$time $text\r\n";
// Returns the number of the current line file_put_contents($log_path, $time . " " . $message);
$currentLine = substr_count($this->upload_info, "\n") - 1;
return $currentLine;
} }
// else if addon function is used, select the line & modify it private function get_log_file(): string
$lines = substr($info, "\n"); // Seperate the string to array in lines {
$lines[$addon] = "$lines[$addon] $text"; // Add the content to the line return join_path(CronUploaderConfig::get_dir(), "uploads.log");
$this->upload_info = implode("\n", $lines); // Put string back together & update
return $addon; // Return line number
} }
/** /**
@ -452,18 +496,7 @@ class CronUploader extends Extension
// Display message // Display message
$page->set_mode(PageMode::DATA); $page->set_mode(PageMode::DATA);
$page->set_type("text/plain"); $page->set_type("text/plain");
$page->set_data($this->upload_info); $page->set_data(implode("\r\n", $this->output_buffer));
// Save log
$log_path = $this->root_dir . "/uploads.log";
if (file_exists($log_path)) {
$prev_content = file_get_contents($log_path);
} else {
$prev_content = "";
}
$content = $prev_content . "\r\n" . $this->upload_info;
file_put_contents($log_path, $content);
} }
} }

3
ext/cron_uploader/style.css Normal file
View file

@ -0,0 +1,3 @@
table.log th {
width: 200px;
}

126
ext/cron_uploader/theme.php Normal file
View file

@ -0,0 +1,126 @@
<?php
class CronUploaderTheme extends Themelet
{
public function display_documentation(bool $running, array $queue_dirinfo, array $uploaded_dirinfo, array $failed_dirinfo,
string $cron_cmd, string $cron_url, ?array $log_entries)
{
global $page;
$info_html = "<b>Information</b>
<br>
<table style='width:470px;'>
" . ($running ? "<tr><td colspan='4'><b style='color:red'>Cron upload is currently running</b></td></tr>" : "") . "
<tr>
<td style='width:90px;'><b>Directory</b></td>
<td style='width:90px;'><b>Files</b></td>
<td style='width:90px;'><b>Size (MB)</b></td>
<td style='width:200px;'><b>Directory Path</b></td>
</tr><tr>
<td>Queue</td>
<td>{$queue_dirinfo['total_files']}</td>
<td>{$queue_dirinfo['total_mb']}</td>
<td>{$queue_dirinfo['path']}</td>
</tr><tr>
<td>Uploaded</td>
<td>{$uploaded_dirinfo['total_files']}</td>
<td>{$uploaded_dirinfo['total_mb']}</td>
<td>{$uploaded_dirinfo['path']}</td>
</tr><tr>
<td>Failed</td>
<td>{$failed_dirinfo['total_files']}</td>
<td>{$failed_dirinfo['total_mb']}</td>
<td>{$failed_dirinfo['path']}</td>
</tr></table>
<br>Cron Command: <input type='text' size='60' value='$cron_cmd'><br>
Create a cron job with the command above.<br/>
Read the documentation if you're not sure what to do.<br>";
$install_html = "
This cron uploader is fairly easy to use but has to be configured first.
<ol>
<li>Install & activate this plugin.</li>
<li>Go to the <a href='".make_link("setup")."'>Board Config</a> and change any settings to match your preference.</li>
<li>Copy the cron command above.</li>
<li>Create a cron job or something else that can open a url on specified times.
<br/>cron is a service that runs commands over and over again on a a schedule. You can set up cron (or any similar tool) to run the command above to trigger the import on whatever schedule you desire.
<br />If you're not sure how to do this, you can give the command to your web host and you can ask them to create the cron job for you.
<br />When you create the cron job, you choose when to upload new images.</li>
</ol>";
$usage_html = "Upload your images you want to be uploaded to the queue directory using your FTP client or other means.
<br />(<b>{$queue_dirinfo['path']}</b>)
<ol>
<li>Any sub-folders will be turned into tags.</li>
<li>If the file name matches \"## - tag1 tag2.png\" the tags will be used.</li>
<li>If both are found, they will all be used.</li>
<li>The character \";\" will be changed into \":\" in any tags.</li>
<li>You can inherit categories by creating a folder that ends with \";\". For instance category;\\tag1 would result in the tag category:tag1. This allows creating a category folder, then creating many subfolders that will use that category.</li>
</ol>
The cron uploader works by importing files from the queue folder whenever this url is visited:
<br/><pre><a href='$cron_url'>$cron_url</a></pre>
<ul>
<li>If an import is already running, another cannot start until it is done.</li>
<li>Each time it runs it will import up to ".CronUploaderConfig::get_count()." file(s). This is controlled from <a href='".make_link("setup")."'>Board Config</a>.</li>
<li>Uploaded images will be moved to the 'uploaded' directory into a subfolder named after the time the import started. It's recommended that you remove everything out of this directory from time to time. If you have admin controls enabled, this can be done from <a href='".make_link("admin")."'>Board Admin</a>.</li>
<li>If you enable the db logging extension, you can view the log output on this screen. Otherwise the log will be written to a file at ".CronUploaderConfig::get_dir().DIRECTORY_SEPARATOR."uploads.log</li>
</ul>
";
$page->set_title("Cron Uploader");
$page->set_heading("Cron Uploader");
$block = new Block("Cron Uploader", $info_html, "main", 10);
$block_install = new Block("Setup Guide", $install_html, "main", 30);
$block_usage= new Block("Usage Guide", $usage_html, "main", 20);
$page->add_block($block);
$page->add_block($block_install);
$page->add_block($block_usage);
if(!empty($log_entries)) {
$log_html = "<table class='log'>";
foreach($log_entries as $entry) {
$log_html .= "<tr><th>{$entry["date_sent"]}</th><td>{$entry["message"]}</td></tr>";
}
$log_html .= "</table>";
$block = new Block("Log", $log_html, "main", 40);
$page->add_block($block);
}
}
public function display_form(array $failed_dirs)
{
global $page, $database;
$link = make_http(make_link("cron_upload"));
$html = "<a href='$link'>Cron uploader documentation</a>";
$html .= make_form(make_link("admin/cron_uploader_restage"));
$html .= "<table class='form'>";
$html .= "<tr><th>Failed dir</th><td><select name='failed_dir' required='required'><option></option>";
foreach ($failed_dirs as $dir) {
$html .= "<option value='$dir'>$dir</option>";
}
$html .= "</select></td></tr>";
$html .= "<tr><td colspan='2'><input type='submit' value='Re-stage files to queue' /></td></tr>";
$html .= "</table></form>";
$html .= make_form(make_link("admin/cron_uploader_clear_queue"), "POST",false,"","return confirm('Are you sure you want to delete everything in the queue folder?');")
."<table class='form'><tr><td>"
."<input type='submit' value='Clear queue folder'></td></tr></table></form>";
$html .= make_form(make_link("admin/cron_uploader_clear_uploaded"), "POST",false,"","return confirm('Are you sure you want to delete everything in the uploaded folder?');")
."<table class='form'><tr><td>"
."<input type='submit' value='Clear uploaded folder'></td></tr></table></form>";
$html .= make_form(make_link("admin/cron_uploader_clear_failed"), "POST",false,"","return confirm('Are you sure you want to delete everything in the failed folder?');")
."<table class='form'><tr><td>"
."<input type='submit' value='Clear failed folder'></td></tr></table></form>";
$html .= "</table>\n";
$page->add_block(new Block("Cron Upload", $html));
}
}