From 40269a6f4a447e387961c87dfd563cee56b9f97f Mon Sep 17 00:00:00 2001 From: matthew Date: Thu, 10 Oct 2019 10:16:15 -0500 Subject: [PATCH 1/3] Cron uploader enhancements and bug fixes --- ext/cron_uploader/config.php | 98 ++++++ ext/cron_uploader/main.php | 637 ++++++++++++++++++----------------- ext/cron_uploader/style.css | 3 + ext/cron_uploader/theme.php | 126 +++++++ 4 files changed, 562 insertions(+), 302 deletions(-) create mode 100644 ext/cron_uploader/config.php create mode 100644 ext/cron_uploader/style.css create mode 100644 ext/cron_uploader/theme.php diff --git a/ext/cron_uploader/config.php b/ext/cron_uploader/config.php new file mode 100644 index 00000000..c749d635 --- /dev/null +++ b/ext/cron_uploader/config.php @@ -0,0 +1,98 @@ +set_default_int(self::COUNT, 1); + $config->set_default_string(self::DIR, data_path(self::DEFAULT_PATH)); + + $upload_key = $config->get_string(self::KEY, ""); + if (empty($upload_key)) { + $upload_key = self::generate_key(); + + $config->set_string(self::KEY, $upload_key); + } + + } + + public static function get_user(): int + { + global $config; + return $config->get_int(self::USER); + } + + public static function set_user(int $value): void + { + global $config; + $config->set_int(self::USER, $value); + } + + + public static function get_key(): string + { + global $config; + return $config->get_string(self::KEY); + } + + public static function set_key(string $value): void + { + global $config; + $config->set_string(self::KEY, $value); + } + + public static function get_count(): int + { + global $config; + return $config->get_int(self::COUNT); + } + + public static function set_count(int $value): int + { + global $config; + $config->get_int(self::COUNT, $value); + } + + public static function get_dir(): string + { + global $config; + $value = $config->get_string(self::DIR); + if (empty($value)) { + $value = data_path("cron_uploader"); + self::set_dir($value); + } + return $value; + } + + public static function set_dir(string $value): void + { + global $config; + $config->set_string(self::DIR, $value); + } + + + /* + * Generates a unique key for the website to prevent unauthorized access. + */ + private static function generate_key() + { + $length = 20; + $characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; + $randomString = ''; + + for ($i = 0; $i < $length; $i++) { + $randomString .= $characters [rand(0, strlen($characters) - 1)]; + } + + return $randomString; + } +} diff --git a/ext/cron_uploader/main.php b/ext/cron_uploader/main.php index 68a85635..98999d29 100644 --- a/ext/cron_uploader/main.php +++ b/ext/cron_uploader/main.php @@ -1,43 +1,31 @@ parent=="system") { + $event->add_nav_link("cron_docs", new Link('cron_upload'), "Cron Upload"); + } + } /** * Checks if the cron upload page has been accessed @@ -45,308 +33,352 @@ class CronUploader extends Extension */ public function onPageRequest(PageRequestEvent $event) { - global $config, $user; + global $user; if ($event->page_matches("cron_upload")) { - $this->upload_key = $config->get_string(self::CONFIG_KEY, ""); - - // If the key is in the url, upload - if ($this->upload_key != "" && $event->get_arg(0) == $this->upload_key) { - // log in as admin - $this->set_dir(); - - $lockfile = fopen($this->root_dir . "/.lock", "w"); - if (!flock($lockfile, LOCK_EX | LOCK_NB)) { - throw new Exception("Cron upload process is already running"); - } - try { - $this->process_upload(); // Start upload - } finally { - flock($lockfile, LOCK_UN); - fclose($lockfile); - } - } elseif ($user->can(Permissions::BULK_ADD)) { - $this->set_dir(); + $key = $event->get_arg(0); + if (!empty($key)) { + $this->process_upload($key); // Start upload + } elseif ($user->is_admin()) { $this->display_documentation(); } } } - private function display_documentation() - { - global $page; - $this->set_dir(); // Determines path to cron_uploader_dir - - - $queue_dir = $this->root_dir . "/" . self::QUEUE_DIR; - $uploaded_dir = $this->root_dir . "/" . self::UPLOADED_DIR; - $failed_dir = $this->root_dir . "/" . self::FAILED_DIR; - - $queue_dirinfo = $this->scan_dir($queue_dir); - $uploaded_dirinfo = $this->scan_dir($uploaded_dir); - $failed_dirinfo = $this->scan_dir($failed_dir); - - $cron_url = make_http(make_link("/cron_upload/" . $this->upload_key)); - $cron_cmd = "curl --silent $cron_url"; - $log_path = $this->root_dir . "/uploads.log"; - - $info_html = "Information -
- - - - - - - - - - - - - - - - - - - - - -
DirectoryFilesSize (MB)Directory Path
Queue{$queue_dirinfo['total_files']}{$queue_dirinfo['total_mb']}
Uploaded{$uploaded_dirinfo['total_files']}{$uploaded_dirinfo['total_mb']}
Failed{$failed_dirinfo['total_files']}{$failed_dirinfo['total_mb']}
- -
Cron Command:
- Create a cron job with the command above.
- Read the documentation if you're not sure what to do.
"; - - $install_html = " - This cron uploader is fairly easy to use but has to be configured first. -
1. Install & activate this plugin. -
-
2. Upload your images you want to be uploaded to the queue directory using your FTP client. -
($queue_dir) -
This also supports directory names to be used as tags. -
-
3. Go to the Board Config to the Cron Uploader menu and copy the Cron Command. -
($cron_cmd) -
-
4. Create a cron job or something else that can open a url on specified times. -
If you're not sure how to do this, you can give the command to your web host and you can ask them to create the cron job for you. -
When you create the cron job, you choose when to upload new images. -
-
5. When the cron command is set up, your image queue will upload x file(s) at the specified times. -
You can see any uploads or failed uploads in the log file. ($log_path) -
Your uploaded images will be moved to the 'uploaded' directory, it's recommended that you remove everything out of this directory from time to time. -
($uploaded_dir) -
-
Whenever the url in that cron job command is opened, a new file will upload from the queue. -
So when you want to manually upload an image, all you have to do is open the link once. -
This link can be found under 'Cron Command' in the board config, just remove the 'wget ' part and only the url remains. -
($cron_url)"; - - $page->set_title("Cron Uploader"); - $page->set_heading("Cron Uploader"); - - $block = new Block("Cron Uploader", $info_html, "main", 10); - $block_install = new Block("Installation Guide", $install_html, "main", 20); - $page->add_block($block); - $page->add_block($block_install); - } - - public function onInitExt(InitExtEvent $event) - { - global $config; - // Set default values - $config->set_default_int(self::CONFIG_COUNT, 1); - $this->set_dir(); - - $this->upload_key = $config->get_string(self::CONFIG_KEY, ""); - if (empty($this->upload_key)) { - $this->upload_key = $this->generate_key(); - - $config->set_string(self::CONFIG_KEY, $this->upload_key); - } - } - public function onSetupBuilding(SetupBuildingEvent $event) { - $this->set_dir(); + global $database; - $cron_url = make_http(make_link("/cron_upload/" . $this->upload_key)); - $cron_cmd = "curl --silent $cron_url"; $documentation_link = make_http(make_link("cron_upload")); - $sb = new SetupBlock("Cron Uploader"); - $sb->add_label("Settings
"); - $sb->add_int_option(self::CONFIG_COUNT, "How many to upload each time"); - $sb->add_text_option(self::CONFIG_DIR, "
Set Cron Uploader root directory
"); + $users = $database->get_pairs("SELECT name, id FROM users UNION ALL SELECT '', null order by name"); - $sb->add_label("
Cron Command:
- Create a cron job with the command above.
- Read the documentation if you're not sure what to do."); + $sb = new SetupBlock("Cron Uploader"); + $sb->start_table(); + $sb->add_int_option(CronUploaderConfig::COUNT, "Upload per run", true); + $sb->add_text_option(CronUploaderConfig::DIR, "Root dir", true); + $sb->add_text_option(CronUploaderConfig::KEY, "Key", true); + $sb->add_choice_option(CronUploaderConfig::USER, $users,"User", true); + $sb->end_table(); + $sb->add_label("Read the documentation for cron setup instructions."); $event->panel->add_block($sb); } - /* - * Generates a unique key for the website to prevent unauthorized access. - */ - private function generate_key() + public function onAdminBuilding(AdminBuildingEvent $event) { - $length = 20; - $characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; - $randomString = ''; + $failed_dir = $this->get_failed_dir(); + $results = get_dir_contents($failed_dir); - for ($i = 0; $i < $length; $i++) { - $randomString .= $characters [rand(0, strlen($characters) - 1)]; + $failed_dirs = []; + foreach ($results as $result) { + $path = join_path($failed_dir, $result); + if (is_dir($path)) { + $failed_dirs[] = $result; + } } - return $randomString; + $this->theme->display_form($failed_dirs); } - /* - * Set the directory for the image queue. If no directory was given, set it to the default directory. - */ - private function set_dir() + public function onAdminAction(AdminActionEvent $event) { - global $config; - // Determine directory (none = default) - - $dir = $config->get_string(self::CONFIG_DIR, ""); - - // Sets new default dir if not in config yet/anymore - if ($dir == "") { - $dir = data_path("cron_uploader"); - $config->set_string(self::CONFIG_DIR, $dir); + $action = $event->action; + switch ($action) { + case "cron_uploader_clear_queue": + $event->redirect = true; + $this->clear_folder(self::QUEUE_DIR); + break; + case "cron_uploader_clear_uploaded": + $event->redirect = true; + $this->clear_folder(self::UPLOADED_DIR); + break; + case "cron_uploader_clear_failed": + $event->redirect = true; + $this->clear_folder(self::FAILED_DIR); + break; + case "cron_uploader_restage": + $event->redirect = true; + if (array_key_exists("failed_dir", $_POST) && !empty($_POST["failed_dir"])) { + $this->restage_folder($_POST["failed_dir"]); + } + break; } + } + + private function restage_folder(string $folder) + { + if (empty($folder)) { + throw new Exception("folder empty"); + } + $queue_dir = $this->get_queue_dir(); + $stage_dir = join_path($this->get_failed_dir(), $folder); + + if (!is_dir($stage_dir)) { + throw new Exception("Could not find $stage_dir"); + } + + $this->prep_root_dir(); + + $results = get_dir_contents($queue_dir); + + if (count($results) > 0) { + flash_message("Queue folder must be empty to re-stage", "error"); + return; + } + + $results = get_dir_contents($stage_dir); + + if (count($results) == 0) { + if(rmdir($stage_dir)===false) { + flash_message("Nothing to stage from $folder, cannot remove folder"); + } else { + flash_message("Nothing to stage from $folder, removing folder"); + } + return; + } + + foreach ($results as $result) { + $original_path = join_path($stage_dir, $result); + $new_path = join_path($queue_dir, $result); + + rename($original_path, $new_path); + } + + flash_message("Re-staged $folder to queue"); + rmdir($stage_dir); + } + + private function clear_folder($folder) + { + $path = join_path(CronUploaderConfig::get_dir(), $folder); + deltree($path); + flash_message("Cleared $path"); + } + + + private function get_cron_url() + { + return make_http(make_link("/cron_upload/" . CronUploaderConfig::get_key())); + } + + private function get_cron_cmd() + { + return "curl --silent " . $this->get_cron_url(); + } + + private function display_documentation() + { + global $database; + + $this->prep_root_dir(); + + $queue_dir = $this->get_queue_dir(); + $uploaded_dir = $this->get_uploaded_dir(); + $failed_dir = $this->get_failed_dir(); + + $queue_dirinfo = scan_dir($queue_dir); + $uploaded_dirinfo = scan_dir($uploaded_dir); + $failed_dirinfo = scan_dir($failed_dir); + + + $running = false; + $lockfile = fopen($this->get_lock_file(), "w"); + try { + if (!flock($lockfile, LOCK_EX | LOCK_NB)) { + $running = true; + } else { + flock($lockfile, LOCK_UN); + } + } finally { + fclose($lockfile); + } + + $logs = []; + if (Extension::is_enabled(LogDatabaseInfo::KEY)) { + $logs = $database->get_all( + "SELECT * FROM score_log WHERE section = :section ORDER BY date_sent DESC LIMIT 100", + ["section" => self::NAME] + ); + } + + $this->theme->display_documentation( + $running, $queue_dirinfo, $uploaded_dirinfo, $failed_dirinfo, + $this->get_cron_cmd(), $this->get_cron_url(), $logs + ); + } + + function get_queue_dir() + { + $dir = CronUploaderConfig::get_dir(); + return join_path($dir, self::QUEUE_DIR); + } + + function get_uploaded_dir() + { + $dir = CronUploaderConfig::get_dir(); + return join_path($dir, self::UPLOADED_DIR); + } + + function get_failed_dir() + { + $dir = CronUploaderConfig::get_dir(); + return join_path($dir, self::FAILED_DIR); + } + + private function prep_root_dir(): string + { + // Determine directory (none = default) + $dir = CronUploaderConfig::get_dir(); // Make the directory if it doesn't exist yet - if (!is_dir($dir . "/" . self::QUEUE_DIR . "/")) { - mkdir($dir . "/" . self::QUEUE_DIR . "/", 0775, true); + if (!is_dir($this->get_queue_dir())) { + mkdir($this->get_queue_dir(), 0775, true); } - if (!is_dir($dir . "/" . self::UPLOADED_DIR . "/")) { - mkdir($dir . "/" . self::UPLOADED_DIR . "/", 0775, true); + if (!is_dir($this->get_uploaded_dir())) { + mkdir($this->get_uploaded_dir(), 0775, true); } - if (!is_dir($dir . "/" . self::FAILED_DIR . "/")) { - mkdir($dir . "/" . self::FAILED_DIR . "/", 0775, true); + if (!is_dir($this->get_failed_dir())) { + mkdir($this->get_failed_dir(), 0775, true); } - $this->root_dir = $dir; return $dir; } - /** - * Returns amount of files & total size of dir. - */ - public function scan_dir(string $path): array + private function get_lock_file(): string { - $bytestotal = 0; - $nbfiles = 0; - - $ite = new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS); - foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) { - $filesize = $cur->getSize(); - $bytestotal += $filesize; - $nbfiles++; - } - - $size_mb = $bytestotal / 1048576; // to mb - $size_mb = number_format($size_mb, 2, '.', ''); - return ['total_files' => $nbfiles, 'total_mb' => $size_mb]; + $root_dir = CronUploaderConfig::get_dir(); + return join_path($root_dir, ".lock"); } /** * Uploads the image & handles everything */ - public function process_upload(int $upload_count = 0): bool + public function process_upload(string $key, ?int $upload_count = null): bool { - global $config, $database; + global $database; - //set_time_limit(0); - - - $output_subdir = date('Ymd-His', time()) . "/"; - $this->generate_image_queue(); - - // Gets amount of imgs to upload - if ($upload_count == 0) { - $upload_count = $config->get_int(self::CONFIG_COUNT, 1); + if ($key!=CronUploaderConfig::get_key()) { + throw new SCoreException("Cron upload key incorrect"); + } + $user_id = CronUploaderConfig::get_user(); + if(empty($user_id)) { + throw new SCoreException("Cron upload user not set"); + } + $user = User::by_id($user_id); + if ($user == null) { + throw new SCoreException("No user found for cron upload user $user_id"); } - // Throw exception if there's nothing in the queue - if (count($this->image_queue) == 0) { - $this->add_upload_info("Your queue is empty so nothing could be uploaded."); - $this->handle_log(); - return false; + send_event(new UserLoginEvent($user)); + $this->log_message(SCORE_LOG_INFO, "Logged in as user {$user->name}"); + + $lockfile = fopen($this->get_lock_file(), "w"); + if (!flock($lockfile, LOCK_EX | LOCK_NB)) { + throw new SCoreException("Cron upload process is already running"); } - // Randomize Images - //shuffle($this->image_queue); + try { + //set_time_limit(0); - $merged = 0; - $added = 0; - $failed = 0; - - // Upload the file(s) - for ($i = 0; $i < $upload_count && sizeof($this->image_queue) > 0; $i++) { - $img = array_pop($this->image_queue); - - $database->beginTransaction(); - try { - $this->add_upload_info("Adding file: {$img[1]} - tags: {$img[2]}"); - $result = $this->add_image($img[0], $img[1], $img[2]); - $database->commit(); - $this->move_uploaded($img[0], $img[1], $output_subdir, false); - if ($result->merged) { - $merged++; - } else { - $added++; - } - } catch (Exception $e) { - $database->rollback(); - $failed++; - $this->move_uploaded($img[0], $img[1], $output_subdir, true); - $msgNumber = $this->add_upload_info("(" . gettype($e) . ") " . $e->getMessage()); - $msgNumber = $this->add_upload_info($e->getTraceAsString()); + // Gets amount of imgs to upload + if ($upload_count == null) { + $upload_count = CronUploaderConfig::get_count(); } + + $output_subdir = date('Ymd-His', time()); + $image_queue = $this->generate_image_queue($upload_count); + + + // Throw exception if there's nothing in the queue + if (count($image_queue) == 0) { + $this->log_message(SCORE_LOG_WARNING, "Your queue is empty so nothing could be uploaded."); + $this->handle_log(); + return false; + } + + // Randomize Images + //shuffle($this->image_queue); + + $merged = 0; + $added = 0; + $failed = 0; + + // Upload the file(s) + for ($i = 0; $i < $upload_count && sizeof($image_queue) > 0; $i++) { + $img = array_pop($image_queue); + + try { + $database->beginTransaction(); + $this->log_message(SCORE_LOG_INFO, "Adding file: {$img[0]} - tags: {$img[2]}"); + $result = $this->add_image($img[0], $img[1], $img[2]); + $database->commit(); + $this->move_uploaded($img[0], $img[1], $output_subdir, false); + if ($result->merged) { + $merged++; + } else { + $added++; + } + } catch (Exception $e) { + try { + $database->rollback(); + } catch (Exception $e) { + } + + $failed++; + $this->move_uploaded($img[0], $img[1], $output_subdir, true); + $this->log_message(SCORE_LOG_ERROR, "(" . gettype($e) . ") " . $e->getMessage()); + $this->log_message(SCORE_LOG_ERROR, $e->getTraceAsString()); + + + } + } + + + $this->log_message(SCORE_LOG_INFO, "Items added: $added"); + $this->log_message(SCORE_LOG_INFO, "Items merged: $merged"); + $this->log_message(SCORE_LOG_INFO, "Items failed: $failed"); + + + // Display upload log + $this->handle_log(); + + return true; + } finally { + flock($lockfile, LOCK_UN); + fclose($lockfile); } - $msgNumber = $this->add_upload_info("Items added: $added"); - $msgNumber = $this->add_upload_info("Items merged: $merged"); - $msgNumber = $this->add_upload_info("Items failed: $failed"); - - // Display & save upload log - $this->handle_log(); - - return true; } - private function move_uploaded($path, $filename, $output_subdir, $corrupt = false) + private function move_uploaded(string $path, string $filename, string $output_subdir, bool $corrupt = false) { - // Create - $newDir = $this->root_dir; + $relativeDir = dirname(substr($path, strlen(CronUploaderConfig::get_dir()) + 7)); - $relativeDir = dirname(substr($path, strlen($this->root_dir) + 7)); + if($relativeDir==".") { + $relativeDir = ""; + } // Determine which dir to move to if ($corrupt) { // Move to corrupt dir - $newDir .= "/" . self::FAILED_DIR . "/" . $output_subdir . $relativeDir; - $info = "ERROR: Image was not uploaded."; + $newDir = join_path($this->get_failed_dir(), $output_subdir, $relativeDir); + $info = "ERROR: Image was not uploaded. "; } else { - $newDir .= "/" . self::UPLOADED_DIR . "/" . $output_subdir . $relativeDir; + $newDir = join_path($this->get_uploaded_dir(), $output_subdir, $relativeDir); $info = "Image successfully uploaded. "; } - $newDir = str_replace("//", "/", $newDir . "/"); + $newDir = str_replace(DIRECTORY_SEPARATOR . DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR, $newDir); if (!is_dir($newDir)) { mkdir($newDir, 0775, true); } + $newFile = join_path($newDir, $filename); // move file to correct dir - rename($path, $newDir . $filename); + rename($path, $newFile); - $this->add_upload_info($info . "Image \"$filename\" moved from queue to \"$newDir\"."); + $this->log_message(SCORE_LOG_INFO, $info . "Image \"$filename\" moved from queue to \"$newDir\"."); } /** @@ -357,7 +389,7 @@ class CronUploader extends Extension assert(file_exists($tmpname)); $tagArray = Tag::explode($tags); - if (count($tagArray)==0) { + if (count($tagArray) == 0) { $tagArray[] = "tagme"; } @@ -377,11 +409,11 @@ class CronUploader extends Extension if ($event->image_id == -1) { throw new Exception("File type not recognised. Filename: {$filename}"); } elseif ($event->merged === true) { - $infomsg = "Image merged. ID: {$event->image_id} Filename: {$filename}"; + $infomsg = "Image merged. ID: {$event->image_id} - Filename: {$filename}"; } else { $infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename}"; } - $msgNumber = $this->add_upload_info($infomsg); + $this->log_message(SCORE_LOG_INFO, $infomsg); // Set tags $img = Image::by_id($event->image_id); @@ -390,18 +422,31 @@ class CronUploader extends Extension return $event; } - private function generate_image_queue(): void + private const PARTIAL_DOWNLOAD_EXTENSIONS = ['crdownload','part']; + + private function is_skippable_file(string $path) { + $info = pathinfo($path); + + if(in_array(strtolower($info['extension']),self::PARTIAL_DOWNLOAD_EXTENSIONS)) { + return true; + } + + return false; + } + + private function generate_image_queue(string $root_dir, ?int $limit = null): array { - $base = $this->root_dir . "/" . self::QUEUE_DIR; + $base = $this->get_queue_dir(); + $output = []; if (!is_dir($base)) { - $this->add_upload_info("Image Queue Directory could not be found at \"$base\"."); - return; + $this->log_message(SCORE_LOG_WARNING, "Image Queue Directory could not be found at \"$base\"."); + return []; } $ite = new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS); foreach (new RecursiveIteratorIterator($ite) as $fullpath => $cur) { - if (!is_link($fullpath) && !is_dir($fullpath)) { + if (!is_link($fullpath) && !is_dir($fullpath) && !$this->is_skippable_file($fullpath)) { $pathinfo = pathinfo($fullpath); $relativePath = substr($fullpath, strlen($base)); @@ -412,34 +457,33 @@ class CronUploader extends Extension 1 => $pathinfo ["basename"], 2 => $tags ]; - array_push($this->image_queue, $img); + $output[] = $img; + if (!empty($limit) && count($output) >= $limit) { + break; + } } } + return $output; } - /** - * Adds a message to the info being published at the end - */ - private function add_upload_info(string $text, int $addon = 0): int + + private function log_message(int $severity, string $message): void { - $info = $this->upload_info; + global $database; + + log_msg(self::NAME, $severity, $message); + $time = "[" . date('Y-m-d H:i:s') . "]"; + $this->output_buffer[] = $time . " " . $message; - // If addon function is not used - if ($addon == 0) { - $this->upload_info .= "$time $text\r\n"; + $log_path = $this->get_log_file(); - // Returns the number of the current line - $currentLine = substr_count($this->upload_info, "\n") - 1; - return $currentLine; - } + file_put_contents($log_path, $time . " " . $message); + } - // else if addon function is used, select the line & modify it - $lines = substr($info, "\n"); // Seperate the string to array in lines - $lines[$addon] = "$lines[$addon] $text"; // Add the content to the line - $this->upload_info = implode("\n", $lines); // Put string back together & update - - return $addon; // Return line number + private function get_log_file(): string + { + return join_path(CronUploaderConfig::get_dir(), "uploads.log"); } /** @@ -452,18 +496,7 @@ class CronUploader extends Extension // Display message $page->set_mode(PageMode::DATA); $page->set_type("text/plain"); - $page->set_data($this->upload_info); - - // Save log - $log_path = $this->root_dir . "/uploads.log"; - - if (file_exists($log_path)) { - $prev_content = file_get_contents($log_path); - } else { - $prev_content = ""; - } - - $content = $prev_content . "\r\n" . $this->upload_info; - file_put_contents($log_path, $content); + $page->set_data(implode("\r\n", $this->output_buffer)); } } + diff --git a/ext/cron_uploader/style.css b/ext/cron_uploader/style.css new file mode 100644 index 00000000..2643a6a1 --- /dev/null +++ b/ext/cron_uploader/style.css @@ -0,0 +1,3 @@ +table.log th { + width: 200px; +} \ No newline at end of file diff --git a/ext/cron_uploader/theme.php b/ext/cron_uploader/theme.php new file mode 100644 index 00000000..8c438d23 --- /dev/null +++ b/ext/cron_uploader/theme.php @@ -0,0 +1,126 @@ +Information +
+ + " . ($running ? "" : "") . " + + + + + + + + + + + + + + + + + + + + +
Cron upload is currently running
DirectoryFilesSize (MB)Directory Path
Queue{$queue_dirinfo['total_files']}{$queue_dirinfo['total_mb']}{$queue_dirinfo['path']}
Uploaded{$uploaded_dirinfo['total_files']}{$uploaded_dirinfo['total_mb']}{$uploaded_dirinfo['path']}
Failed{$failed_dirinfo['total_files']}{$failed_dirinfo['total_mb']}{$failed_dirinfo['path']}
+ +
Cron Command:
+ Create a cron job with the command above.
+ Read the documentation if you're not sure what to do.
"; + + $install_html = " + This cron uploader is fairly easy to use but has to be configured first. +
    +
  1. Install & activate this plugin.
    2. +
  2. Go to the Board Config and change any settings to match your preference.
    3. +
  3. Copy the cron command above.
    4. +
  4. Create a cron job or something else that can open a url on specified times. +
    cron is a service that runs commands over and over again on a a schedule. You can set up cron (or any similar tool) to run the command above to trigger the import on whatever schedule you desire. +
    If you're not sure how to do this, you can give the command to your web host and you can ask them to create the cron job for you. +
    When you create the cron job, you choose when to upload new images.
    5. +
"; + + $usage_html = "Upload your images you want to be uploaded to the queue directory using your FTP client or other means. +
({$queue_dirinfo['path']}) +
    +
  1. Any sub-folders will be turned into tags.
    2. +
  2. If the file name matches \"## - tag1 tag2.png\" the tags will be used.
    3. +
  3. If both are found, they will all be used.
    4. +
  4. The character \";\" will be changed into \":\" in any tags.
    5. +
  5. You can inherit categories by creating a folder that ends with \";\". For instance category;\\tag1 would result in the tag category:tag1. This allows creating a category folder, then creating many subfolders that will use that category.
    6. +
+ The cron uploader works by importing files from the queue folder whenever this url is visited: + 
$cron_url
+ + + "; + + $page->set_title("Cron Uploader"); + $page->set_heading("Cron Uploader"); + + $block = new Block("Cron Uploader", $info_html, "main", 10); + $block_install = new Block("Setup Guide", $install_html, "main", 30); + $block_usage= new Block("Usage Guide", $usage_html, "main", 20); + $page->add_block($block); + $page->add_block($block_install); + $page->add_block($block_usage); + + if(!empty($log_entries)) { + $log_html = ""; + foreach($log_entries as $entry) { + $log_html .= ""; + } + $log_html .= "
{$entry["date_sent"]}{$entry["message"]}
"; + $block = new Block("Log", $log_html, "main", 40); + $page->add_block($block); + } + } + + public function display_form(array $failed_dirs) + { + global $page, $database; + + $link = make_http(make_link("cron_upload")); + $html = "Cron uploader documentation"; + + $html .= make_form(make_link("admin/cron_uploader_restage")); + $html .= ""; + $html .= ""; + $html .= ""; + $html .= "
Failed dir
"; + + $html .= make_form(make_link("admin/cron_uploader_clear_queue"), "POST",false,"","return confirm('Are you sure you want to delete everything in the queue folder?');") + ."
" + ."
"; + $html .= make_form(make_link("admin/cron_uploader_clear_uploaded"), "POST",false,"","return confirm('Are you sure you want to delete everything in the uploaded folder?');") + ."
" + ."
"; + $html .= make_form(make_link("admin/cron_uploader_clear_failed"), "POST",false,"","return confirm('Are you sure you want to delete everything in the failed folder?');") + ."
" + ."
"; + $html .= "\n"; + $page->add_block(new Block("Cron Upload", $html)); + } +} From 92a0afc15ef1f9e43159d273ac332adca6ada99c Mon Sep 17 00:00:00 2001 From: Matthew Barbour Date: Thu, 10 Oct 2019 10:25:37 -0500 Subject: [PATCH 2/3] Supporting function for cron uploader changes --- core/util.php | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/core/util.php b/core/util.php index 70b5f2ea..b79a2238 100644 --- a/core/util.php +++ b/core/util.php @@ -350,6 +350,51 @@ function join_url(string $base, string ...$paths) return $output; } +function get_dir_contents(string $dir): array +{ + if(empty($dir)) { + throw new Exception("dir required"); + } + if(!is_dir($dir)) { + return []; + } + $results = array_diff( + scandir( + $dir), + ['..', '.']); + + return $results; +} + +/** + * Returns amount of files & total size of dir. + */ +function scan_dir(string $path): array +{ + $bytestotal = 0; + $nbfiles = 0; + + $ite = new RecursiveDirectoryIterator( + $path, + FilesystemIterator::KEY_AS_PATHNAME | + FilesystemIterator::CURRENT_AS_FILEINFO | + FilesystemIterator::SKIP_DOTS); + foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) { + try { + $filesize = $cur->getSize(); + $bytestotal += $filesize; + $nbfiles++; + } catch (RuntimeException $e) { + // This usually just means that the file got eaten by the import + continue; + } + } + + $size_mb = $bytestotal / 1048576; // to mb + $size_mb = number_format($size_mb, 2, '.', ''); + return ['path' => $path, 'total_files' => $nbfiles, 'total_mb' => $size_mb]; +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Debugging functions * From d605e0e572bf8612aa2a88d77f606ef44c199617 Mon Sep 17 00:00:00 2001 From: Matthew Barbour Date: Thu, 17 Oct 2019 14:22:33 -0500 Subject: [PATCH 3/3] Added cron_admin permission --- core/permissions.php | 1 + core/userclass.php | 2 ++ ext/cron_uploader/main.php | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/permissions.php b/core/permissions.php index bd60b0c0..e546236b 100644 --- a/core/permissions.php +++ b/core/permissions.php @@ -80,4 +80,5 @@ abstract class Permissions public const NOTES_ADMIN = "notes_admin"; public const POOLS_ADMIN = "pools_admin"; public const TIPS_ADMIN = "tips_admin"; + public const CRON_ADMIN = "cron_admin"; } diff --git a/core/userclass.php b/core/userclass.php index 5c60c9a0..2207fa63 100644 --- a/core/userclass.php +++ b/core/userclass.php @@ -150,6 +150,7 @@ new UserClass("base", null, [ Permissions::NOTES_ADMIN => false, Permissions::POOLS_ADMIN => false, Permissions::TIPS_ADMIN => false, + Permissions::CRON_ADMIN => false, ]); new UserClass("anonymous", "base", [ @@ -226,6 +227,7 @@ new UserClass("admin", "base", [ Permissions::NOTES_ADMIN => true, Permissions::POOLS_ADMIN => true, Permissions::TIPS_ADMIN => true, + Permissions::CRON_ADMIN => true, ]); new UserClass("hellbanned", "user", [ diff --git a/ext/cron_uploader/main.php b/ext/cron_uploader/main.php index 98999d29..38ff3e8c 100644 --- a/ext/cron_uploader/main.php +++ b/ext/cron_uploader/main.php @@ -39,7 +39,7 @@ class CronUploader extends Extension $key = $event->get_arg(0); if (!empty($key)) { $this->process_upload($key); // Start upload - } elseif ($user->is_admin()) { + } elseif ($user->can(Permissions::CRON_ADMIN)) { $this->display_documentation(); } }