badmanners/shimmie2
badmanners/shimmie2
Archived
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Converted cron_upload to be able to run per-user, using user API keys

Browse source
This commit is contained in:
Matthew Barbour 2020-10-08 17:14:15 -05:00
parent e82b9ea811
commit e7d11f2310
6 changed files with 157 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/permissions.php
View file

@ -103,6 +103,7 @@ abstract class Permissions
public const SET_PRIVATE_IMAGE = "set_private_image";
public const SET_OTHERS_PRIVATE_IMAGES = "set_others_private_images";
public const CRON_RUN = "cron_run";
public const BULK_IMPORT = "bulk_import";
public const BULK_EXPORT = "bulk_export";
public const BULK_DOWNLOAD = "bulk_download";

2
core/userclass.php
View file

@ -203,6 +203,8 @@ new UserClass("admin", "base", [
Permissions::APPROVE_IMAGE => true,
Permissions::APPROVE_COMMENT => true,
Permissions::CRON_RUN =>true,
Permissions::BULK_IMPORT =>true,
Permissions::BULK_EXPORT =>true,
Permissions::BULK_DOWNLOAD => true,

58
ext/cron_uploader/config.php
View file

@ -5,66 +5,8 @@ abstract class CronUploaderConfig
{
public const DEFAULT_PATH = "cron_uploader";
public const KEY = "cron_uploader_key";
public const DIR = "cron_uploader_dir";
public const USER = "cron_uploader_user";
public const STOP_ON_ERROR = "cron_uploader_stop_on_error";
public const INCLUDE_ALL_LOGS = "cron_uploader_include_all_logs";
public const LOG_LEVEL = "cron_uploader_log_level";
public static function set_defaults(): void
{
global $config;
$config->set_default_string(self::DIR, data_path(self::DEFAULT_PATH));
$config->set_default_bool(self::INCLUDE_ALL_LOGS, false);
$config->set_default_bool(self::STOP_ON_ERROR, false);
$config->set_default_int(self::LOG_LEVEL, SCORE_LOG_INFO);
$upload_key = $config->get_string(self::KEY, "");
if (empty($upload_key)) {
$upload_key = generate_key();
$config->set_string(self::KEY, $upload_key);
}
}
public static function get_user(): int
{
global $config;
return $config->get_int(self::USER);
}
public static function set_user(int $value): void
{
global $config;
$config->set_int(self::USER, $value);
}
public static function get_key(): string
{
global $config;
return $config->get_string(self::KEY);
}
public static function set_key(string $value): void
{
global $config;
$config->set_string(self::KEY, $value);
}
public static function get_dir(): string
{
global $config;
$value = $config->get_string(self::DIR);
if (empty($value)) {
$value = data_path("cron_uploader");
self::set_dir($value);
}
return $value;
}
public static function set_dir(string $value): void
{
global $config;
$config->set_string(self::DIR, $value);
}
}

143
ext/cron_uploader/main.php
View file

@ -17,12 +17,40 @@ class CronUploader extends Extension
private static $IMPORT_RUNNING = false;
public function onInitExt(InitExtEvent $event)
public function onInitUserConfig(InitUserConfigEvent $event)
{
// Set default values
CronUploaderConfig::set_defaults();
$event->user_config->set_default_string(
CronUploaderConfig::DIR,
data_path(CronUploaderConfig::DEFAULT_PATH.DIRECTORY_SEPARATOR.$event->user->name)
);
$event->user_config->set_default_bool(CronUploaderConfig::INCLUDE_ALL_LOGS, false);
$event->user_config->set_default_bool(CronUploaderConfig::STOP_ON_ERROR, false);
$event->user_config->set_default_int(CronUploaderConfig::LOG_LEVEL, SCORE_LOG_INFO);
}
public function onUserOptionsBuilding(UserOptionsBuildingEvent $event)
{
if ($event->user->can(Permissions::CRON_ADMIN)) {
$documentation_link = make_http(make_link("cron_upload"));
$sb = $event->panel->create_new_block("Cron Uploader");
$sb->start_table();
$sb->add_text_option(CronUploaderConfig::DIR, "Root dir", true);
$sb->add_bool_option(CronUploaderConfig::STOP_ON_ERROR, "Stop On Error", true);
$sb->add_choice_option(CronUploaderConfig::LOG_LEVEL, [
LOGGING_LEVEL_NAMES[SCORE_LOG_DEBUG] => SCORE_LOG_DEBUG,
LOGGING_LEVEL_NAMES[SCORE_LOG_INFO] => SCORE_LOG_INFO,
LOGGING_LEVEL_NAMES[SCORE_LOG_WARNING] => SCORE_LOG_WARNING,
LOGGING_LEVEL_NAMES[SCORE_LOG_ERROR] => SCORE_LOG_ERROR,
LOGGING_LEVEL_NAMES[SCORE_LOG_CRITICAL] => SCORE_LOG_CRITICAL,
], "Output Log Level: ", true);
$sb->add_bool_option(CronUploaderConfig::INCLUDE_ALL_LOGS, "Include All Logs", true);
$sb->end_table();
$sb->add_label("<a href='$documentation_link'>Read the documentation</a> for cron setup instructions.");
}
}
public function onPageSubNavBuilding(PageSubNavBuildingEvent $event)
{
if ($event->parent=="system") {
@ -39,40 +67,14 @@ class CronUploader extends Extension
global $user;
if ($event->page_matches("cron_upload")) {
if ($event->count_args() == 1) {
$this->process_upload($event->get_arg(0)); // Start upload
} elseif ($user->can(Permissions::CRON_ADMIN)) {
if ($event->count_args() == 1 && $event->get_arg(0) =="run") {
$this->process_upload(); // Start upload
} elseif ($user->can(Permissions::CRON_RUN)) {
$this->display_documentation();
}
}
}
public function onSetupBuilding(SetupBuildingEvent $event)
{
global $database;
$documentation_link = make_http(make_link("cron_upload"));
$users = $database->get_pairs("SELECT name, id FROM users UNION ALL SELECT '', null order by name");
$sb = $event->panel->create_new_block("Cron Uploader");
$sb->start_table();
$sb->add_text_option(CronUploaderConfig::DIR, "Root dir", true);
$sb->add_text_option(CronUploaderConfig::KEY, "Key", true);
$sb->add_choice_option(CronUploaderConfig::USER, $users, "User", true);
$sb->add_bool_option(CronUploaderConfig::STOP_ON_ERROR, "Stop On Error", true);
$sb->add_choice_option(CronUploaderConfig::LOG_LEVEL, [
LOGGING_LEVEL_NAMES[SCORE_LOG_DEBUG] => SCORE_LOG_DEBUG,
LOGGING_LEVEL_NAMES[SCORE_LOG_INFO] => SCORE_LOG_INFO,
LOGGING_LEVEL_NAMES[SCORE_LOG_WARNING] => SCORE_LOG_WARNING,
LOGGING_LEVEL_NAMES[SCORE_LOG_ERROR] => SCORE_LOG_ERROR,
LOGGING_LEVEL_NAMES[SCORE_LOG_CRITICAL] => SCORE_LOG_CRITICAL,
], "Output Log Level: ", true);
$sb->add_bool_option(CronUploaderConfig::INCLUDE_ALL_LOGS, "Include All Logs", true);
$sb->end_table();
$sb->add_label("<a href='$documentation_link'>Read the documentation</a> for cron setup instructions.");
}
public function onAdminBuilding(AdminBuildingEvent $event)
{
$failed_dir = $this->get_failed_dir();
@ -116,10 +118,10 @@ class CronUploader extends Extension
public function onLog(LogEvent $event)
{
global $config;
$all = $config->get_bool(CronUploaderConfig::INCLUDE_ALL_LOGS);
global $user_config;
$all = $user_config->get_bool(CronUploaderConfig::INCLUDE_ALL_LOGS);
if (self::$IMPORT_RUNNING &&
$event->priority >= $config->get_int(CronUploaderConfig::LOG_LEVEL) &&
$event->priority >= $user_config->get_int(CronUploaderConfig::LOG_LEVEL) &&
($event->section==self::NAME || $all)
) {
$output = "[" . date('Y-m-d H:i:s') . "] " . ($all ? '['. $event->section .'] ' :'') . "[" . LOGGING_LEVEL_NAMES[$event->priority] . "] " . $event->message ;
@ -191,8 +193,8 @@ class CronUploader extends Extension
private function clear_folder($folder)
{
global $page;
$path = join_path(CronUploaderConfig::get_dir(), $folder);
global $page, $user_config;
$path = join_path($user_config->get_string(CronUploaderConfig::DIR), $folder);
deltree($path);
$page->flash("Cleared $path");
}
@ -200,7 +202,11 @@ class CronUploader extends Extension
private function get_cron_url()
{
return make_http(make_link("/cron_upload/" . CronUploaderConfig::get_key()));
global $user_config;
$user_api_key = $user_config->get_string(UserConfig::API_KEY);
return make_http(make_link("/cron_upload/run", "api_key=".urlencode($user_api_key)));
}
private function get_cron_cmd()
@ -256,26 +262,34 @@ class CronUploader extends Extension
public function get_queue_dir()
{
$dir = CronUploaderConfig::get_dir();
global $user_config;
$dir = $user_config->get_string(CronUploaderConfig::DIR);
return join_path($dir, self::QUEUE_DIR);
}
public function get_uploaded_dir()
{
$dir = CronUploaderConfig::get_dir();
global $user_config;
$dir = $user_config->get_string(CronUploaderConfig::DIR);
return join_path($dir, self::UPLOADED_DIR);
}
public function get_failed_dir()
{
$dir = CronUploaderConfig::get_dir();
global $user_config;
$dir = $user_config->get_string(CronUploaderConfig::DIR);
return join_path($dir, self::FAILED_DIR);
}
private function prep_root_dir(): string
{
global $user_config;
// Determine directory (none = default)
$dir = CronUploaderConfig::get_dir();
$dir = $user_config->get_string(CronUploaderConfig::DIR);
// Make the directory if it doesn't exist yet
if (!is_dir($this->get_queue_dir())) {
@ -293,35 +307,36 @@ class CronUploader extends Extension
private function get_lock_file(): string
{
$root_dir = CronUploaderConfig::get_dir();
global $user_config;
$root_dir = $user_config->get_string(CronUploaderConfig::DIR);
return join_path($root_dir, ".lock");
}
/**
* Uploads the image & handles everything
*/
public function process_upload(string $key, ?int $upload_count = null): bool
public function process_upload(): bool
{
global $database, $config, $_shm_load_start;
global $database, $user, $user_config, $config, $_shm_load_start;
$max_time = intval(ini_get('max_execution_time'))*.8;
$this->set_headers();
if ($key!=CronUploaderConfig::get_key()) {
throw new SCoreException("Cron upload key incorrect");
}
$user_id = CronUploaderConfig::get_user();
if (empty($user_id)) {
throw new SCoreException("Cron upload user not set");
}
$my_user = User::by_id($user_id);
if ($my_user == null) {
throw new SCoreException("No user found for cron upload user $user_id");
if (!$config->get_bool(UserConfig::ENABLE_API_KEYS)) {
throw new SCoreException("User API keys are note enabled. Please enable them for the cron upload functionality to work.");
}
send_event(new UserLoginEvent($my_user));
$this->log_message(SCORE_LOG_INFO, "Logged in as user {$my_user->name}");
if ($user->is_anonymous()) {
throw new SCoreException("User not present. Please specify the api_key for the user to run cron upload as.");
}
$this->log_message(SCORE_LOG_INFO, "Logged in as user {$user->name}");
if (!$user->can(Permissions::CRON_RUN)) {
throw new SCoreException("User does not have permission to run cron upload");
}
$lockfile = fopen($this->get_lock_file(), "w");
if (!flock($lockfile, LOCK_EX | LOCK_NB)) {
@ -333,7 +348,7 @@ class CronUploader extends Extension
//set_time_limit(0);
$output_subdir = date('Ymd-His', time());
$image_queue = $this->generate_image_queue(CronUploaderConfig::get_dir());
$image_queue = $this->generate_image_queue($user_config->get_string(CronUploaderConfig::DIR));
// Randomize Images
//shuffle($this->image_queue);
@ -372,7 +387,7 @@ class CronUploader extends Extension
$failed++;
$this->log_message(SCORE_LOG_ERROR, "(" . gettype($e) . ") " . $e->getMessage());
$this->log_message(SCORE_LOG_ERROR, $e->getTraceAsString());
if ($config->get_bool(CronUploaderConfig::STOP_ON_ERROR)) {
if ($user_config->get_bool(CronUploaderConfig::STOP_ON_ERROR)) {
break;
} else {
$this->move_uploaded($img[0], $img[1], $output_subdir, true);
@ -401,7 +416,9 @@ class CronUploader extends Extension
private function move_uploaded(string $path, string $filename, string $output_subdir, bool $corrupt = false)
{
$rootDir = CronUploaderConfig::get_dir();
global $user_config;
$rootDir = $user_config->get_string(CronUploaderConfig::DIR);
$rootLength = strlen($rootDir);
if ($rootDir[$rootLength-1]=="/"||$rootDir[$rootLength-1]=="\\") {
$rootLength--;
@ -538,7 +555,11 @@ class CronUploader extends Extension
private function get_log_file(): string
{
return join_path(CronUploaderConfig::get_dir(), "uploads.log");
global $user_config;
$dir = $user_config->get_string(CronUploaderConfig::DIR);
return join_path($dir, "uploads.log");
}
private function set_headers(): void

7
ext/cron_uploader/script.js Normal file
View file

@ -0,0 +1,7 @@
function copyInputToClipboard(inputId) {
// Referenced from https://www.w3schools.com/howto/howto_js_copy_clipboard.asp
let source = document.getElementById(inputId);
source.select();
source.setSelectionRange(0, 99999); /*For mobile devices*/
document.execCommand("copy");
}

73
ext/cron_uploader/theme.php
View file

@ -1,5 +1,18 @@
<?php declare(strict_types=1);
use function MicroHTML\LABEL;
use function MicroHTML\TABLE;
use function MicroHTML\TBODY;
use function MicroHTML\TFOOT;
use function MicroHTML\TR;
use function MicroHTML\TH;
use function MicroHTML\TD;
use function MicroHTML\INPUT;
use function MicroHTML\rawHTML;
use function MicroHTML\emptyHTML;
use function MicroHTML\SELECT;
use function MicroHTML\OPTION;
class CronUploaderTheme extends Themelet
{
public function display_documentation(
@ -11,11 +24,19 @@ class CronUploaderTheme extends Themelet
string $cron_url,
?array $log_entries
) {
global $page;
global $page, $config, $user_config;
$info_html = "";
$page->set_title("Cron Uploader");
$page->set_heading("Cron Uploader");
$info_html = "<b>Information</b>
if (!$config->get_bool(UserConfig::ENABLE_API_KEYS)) {
$info_html .= "<b style='color:red'>THIS EXTENSION REQUIRES USER API KEYS TO BE ENABLED IN <a href=''>BOARD ADMIN</a></b>";
} else {
}
$info_html .= "<b>Information</b>
<br>
<table style='width:470px;'>
" . ($running ? "<tr><td colspan='4'><b style='color:red'>Cron upload is currently running</b></td></tr>" : "") . "
@ -41,9 +62,13 @@ class CronUploaderTheme extends Themelet
<td>{$failed_dirinfo['path']}</td>
</tr></table>
<br>Cron Command: <input type='text' size='60' value='$cron_cmd'><br>
Create a cron job with the command above.<br/>
Read the documentation if you're not sure what to do.<br>";
<div>Cron Command: <input type='text' size='60' value='$cron_cmd' id='cron_command'>
<button onclick='copyInputToClipboard(\"cron_command\")'>Copy</button></div>
<div>Create a cron job with the command above.
Read the documentation if you're not sure what to do.</div>
<div>URL: <input type='text' size='60' value='$cron_url' id='cron_url'>
<button onclick='copyInputToClipboard(\"cron_url\")'>Copy</button></div>";
$install_html = "
This cron uploader is fairly easy to use but has to be configured first.
@ -76,12 +101,10 @@ class CronUploaderTheme extends Themelet
<li>If an import is already running, another cannot start until it is done.</li>
<li>Each time it runs it will import for up to ".number_format($max_time)." seconds. This is controlled by the PHP max execution time.</li>
<li>Uploaded images will be moved to the 'uploaded' directory into a subfolder named after the time the import started. It's recommended that you remove everything out of this directory from time to time. If you have admin controls enabled, this can be done from <a href='".make_link("admin")."'>Board Admin</a>.</li>
<li>If you enable the db logging extension, you can view the log output on this screen. Otherwise the log will be written to a file at ".CronUploaderConfig::get_dir().DIRECTORY_SEPARATOR."uploads.log</li>
<li>If you enable the db logging extension, you can view the log output on this screen. Otherwise the log will be written to a file at ".$user_config->get_string(CronUploaderConfig::DIR).DIRECTORY_SEPARATOR."uploads.log</li>
</ul>
";
$page->set_title("Cron Uploader");
$page->set_heading("Cron Uploader");
$block = new Block("Cron Uploader", $info_html, "main", 10);
$block_install = new Block("Setup Guide", $install_html, "main", 30);
@ -101,6 +124,40 @@ class CronUploaderTheme extends Themelet
}
}
public function get_user_options(string $dir, bool $stop_on_error, int $log_level, bool $all_logs): string
{
$form = SHM_SIMPLE_FORM(
"user_admin/cron_uploader",
TABLE(
["class"=>"form"],
TBODY(
TR(
TH("Cron Uploader")
),
TR(
TH("Root dir"),
TD(INPUT(["type"=>'text', "name"=>'name', "required"=>true]))
),
TR(
TH(),
TD(
LABEL(INPUT(["type"=>'checkbox', "name"=>'stop_on_error']), "Stop On Error")
)
),
TR(
TH(rawHTML("Repeat&nbsp;Password")),
TD(INPUT(["type"=>'password', "name"=>'pass2', "required"=>true]))
)
),
TFOOT(
TR(TD(["colspan"=>"2"], INPUT(["type"=>"submit", "value"=>"Save Settings"])))
)
)
);
$html = emptyHTML($form);
return (string)$html;
}
public function display_form(array $failed_dirs)
{
global $page;