From d64603674e654dab7f746a3747a766fd940acf83 Mon Sep 17 00:00:00 2001 From: matthew Date: Fri, 5 Jul 2019 10:24:46 -0500 Subject: [PATCH] Added ability to use generators with database queries. Adapted bulk actions to use generators. --- core/database.php | 26 ++++++++- core/imageboard/image.php | 112 +++++++++++++++----------------------- core/imageboard/misc.php | 5 ++ ext/bulk_actions/main.php | 92 +++++++++++++------------------ ext/rating/main.php | 7 +-- ext/regen_thumb/main.php | 7 +-- ext/transcode/main.php | 8 +-- ext/trash/main.php | 4 +- 8 files changed, 119 insertions(+), 142 deletions(-) diff --git a/core/database.php b/core/database.php index 23af8bcb..31735a9d 100644 --- a/core/database.php +++ b/core/database.php @@ -249,6 +249,17 @@ class Database return $data; } + /** + * Execute an SQL query and return a iterable object for use with generators. + */ + public function get_all_iterable(string $query, array $args=[]): PDOStatement + { + $_start = microtime(true); + $data = $this->execute($query, $args); + $this->count_time("get_all_iterable", $_start); + return $data; + } + /** * Execute an SQL query and return a single row. */ @@ -276,7 +287,20 @@ class Database } /** - * Execute an SQL query and return the the first row => the second row. + * Execute an SQL query and return the first column of each row as a single iterable object. + */ + public function get_col_iterable(string $query, array $args=[]): Generator + { + $_start = microtime(true); + $stmt = $this->execute($query, $args); + $this->count_time("get_col_iterable", $_start); + foreach ($stmt as $row) { + yield $row[0]; + } + } + + /** + * Execute an SQL query and return the the first column => the second column. */ public function get_pairs(string $query, array $args=[]): array { diff --git a/core/imageboard/image.php b/core/imageboard/image.php index d90e7ed3..a532201f 100644 --- a/core/imageboard/image.php +++ b/core/imageboard/image.php @@ -10,7 +10,6 @@ */ class Image { - public const DATA_DIR = "data"; public const IMAGE_DIR = "images"; public const THUMBNAIL_DIR = "thumbs"; @@ -104,6 +103,43 @@ class Image } } + + private static function find_images_internal(int $start = 0, ?int $limit = null, array $tags=[]): iterable + { + global $database, $user, $config; + + if ($start < 0) { + $start = 0; + } + if ($limit!=null && $limit < 1) { + $limit = 1; + } + + if (SPEED_HAX) { + if (!$user->can("big_search") and count($tags) > 3) { + throw new SCoreException("Anonymous users may only search for up to 3 tags at a time"); + } + } + + list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags); + + $result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit); + if (!$result) { + $querylet = Image::build_search_querylet($tag_conditions, $img_conditions); + $querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order")))); + if($limit!=null) { + $querylet->append(new Querylet(" LIMIT :limit ", ["limit" => $limit])); + } + $querylet->append(new Querylet(" OFFSET :offset ", ["offset"=>$start])); + #var_dump($querylet->sql); var_dump($querylet->variables); + $result = $database->get_all_iterable($querylet->sql, $querylet->variables); + } + + Image::$order_sql = null; + + return $result; + } + /** * Search for an array of images * @@ -112,82 +148,24 @@ class Image */ public static function find_images(int $start, int $limit, array $tags=[]): array { - global $database, $user, $config; + $result = self::find_images_internal($start, $limit, $tags); $images = []; - - if ($start < 0) { - $start = 0; - } - if ($limit < 1) { - $limit = 1; - } - - if (SPEED_HAX) { - if (!$user->can("big_search") and count($tags) > 3) { - throw new SCoreException("Anonymous users may only search for up to 3 tags at a time"); - } - } - - list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags); - - $result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit); - if (!$result) { - $querylet = Image::build_search_querylet($tag_conditions, $img_conditions); - $querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order")))); - $querylet->append(new Querylet(" LIMIT :limit OFFSET :offset", ["limit"=>$limit, "offset"=>$start])); - #var_dump($querylet->sql); var_dump($querylet->variables); - $result = $database->execute($querylet->sql, $querylet->variables); - } - - while ($row = $result->fetch()) { + foreach ($result as $row) { $images[] = new Image($row); } - Image::$order_sql = null; return $images; } /** - * Search for an array of image IDs - * - * #param string[] $tags - * #return int[] + * Search for an array of images, returning a iterable object of Image */ - public static function find_image_ids(int $start, int $limit, array $tags=[]): array + public static function find_images_iterable(int $start = 0, ?int $limit = null, array $tags=[]): Generator { - global $database, $user, $config; - - $images = []; - - if ($start < 0) { - $start = 0; + $result = self::find_images_internal($start, $limit, $tags); + foreach ($result as $row) { + yield new Image($row); } - if ($limit < 1) { - $limit = 1; - } - - if (SPEED_HAX) { - if (!$user->can("big_search") and count($tags) > 3) { - throw new SCoreException("Anonymous users may only search for up to 3 tags at a time"); - } - } - - list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags); - - $result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit); - if (!$result) { - $querylet = Image::build_search_querylet($tag_conditions, $img_conditions); - $querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order")))); - $querylet->append(new Querylet(" LIMIT :limit OFFSET :offset", ["limit"=>$limit, "offset"=>$start])); - #var_dump($querylet->sql); var_dump($querylet->variables); - $result = $database->execute($querylet->sql, $querylet->variables); - } - - while ($row = $result->fetch()) { - $images[] = $row["id"]; - } - Image::$order_sql = null; - return $images; } /* @@ -220,7 +198,7 @@ class Image return null; } - public static function get_accelerated_result(array $tag_conditions, array $img_conditions, int $offset, int $limit): ?PDOStatement + public static function get_accelerated_result(array $tag_conditions, array $img_conditions, int $offset, ?int $limit): ?PDOStatement { if (!SEARCH_ACCEL || !empty($img_conditions)) { return null; diff --git a/core/imageboard/misc.php b/core/imageboard/misc.php index 865a1c65..1614ca70 100644 --- a/core/imageboard/misc.php +++ b/core/imageboard/misc.php @@ -501,3 +501,8 @@ function is_animated_gif(String $image_filename) } return ($is_anim_gif == 0); } + +function image_to_id(Image $image): int +{ + return $image->id; +} diff --git a/ext/bulk_actions/main.php b/ext/bulk_actions/main.php index f9d1aaab..93d80480 100644 --- a/ext/bulk_actions/main.php +++ b/ext/bulk_actions/main.php @@ -51,7 +51,7 @@ class BulkActionEvent extends Event /** @var PageRequestEvent */ public $page_request; - public function __construct(String $action, PageRequestEvent $pageRequestEvent, array $items) + public function __construct(String $action, PageRequestEvent $pageRequestEvent, Generator $items) { $this->action = $action; $this->page_request = $pageRequestEvent; @@ -154,36 +154,20 @@ class BulkActions extends Extension $action = $_POST['bulk_action']; - $items = []; + $items = null; if (isset($_POST['bulk_selected_ids']) && $_POST['bulk_selected_ids'] != "") { $data = json_decode($_POST['bulk_selected_ids']); - if (is_array($data)) { - foreach ($data as $id) { - if (is_numeric($id)) { - array_push($items, int_escape($id)); - } - } + if (is_array($data)&&!empty($data)) { + $items = $this->yield_items($data); } } elseif (isset($_POST['bulk_query']) && $_POST['bulk_query'] != "") { $query = $_POST['bulk_query']; if ($query != null && $query != "") { - $n = 0; - $tags = Tag::explode($query); - while (true) { - $results = Image::find_image_ids($n, 100, $tags); - if (count($results) == 0) { - break; - } - - reset($results); // rewind to first element in array. - $items = array_merge($items, $results); - $n += count($results); - } + $items = $this->yield_search_results($query); } } - if (sizeof($items) > 0) { - reset($items); // rewind to first element in array. + if (is_iterable($items)) { $newEvent = new BulkActionEvent($action, $event, $items); send_event($newEvent); } @@ -197,21 +181,34 @@ class BulkActions extends Extension } } + private function yield_items(array $data): Generator + { + foreach ($data as $id) { + if (is_numeric($id)) { + $image = Image::by_id($id); + if($image!=null) { + yield $image; + } + } + } + } + + private function yield_search_results(string $query): Generator + { + $tags = Tag::explode($query); + return Image::find_images_iterable(0, null, $tags); + } + private function sort_blocks($a, $b) { return $a["position"] - $b["position"]; } - private function delete_items(array $items): int + private function delete_items(iterable $items): int { $total = 0; - foreach ($items as $id) { + foreach ($items as $image) { try { - $image = Image::by_id($id); - if ($image==null) { - continue; - } - if (class_exists("ImageBan") && isset($_POST['bulk_ban_reason'])) { $reason = $_POST['bulk_ban_reason']; if ($reason) { @@ -221,13 +218,13 @@ class BulkActions extends Extension send_event(new ImageDeletionEvent($image)); $total++; } catch (Exception $e) { - flash_message("Error while removing $id: " . $e->getMessage(), "error"); + flash_message("Error while removing {$image->id}: " . $e->getMessage(), "error"); } } return $total; } - private function tag_items(array $items, string $tags, bool $replace): int + private function tag_items(iterable $items, string $tags, bool $replace): int { $tags = Tag::explode($tags); @@ -243,28 +240,21 @@ class BulkActions extends Extension $total = 0; if ($replace) { - foreach ($items as $id) { - $image = Image::by_id($id); - if ($image==null) { - continue; - } - + foreach ($items as $image) { send_event(new TagSetEvent($image, $tags)); $total++; } } else { - foreach ($items as $id) { - $image = Image::by_id($id); - if ($image==null) { - continue; - } + foreach ($items as $image) { + $img_tags = array_map("strtolower",$image->get_tag_array()); - $img_tags = []; if (!empty($neg_tag_array)) { - $img_tags = array_merge($pos_tag_array, $image->get_tag_array()); + $neg_tag_array = array_map("strtolower",$neg_tag_array); + + $img_tags = array_merge($pos_tag_array, $img_tags); $img_tags = array_diff($img_tags, $neg_tag_array); } else { - $img_tags = array_merge($tags, $image->get_tag_array()); + $img_tags = array_merge($tags, $img_tags); } send_event(new TagSetEvent($image, $img_tags)); $total++; @@ -274,23 +264,17 @@ class BulkActions extends Extension return $total; } - private function set_source(array $items, String $source): int + private function set_source(iterable $items, String $source): int { $total = 0; - foreach ($items as $id) { + foreach ($items as $image) { try { - $image = Image::by_id($id); - if ($image==null) { - continue; - } - send_event(new SourceSetEvent($image, $source)); $total++; } catch (Exception $e) { - flash_message("Error while setting source for $id: " . $e->getMessage(), "error"); + flash_message("Error while setting source for {$image->id}: " . $e->getMessage(), "error"); } } - return $total; } } diff --git a/ext/rating/main.php b/ext/rating/main.php index 30988977..7b43d914 100644 --- a/ext/rating/main.php +++ b/ext/rating/main.php @@ -186,12 +186,7 @@ class Ratings extends Extension if ($user->can("bulk_edit_image_rating")) { $rating = $_POST['bulk_rating']; $total = 0; - foreach ($event->items as $id) { - $image = Image::by_id($id); - if ($image==null) { - continue; - } - + foreach ($event->items as $image) { send_event(new RatingSetEvent($image, $rating)); $total++; } diff --git a/ext/regen_thumb/main.php b/ext/regen_thumb/main.php index 70e00d1b..637febbf 100644 --- a/ext/regen_thumb/main.php +++ b/ext/regen_thumb/main.php @@ -87,12 +87,7 @@ class RegenThumb extends Extension } $total = 0; - foreach ($event->items as $id) { - $image = Image::by_id($id); - if ($image==null) { - continue; - } - + foreach ($event->items as $image) { if ($this->regenerate_thumbnail($image, $force)) { $total++; } diff --git a/ext/transcode/main.php b/ext/transcode/main.php index bb1e3ada..9c8d3185 100644 --- a/ext/transcode/main.php +++ b/ext/transcode/main.php @@ -246,14 +246,10 @@ class TranscodeImage extends Extension if ($user->is_admin()) { $format = $_POST['transcode_format']; $total = 0; - foreach ($event->items as $id) { + foreach ($event->items as $image) { try { $database->beginTransaction(); - $image = Image::by_id($id); - if ($image==null) { - continue; - } - + $this->transcode_and_replace_image($image, $format); // If a subsequent transcode fails, the database need to have everything about the previous transcodes recorded already, // otherwise the image entries will be stuck pointing to missing image files diff --git a/ext/trash/main.php b/ext/trash/main.php index 92adbf7b..89be2aee 100644 --- a/ext/trash/main.php +++ b/ext/trash/main.php @@ -136,8 +136,8 @@ class Trash extends Extension case "bulk_trash_restore": if ($user->can("view_trash")) { $total = 0; - foreach ($event->items as $id) { - self::set_trash($id, false); + foreach ($event->items as $image) { + self::set_trash($image->id, false); $total++; } flash_message("Restored $total items from trash");