badmanners/shimmie2
badmanners/shimmie2
Archived
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Changed omitted related tags (like tagme) to be powered by a setting, allowing any performance-impacting (or just not useful in this context) tags to be omitted

Browse source 
Further tag list performance improvements
TagListConfig constants
This commit is contained in:
Matthew Barbour 2019-07-13 21:18:45 -05:00 committed by Shish
parent 8606c70437
commit 88e6e68d79
5 changed files with 226 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/imageboard/tag.php
View file

@ -110,6 +110,7 @@ class Tag
$term = str_replace('_', '\_', $term);
$term = str_replace('%', '\%', $term);
$term = str_replace('*', '%', $term);
$term = str_replace("?", "_", $term);
return $term;
}
}

6
ext/ouroboros_api/main.php
View file

@ -585,7 +585,7 @@ class OuroborosAPI extends Extension
ORDER BY SCORE_STRNORM(substr(tag, 1, 1)) LIMIT :start, :max_items
"
),
['tags_min' => $config->get_int('tags_min'), 'start' => $start, 'max_items' => $limit]
['tags_min' => $config->get_int(TagListConfig::TAGS_MIN), 'start' => $start, 'max_items' => $limit]
);
break;
case 'count':
@ -596,7 +596,7 @@ class OuroborosAPI extends Extension
WHERE count >= :tags_min
ORDER BY count DESC, tag ASC LIMIT :start, :max_items
",
['tags_min' => $config->get_int('tags_min'), 'start' => $start, 'max_items' => $limit]
['tags_min' => $config->get_int(TagListConfig::TAGS_MIN), 'start' => $start, 'max_items' => $limit]
);
break;
case 'date':
@ -607,7 +607,7 @@ class OuroborosAPI extends Extension
WHERE count >= :tags_min
ORDER BY count DESC, tag ASC LIMIT :start, :max_items
",
['tags_min' => $config->get_int('tags_min'), 'start' => $start, 'max_items' => $limit]
['tags_min' => $config->get_int(TagListConfig::TAGS_MIN), 'start' => $start, 'max_items' => $limit]
);
break;
}

31
ext/tag_list/config.php Normal file
View file

@ -0,0 +1,31 @@
<?php
class TagListConfig
{
public const LENGTH = "tag_list_length";
public const POPULAR_TAG_LIST_LENGTH = "popular_tag_list_length";
public CONSt TAGS_MIN = "tags_min";
public const INFO_LINK = "info_link";
public const IMAGE_TYPE = "tag_list_image_type";
public const RELATED_SORT = "tag_list_related_sort";
public const POPULAR_SORT = "tag_list_popular_sort";
public const PAGES = "tag_list_pages";
public const OMIT_TAGS = "tag_list_omit_tags";
public const TYPE_RELATED = "related";
public const TYPE_TAGS= "tags";
public const TYPE_CHOICES = [
"Image's tags only" => TagListConfig::TYPE_TAGS,
"Show related" => TagListConfig::TYPE_RELATED
];
public const SORT_ALPHABETICAL = "alphabetical";
public const SORT_TAG_COUNT = "tagcount";
public const SORT_CHOICES = [
"Tag Count" => TagListConfig::SORT_TAG_COUNT,
"Alphabetical" => TagListConfig::SORT_ALPHABETICAL
];
}

258
ext/tag_list/main.php
View file

@ -1,18 +1,21 @@
<?php
require_once "config.php";
class TagList extends Extension
{
public function onInitExt(InitExtEvent $event)
{
global $config;
$config->set_default_int("tag_list_length", 15);
$config->set_default_int("popular_tag_list_length", 15);
$config->set_default_int("tags_min", 3);
$config->set_default_string("info_link", 'http://en.wikipedia.org/wiki/$tag');
$config->set_default_string("tag_list_image_type", 'related');
$config->set_default_string("tag_list_related_sort", 'alphabetical');
$config->set_default_string("tag_list_popular_sort", 'tagcount');
$config->set_default_bool("tag_list_pages", false);
$config->set_default_int(TagListConfig::LENGTH, 15);
$config->set_default_int(TagListConfig::POPULAR_TAG_LIST_LENGTH, 15);
$config->set_default_int(TagListConfig::TAGS_MIN, 3);
$config->set_default_string(TagListConfig::INFO_LINK, 'http://en.wikipedia.org/wiki/$tag');
$config->set_default_string(TagListConfig::OMIT_TAGS, 'tagme*');
$config->set_default_string(TagListConfig::IMAGE_TYPE, TagListConfig::TYPE_RELATED);
$config->set_default_string(TagListConfig::RELATED_SORT, TagListConfig::SORT_ALPHABETICAL);
$config->set_default_string(TagListConfig::POPULAR_SORT, TagListConfig::SORT_TAG_COUNT);
$config->set_default_bool(TagListConfig::PAGES, false);
}
public function onPageRequest(PageRequestEvent $event)
@ -78,7 +81,7 @@ class TagList extends Extension
public function onPostListBuilding(PostListBuildingEvent $event)
{
global $config, $page;
if ($config->get_int('tag_list_length') > 0) {
if ($config->get_int(TagListConfig::LENGTH) > 0) {
if (!empty($event->search_terms)) {
$this->add_refine_block($page, $event->search_terms);
} else {
@ -105,8 +108,8 @@ class TagList extends Extension
public function onDisplayingImage(DisplayingImageEvent $event)
{
global $config, $page;
if ($config->get_int('tag_list_length') > 0) {
if ($config->get_string('tag_list_image_type') == 'related') {
if ($config->get_int(TagListConfig::LENGTH) > 0) {
if ($config->get_string(TagListConfig::IMAGE_TYPE) == TagListConfig::TYPE_RELATED) {
$this->add_related_block($page, $event->image);
} else {
if (class_exists("TagCategories") and $config->get_bool(TagCategoriesConfig::SPLIT_ON_VIEW)) {
@ -121,30 +124,30 @@ class TagList extends Extension
public function onSetupBuilding(SetupBuildingEvent $event)
{
$sb = new SetupBlock("Tag Map Options");
$sb->add_int_option("tags_min", "Only show tags used at least ");
$sb->add_int_option(TagListConfig::TAGS_MIN, "Only show tags used at least ");
$sb->add_label(" times");
$sb->add_bool_option("tag_list_pages", "<br>Paged tag lists: ");
$sb->add_bool_option(TagListConfig::PAGES, "<br>Paged tag lists: ");
$event->panel->add_block($sb);
$sb = new SetupBlock("Popular / Related Tag List");
$sb->add_int_option("tag_list_length", "Show top ");
$sb->add_int_option(TagListConfig::LENGTH, "Show top ");
$sb->add_label(" related tags");
$sb->add_int_option("popular_tag_list_length", "<br>Show top ");
$sb->add_int_option(TagListConfig::POPULAR_TAG_LIST_LENGTH, "<br>Show top ");
$sb->add_label(" popular tags");
$sb->add_text_option("info_link", "<br>Tag info link: ");
$sb->add_choice_option("tag_list_image_type", [
"Image's tags only" => "tags",
"Show related" => "related"
], "<br>Image tag list: ");
$sb->add_choice_option("tag_list_related_sort", [
"Tag Count" => "tagcount",
"Alphabetical" => "alphabetical"
], "<br>Sort related list by: ");
$sb->add_choice_option("tag_list_popular_sort", [
"Tag Count" => "tagcount",
"Alphabetical" => "alphabetical"
], "<br>Sort popular list by: ");
$sb->add_bool_option("tag_list_numbers", "<br>Show tag counts: ");
$sb->start_table();
$sb->add_text_option(TagListConfig::INFO_LINK, "Tag info link", true);
$sb->add_text_option(TagListConfig::OMIT_TAGS, "Omit tags", true);
$sb->add_choice_option(TagListConfig::IMAGE_TYPE,
TagListConfig::TYPE_CHOICES,
"Image tag list", true);
$sb->add_choice_option(TagListConfig::RELATED_SORT,
TagListConfig::SORT_CHOICES,
"Sort related list by", true);
$sb->add_choice_option(TagListConfig::POPULAR_SORT,
TagListConfig::SORT_CHOICES,
"Sort popular list by", true);
$sb->add_bool_option("tag_list_numbers", "Show tag counts", true);
$sb->end_table();
$event->panel->add_block($sb);
}
// }}}
@ -165,17 +168,56 @@ class TagList extends Extension
return int_escape($_GET['mincount']);
} else {
global $config;
return $config->get_int('tags_min'); // get the default.
return $config->get_int(TagListConfig::TAGS_MIN); // get the default.
}
}
private static function get_omitted_tags(): array
{
global $config, $database;
$tags_config = $config->get_string(TagListConfig::OMIT_TAGS);
$results = $database->cache->get("tag_list_omitted_tags:".$tags_config);
if($results==null) {
$results = [];
$tags = explode(" ", $tags_config);
if (empty($tags)) {
return [];
}
$where = [];
$args = [];
$i = 0;
foreach ($tags as $tag) {
$i++;
$arg = "tag$i";
$args[$arg] = Tag::sqlify($tag);
if (strpos($tag, '*') === false
&& strpos($tag, '?') === false) {
$where[] = " tag = :$arg ";
} else {
$where[] = " tag LIKE :$arg ";
}
}
$results = $database->get_col("SELECT id FROM tags WHERE " . implode(" OR ", $where), $args);
$database->cache->set("tag_list_omitted_tags:" . $tags_config, $results, 600);
}
return $results;
}
private function get_starts_with(): string
{
global $config;
if (isset($_GET['starts_with'])) {
return $_GET['starts_with'] . "%";
} else {
if ($config->get_bool("tag_list_pages")) {
if ($config->get_bool(TagListConfig::PAGES)) {
return "a%";
} else {
return "%";
@ -244,7 +286,7 @@ class TagList extends Extension
"), ["tags_min"=>$tags_min, "tags_min2"=>$tags_min, "starts_with"=>$starts_with]);
$html = "";
if ($config->get_bool("tag_list_pages")) {
if ($config->get_bool(TagListConfig::PAGES)) {
$html .= $this->build_az();
}
foreach ($tag_data as $row) {
@ -287,7 +329,7 @@ class TagList extends Extension
"), ["tags_min"=>$tags_min, "starts_with"=>$starts_with]);
$html = "";
if ($config->get_bool("tag_list_pages")) {
if ($config->get_bool(TagListConfig::PAGES)) {
$html .= $this->build_az();
}
@ -407,27 +449,34 @@ class TagList extends Extension
}
// }}}
// blocks {{{
private function add_related_block(Page $page, Image $image)
private function add_related_block(Page $page, Image $image): void
{
global $database, $config;
$query = "
SELECT t3.tag AS tag, t3.count AS calc_count, it3.tag_id
FROM image_tags AS it1 -- Starting image's tags
INNER JOIN tags AS t1 ON t1.id = it1.tag_id AND t1.tag NOT LIKE 'tagme%'
-- Get images with the same tags as the starting image
INNER JOIN image_tags AS it2 ON it1.tag_id=it2.tag_id
-- Get the tags from those other images except the same as the starting tags
INNER JOIN image_tags AS it3 ON it2.image_id=it3.image_id
LEFT JOIN image_tags it4 ON it4.image_id = it1.image_id AND it4.tag_id = it3.tag_id
INNER JOIN tags AS t3 ON t3.id = it3.tag_id AND t3.tag NOT LIKE 'tagme%'
WHERE
it1.image_id=:image_id
GROUP BY it3.tag_id, t3.tag, t3.count
ORDER BY calc_count DESC
$omitted_tags = self::get_omitted_tags();
$starting_tags = $database->get_col("SELECT tag_id FROM image_tags WHERE image_id = :image_id",["image_id" => $image->id]);
$starting_tags = array_diff($starting_tags,$omitted_tags);
if(count($starting_tags) === 0) {
// No valid starting tags, so can't look anything up
return;
}
$query = "SELECT tags.* FROM tags INNER JOIN (
SELECT it2.tag_id
FROM image_tags AS it1
INNER JOIN image_tags AS it2 ON it1.image_id=it2.image_id
AND it2.tag_id NOT IN (".implode(",",array_merge($omitted_tags,$starting_tags)).")
WHERE
it1.tag_id IN (".implode(",",$starting_tags).")
GROUP BY it2.tag_id
) A ON A.tag_id = tags.id
ORDER BY count DESC
LIMIT :tag_list_length
";
$args = ["image_id" => $image->id, "tag_list_length" => $config->get_int('tag_list_length')];
$args = ["tag_list_length" => $config->get_int(TagListConfig::LENGTH)];
$tags = $database->get_all($query, $args);
if (count($tags) > 0) {
@ -440,11 +489,11 @@ class TagList extends Extension
global $database;
$query = "
SELECT tags.tag, tags.count as calc_count
SELECT tags.tag, tags.count
FROM tags, image_tags
WHERE tags.id = image_tags.tag_id
AND image_tags.image_id = :image_id
ORDER BY calc_count DESC
ORDER BY tags.count DESC
";
$args = ["image_id"=>$image->id];
@ -459,11 +508,11 @@ class TagList extends Extension
global $database;
$query = "
SELECT tags.tag, tags.count as calc_count
SELECT tags.tag, tags.count
FROM tags, image_tags
WHERE tags.id = image_tags.tag_id
AND image_tags.image_id = :image_id
ORDER BY calc_count DESC
ORDER BY tags.count DESC
";
$args = ["image_id"=>$image->id];
@ -479,16 +528,32 @@ class TagList extends Extension
$tags = $database->cache->get("popular_tags");
if (empty($tags)) {
$query = "
SELECT tag, count as calc_count
FROM tags
WHERE count > 0
ORDER BY count DESC
LIMIT :popular_tag_list_length
";
$args = ["popular_tag_list_length"=>$config->get_int('popular_tag_list_length')];
$omitted_tags = self::get_omitted_tags();
if(empty($omitted_tags)) {
$query = "
SELECT tag, count
FROM tags
WHERE count > 0
ORDER BY count DESC
LIMIT :popular_tag_list_length
";
} else {
$query = "
SELECT tag, count
FROM tags
WHERE count > 0
AND id NOT IN (".(implode(",", $omitted_tags)).")
ORDER BY count DESC
LIMIT :popular_tag_list_length
";
}
$args = ["popular_tag_list_length"=>$config->get_int(TagListConfig::POPULAR_TAG_LIST_LENGTH)];
$tags = $database->get_all($query, $args);
$database->cache->set("popular_tags", $tags, 600);
}
if (count($tags) > 0) {
@ -507,6 +572,20 @@ class TagList extends Extension
return;
}
$wild_tags = $search;
$related_tags = self::get_related_tags($search, $config->get_int(TagListConfig::LENGTH));
if (!empty($related_tags)) {
$this->theme->display_refine_block($page, $related_tags, $wild_tags);
}
}
public static function get_related_tags(array $search, int $limit): array
{
global $config, $database;
$wild_tags = $search;
$str_search = Tag::implode($search);
$related_tags = $database->cache->get("related_tags:$str_search");
@ -514,53 +593,66 @@ class TagList extends Extension
if (empty($related_tags)) {
// $search_tags = array();
$tag_id_array = [];
$starting_tags = [];
$tags_ok = true;
foreach ($wild_tags as $tag) {
if ($tag[0] == "-" || strpos($tag, "tagme")===0) {
if ($tag[0] == "-") {
continue;
}
$tag = str_replace("*", "%", $tag);
$tag = str_replace("?", "_", $tag);
$tag = Tag::sqlify($tag);
$tag_ids = $database->get_col("SELECT id FROM tags WHERE tag LIKE :tag AND count < 25000", ["tag" => $tag]);
// $search_tags = array_merge($search_tags,
// $database->get_col("SELECT tag FROM tags WHERE tag LIKE :tag", array("tag"=>$tag)));
$tag_id_array = array_merge($tag_id_array, $tag_ids);
$starting_tags = array_merge($starting_tags, $tag_ids);
$tags_ok = count($tag_ids) > 0;
if (!$tags_ok) {
break;
}
}
$tag_id_list = join(', ', $tag_id_array);
if (count($tag_id_array) > 5 || count($tag_id_array) == 0) {
return;
if (count($starting_tags) > 5 || count($starting_tags) === 0) {
return [];
}
$omitted_tags = self::get_omitted_tags();
$starting_tags = array_diff($starting_tags,$omitted_tags);
if(count($starting_tags) === 0) {
// No valid starting tags, so can't look anything up
return [];
}
if ($tags_ok) {
$query = "
SELECT t2.tag AS tag, COUNT(it2.image_id) AS calc_count
$query = "SELECT t.tag, A.calc_count AS count FROM tags t INNER JOIN (
SELECT it2.tag_id, COUNT(it2.image_id) AS calc_count
FROM image_tags AS it1 -- Got other images with the same tags
-- Get the tags from those images, except those the same as the starting tags
INNER JOIN image_tags AS it2 ON it1.image_id=it2.image_id AND it2.tag_id NOT IN($tag_id_list)
-- And filter out anything starting with tagme
INNER JOIN tags AS t2 ON it2.tag_id = t2.id AND t2.tag NOT LIKE 'tagme%'
INNER JOIN image_tags AS it2 ON it1.image_id=it2.image_id
-- And filter out unwanted tags
AND it2.tag_id NOT IN (".implode(",",array_merge($omitted_tags,$starting_tags)).")
WHERE
it1.tag_id IN($tag_id_list)
GROUP BY t2.tag
ORDER BY calc_count
it1.tag_id IN (".implode(",",$starting_tags).")
GROUP BY it2.tag_id) A ON A.tag_id = t.id
ORDER BY A.calc_count
DESC LIMIT :limit
";
$args = ["limit"=>$config->get_int('tag_list_length')];
$args = ["limit" => $limit];
$related_tags = $database->get_all($query, $args);
$database->cache->set("related_tags:$str_search", $related_tags, 60*60);
$database->cache->set("related_tags:$str_search", $related_tags, 60 * 60);
}
}
if ($related_tags === false) {
return [];
} else {
return $related_tags;
}
if (!empty($related_tags)) {
$this->theme->display_refine_block($page, $related_tags, $wild_tags);
}
}
// }}}
}

32
ext/tag_list/theme.php
View file

@ -38,7 +38,7 @@ class TagListTheme extends Themelet
{
global $config;
$tag_info_link_is_visible = !is_null($config->get_string('info_link'));
$tag_info_link_is_visible = !is_null($config->get_string(TagListConfig::INFO_LINK));
$tag_count_is_visible = $config->get_bool("tag_list_numbers");
return '
@ -68,7 +68,7 @@ class TagListTheme extends Themelet
{
global $config;
if ($config->get_string('tag_list_related_sort') == 'alphabetical') {
if ($config->get_string(TagListConfig::RELATED_SORT) == TagListConfig::SORT_ALPHABETICAL) {
asort($tag_infos);
}
@ -117,12 +117,12 @@ class TagListTheme extends Themelet
$page->add_block(new Block($category_display_name, $tag_categories_html[$category], "left", 9));
}
if ($config->get_string('tag_list_image_type')=="tags") {
if ($main_html != null) {
$page->add_block(new Block("Tags", $main_html, "left", 10));
}
} else {
$page->add_block(new Block("Related Tags", $main_html, "left", 10));
if ($main_html != null) {
if ($config->get_string(TagListConfig::IMAGE_TYPE)==TagListConfig::TYPE_TAGS) {
$page->add_block(new Block("Tags", $main_html, "left", 10));
} else {
$page->add_block(new Block("Related Tags", $main_html, "left", 10));
}
}
}
@ -134,7 +134,7 @@ class TagListTheme extends Themelet
*/
private function get_tag_list_html($tag_infos, $sort)
{
if ($sort == 'alphabetical') {
if ($sort == TagListConfig::SORT_ALPHABETICAL) {
asort($tag_infos);
}
@ -170,10 +170,10 @@ class TagListTheme extends Themelet
$main_html = $this->get_tag_list_html(
$tag_infos,
$config->get_string('tag_list_related_sort')
$config->get_string(TagListConfig::RELATED_SORT)
);
if ($config->get_string('tag_list_image_type')=="tags") {
if ($config->get_string(TagListConfig::IMAGE_TYPE)==TagListConfig::TYPE_TAGS) {
$page->add_block(new Block("Tags", $main_html, "left", 10));
} else {
$page->add_block(new Block("Related Tags", $main_html, "left", 10));
@ -193,7 +193,7 @@ class TagListTheme extends Themelet
$main_html = $this->get_tag_list_html(
$tag_infos,
$config->get_string('tag_list_popular_sort')
$config->get_string(TagListConfig::POPULAR_SORT)
);
$main_html .= "&nbsp;<br><a class='more' href='".make_link("tags")."'>Full List</a>\n";
@ -213,7 +213,7 @@ class TagListTheme extends Themelet
$main_html = $this->get_tag_list_html(
$tag_infos,
$config->get_string('tag_list_popular_sort')
$config->get_string(TagListConfig::POPULAR_SORT)
);
$main_html .= "&nbsp;<br><a class='more' href='".make_link("tags")."'>Full List</a>\n";
@ -242,10 +242,10 @@ class TagListTheme extends Themelet
}
$h_tag_no_underscores = str_replace("_", " ", $h_tag);
$count = $row['calc_count'];
$count = $row['count'];
// if($n++) $display_html .= "\n<br/>";
if (!is_null($config->get_string('info_link'))) {
$link = html_escape(str_replace('$tag', url_escape($tag), $config->get_string('info_link')));
if (!is_null($config->get_string(TagListConfig::INFO_LINK))) {
$link = html_escape(str_replace('$tag', url_escape($tag), $config->get_string(TagListConfig::INFO_LINK)));
$display_html .= '<td class="tag_info_link_cell"> <a class="tag_info_link'.$tag_category_css.'" '.$tag_category_style.'href="'.$link.'">?</a></td>';
}
$link = $this->tag_link($row['tag']);