badmanners/shimmie2
badmanners/shimmie2
Archived
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/develop' into path_to_tags_enhancements

Browse source
This commit is contained in:
matthew 2019-06-19 20:21:47 -05:00
commit 27574cad76
5 changed files with 125 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

178
core/imageboard/image.php
View file

@ -125,13 +125,11 @@ class Image
}
}
$result = null;
if (SEARCH_ACCEL) {
$result = Image::get_accelerated_result($tags, $start, $limit);
}
list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
$result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit);
if (!$result) {
$querylet = Image::build_search_querylet($tags);
$querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
$querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order"))));
$querylet->append(new Querylet(" LIMIT :limit OFFSET :offset", ["limit"=>$limit, "offset"=>$start]));
#var_dump($querylet->sql); var_dump($querylet->variables);
@ -170,13 +168,11 @@ class Image
}
}
$result = null;
if (SEARCH_ACCEL) {
$result = Image::get_accelerated_result($tags, $start, $limit);
}
list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
$result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit);
if (!$result) {
$querylet = Image::build_search_querylet($tags);
$querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
$querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order"))));
$querylet->append(new Querylet(" LIMIT :limit OFFSET :offset", ["limit"=>$limit, "offset"=>$start]));
#var_dump($querylet->sql); var_dump($querylet->variables);
@ -193,7 +189,7 @@ class Image
/*
* Accelerator stuff
*/
public static function get_acceleratable(array $tags): ?array
public static function get_acceleratable(array $tag_conditions): ?array
{
$ret = [
"yays" => [],
@ -201,16 +197,17 @@ class Image
];
$yays = 0;
$nays = 0;
foreach ($tags as $tag) {
if (!preg_match("/^-?[a-zA-Z0-9_'-]+$/", $tag)) {
foreach ($tag_conditions as $tq) {
if (strpos($tq->tag, "*") !== false) {
// can't deal with wildcards
return null;
}
if ($tag[0] == "-") {
$nays++;
$ret["nays"][] = substr($tag, 1);
} else {
if ($tq->positive) {
$yays++;
$ret["yays"][] = $tag;
$ret["yays"][] = $tq->tag;
} else {
$nays++;
$ret["nays"][] = $tq->tag;
}
}
if ($yays > 1 || $nays > 0) {
@ -219,11 +216,15 @@ class Image
return null;
}
public static function get_accelerated_result(array $tags, int $offset, int $limit): ?PDOStatement
public static function get_accelerated_result(array $tag_conditions, array $img_conditions, int $offset, int $limit): ?PDOStatement
{
if (!SEARCH_ACCEL || !empty($img_conditions)) {
return null;
}
global $database;
$req = Image::get_acceleratable($tags);
$req = Image::get_acceleratable($tag_conditions);
if (!$req) {
return null;
}
@ -240,9 +241,13 @@ class Image
return $result;
}
public static function get_accelerated_count(array $tags): ?int
public static function get_accelerated_count(array $tag_conditions, array $img_conditions): ?int
{
$req = Image::get_acceleratable($tags);
if (!SEARCH_ACCEL || !empty($img_conditions)) {
return null;
}
$req = Image::get_acceleratable($tag_conditions);
if (!$req) {
return null;
}
@ -295,9 +300,10 @@ class Image
["tag"=>$tags[0]]
);
} else {
$total = Image::get_accelerated_count($tags);
list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
$total = Image::get_accelerated_count($tag_conditions, $img_conditions);
if (is_null($total)) {
$querylet = Image::build_search_querylet($tags);
$querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
$total = $database->get_one("SELECT COUNT(*) AS cnt FROM ($querylet->sql) AS tbl", $querylet->variables);
}
}
@ -318,6 +324,49 @@ class Image
return ceil(Image::count_images($tags) / $config->get_int('index_images'));
}
private static function terms_to_conditions(array $terms): array
{
$tag_conditions = [];
$img_conditions = [];
/*
* Turn a bunch of strings into a bunch of TagCondition
* and ImgCondition objects
*/
$stpe = new SearchTermParseEvent(null, $terms);
send_event($stpe);
if ($stpe->is_querylet_set()) {
foreach ($stpe->get_querylets() as $querylet) {
$img_conditions[] = new ImgCondition($querylet, true);
}
}
foreach ($terms as $term) {
$positive = true;
if (is_string($term) && !empty($term) && ($term[0] == '-')) {
$positive = false;
$term = substr($term, 1);
}
if (strlen($term) === 0) {
continue;
}
$stpe = new SearchTermParseEvent($term, $terms);
send_event($stpe);
if ($stpe->is_querylet_set()) {
foreach ($stpe->get_querylets() as $querylet) {
$img_conditions[] = new ImgCondition($querylet, $positive);
}
} else {
// if the whole match is wild, skip this
if (str_replace("*", "", $term) != "") {
$tag_conditions[] = new TagCondition($term, $positive);
}
}
}
return [$tag_conditions, $img_conditions];
}
/*
* Accessors & mutators
*/
@ -352,7 +401,8 @@ class Image
');
} else {
$tags[] = 'id'. $gtlt . $this->id;
$querylet = Image::build_search_querylet($tags);
list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
$querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
$querylet->append_sql(' ORDER BY images.id '.$dir.' LIMIT 1');
$row = $database->get_row($querylet->sql, $querylet->variables);
}
@ -813,59 +863,19 @@ class Image
/**
* #param string[] $terms
*/
private static function build_search_querylet(array $terms): Querylet
private static function build_search_querylet(array $tag_conditions, array $img_conditions): Querylet
{
global $database;
$tag_querylets = [];
$img_querylets = [];
$positive_tag_count = 0;
$negative_tag_count = 0;
/*
* Turn a bunch of strings into a bunch of TagQuerylet
* and ImgQuerylet objects
*/
$stpe = new SearchTermParseEvent(null, $terms);
send_event($stpe);
if ($stpe->is_querylet_set()) {
foreach ($stpe->get_querylets() as $querylet) {
$img_querylets[] = new ImgQuerylet($querylet, true);
}
}
foreach ($terms as $term) {
$positive = true;
if (is_string($term) && !empty($term) && ($term[0] == '-')) {
$positive = false;
$term = substr($term, 1);
}
if (strlen($term) === 0) {
continue;
}
$stpe = new SearchTermParseEvent($term, $terms);
send_event($stpe);
if ($stpe->is_querylet_set()) {
foreach ($stpe->get_querylets() as $querylet) {
$img_querylets[] = new ImgQuerylet($querylet, $positive);
}
} else {
// if the whole match is wild, skip this;
// if not, translate into SQL
if (str_replace("*", "", $term) != "") {
$term = str_replace('_', '\_', $term);
$term = str_replace('%', '\%', $term);
$term = str_replace('*', '%', $term);
$tag_querylets[] = new TagQuerylet($term, $positive);
if ($positive) {
foreach ($tag_conditions as $tq) {
if ($tq->positive) {
$positive_tag_count++;
} else {
$negative_tag_count++;
}
}
}
}
/*
* Turn a bunch of Querylet objects into a base query
@ -902,15 +912,15 @@ class Image
GROUP BY images.id
) AS images
WHERE 1=1
"), ["tag"=>$tag_querylets[0]->tag]);
"), ["tag"=>Tag::sqlify($tag_conditions[0]->tag)]);
}
// more than one positive tag, or more than zero negative tags
else {
if ($database->get_driver_name() === "mysql") {
$query = Image::build_ugly_search_querylet($tag_querylets);
$query = Image::build_ugly_search_querylet($tag_conditions);
} else {
$query = Image::build_accurate_search_querylet($tag_querylets);
$query = Image::build_accurate_search_querylet($tag_conditions);
}
}
@ -918,11 +928,11 @@ class Image
* Merge all the image metadata searches into one generic querylet
* and append to the base querylet with "AND blah"
*/
if (!empty($img_querylets)) {
if (!empty($img_conditions)) {
$n = 0;
$img_sql = "";
$img_vars = [];
foreach ($img_querylets as $iq) {
foreach ($img_conditions as $iq) {
if ($n++ > 0) {
$img_sql .= " AND";
}
@ -960,23 +970,23 @@ class Image
* All the subqueries are executed every time for every row in the
* images table. Yes, MySQL does suck this much.
*
* #param TagQuerylet[] $tag_querylets
* #param TagQuerylet[] $tag_conditions
*/
private static function build_accurate_search_querylet(array $tag_querylets): Querylet
private static function build_accurate_search_querylet(array $tag_conditions): Querylet
{
global $database;
$positive_tag_id_array = [];
$negative_tag_id_array = [];
foreach ($tag_querylets as $tq) {
foreach ($tag_conditions as $tq) {
$tag_ids = $database->get_col(
$database->scoreql_to_sql("
SELECT id
FROM tags
WHERE SCORE_STRNORM(tag) LIKE SCORE_STRNORM(:tag)
"),
["tag" => $tq->tag]
["tag" => Tag::sqlify($tq->tag)]
);
if ($tq->positive) {
$positive_tag_id_array = array_merge($positive_tag_id_array, $tag_ids);
@ -1022,14 +1032,14 @@ class Image
* this function exists because mysql is a turd, see the docs for
* build_accurate_search_querylet() for a full explanation
*
* #param TagQuerylet[] $tag_querylets
* #param TagQuerylet[] $tag_conditions
*/
private static function build_ugly_search_querylet(array $tag_querylets): Querylet
private static function build_ugly_search_querylet(array $tag_conditions): Querylet
{
global $database;
$positive_tag_count = 0;
foreach ($tag_querylets as $tq) {
foreach ($tag_conditions as $tq) {
if ($tq->positive) {
$positive_tag_count++;
}
@ -1049,24 +1059,24 @@ class Image
// merge all the tag querylets into one generic one
$sql = "0";
$terms = [];
foreach ($tag_querylets as $tq) {
foreach ($tag_conditions as $tq) {
$sign = $tq->positive ? "+" : "-";
$sql .= ' '.$sign.' IF(SUM(tag LIKE :tag'.Image::$tag_n.'), 1, 0)';
$terms['tag'.Image::$tag_n] = $tq->tag;
$terms['tag'.Image::$tag_n] = Tag::sqlify($tq->tag);
Image::$tag_n++;
}
$tag_search = new Querylet($sql, $terms);
$tag_id_array = [];
foreach ($tag_querylets as $tq) {
foreach ($tag_conditions as $tq) {
$tag_ids = $database->get_col(
$database->scoreql_to_sql("
SELECT id
FROM tags
WHERE SCORE_STRNORM(tag) LIKE SCORE_STRNORM(:tag)
"),
["tag" => $tq->tag]
["tag" => Tag::sqlify($tq->tag)]
);
$tag_id_array = array_merge($tag_id_array, $tag_ids);

4
core/imageboard/search.php
View file

@ -29,7 +29,7 @@ class Querylet
}
}
class TagQuerylet
class TagCondition
{
/** @var string */
public $tag;
@ -43,7 +43,7 @@ class TagQuerylet
}
}
class ImgQuerylet
class ImgCondition
{
/** @var Querylet */
public $qlet;

8
core/imageboard/tag.php
View file

@ -100,4 +100,12 @@ class Tag
return $tag_array;
}
public static function sqlify(string $term): string
{
$term = str_replace('_', '\_', $term);
$term = str_replace('%', '\%', $term);
$term = str_replace('*', '%', $term);
return $term;
}
}

9
ext/view/main.php
View file

@ -126,6 +126,15 @@ class ViewImage extends Extension
$page->set_mode("redirect");
$page->set_redirect(make_link("post/view/{$image->id}", $query));
} elseif ($event->page_matches("post/view")) {
if (!is_numeric($event->get_arg(0))) {
// For some reason there exists some very broken mobile client
// who follows up every request to '/post/view/123' with
// '/post/view/12300000000000Image 123: tags' which spams the
// database log with 'integer out of range'
$this->theme->display_error(404, "Image not found", "Invalid image ID");
return;
}
$image_id = int_escape($event->get_arg(0));
$image = Image::by_id($image_id);

10
ext/wiki/main.php
View file

@ -213,7 +213,7 @@ class Wiki extends Extension
return false;
}
private function get_page(string $title, int $revision=-1): WikiPage
private function get_page(string $title): WikiPage
{
global $database;
// first try and get the actual page
@ -222,7 +222,9 @@ class Wiki extends Extension
SELECT *
FROM wiki_pages
WHERE SCORE_STRNORM(title) LIKE SCORE_STRNORM(:title)
ORDER BY revision DESC"),
ORDER BY revision DESC
LIMIT 1
"),
["title"=>$title]
);
@ -232,7 +234,9 @@ class Wiki extends Extension
SELECT *
FROM wiki_pages
WHERE title LIKE :title
ORDER BY revision DESC", ["title"=>"wiki:default"]);
ORDER BY revision DESC
LIMIT 1
", ["title"=>"wiki:default"]);
// fall further back to manual
if (empty($row)) {