[core] Database::with_savepoint()

This commit is contained in:
Shish 2024-01-09 21:59:24 +00:00
parent b2f67363a1
commit 7b9201cb42
11 changed files with 111 additions and 134 deletions

View file

@ -122,6 +122,19 @@ class Database
} }
} }
public function with_savepoint(callable $callback, string $name = "sp"): mixed
{
try {
$this->execute("SAVEPOINT $name");
$ret = $callback();
$this->execute("RELEASE SAVEPOINT $name");
return $ret;
} catch (\Exception $e) {
$this->execute("ROLLBACK TO SAVEPOINT $name");
throw $e;
}
}
private function get_engine(): DBEngine private function get_engine(): DBEngine
{ {
if (is_null($this->engine)) { if (is_null($this->engine)) {

View file

@ -25,18 +25,20 @@ function add_dir(string $base, ?array $extra_tags = []): array
$tags = array_merge(path_to_tags($short_path), $extra_tags); $tags = array_merge(path_to_tags($short_path), $extra_tags);
try { try {
$database->execute("SAVEPOINT upload"); $more_results = $database->with_savepoint(function () use ($full_path, $filename, $tags) {
$dae = send_event(new DataUploadEvent($full_path, [ $dae = send_event(new DataUploadEvent($full_path, [
'filename' => pathinfo($filename, PATHINFO_BASENAME), 'filename' => pathinfo($filename, PATHINFO_BASENAME),
'tags' => $tags, 'tags' => $tags,
'source' => null, 'source' => null,
])); ]));
foreach($dae->images as $image) { $results = [];
$results[] = new UploadSuccess($filename, $image->id); foreach($dae->images as $image) {
} $results[] = new UploadSuccess($filename, $image->id);
$database->execute("RELEASE SAVEPOINT upload"); }
return $results;
});
$results = array_merge($results, $more_results);
} catch (UploadException $ex) { } catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$results[] = new UploadError($filename, $ex->getMessage()); $results[] = new UploadError($filename, $ex->getMessage());
} }
} }

View file

@ -30,17 +30,13 @@ class BulkImportExport extends DataHandlerExtension
$skipped = 0; $skipped = 0;
$failed = 0; $failed = 0;
$database->commit();
while (!empty($json_data)) { while (!empty($json_data)) {
$item = array_pop($json_data); $item = array_pop($json_data);
$database->begin_transaction();
try { try {
$image = Image::by_hash($item->hash); $image = Image::by_hash($item->hash);
if ($image != null) { if ($image != null) {
$skipped++; $skipped++;
log_info(BulkImportExportInfo::KEY, "Post $item->hash already present, skipping"); log_info(BulkImportExportInfo::KEY, "Post $item->hash already present, skipping");
$database->commit();
continue; continue;
} }
@ -52,34 +48,29 @@ class BulkImportExport extends DataHandlerExtension
file_put_contents($tmpfile, $stream); file_put_contents($tmpfile, $stream);
$images = send_event(new DataUploadEvent($tmpfile, [ $database->with_savepoint(function () use ($item, $tmpfile, $event) {
'filename' => pathinfo($item->filename, PATHINFO_BASENAME), $images = send_event(new DataUploadEvent($tmpfile, [
'tags' => $item->new_tags, 'filename' => pathinfo($item->filename, PATHINFO_BASENAME),
'source' => null, 'tags' => $item->new_tags,
]))->images; 'source' => null,
]))->images;
if (count($images) == 0) { if (count($images) == 0) {
throw new SCoreException("Unable to import file $item->hash"); throw new SCoreException("Unable to import file $item->hash");
}
foreach ($images as $image) {
$event->images[] = $image;
if ($item->source != null) {
$image->set_source($item->source);
} }
send_event(new BulkImportEvent($image, $item)); foreach ($images as $image) {
} $event->images[] = $image;
if ($item->source != null) {
$image->set_source($item->source);
}
send_event(new BulkImportEvent($image, $item));
}
});
$database->commit();
$total++; $total++;
} catch (\Exception $ex) { } catch (\Exception $ex) {
$failed++; $failed++;
try {
$database->rollBack();
} catch (\Exception $ex2) {
log_error(BulkImportExportInfo::KEY, "Could not roll back transaction: " . $ex2->getMessage(), "Could not import " . $item->hash . ": " . $ex->getMessage());
}
log_error(BulkImportExportInfo::KEY, "Could not import " . $item->hash . ": " . $ex->getMessage(), "Could not import " . $item->hash . ": " . $ex->getMessage()); log_error(BulkImportExportInfo::KEY, "Could not import " . $item->hash . ": " . $ex->getMessage(), "Could not import " . $item->hash . ": " . $ex->getMessage());
continue;
} finally { } finally {
if (!empty($tmpfile) && is_file($tmpfile)) { if (!empty($tmpfile) && is_file($tmpfile)) {
unlink($tmpfile); unlink($tmpfile);
@ -98,8 +89,6 @@ class BulkImportExport extends DataHandlerExtension
} }
} }
public function onBulkActionBlockBuilding(BulkActionBlockBuildingEvent $event) public function onBulkActionBlockBuilding(BulkActionBlockBuildingEvent $event)
{ {
global $user; global $user;
@ -150,6 +139,7 @@ class BulkImportExport extends DataHandlerExtension
} }
} }
} }
// we don't actually do anything, just accept one upload and spawn several // we don't actually do anything, just accept one upload and spawn several
protected function media_check_properties(MediaCheckPropertiesEvent $event): void protected function media_check_properties(MediaCheckPropertiesEvent $event): void
{ {

View file

@ -372,27 +372,18 @@ class CronUploader extends Extension
$this->log_message(SCORE_LOG_DEBUG, "Max run time remaining: $remaining"); $this->log_message(SCORE_LOG_DEBUG, "Max run time remaining: $remaining");
} }
try { try {
$database->begin_transaction(); $result = $database->with_savepoint(function () use ($img, $output_subdir) {
$this->log_message(SCORE_LOG_INFO, "Adding file: {$img[0]} - tags: {$img[2]}"); $this->log_message(SCORE_LOG_INFO, "Adding file: {$img[0]} - tags: {$img[2]}");
$result = $this->add_image($img[0], $img[1], $img[2]); $result = $this->add_image($img[0], $img[1], $img[2]);
if ($database->is_transaction_open()) { $this->move_uploaded($img[0], $img[1], $output_subdir, false);
$database->commit(); return $result;
} });
$this->move_uploaded($img[0], $img[1], $output_subdir, false);
if ($result->merged) { if ($result->merged) {
$merged++; $merged++;
} else { } else {
$added++; $added++;
} }
} catch (\Exception $e) { } catch (\Exception $e) {
try {
if ($database->is_transaction_open()) {
$database->rollback();
}
} catch (\Exception $e) {
// rollback failed, let's just log things and die
}
$failed++; $failed++;
$this->log_message(SCORE_LOG_ERROR, "(" . gettype($e) . ") " . $e->getMessage()); $this->log_message(SCORE_LOG_ERROR, "(" . gettype($e) . ") " . $e->getMessage());
$this->log_message(SCORE_LOG_ERROR, $e->getTraceAsString()); $this->log_message(SCORE_LOG_ERROR, $e->getTraceAsString());

View file

@ -342,17 +342,19 @@ class DanbooruApi extends Extension
//log_debug("danbooru_api", "upload($filename): fileinfo(".var_export($fileinfo,TRUE)."), metadata(".var_export($metadata,TRUE).")..."); //log_debug("danbooru_api", "upload($filename): fileinfo(".var_export($fileinfo,TRUE)."), metadata(".var_export($metadata,TRUE).")...");
try { try {
$database->execute("SAVEPOINT upload"); $newimg = $database->with_savepoint(function () use ($file, $filename, $posttags, $source) {
// Fire off an event which should process the new file and add it to the db // Fire off an event which should process the new file and add it to the db
$dae = send_event(new DataUploadEvent($file, [ $dae = send_event(new DataUploadEvent($file, [
'filename' => pathinfo($filename, PATHINFO_BASENAME), 'filename' => pathinfo($filename, PATHINFO_BASENAME),
'tags' => $posttags, 'tags' => $posttags,
'source' => $source, 'source' => $source,
])); ]));
//log_debug("danbooru_api", "send_event(".var_export($nevent,TRUE).")");
// If it went ok, grab the id for the newly uploaded image and pass it in the header
return $dae->images[0];
});
//log_debug("danbooru_api", "send_event(".var_export($nevent,TRUE).")");
// If it went ok, grab the id for the newly uploaded image and pass it in the header
$newimg = $dae->images[0];
$newid = make_link("post/view/" . $newimg->id); $newid = make_link("post/view/" . $newimg->id);
if ($danboorup_kludge) { if ($danboorup_kludge) {
$newid = make_http($newid); $newid = make_http($newid);
@ -364,9 +366,7 @@ class DanbooruApi extends Extension
} else { } else {
$page->add_http_header("Location: $newid"); $page->add_http_header("Location: $newid");
} }
$database->execute("RELEASE SAVEPOINT upload"); } catch (UploadException $ex) {
} catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$page->set_code(409); $page->set_code(409);
$page->add_http_header("X-Danbooru-Errors: exception - " . $ex->getMessage()); $page->add_http_header("X-Danbooru-Errors: exception - " . $ex->getMessage());
} }

View file

@ -383,18 +383,12 @@ class OuroborosAPI extends Extension
} }
$meta['extension'] = pathinfo($meta['filename'], PATHINFO_EXTENSION); $meta['extension'] = pathinfo($meta['filename'], PATHINFO_EXTENSION);
try { try {
$database->execute("SAVEPOINT upload"); $image = $database->with_savepoint(function () use ($meta) {
$dae = send_event(new DataUploadEvent($meta['file'], $meta)); $dae = send_event(new DataUploadEvent($meta['file'], $meta));
$image = $dae->images[0]; return $dae->images[0];
if (!is_null($image)) { });
$this->sendResponse(200, make_link('post/view/' . $image->id), true); $this->sendResponse(200, make_link('post/view/' . $image->id), true);
} else {
// Fail, unsupported file?
$this->sendResponse(500, 'Unknown error');
}
$database->execute("RELEASE SAVEPOINT upload");
} catch (UploadException $e) { } catch (UploadException $e) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
// Cleanup in case shit hit the fan // Cleanup in case shit hit the fan
$this->sendResponse(500, $e->getMessage()); $this->sendResponse(500, $e->getMessage());
} }

View file

@ -343,27 +343,24 @@ class Pools extends Extension
break; break;
case "reverse": case "reverse":
if ($this->have_permission($user, $pool)) { if ($this->have_permission($user, $pool)) {
$result = $database->execute( $database->with_savepoint(function () use ($pool_id) {
"SELECT image_id FROM pool_images WHERE pool_id=:pid ORDER BY image_order DESC", global $database;
["pid" => $pool_id] $result = $database->execute(
); "SELECT image_id FROM pool_images WHERE pool_id=:pid ORDER BY image_order DESC",
$image_order = 1; ["pid" => $pool_id]
try { );
$database->begin_transaction(); $image_order = 1;
while ($row = $result->fetch()) { while ($row = $result->fetch()) {
$database->execute( $database->execute(
" "
UPDATE pool_images UPDATE pool_images
SET image_order=:ord SET image_order=:ord
WHERE pool_id = :pid AND image_id = :iid", WHERE pool_id = :pid AND image_id = :iid",
["ord" => $image_order, "pid" => $pool_id, "iid" => (int)$row['image_id']] ["ord" => $image_order, "pid" => $pool_id, "iid" => (int)$row['image_id']]
); );
$image_order = $image_order + 1; $image_order = $image_order + 1;
} }
$database->commit(); });
} catch (\Exception $e) {
$database->rollback();
}
$page->set_mode(PageMode::REDIRECT); $page->set_mode(PageMode::REDIRECT);
$page->set_redirect(make_link("pool/view/" . $pool_id)); $page->set_redirect(make_link("pool/view/" . $pool_id));
} else { } else {

View file

@ -355,6 +355,9 @@ class TagEdit extends Extension
$last_id = $image->id; $last_id = $image->id;
} }
if ($commit) { if ($commit) {
// Mass tag edit can take longer than the page timeout,
// so we need to commit periodically to save what little
// work we've done and avoid starting from scratch.
$database->commit(); $database->commit();
$database->begin_transaction(); $database->begin_transaction();
} }

View file

@ -294,24 +294,17 @@ class TranscodeImage extends Extension
$size_difference = 0; $size_difference = 0;
foreach ($event->items as $image) { foreach ($event->items as $image) {
try { try {
$database->begin_transaction(); $before_size = $image->filesize;
$database->with_savepoint(function () use ($image, $mime) {
$before_size = $image->filesize; $this->transcode_and_replace_image($image, $mime);
});
$this->transcode_and_replace_image($image, $mime);
// If a subsequent transcode fails, the database needs to have everything about the previous // If a subsequent transcode fails, the database needs to have everything about the previous
// transcodes recorded already, otherwise the image entries will be stuck pointing to // transcodes recorded already, otherwise the image entries will be stuck pointing to
// missing image files // missing image files
$database->commit();
$total++; $total++;
$size_difference += ($before_size - $image->filesize); $size_difference += ($before_size - $image->filesize);
} catch (\Exception $e) { } catch (\Exception $e) {
log_error("transcode", "Error while bulk transcode on item {$image->id} to $mime: ".$e->getMessage()); log_error("transcode", "Error while bulk transcode on item {$image->id} to $mime: ".$e->getMessage());
try {
$database->rollback();
} catch (\Exception $e) {
// is this safe? o.o
}
} }
} }
if ($size_difference > 0) { if ($size_difference > 0) {

View file

@ -155,23 +155,17 @@ class TranscodeVideo extends Extension
$total = 0; $total = 0;
foreach ($event->items as $image) { foreach ($event->items as $image) {
try { try {
$database->begin_transaction();
$transcoded = $this->transcode_and_replace_video($image, $format);
// If a subsequent transcode fails, the database needs to have everything about the previous // If a subsequent transcode fails, the database needs to have everything about the previous
// transcodes recorded already, otherwise the image entries will be stuck pointing to // transcodes recorded already, otherwise the image entries will be stuck pointing to
// missing image files // missing image files
$database->commit(); $transcoded = $database->with_savepoint(function () use ($image, $format) {
return $this->transcode_and_replace_video($image, $format);
});
if ($transcoded) { if ($transcoded) {
$total++; $total++;
} }
} catch (\Exception $e) { } catch (\Exception $e) {
log_error("transcode_video", "Error while bulk transcode on item {$image->id} to $format: ".$e->getMessage()); log_error("transcode_video", "Error while bulk transcode on item {$image->id} to $format: ".$e->getMessage());
try {
$database->rollback();
} catch (\Exception $e) {
// is this safe? o.o
}
} }
} }
$page->flash("Transcoded $total items"); $page->flash("Transcoded $total items");

View file

@ -337,27 +337,26 @@ class Upload extends Extension
continue; continue;
} }
try { try {
$database->execute("SAVEPOINT upload");
// check if the upload was successful // check if the upload was successful
if ($error !== UPLOAD_ERR_OK) { if ($error !== UPLOAD_ERR_OK) {
throw new UploadException($this->upload_error_message($error)); throw new UploadException($this->upload_error_message($error));
} }
$event = new DataUploadEvent($tmp_name, [ $new_images = $database->with_savepoint(function () use ($tmp_name, $name, $tags, $source) {
'filename' => pathinfo($name, PATHINFO_BASENAME), $event = send_event(new DataUploadEvent($tmp_name, [
'tags' => $tags, 'filename' => pathinfo($name, PATHINFO_BASENAME),
'source' => $source, 'tags' => $tags,
]); 'source' => $source,
send_event($event); ]));
if (count($event->images) == 0) { if (count($event->images) == 0) {
throw new UploadException("MIME type not supported: " . $event->mime); throw new UploadException("MIME type not supported: " . $event->mime);
} }
foreach($event->images as $image) { return $event->images;
});
foreach($new_images as $image) {
$results[] = new UploadSuccess($name, $image->id); $results[] = new UploadSuccess($name, $image->id);
} }
$database->execute("RELEASE SAVEPOINT upload");
} catch (UploadException $ex) { } catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$results[] = new UploadError($name, $ex->getMessage()); $results[] = new UploadError($name, $ex->getMessage());
} }
} }
@ -376,7 +375,6 @@ class Upload extends Extension
$tmp_filename = tempnam(ini_get('upload_tmp_dir'), "shimmie_transload"); $tmp_filename = tempnam(ini_get('upload_tmp_dir'), "shimmie_transload");
try { try {
$database->execute("SAVEPOINT upload");
// Fetch file // Fetch file
$headers = fetch_url($url, $tmp_filename); $headers = fetch_url($url, $tmp_filename);
if (is_null($headers)) { if (is_null($headers)) {
@ -404,18 +402,20 @@ class Upload extends Extension
$metadata['rating'] = strtolower($_GET['rating'])[0]; $metadata['rating'] = strtolower($_GET['rating'])[0];
} }
// Upload file $new_images = $database->with_savepoint(function () use ($tmp_filename, $metadata) {
$event = new DataUploadEvent($tmp_filename, $metadata); $event = send_event(new DataUploadEvent($tmp_filename, $metadata));
send_event($event); if (count($event->images) == 0) {
if (count($event->images) == 0) { throw new UploadException("File type not supported: " . $event->mime);
throw new UploadException("File type not supported: " . $event->mime); }
} if (count($event->images) == 0) {
foreach($event->images as $image) { throw new UploadException("File type not supported: " . $event->mime);
}
return $event->images;
});
foreach($new_images as $image) {
$results[] = new UploadSuccess($url, $image->id); $results[] = new UploadSuccess($url, $image->id);
} }
$database->execute("RELEASE SAVEPOINT upload");
} catch (UploadException $ex) { } catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$results[] = new UploadError($url, $ex->getMessage()); $results[] = new UploadError($url, $ex->getMessage());
} finally { } finally {
if (file_exists($tmp_filename)) { if (file_exists($tmp_filename)) {