badmanners/shimmie2
badmanners/shimmie2
Archived
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[core] Database::with_savepoint()

Browse source
This commit is contained in:
Shish 2024-01-09 21:59:24 +00:00
parent b2f67363a1
commit 7b9201cb42
11 changed files with 111 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
core/database.php
View file

@ -122,6 +122,19 @@ class Database
}
}
public function with_savepoint(callable $callback, string $name = "sp"): mixed
{
try {
$this->execute("SAVEPOINT $name");
$ret = $callback();
$this->execute("RELEASE SAVEPOINT $name");
return $ret;
} catch (\Exception $e) {
$this->execute("ROLLBACK TO SAVEPOINT $name");
throw $e;
}
}
private function get_engine(): DBEngine
{
if (is_null($this->engine)) {

24
core/imageboard/misc.php
View file

@ -25,18 +25,20 @@ function add_dir(string $base, ?array $extra_tags = []): array
$tags = array_merge(path_to_tags($short_path), $extra_tags);
try {
$database->execute("SAVEPOINT upload");
$dae = send_event(new DataUploadEvent($full_path, [
'filename' => pathinfo($filename, PATHINFO_BASENAME),
'tags' => $tags,
'source' => null,
]));
foreach($dae->images as $image) {
$results[] = new UploadSuccess($filename, $image->id);
}
$database->execute("RELEASE SAVEPOINT upload");
$more_results = $database->with_savepoint(function () use ($full_path, $filename, $tags) {
$dae = send_event(new DataUploadEvent($full_path, [
'filename' => pathinfo($filename, PATHINFO_BASENAME),
'tags' => $tags,
'source' => null,
]));
$results = [];
foreach($dae->images as $image) {
$results[] = new UploadSuccess($filename, $image->id);
}
return $results;
});
$results = array_merge($results, $more_results);
} catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$results[] = new UploadError($filename, $ex->getMessage());
}
}

44
ext/bulk_import_export/main.php
View file

@ -30,17 +30,13 @@ class BulkImportExport extends DataHandlerExtension
$skipped = 0;
$failed = 0;
$database->commit();
while (!empty($json_data)) {
$item = array_pop($json_data);
$database->begin_transaction();
try {
$image = Image::by_hash($item->hash);
if ($image != null) {
$skipped++;
log_info(BulkImportExportInfo::KEY, "Post $item->hash already present, skipping");
$database->commit();
continue;
}
@ -52,34 +48,29 @@ class BulkImportExport extends DataHandlerExtension
file_put_contents($tmpfile, $stream);
$images = send_event(new DataUploadEvent($tmpfile, [
'filename' => pathinfo($item->filename, PATHINFO_BASENAME),
'tags' => $item->new_tags,
'source' => null,
]))->images;
$database->with_savepoint(function () use ($item, $tmpfile, $event) {
$images = send_event(new DataUploadEvent($tmpfile, [
'filename' => pathinfo($item->filename, PATHINFO_BASENAME),
'tags' => $item->new_tags,
'source' => null,
]))->images;
if (count($images) == 0) {
throw new SCoreException("Unable to import file $item->hash");
}
foreach ($images as $image) {
$event->images[] = $image;
if ($item->source != null) {
$image->set_source($item->source);
if (count($images) == 0) {
throw new SCoreException("Unable to import file $item->hash");
}
send_event(new BulkImportEvent($image, $item));
}
foreach ($images as $image) {
$event->images[] = $image;
if ($item->source != null) {
$image->set_source($item->source);
}
send_event(new BulkImportEvent($image, $item));
}
});
$database->commit();
$total++;
} catch (\Exception $ex) {
$failed++;
try {
$database->rollBack();
} catch (\Exception $ex2) {
log_error(BulkImportExportInfo::KEY, "Could not roll back transaction: " . $ex2->getMessage(), "Could not import " . $item->hash . ": " . $ex->getMessage());
}
log_error(BulkImportExportInfo::KEY, "Could not import " . $item->hash . ": " . $ex->getMessage(), "Could not import " . $item->hash . ": " . $ex->getMessage());
continue;
} finally {
if (!empty($tmpfile) && is_file($tmpfile)) {
unlink($tmpfile);
@ -98,8 +89,6 @@ class BulkImportExport extends DataHandlerExtension
}
}
public function onBulkActionBlockBuilding(BulkActionBlockBuildingEvent $event)
{
global $user;
@ -150,6 +139,7 @@ class BulkImportExport extends DataHandlerExtension
}
}
}
// we don't actually do anything, just accept one upload and spawn several
protected function media_check_properties(MediaCheckPropertiesEvent $event): void
{

21
ext/cron_uploader/main.php
View file

@ -372,27 +372,18 @@ class CronUploader extends Extension
$this->log_message(SCORE_LOG_DEBUG, "Max run time remaining: $remaining");
}
try {
$database->begin_transaction();
$this->log_message(SCORE_LOG_INFO, "Adding file: {$img[0]} - tags: {$img[2]}");
$result = $this->add_image($img[0], $img[1], $img[2]);
if ($database->is_transaction_open()) {
$database->commit();
}
$this->move_uploaded($img[0], $img[1], $output_subdir, false);
$result = $database->with_savepoint(function () use ($img, $output_subdir) {
$this->log_message(SCORE_LOG_INFO, "Adding file: {$img[0]} - tags: {$img[2]}");
$result = $this->add_image($img[0], $img[1], $img[2]);
$this->move_uploaded($img[0], $img[1], $output_subdir, false);
return $result;
});
if ($result->merged) {
$merged++;
} else {
$added++;
}
} catch (\Exception $e) {
try {
if ($database->is_transaction_open()) {
$database->rollback();
}
} catch (\Exception $e) {
// rollback failed, let's just log things and die
}
$failed++;
$this->log_message(SCORE_LOG_ERROR, "(" . gettype($e) . ") " . $e->getMessage());
$this->log_message(SCORE_LOG_ERROR, $e->getTraceAsString());

26
ext/danbooru_api/main.php
View file

@ -342,17 +342,19 @@ class DanbooruApi extends Extension
//log_debug("danbooru_api", "upload($filename): fileinfo(".var_export($fileinfo,TRUE)."), metadata(".var_export($metadata,TRUE).")...");
try {
$database->execute("SAVEPOINT upload");
// Fire off an event which should process the new file and add it to the db
$dae = send_event(new DataUploadEvent($file, [
'filename' => pathinfo($filename, PATHINFO_BASENAME),
'tags' => $posttags,
'source' => $source,
]));
$newimg = $database->with_savepoint(function () use ($file, $filename, $posttags, $source) {
// Fire off an event which should process the new file and add it to the db
$dae = send_event(new DataUploadEvent($file, [
'filename' => pathinfo($filename, PATHINFO_BASENAME),
'tags' => $posttags,
'source' => $source,
]));
//log_debug("danbooru_api", "send_event(".var_export($nevent,TRUE).")");
// If it went ok, grab the id for the newly uploaded image and pass it in the header
return $dae->images[0];
});
//log_debug("danbooru_api", "send_event(".var_export($nevent,TRUE).")");
// If it went ok, grab the id for the newly uploaded image and pass it in the header
$newimg = $dae->images[0];
$newid = make_link("post/view/" . $newimg->id);
if ($danboorup_kludge) {
$newid = make_http($newid);
@ -364,9 +366,7 @@ class DanbooruApi extends Extension
} else {
$page->add_http_header("Location: $newid");
}
$database->execute("RELEASE SAVEPOINT upload");
} catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
} catch (UploadException $ex) {
$page->set_code(409);
$page->add_http_header("X-Danbooru-Errors: exception - " . $ex->getMessage());
}

16
ext/ouroboros_api/main.php
View file

@ -383,18 +383,12 @@ class OuroborosAPI extends Extension
}
$meta['extension'] = pathinfo($meta['filename'], PATHINFO_EXTENSION);
try {
$database->execute("SAVEPOINT upload");
$dae = send_event(new DataUploadEvent($meta['file'], $meta));
$image = $dae->images[0];
if (!is_null($image)) {
$this->sendResponse(200, make_link('post/view/' . $image->id), true);
} else {
// Fail, unsupported file?
$this->sendResponse(500, 'Unknown error');
}
$database->execute("RELEASE SAVEPOINT upload");
$image = $database->with_savepoint(function () use ($meta) {
$dae = send_event(new DataUploadEvent($meta['file'], $meta));
return $dae->images[0];
});
$this->sendResponse(200, make_link('post/view/' . $image->id), true);
} catch (UploadException $e) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
// Cleanup in case shit hit the fan
$this->sendResponse(500, $e->getMessage());
}

25
ext/pools/main.php
View file

@ -343,27 +343,24 @@ class Pools extends Extension
break;
case "reverse":
if ($this->have_permission($user, $pool)) {
$result = $database->execute(
"SELECT image_id FROM pool_images WHERE pool_id=:pid ORDER BY image_order DESC",
["pid" => $pool_id]
);
$image_order = 1;
try {
$database->begin_transaction();
$database->with_savepoint(function () use ($pool_id) {
global $database;
$result = $database->execute(
"SELECT image_id FROM pool_images WHERE pool_id=:pid ORDER BY image_order DESC",
["pid" => $pool_id]
);
$image_order = 1;
while ($row = $result->fetch()) {
$database->execute(
"
UPDATE pool_images
SET image_order=:ord
WHERE pool_id = :pid AND image_id = :iid",
UPDATE pool_images
SET image_order=:ord
WHERE pool_id = :pid AND image_id = :iid",
["ord" => $image_order, "pid" => $pool_id, "iid" => (int)$row['image_id']]
);
$image_order = $image_order + 1;
}
$database->commit();
} catch (\Exception $e) {
$database->rollback();
}
});
$page->set_mode(PageMode::REDIRECT);
$page->set_redirect(make_link("pool/view/" . $pool_id));
} else {

3
ext/tag_edit/main.php
View file

@ -355,6 +355,9 @@ class TagEdit extends Extension
$last_id = $image->id;
}
if ($commit) {
// Mass tag edit can take longer than the page timeout,
// so we need to commit periodically to save what little
// work we've done and avoid starting from scratch.
$database->commit();
$database->begin_transaction();
}

15
ext/transcode/main.php
View file

@ -294,24 +294,17 @@ class TranscodeImage extends Extension
$size_difference = 0;
foreach ($event->items as $image) {
try {
$database->begin_transaction();
$before_size = $image->filesize;
$this->transcode_and_replace_image($image, $mime);
$before_size = $image->filesize;
$database->with_savepoint(function () use ($image, $mime) {
$this->transcode_and_replace_image($image, $mime);
});
// If a subsequent transcode fails, the database needs to have everything about the previous
// transcodes recorded already, otherwise the image entries will be stuck pointing to
// missing image files
$database->commit();
$total++;
$size_difference += ($before_size - $image->filesize);
} catch (\Exception $e) {
log_error("transcode", "Error while bulk transcode on item {$image->id} to $mime: ".$e->getMessage());
try {
$database->rollback();
} catch (\Exception $e) {
// is this safe? o.o
}
}
}
if ($size_difference > 0) {

12
ext/transcode_video/main.php
View file

@ -155,23 +155,17 @@ class TranscodeVideo extends Extension
$total = 0;
foreach ($event->items as $image) {
try {
$database->begin_transaction();
$transcoded = $this->transcode_and_replace_video($image, $format);
// If a subsequent transcode fails, the database needs to have everything about the previous
// transcodes recorded already, otherwise the image entries will be stuck pointing to
// missing image files
$database->commit();
$transcoded = $database->with_savepoint(function () use ($image, $format) {
return $this->transcode_and_replace_video($image, $format);
});
if ($transcoded) {
$total++;
}
} catch (\Exception $e) {
log_error("transcode_video", "Error while bulk transcode on item {$image->id} to $format: ".$e->getMessage());
try {
$database->rollback();
} catch (\Exception $e) {
// is this safe? o.o
}
}
}
$page->flash("Transcoded $total items");

46
ext/upload/main.php
View file

@ -337,27 +337,26 @@ class Upload extends Extension
continue;
}
try {
$database->execute("SAVEPOINT upload");
// check if the upload was successful
if ($error !== UPLOAD_ERR_OK) {
throw new UploadException($this->upload_error_message($error));
}
$event = new DataUploadEvent($tmp_name, [
'filename' => pathinfo($name, PATHINFO_BASENAME),
'tags' => $tags,
'source' => $source,
]);
send_event($event);
if (count($event->images) == 0) {
throw new UploadException("MIME type not supported: " . $event->mime);
}
foreach($event->images as $image) {
$new_images = $database->with_savepoint(function () use ($tmp_name, $name, $tags, $source) {
$event = send_event(new DataUploadEvent($tmp_name, [
'filename' => pathinfo($name, PATHINFO_BASENAME),
'tags' => $tags,
'source' => $source,
]));
if (count($event->images) == 0) {
throw new UploadException("MIME type not supported: " . $event->mime);
}
return $event->images;
});
foreach($new_images as $image) {
$results[] = new UploadSuccess($name, $image->id);
}
$database->execute("RELEASE SAVEPOINT upload");
} catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$results[] = new UploadError($name, $ex->getMessage());
}
}
@ -376,7 +375,6 @@ class Upload extends Extension
$tmp_filename = tempnam(ini_get('upload_tmp_dir'), "shimmie_transload");
try {
$database->execute("SAVEPOINT upload");
// Fetch file
$headers = fetch_url($url, $tmp_filename);
if (is_null($headers)) {
@ -404,18 +402,20 @@ class Upload extends Extension
$metadata['rating'] = strtolower($_GET['rating'])[0];
}
// Upload file
$event = new DataUploadEvent($tmp_filename, $metadata);
send_event($event);
if (count($event->images) == 0) {
throw new UploadException("File type not supported: " . $event->mime);
}
foreach($event->images as $image) {
$new_images = $database->with_savepoint(function () use ($tmp_filename, $metadata) {
$event = send_event(new DataUploadEvent($tmp_filename, $metadata));
if (count($event->images) == 0) {
throw new UploadException("File type not supported: " . $event->mime);
}
if (count($event->images) == 0) {
throw new UploadException("File type not supported: " . $event->mime);
}
return $event->images;
});
foreach($new_images as $image) {
$results[] = new UploadSuccess($url, $image->id);
}
$database->execute("RELEASE SAVEPOINT upload");
} catch (UploadException $ex) {
$database->execute("ROLLBACK TO SAVEPOINT upload");
$results[] = new UploadError($url, $ex->getMessage());
} finally {
if (file_exists($tmp_filename)) {