mirror of
https://github.com/Horizontal42/summarum.git
synced 2026-11-11 02:12:00 +03:00
Optimize data directory backup migration via concurrent chunked copying
Co-authored-by: Horizontal42 <75383519+Horizontal42@users.noreply.github.com>
This commit is contained in:
parent
c958f30662
commit
64eb393c37
1 changed files with 31 additions and 2 deletions
|
|
@ -569,6 +569,7 @@ fn migrate_data_dir(
|
||||||
}
|
}
|
||||||
// merge backups (skip files that already exist in the target)
|
// merge backups (skip files that already exist in the target)
|
||||||
let mut all_copied = true;
|
let mut all_copied = true;
|
||||||
|
let mut files_to_copy = Vec::new();
|
||||||
for sub in ["backups", "backups/deleted"] {
|
for sub in ["backups", "backups/deleted"] {
|
||||||
let from = old.join(sub);
|
let from = old.join(sub);
|
||||||
let to = new.join(sub);
|
let to = new.join(sub);
|
||||||
|
|
@ -576,10 +577,38 @@ fn migrate_data_dir(
|
||||||
if let Ok(entries) = fs::read_dir(&from) {
|
if let Ok(entries) = fs::read_dir(&from) {
|
||||||
for entry in entries.flatten() {
|
for entry in entries.flatten() {
|
||||||
let dest = to.join(entry.file_name());
|
let dest = to.join(entry.file_name());
|
||||||
if entry.path().is_file() && !dest.exists() && fs::copy(entry.path(), dest).is_err()
|
if entry.path().is_file() && !dest.exists() {
|
||||||
{
|
files_to_copy.push((entry.path(), dest));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !files_to_copy.is_empty() {
|
||||||
|
let thread_count = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4);
|
||||||
|
let chunk_size = (files_to_copy.len() / thread_count).max(1);
|
||||||
|
let chunks: Vec<_> = files_to_copy.chunks(chunk_size).map(|c| c.to_vec()).collect();
|
||||||
|
|
||||||
|
let mut handles = Vec::new();
|
||||||
|
for chunk in chunks {
|
||||||
|
handles.push(std::thread::spawn(move || {
|
||||||
|
let mut chunk_all_copied = true;
|
||||||
|
for (src, dest) in chunk {
|
||||||
|
if fs::copy(&src, &dest).is_err() {
|
||||||
|
chunk_all_copied = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chunk_all_copied
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
for handle in handles {
|
||||||
|
if let Ok(chunk_all_copied) = handle.join() {
|
||||||
|
if !chunk_all_copied {
|
||||||
all_copied = false;
|
all_copied = false;
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
all_copied = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue