1mod atomic_output;
4mod piece_length;
5mod progress;
6
7pub use atomic_output::write_atomic;
8pub use piece_length::{
9 PIECE_LENGTH_POLICY_ID, PieceLengthMode, automatic_piece_length, validate_piece_length,
10};
11pub use progress::{CancellationToken, HashProgress, NoProgress, ProgressSink};
12
13use std::collections::HashSet;
14use std::ffi::OsStr;
15use std::fs;
16use std::path::{Path, PathBuf};
17use std::time::SystemTime;
18
19use globset::{Glob, GlobSet, GlobSetBuilder};
20use sha1::Digest as _;
21
22use crate::metadata::{
23 MetadataText, NodeHost, TrackerTier, WebSeed, validate_creation_date, validate_nodes,
24 validate_tracker_tiers, validate_web_seeds,
25};
26use crate::{Error, Result};
27
28const HASH_READ_BUFFER_LENGTH: usize = 64 * 1024;
29
30#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
32pub enum HiddenPolicy {
33 #[default]
35 Include,
36 Exclude,
38}
39
40#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
42pub enum SymlinkPolicy {
43 #[default]
45 Reject,
46 Skip,
48 Follow,
50}
51
52#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
54pub enum SpecialFilePolicy {
55 #[default]
57 Reject,
58 Skip,
60}
61
62#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
64pub enum EmptyFilePolicy {
65 #[default]
67 Include,
68 Exclude,
70}
71
72#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
74pub enum EmptyDirectoryPolicy {
75 #[default]
77 Ignore,
78 Reject,
80}
81
82#[derive(Clone, Debug, Default, Eq, PartialEq)]
84pub enum RootName {
85 #[default]
87 Automatic,
88 Override(Vec<u8>),
90}
91
92#[derive(Clone, Debug)]
94pub struct ManifestOptions {
95 hidden: HiddenPolicy,
96 symlinks: SymlinkPolicy,
97 special_files: SpecialFilePolicy,
98 empty_files: EmptyFilePolicy,
99 empty_directories: EmptyDirectoryPolicy,
100 include: GlobSet,
101 include_is_empty: bool,
102 exclude: GlobSet,
103 exclude_is_empty: bool,
104 include_raw_paths: Vec<Vec<Vec<u8>>>,
105 exclude_raw_paths: Vec<Vec<Vec<u8>>>,
106 root_name: RootName,
107}
108
109impl Default for ManifestOptions {
110 fn default() -> Self {
111 Self::builder().build().expect("empty glob sets are valid")
112 }
113}
114
115impl ManifestOptions {
116 #[must_use]
118 pub fn builder() -> ManifestOptionsBuilder {
119 ManifestOptionsBuilder::default()
120 }
121}
122
123#[derive(Clone, Debug, Default)]
125pub struct ManifestOptionsBuilder {
126 hidden: HiddenPolicy,
127 symlinks: SymlinkPolicy,
128 special_files: SpecialFilePolicy,
129 empty_files: EmptyFilePolicy,
130 empty_directories: EmptyDirectoryPolicy,
131 include: Vec<String>,
132 exclude: Vec<String>,
133 include_raw_paths: Vec<Vec<Vec<u8>>>,
134 exclude_raw_paths: Vec<Vec<Vec<u8>>>,
135 root_name: RootName,
136}
137
138impl ManifestOptionsBuilder {
139 #[must_use]
141 pub const fn hidden(mut self, policy: HiddenPolicy) -> Self {
142 self.hidden = policy;
143 self
144 }
145
146 #[must_use]
148 pub const fn symlinks(mut self, policy: SymlinkPolicy) -> Self {
149 self.symlinks = policy;
150 self
151 }
152
153 #[must_use]
155 pub const fn special_files(mut self, policy: SpecialFilePolicy) -> Self {
156 self.special_files = policy;
157 self
158 }
159
160 #[must_use]
162 pub const fn empty_files(mut self, policy: EmptyFilePolicy) -> Self {
163 self.empty_files = policy;
164 self
165 }
166
167 #[must_use]
169 pub const fn empty_directories(mut self, policy: EmptyDirectoryPolicy) -> Self {
170 self.empty_directories = policy;
171 self
172 }
173
174 #[must_use]
176 pub fn include(mut self, patterns: impl IntoIterator<Item = impl Into<String>>) -> Self {
177 self.include = patterns.into_iter().map(Into::into).collect();
178 self
179 }
180
181 #[must_use]
183 pub fn exclude(mut self, patterns: impl IntoIterator<Item = impl Into<String>>) -> Self {
184 self.exclude = patterns.into_iter().map(Into::into).collect();
185 self
186 }
187
188 #[must_use]
190 pub fn include_raw_paths(mut self, paths: impl IntoIterator<Item = Vec<Vec<u8>>>) -> Self {
191 self.include_raw_paths = paths.into_iter().collect();
192 self
193 }
194
195 #[must_use]
197 pub fn exclude_raw_paths(mut self, paths: impl IntoIterator<Item = Vec<Vec<u8>>>) -> Self {
198 self.exclude_raw_paths = paths.into_iter().collect();
199 self
200 }
201
202 #[must_use]
204 pub fn root_name(mut self, root_name: RootName) -> Self {
205 self.root_name = root_name;
206 self
207 }
208
209 pub fn build(self) -> Result<ManifestOptions> {
215 let include_is_empty = self.include.is_empty();
216 let exclude_is_empty = self.exclude.is_empty();
217 let include = compile_globs(&self.include)?;
218 let exclude = compile_globs(&self.exclude)?;
219 if let RootName::Override(name) = &self.root_name {
220 validate_component(name, "root name")?;
221 }
222 Ok(ManifestOptions {
223 hidden: self.hidden,
224 symlinks: self.symlinks,
225 special_files: self.special_files,
226 empty_files: self.empty_files,
227 empty_directories: self.empty_directories,
228 include,
229 include_is_empty,
230 exclude,
231 exclude_is_empty,
232 include_raw_paths: self.include_raw_paths,
233 exclude_raw_paths: self.exclude_raw_paths,
234 root_name: self.root_name,
235 })
236 }
237}
238
239#[derive(Clone, Debug, Eq, PartialEq)]
241pub struct ManifestEntry {
242 source_path: PathBuf,
243 torrent_path: Vec<Vec<u8>>,
244 length: u64,
245 modified: Option<SystemTime>,
246 snapshot: FileSnapshot,
247}
248
249#[derive(Clone, Debug, Eq, PartialEq)]
250struct FileSnapshot {
251 is_file: bool,
252 length: u64,
253 modified: Option<SystemTime>,
254 created: Option<SystemTime>,
255 identity: Option<file_id::FileId>,
256 #[cfg(unix)]
257 change_seconds: i64,
258 #[cfg(unix)]
259 change_nanoseconds: i64,
260}
261
262impl FileSnapshot {
263 fn from_path(path: &Path, metadata: &fs::Metadata) -> Result<Self> {
264 #[cfg(unix)]
265 use std::os::unix::fs::MetadataExt as _;
266
267 Ok(Self {
268 is_file: metadata.is_file(),
269 length: metadata.len(),
270 modified: metadata.modified().ok(),
271 created: metadata.created().ok(),
272 identity: Some(file_id::get_file_id(path).map_err(|source| Error::io(path, source))?),
273 #[cfg(unix)]
274 change_seconds: metadata.ctime(),
275 #[cfg(unix)]
276 change_nanoseconds: metadata.ctime_nsec(),
277 })
278 }
279
280 #[cfg(test)]
281 fn limited(length: u64, modified: Option<SystemTime>) -> Self {
282 Self {
283 is_file: true,
284 length,
285 modified,
286 created: None,
287 identity: None,
288 #[cfg(unix)]
289 change_seconds: 0,
290 #[cfg(unix)]
291 change_nanoseconds: 0,
292 }
293 }
294
295 fn matches_metadata(&self, metadata: &fs::Metadata) -> bool {
296 #[cfg(unix)]
297 use std::os::unix::fs::MetadataExt as _;
298
299 if self.is_limited() {
300 return metadata.is_file() == self.is_file
301 && metadata.len() == self.length
302 && self
303 .modified
304 .is_none_or(|modified| metadata.modified().ok() == Some(modified));
305 }
306 metadata.is_file() == self.is_file
307 && metadata.len() == self.length
308 && metadata.modified().ok() == self.modified
309 && metadata.created().ok() == self.created
310 && {
311 #[cfg(unix)]
312 {
313 metadata.ctime() == self.change_seconds
314 && metadata.ctime_nsec() == self.change_nanoseconds
315 }
316 #[cfg(not(unix))]
317 {
318 true
319 }
320 }
321 }
322
323 fn is_limited(&self) -> bool {
324 self.identity.is_none()
325 }
326}
327
328impl ManifestEntry {
329 #[must_use]
331 pub fn source_path(&self) -> &Path {
332 &self.source_path
333 }
334
335 #[must_use]
337 pub fn torrent_path(&self) -> &[Vec<u8>] {
338 &self.torrent_path
339 }
340
341 #[must_use]
343 pub const fn length(&self) -> u64 {
344 self.length
345 }
346
347 #[must_use]
349 pub const fn modified(&self) -> Option<SystemTime> {
350 self.modified
351 }
352
353 #[cfg(test)]
355 #[must_use]
356 pub(crate) fn for_test(
357 source: impl Into<PathBuf>,
358 torrent_path: Vec<Vec<u8>>,
359 length: u64,
360 ) -> Self {
361 Self {
362 source_path: source.into(),
363 torrent_path,
364 length,
365 modified: None,
366 snapshot: FileSnapshot::limited(length, None),
367 }
368 }
369
370 #[cfg(test)]
372 #[must_use]
373 pub(crate) fn from_snapshot(
374 source: impl Into<PathBuf>,
375 torrent_path: Vec<Vec<u8>>,
376 length: u64,
377 modified: Option<SystemTime>,
378 ) -> Self {
379 Self {
380 source_path: source.into(),
381 torrent_path,
382 length,
383 modified,
384 snapshot: FileSnapshot::limited(length, modified),
385 }
386 }
387}
388
389#[derive(Clone, Debug, Eq, PartialEq)]
391pub struct PayloadManifest {
392 root_name: Vec<u8>,
393 entries: Vec<ManifestEntry>,
394 total_length: u64,
395}
396
397impl PayloadManifest {
398 #[must_use]
400 pub fn root_name(&self) -> &[u8] {
401 &self.root_name
402 }
403
404 #[must_use]
406 pub fn entries(&self) -> &[ManifestEntry] {
407 &self.entries
408 }
409
410 #[must_use]
412 pub const fn total_length(&self) -> u64 {
413 self.total_length
414 }
415
416 #[must_use]
418 pub fn relative_snapshot(&self) -> Vec<(Vec<Vec<u8>>, u64)> {
419 self.entries
420 .iter()
421 .map(|entry| (entry.torrent_path.clone(), entry.length))
422 .collect()
423 }
424}
425
426pub fn scan_manifest(path: impl AsRef<Path>, options: &ManifestOptions) -> Result<PayloadManifest> {
438 let path = path.as_ref();
439 let metadata = fs::symlink_metadata(path).map_err(|source| Error::io(path, source))?;
440 let automatic_name = path
441 .file_name()
442 .ok_or_else(|| Error::metainfo_field("root name", "input has no file name"))?;
443 let root_name = match &options.root_name {
444 RootName::Automatic => os_component_bytes(automatic_name)?,
445 RootName::Override(name) => name.clone(),
446 };
447 validate_component(&root_name, "root name")?;
448 let canonical_root = path
449 .canonicalize()
450 .map_err(|source| Error::io(path, source))?;
451 let mut entries = Vec::new();
452 let mut visited = HashSet::new();
453 if metadata.file_type().is_symlink() {
454 match options.symlinks {
455 SymlinkPolicy::Reject => {
456 return Err(Error::unsupported(format!(
457 "symbolic link encountered: {}",
458 path.display()
459 )));
460 }
461 SymlinkPolicy::Skip => {
462 return Err(Error::unsupported(
463 "top-level symbolic link cannot produce an empty manifest",
464 ));
465 }
466 SymlinkPolicy::Follow => {
467 let target = fs::metadata(path).map_err(|source| Error::io(path, source))?;
468 if target.is_file() {
469 collect_file(
470 path,
471 vec![root_name.clone()],
472 options,
473 &mut entries,
474 &target,
475 )?;
476 } else if target.is_dir() {
477 visited.insert(canonical_root.clone());
478 walk_directory(
479 path,
480 &[],
481 &canonical_root,
482 options,
483 &mut entries,
484 &mut visited,
485 )?;
486 } else {
487 handle_special(path, options)?;
488 }
489 }
490 }
491 } else if metadata.is_file() {
492 collect_file(
493 path,
494 vec![root_name.clone()],
495 options,
496 &mut entries,
497 &metadata,
498 )?;
499 } else if metadata.is_dir() {
500 visited.insert(canonical_root.clone());
501 walk_directory(
502 path,
503 &[],
504 &canonical_root,
505 options,
506 &mut entries,
507 &mut visited,
508 )?;
509 } else {
510 handle_special(path, options)?;
511 }
512 let entries = sort_manifest_entries(entries)?;
513 let total_length = entries.iter().try_fold(0_u64, |total, entry| {
514 total.checked_add(entry.length).ok_or_else(|| {
515 Error::metainfo_field("manifest length", "total payload length overflowed")
516 })
517 })?;
518 Ok(PayloadManifest {
519 root_name,
520 entries,
521 total_length,
522 })
523}
524
525pub fn sort_manifest_entries(mut entries: Vec<ManifestEntry>) -> Result<Vec<ManifestEntry>> {
531 entries.sort_by(|left, right| left.torrent_path.cmp(&right.torrent_path));
532 crate::metainfo::validate_torrent_path_graph(
533 entries
534 .iter()
535 .map(|entry| entry.torrent_path.iter().map(Vec::as_slice).collect())
536 .collect(),
537 "manifest path",
538 )?;
539 Ok(entries)
540}
541
542#[cfg(test)]
543mod manifest_sort_tests {
544 use super::{ManifestEntry, sort_manifest_entries};
545 use crate::ErrorCategory;
546
547 #[test]
548 fn rejects_duplicate_and_prefix_collisions() {
549 let duplicate = vec![
550 ManifestEntry::for_test("one", vec![b"same".to_vec()], 1),
551 ManifestEntry::for_test("two", vec![b"same".to_vec()], 2),
552 ];
553 assert_eq!(
554 sort_manifest_entries(duplicate).unwrap_err().category(),
555 ErrorCategory::Metainfo
556 );
557
558 let prefix = vec![
559 ManifestEntry::for_test("one", vec![b"a".to_vec()], 1),
560 ManifestEntry::for_test("two", vec![b"a".to_vec(), b"b".to_vec()], 2),
561 ];
562 assert_eq!(
563 sort_manifest_entries(prefix).unwrap_err().field(),
564 Some("manifest path")
565 );
566 }
567}
568
569fn walk_directory(
570 directory: &Path,
571 relative: &[Vec<u8>],
572 canonical_root: &Path,
573 options: &ManifestOptions,
574 entries: &mut Vec<ManifestEntry>,
575 visited: &mut HashSet<PathBuf>,
576) -> Result<bool> {
577 let children = fs::read_dir(directory)
578 .map_err(|source| Error::io(directory, source))?
579 .collect::<std::result::Result<Vec<_>, _>>()
580 .map_err(|source| Error::io(directory, source))?;
581 let mut children = children
582 .into_iter()
583 .map(|entry| Ok((os_component_bytes(&entry.file_name())?, entry)))
584 .collect::<Result<Vec<_>>>()?;
585 children.sort_by(|(left, _), (right, _)| left.cmp(right));
586 let mut represented = false;
587 for (component, child) in children {
588 validate_component(&component, "manifest path")?;
589 if options.hidden == HiddenPolicy::Exclude && component.starts_with(b".") {
590 continue;
591 }
592 let mut child_relative = relative.to_owned();
593 child_relative.push(component);
594 let child_path = child.path();
595 let metadata =
596 fs::symlink_metadata(&child_path).map_err(|source| Error::io(&child_path, source))?;
597 if metadata.file_type().is_symlink() {
598 represented |= handle_symlink(
599 &child_path,
600 child_relative,
601 canonical_root,
602 options,
603 entries,
604 visited,
605 )?;
606 } else if metadata.is_dir() {
607 let canonical = child_path
608 .canonicalize()
609 .map_err(|source| Error::io(&child_path, source))?;
610 if !visited.insert(canonical.clone()) {
611 return Err(Error::unsupported("directory cycle during manifest scan"));
612 }
613 let child_represented = walk_directory(
614 &child_path,
615 &child_relative,
616 canonical_root,
617 options,
618 entries,
619 visited,
620 )?;
621 visited.remove(&canonical);
622 if !child_represented && options.empty_directories == EmptyDirectoryPolicy::Reject {
623 return Err(Error::unsupported(format!(
624 "empty directory cannot be represented: {}",
625 child_path.display()
626 )));
627 }
628 represented |= child_represented;
629 } else if metadata.is_file() {
630 represented |= collect_file(&child_path, child_relative, options, entries, &metadata)?;
631 } else {
632 handle_special(&child_path, options)?;
633 }
634 }
635 Ok(represented)
636}
637
638fn handle_symlink(
639 path: &Path,
640 relative: Vec<Vec<u8>>,
641 canonical_root: &Path,
642 options: &ManifestOptions,
643 entries: &mut Vec<ManifestEntry>,
644 visited: &mut HashSet<PathBuf>,
645) -> Result<bool> {
646 match options.symlinks {
647 SymlinkPolicy::Reject => Err(Error::unsupported(format!(
648 "symbolic link encountered: {}",
649 path.display()
650 ))),
651 SymlinkPolicy::Skip => Ok(false),
652 SymlinkPolicy::Follow => {
653 let canonical = path
654 .canonicalize()
655 .map_err(|source| Error::io(path, source))?;
656 if !canonical.starts_with(canonical_root) {
657 return Err(Error::unsupported(format!(
658 "symbolic link escapes payload root: {}",
659 path.display()
660 )));
661 }
662 let metadata = fs::metadata(path).map_err(|source| Error::io(path, source))?;
663 if metadata.is_file() {
664 collect_file(path, relative, options, entries, &metadata)
665 } else if metadata.is_dir() {
666 if !visited.insert(canonical.clone()) {
667 return Err(Error::unsupported("symbolic link directory cycle"));
668 }
669 let represented =
670 walk_directory(path, &relative, canonical_root, options, entries, visited)?;
671 visited.remove(&canonical);
672 Ok(represented)
673 } else {
674 handle_special(path, options)?;
675 Ok(false)
676 }
677 }
678 }
679}
680
681fn collect_file(
682 path: &Path,
683 relative: Vec<Vec<u8>>,
684 options: &ManifestOptions,
685 entries: &mut Vec<ManifestEntry>,
686 before: &fs::Metadata,
687) -> Result<bool> {
688 if !options.include_is_empty || !options.exclude_is_empty {
689 let mut match_path = String::new();
690 for (index, component) in relative.iter().enumerate() {
691 if index != 0 {
692 match_path.push('/');
693 }
694 match_path.push_str(std::str::from_utf8(component).map_err(|_| {
695 Error::metainfo_field(
696 "manifest filter",
697 "text glob patterns require UTF-8 payload paths; use raw path filters",
698 )
699 })?);
700 }
701 if (!options.include_is_empty && !options.include.is_match(&match_path))
702 || (!options.exclude_is_empty && options.exclude.is_match(&match_path))
703 {
704 return Ok(false);
705 }
706 }
707 if (!options.include_raw_paths.is_empty() && !options.include_raw_paths.contains(&relative))
708 || options.exclude_raw_paths.contains(&relative)
709 {
710 return Ok(false);
711 }
712 if before.len() == 0 && options.empty_files == EmptyFilePolicy::Exclude {
713 return Ok(false);
714 }
715 let snapshot = FileSnapshot::from_path(path, before)?;
716 let modified = snapshot.modified;
717 let after = fs::metadata(path).map_err(|source| Error::io(path, source))?;
718 if !snapshot.matches_metadata(&after) || !snapshot_matches_path(&snapshot, path)? {
719 return Err(Error::unsupported(format!(
720 "file changed during manifest scan: {}",
721 path.display()
722 )));
723 }
724 entries.push(ManifestEntry {
725 source_path: path.to_path_buf(),
726 torrent_path: relative,
727 length: before.len(),
728 modified,
729 snapshot,
730 });
731 Ok(true)
732}
733
734fn handle_special(path: &Path, options: &ManifestOptions) -> Result<()> {
735 match options.special_files {
736 SpecialFilePolicy::Reject => Err(Error::unsupported(format!(
737 "special file encountered: {}",
738 path.display()
739 ))),
740 SpecialFilePolicy::Skip => Ok(()),
741 }
742}
743
744fn compile_globs(patterns: &[String]) -> Result<GlobSet> {
745 let mut builder = GlobSetBuilder::new();
746 for pattern in patterns {
747 builder.add(
748 Glob::new(pattern)
749 .map_err(|error| Error::metainfo_field("manifest glob", error.to_string()))?,
750 );
751 }
752 builder
753 .build()
754 .map_err(|error| Error::metainfo_field("manifest glob", error.to_string()))
755}
756
757fn validate_component(component: &[u8], field: &'static str) -> Result<()> {
758 if component.is_empty()
759 || component == b"."
760 || component == b".."
761 || component.contains(&b'/')
762 || component.contains(&b'\\')
763 || component.contains(&0)
764 {
765 return Err(Error::metainfo_field(
766 field,
767 "unsafe torrent path component",
768 ));
769 }
770 Ok(())
771}
772
773#[cfg(unix)]
774#[allow(clippy::unnecessary_wraps)]
775fn os_component_bytes(component: &OsStr) -> Result<Vec<u8>> {
776 use std::os::unix::ffi::OsStrExt as _;
777 Ok(component.as_bytes().to_vec())
778}
779
780#[cfg(not(unix))]
781fn os_component_bytes(component: &OsStr) -> Result<Vec<u8>> {
782 component
783 .to_str()
784 .map(|text| text.as_bytes().to_vec())
785 .ok_or_else(|| Error::unsupported("filesystem name is not valid Unicode on this platform"))
786}
787
788#[derive(Clone, Debug, Eq, PartialEq)]
790pub struct V1HashResult {
791 pieces: Vec<[u8; 20]>,
792 total_bytes: u64,
793}
794
795impl V1HashResult {
796 #[must_use]
798 pub fn pieces(&self) -> &[[u8; 20]] {
799 &self.pieces
800 }
801
802 #[must_use]
804 pub fn piece_count(&self) -> usize {
805 self.pieces.len()
806 }
807
808 #[must_use]
810 pub const fn total_bytes(&self) -> u64 {
811 self.total_bytes
812 }
813
814 #[must_use]
816 pub fn concatenated_pieces(&self) -> Vec<u8> {
817 self.pieces.iter().flatten().copied().collect()
818 }
819}
820
821pub fn hash_v1_sequential(
831 entries: &[ManifestEntry],
832 piece_length: u64,
833 cancellation: &CancellationToken,
834 progress: &impl ProgressSink,
835) -> Result<V1HashResult> {
836 if piece_length == 0 {
837 return Err(Error::metainfo_field("piece length", "must be positive"));
838 }
839 let piece_length = usize::try_from(piece_length)
840 .map_err(|_| Error::metainfo_field("piece length", "cannot fit in memory size"))?;
841 let total_bytes = entries.iter().try_fold(0_u64, |total, entry| {
842 total.checked_add(entry.length).ok_or_else(|| {
843 Error::metainfo_field("manifest length", "total payload length overflowed")
844 })
845 })?;
846 let mut pieces = Vec::new();
847 let mut piece = Vec::with_capacity(piece_length);
848 let mut buffer = vec![0_u8; HASH_READ_BUFFER_LENGTH];
849 let mut bytes_hashed = 0_u64;
850 for entry in entries {
851 cancellation.check()?;
852 let mut file = open_snapshot(entry)?;
853 let mut remaining = entry.length;
854 while remaining > 0 {
855 cancellation.check()?;
856 let available = piece_length - piece.len();
857 let read_limit = usize::try_from(remaining)
858 .unwrap_or(usize::MAX)
859 .min(available)
860 .min(buffer.len());
861 let read = std::io::Read::read(&mut file, &mut buffer[..read_limit])
862 .map_err(|source| Error::io(&entry.source_path, source))?;
863 if read == 0 {
864 return Err(Error::io(
865 &entry.source_path,
866 std::io::Error::new(
867 std::io::ErrorKind::UnexpectedEof,
868 "file became shorter than manifest snapshot",
869 ),
870 ));
871 }
872 piece.extend_from_slice(&buffer[..read]);
873 let read_u64 = u64::try_from(read)
874 .map_err(|_| Error::metainfo_field("hashing", "read size cannot be represented"))?;
875 remaining -= read_u64;
876 bytes_hashed += read_u64;
877 if piece.len() == piece_length {
878 pieces.push(sha1::Sha1::digest(&piece).into());
879 piece.clear();
880 }
881 progress.on_progress(HashProgress {
882 bytes_hashed,
883 total_bytes,
884 pieces_hashed: u64::try_from(pieces.len()).unwrap_or(u64::MAX),
885 });
886 }
887 let mut extra = [0_u8; 1];
888 if std::io::Read::read(&mut file, &mut extra)
889 .map_err(|source| Error::io(&entry.source_path, source))?
890 != 0
891 {
892 return Err(Error::io(
893 &entry.source_path,
894 std::io::Error::new(
895 std::io::ErrorKind::InvalidData,
896 "file became longer than manifest snapshot",
897 ),
898 ));
899 }
900 verify_open_snapshot(entry, &file)?;
901 }
902 if !piece.is_empty() {
903 pieces.push(sha1::Sha1::digest(&piece).into());
904 progress.on_progress(HashProgress {
905 bytes_hashed,
906 total_bytes,
907 pieces_hashed: u64::try_from(pieces.len()).unwrap_or(u64::MAX),
908 });
909 }
910 Ok(V1HashResult {
911 pieces,
912 total_bytes,
913 })
914}
915
916#[derive(Clone, Copy, Debug, Eq, PartialEq)]
918pub struct ParallelHashOptions {
919 workers: usize,
920 queue_capacity: usize,
921}
922
923impl ParallelHashOptions {
924 pub fn new(workers: usize, queue_capacity: usize) -> Result<Self> {
930 if workers == 0 {
931 return Err(Error::metainfo_field("threads", "must be positive"));
932 }
933 if queue_capacity == 0 {
934 return Err(Error::metainfo_field(
935 "hash queue capacity",
936 "must be positive",
937 ));
938 }
939 Ok(Self {
940 workers,
941 queue_capacity,
942 })
943 }
944
945 #[must_use]
947 pub const fn workers(self) -> usize {
948 self.workers
949 }
950
951 #[must_use]
953 pub const fn queue_capacity(self) -> usize {
954 self.queue_capacity
955 }
956
957 #[must_use]
959 pub fn automatic() -> Self {
960 let parallelism = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
961 let workers = automatic_v1_workers(parallelism);
962 Self {
963 workers,
964 queue_capacity: workers,
965 }
966 }
967}
968
969const fn automatic_v1_workers(parallelism: usize) -> usize {
970 if parallelism <= 1 { 1 } else { 2 }
971}
972
973#[derive(Debug)]
974struct PieceJob {
975 sequence: usize,
976 bytes: Vec<u8>,
977}
978
979#[derive(Debug)]
980struct PieceHash {
981 sequence: usize,
982 digest: Result<[u8; 20]>,
983 bytes: Vec<u8>,
984}
985
986pub fn hash_v1_parallel(
998 entries: &[ManifestEntry],
999 piece_length: u64,
1000 options: ParallelHashOptions,
1001 cancellation: &CancellationToken,
1002 progress: &impl ProgressSink,
1003) -> Result<V1HashResult> {
1004 hash_v1_parallel_inner(
1005 entries,
1006 piece_length,
1007 options,
1008 cancellation,
1009 progress,
1010 None,
1011 None,
1012 )
1013}
1014
1015#[allow(clippy::too_many_arguments)]
1016fn hash_v1_parallel_inner(
1017 entries: &[ManifestEntry],
1018 piece_length: u64,
1019 options: ParallelHashOptions,
1020 cancellation: &CancellationToken,
1021 progress: &impl ProgressSink,
1022 fail_sequence: Option<usize>,
1023 slow_sequence: Option<usize>,
1024) -> Result<V1HashResult> {
1025 let (piece_length, piece_length_u64, total_bytes, piece_count) =
1026 v1_parallel_shape(entries, piece_length)?;
1027 if piece_count == 0 {
1028 return Ok(V1HashResult {
1029 pieces: Vec::new(),
1030 total_bytes,
1031 });
1032 }
1033
1034 let buffer_count = options
1035 .workers
1036 .checked_add(options.queue_capacity)
1037 .ok_or_else(|| Error::metainfo_field("hash buffers", "count overflowed"))?;
1038 let (buffer_tx, buffer_rx) = std::sync::mpsc::sync_channel(buffer_count);
1039 for _ in 0..buffer_count {
1040 buffer_tx
1041 .send(Vec::with_capacity(piece_length))
1042 .map_err(|_| Error::unsupported("cannot initialize hash buffer pool"))?;
1043 }
1044 let (job_tx, job_rx) = std::sync::mpsc::sync_channel(options.queue_capacity);
1045 let job_rx = std::sync::Arc::new(std::sync::Mutex::new(job_rx));
1046 let (result_tx, result_rx) = std::sync::mpsc::channel();
1047
1048 std::thread::scope(|scope| {
1049 let producer = scope.spawn(move || {
1050 produce_v1_pieces(entries, piece_length, cancellation, &buffer_rx, &job_tx)
1051 });
1052 for _ in 0..options.workers {
1053 let receiver = std::sync::Arc::clone(&job_rx);
1054 let sender = result_tx.clone();
1055 scope.spawn(move || {
1056 hash_piece_jobs(&receiver, &sender, fail_sequence, slow_sequence);
1057 });
1058 }
1059 drop(result_tx);
1060
1061 let mut pieces = vec![[0_u8; 20]; piece_count];
1062 let mut received = 0_usize;
1063 let mut completed = vec![false; piece_count];
1064 let mut next_progress = 0_usize;
1065 let mut worker_error = None;
1066 for result in result_rx {
1067 match result.digest {
1068 Ok(digest) if result.sequence < pieces.len() => {
1069 pieces[result.sequence] = digest;
1070 completed[result.sequence] = true;
1071 received += 1;
1072 while next_progress < completed.len() && completed[next_progress] {
1073 next_progress += 1;
1074 progress.on_progress(HashProgress {
1075 bytes_hashed: total_bytes.min(
1076 u64::try_from(next_progress)
1077 .unwrap_or(u64::MAX)
1078 .saturating_mul(piece_length_u64),
1079 ),
1080 total_bytes,
1081 pieces_hashed: u64::try_from(next_progress).unwrap_or(u64::MAX),
1082 });
1083 }
1084 }
1085 Ok(_) => {
1086 cancellation.cancel();
1087 worker_error.get_or_insert_with(|| {
1088 Error::metainfo_field(
1089 "piece sequence",
1090 "worker returned out-of-range index",
1091 )
1092 });
1093 }
1094 Err(error) => {
1095 cancellation.cancel();
1096 worker_error.get_or_insert(error);
1097 }
1098 }
1099 let mut bytes = result.bytes;
1100 bytes.clear();
1101 let _ = buffer_tx.try_send(bytes);
1102 }
1103 let producer_result = producer
1104 .join()
1105 .map_err(|_| Error::unsupported("v1 piece reader thread panicked"))?;
1106 if let Some(error) = worker_error {
1107 return Err(error);
1108 }
1109 producer_result?;
1110 cancellation.check()?;
1111 if received != piece_count {
1112 return Err(Error::metainfo_field(
1113 "piece count",
1114 format!("expected {piece_count} worker results, received {received}"),
1115 ));
1116 }
1117 Ok(V1HashResult {
1118 pieces,
1119 total_bytes,
1120 })
1121 })
1122}
1123
1124fn v1_parallel_shape(
1125 entries: &[ManifestEntry],
1126 piece_length: u64,
1127) -> Result<(usize, u64, u64, usize)> {
1128 if piece_length == 0 {
1129 return Err(Error::metainfo_field("piece length", "must be positive"));
1130 }
1131 let piece_length_usize = usize::try_from(piece_length)
1132 .map_err(|_| Error::metainfo_field("piece length", "cannot fit in memory size"))?;
1133 let total_bytes = entries.iter().try_fold(0_u64, |total, entry| {
1134 total.checked_add(entry.length).ok_or_else(|| {
1135 Error::metainfo_field("manifest length", "total payload length overflowed")
1136 })
1137 })?;
1138 let piece_count = usize::try_from(total_bytes.div_ceil(piece_length))
1139 .map_err(|_| Error::metainfo_field("piece count", "cannot fit in memory size"))?;
1140 Ok((piece_length_usize, piece_length, total_bytes, piece_count))
1141}
1142
1143fn produce_v1_pieces(
1144 entries: &[ManifestEntry],
1145 piece_length: usize,
1146 cancellation: &CancellationToken,
1147 buffer_rx: &std::sync::mpsc::Receiver<Vec<u8>>,
1148 job_tx: &std::sync::mpsc::SyncSender<PieceJob>,
1149) -> Result<()> {
1150 let mut sequence = 0_usize;
1151 let mut piece = buffer_rx
1152 .recv()
1153 .map_err(|_| Error::unsupported("hash buffer pool disconnected"))?;
1154 let mut read_buffer = vec![0_u8; HASH_READ_BUFFER_LENGTH];
1155 for entry in entries {
1156 cancellation.check()?;
1157 let mut file = open_snapshot(entry)?;
1158 let mut remaining = entry.length;
1159 while remaining > 0 {
1160 cancellation.check()?;
1161 let available = piece_length - piece.len();
1162 let read_limit = usize::try_from(remaining)
1163 .unwrap_or(usize::MAX)
1164 .min(available)
1165 .min(read_buffer.len());
1166 let read = std::io::Read::read(&mut file, &mut read_buffer[..read_limit])
1167 .map_err(|source| Error::io(&entry.source_path, source))?;
1168 if read == 0 {
1169 return Err(Error::io(
1170 &entry.source_path,
1171 std::io::Error::new(
1172 std::io::ErrorKind::UnexpectedEof,
1173 "file became shorter than manifest snapshot",
1174 ),
1175 ));
1176 }
1177 piece.extend_from_slice(&read_buffer[..read]);
1178 remaining -= u64::try_from(read)
1179 .map_err(|_| Error::metainfo_field("hashing", "read size cannot be represented"))?;
1180 if piece.len() == piece_length {
1181 job_tx
1182 .send(PieceJob {
1183 sequence,
1184 bytes: piece,
1185 })
1186 .map_err(|_| Error::unsupported("hash workers disconnected"))?;
1187 sequence += 1;
1188 piece = buffer_rx
1189 .recv()
1190 .map_err(|_| Error::unsupported("hash buffer pool disconnected"))?;
1191 }
1192 }
1193 let mut extra = [0_u8; 1];
1194 if std::io::Read::read(&mut file, &mut extra)
1195 .map_err(|source| Error::io(&entry.source_path, source))?
1196 != 0
1197 {
1198 return Err(Error::io(
1199 &entry.source_path,
1200 std::io::Error::new(
1201 std::io::ErrorKind::InvalidData,
1202 "file became longer than manifest snapshot",
1203 ),
1204 ));
1205 }
1206 verify_open_snapshot(entry, &file)?;
1207 }
1208 if !piece.is_empty() {
1209 job_tx
1210 .send(PieceJob {
1211 sequence,
1212 bytes: piece,
1213 })
1214 .map_err(|_| Error::unsupported("hash workers disconnected"))?;
1215 }
1216 Ok(())
1217}
1218
1219fn hash_piece_jobs(
1220 receiver: &std::sync::Arc<std::sync::Mutex<std::sync::mpsc::Receiver<PieceJob>>>,
1221 sender: &std::sync::mpsc::Sender<PieceHash>,
1222 fail_sequence: Option<usize>,
1223 slow_sequence: Option<usize>,
1224) {
1225 loop {
1226 let job = match receiver.lock() {
1227 Ok(receiver) => receiver.recv(),
1228 Err(_) => return,
1229 };
1230 let Ok(job) = job else {
1231 return;
1232 };
1233 if slow_sequence == Some(job.sequence) {
1234 std::thread::sleep(std::time::Duration::from_millis(10));
1235 }
1236 let digest = if fail_sequence == Some(job.sequence) {
1237 Err(Error::io(
1238 format!("piece-{}", job.sequence),
1239 std::io::Error::other("injected worker failure"),
1240 ))
1241 } else {
1242 Ok(sha1::Sha1::digest(&job.bytes).into())
1243 };
1244 if sender
1245 .send(PieceHash {
1246 sequence: job.sequence,
1247 digest,
1248 bytes: job.bytes,
1249 })
1250 .is_err()
1251 {
1252 return;
1253 }
1254 }
1255}
1256
1257pub const V2_BLOCK_LENGTH: usize = 16 * 1024;
1259
1260#[derive(Clone, Debug, Eq, PartialEq)]
1262pub struct V2HashResult {
1263 pieces_root: Option<[u8; 32]>,
1264 piece_layer: Vec<[u8; 32]>,
1265 total_bytes: u64,
1266}
1267
1268impl V2HashResult {
1269 #[must_use]
1271 pub const fn pieces_root(&self) -> Option<&[u8; 32]> {
1272 self.pieces_root.as_ref()
1273 }
1274
1275 #[must_use]
1277 pub fn piece_layer(&self) -> &[[u8; 32]] {
1278 &self.piece_layer
1279 }
1280
1281 #[must_use]
1283 pub const fn total_bytes(&self) -> u64 {
1284 self.total_bytes
1285 }
1286}
1287
1288#[must_use]
1290pub fn v2_zero_hash(level: usize) -> [u8; 32] {
1291 let mut hash = [0; 32];
1292 for _ in 0..level {
1293 hash = hash_v2_pair(hash, hash);
1294 }
1295 hash
1296}
1297
1298pub fn hash_v2_file_sequential(
1305 entry: &ManifestEntry,
1306 piece_length: u64,
1307 cancellation: &CancellationToken,
1308 progress: &impl ProgressSink,
1309) -> Result<V2HashResult> {
1310 let mut file = open_snapshot(entry)?;
1311 let result = hash_v2_open_file_sequential(
1312 &mut file,
1313 &entry.source_path,
1314 entry.length,
1315 piece_length,
1316 cancellation,
1317 progress,
1318 )?;
1319 verify_open_snapshot(entry, &file)?;
1320 Ok(result)
1321}
1322
1323pub(crate) fn hash_v2_open_file_sequential(
1324 file: &mut fs::File,
1325 source_path: &Path,
1326 length: u64,
1327 piece_length: u64,
1328 cancellation: &CancellationToken,
1329 progress: &impl ProgressSink,
1330) -> Result<V2HashResult> {
1331 let piece_length = crate::metainfo::validate_v2_piece_length(piece_length)?;
1332 let blocks_per_piece = usize::try_from(piece_length / V2_BLOCK_LENGTH as u64)
1333 .map_err(|_| Error::metainfo_field("piece length", "cannot be represented"))?;
1334 cancellation.check()?;
1335 let mut buffer = [0_u8; V2_BLOCK_LENGTH];
1336 let mut file_tree = MerkleAccumulator::default();
1337 let mut piece_tree = MerkleAccumulator::default();
1338 let mut piece_layer = Vec::new();
1339 let mut blocks_in_piece = 0_usize;
1340 let mut bytes_hashed = 0_u64;
1341
1342 loop {
1343 cancellation.check()?;
1344 let mut block_length = 0_usize;
1345 while block_length < buffer.len() {
1346 let read = std::io::Read::read(&mut *file, &mut buffer[block_length..])
1347 .map_err(|source| Error::io(source_path, source))?;
1348 if read == 0 {
1349 break;
1350 }
1351 block_length += read;
1352 }
1353 if block_length == 0 {
1354 break;
1355 }
1356 let block_hash: [u8; 32] = sha2::Sha256::digest(&buffer[..block_length]).into();
1357 file_tree.push_leaf(block_hash);
1358 piece_tree.push_leaf(block_hash);
1359 blocks_in_piece += 1;
1360 bytes_hashed = bytes_hashed
1361 .checked_add(u64::try_from(block_length).unwrap_or(u64::MAX))
1362 .ok_or_else(|| Error::metainfo_field("hashing", "byte count overflowed"))?;
1363 if blocks_in_piece == blocks_per_piece {
1364 piece_layer.push(piece_tree.finish_power_of_two(blocks_per_piece));
1365 piece_tree = MerkleAccumulator::default();
1366 blocks_in_piece = 0;
1367 }
1368 progress.on_progress(HashProgress {
1369 bytes_hashed,
1370 total_bytes: length,
1371 pieces_hashed: u64::try_from(piece_layer.len()).unwrap_or(u64::MAX),
1372 });
1373 if block_length < buffer.len() {
1374 break;
1375 }
1376 }
1377 cancellation.check()?;
1378 if bytes_hashed != length {
1379 return Err(Error::io(
1380 source_path,
1381 std::io::Error::new(
1382 std::io::ErrorKind::InvalidData,
1383 "file length differs from manifest snapshot",
1384 ),
1385 ));
1386 }
1387 if bytes_hashed == 0 {
1388 return Ok(V2HashResult {
1389 pieces_root: None,
1390 piece_layer: Vec::new(),
1391 total_bytes: 0,
1392 });
1393 }
1394 if blocks_in_piece != 0 && bytes_hashed > piece_length {
1395 piece_layer.push(piece_tree.finish_power_of_two(blocks_per_piece));
1396 }
1397 if bytes_hashed <= piece_length {
1398 piece_layer.clear();
1399 }
1400 progress.on_progress(HashProgress {
1401 bytes_hashed,
1402 total_bytes: length,
1403 pieces_hashed: u64::try_from(piece_layer.len()).unwrap_or(u64::MAX),
1404 });
1405 cancellation.check()?;
1406 let pieces_root = file_tree.finish_next_power_of_two();
1407 Ok(V2HashResult {
1408 pieces_root: Some(pieces_root),
1409 piece_layer,
1410 total_bytes: bytes_hashed,
1411 })
1412}
1413
1414#[derive(Default)]
1415struct MerkleAccumulator {
1416 levels: Vec<Option<[u8; 32]>>,
1417 leaves: usize,
1418}
1419
1420impl MerkleAccumulator {
1421 fn push_leaf(&mut self, hash: [u8; 32]) {
1422 self.push_subtree(hash, 0);
1423 }
1424
1425 fn push_subtree(&mut self, mut hash: [u8; 32], mut level: usize) {
1426 self.leaves += 1_usize << level;
1427 loop {
1428 if self.levels.len() <= level {
1429 self.levels.resize(level + 1, None);
1430 }
1431 if let Some(left) = self.levels[level].take() {
1432 hash = hash_v2_pair(left, hash);
1433 level += 1;
1434 } else {
1435 self.levels[level] = Some(hash);
1436 return;
1437 }
1438 }
1439 }
1440
1441 fn finish_next_power_of_two(mut self) -> [u8; 32] {
1442 let target = self.leaves.next_power_of_two();
1443 self.pad_to(target);
1444 self.root(target)
1445 }
1446
1447 fn finish_power_of_two(mut self, target: usize) -> [u8; 32] {
1448 self.pad_to(target);
1449 self.root(target)
1450 }
1451
1452 fn pad_to(&mut self, target: usize) {
1453 while self.leaves < target {
1454 let remaining = target - self.leaves;
1455 let alignment_level = self.leaves.trailing_zeros() as usize;
1456 let remaining_level = usize::BITS as usize - 1 - remaining.leading_zeros() as usize;
1457 let level = alignment_level.min(remaining_level);
1458 self.push_subtree(v2_zero_hash(level), level);
1459 }
1460 }
1461
1462 fn root(mut self, target: usize) -> [u8; 32] {
1463 let level = target.trailing_zeros() as usize;
1464 self.levels[level]
1465 .take()
1466 .expect("a completed power-of-two tree has one root")
1467 }
1468}
1469
1470fn hash_v2_pair(left: [u8; 32], right: [u8; 32]) -> [u8; 32] {
1471 use sha2::Digest as _;
1472 let mut hasher = sha2::Sha256::new();
1473 hasher.update(left);
1474 hasher.update(right);
1475 hasher.finalize().into()
1476}
1477
1478fn hash_v2_files_parallel(
1479 entries: &[ManifestEntry],
1480 piece_length: u64,
1481 workers: usize,
1482 cancellation: &CancellationToken,
1483 progress: &impl ProgressSink,
1484) -> Result<Vec<V2HashResult>> {
1485 hash_v2_files_parallel_inner(entries, piece_length, workers, cancellation, progress, None)
1486}
1487
1488fn hash_v2_files_parallel_inner(
1489 entries: &[ManifestEntry],
1490 piece_length: u64,
1491 workers: usize,
1492 cancellation: &CancellationToken,
1493 progress: &impl ProgressSink,
1494 worker_counts: Option<&WorkerCounts>,
1495) -> Result<Vec<V2HashResult>> {
1496 let total_bytes = entries.iter().try_fold(0_u64, |total, entry| {
1497 total
1498 .checked_add(entry.length())
1499 .ok_or_else(|| Error::metainfo_field("progress", "byte count overflowed"))
1500 })?;
1501 let next = std::sync::Arc::new(std::sync::Mutex::new(0_usize));
1502 let (sender, receiver) = std::sync::mpsc::sync_channel(workers);
1503 std::thread::scope(|scope| {
1504 for _ in 0..workers.min(entries.len()) {
1505 let next = std::sync::Arc::clone(&next);
1506 let sender = sender.clone();
1507 scope.spawn(move || {
1508 loop {
1509 let index = match next.lock() {
1510 Ok(mut next) if *next < entries.len() => {
1511 let index = *next;
1512 *next += 1;
1513 index
1514 }
1515 _ => return,
1516 };
1517 let _active = ActiveV2Worker::new(worker_counts);
1518 let result = hash_v2_file_sequential(
1519 &entries[index],
1520 piece_length,
1521 cancellation,
1522 &NoProgress,
1523 );
1524 if result.is_err() {
1525 cancellation.cancel();
1526 }
1527 if sender.send((index, result)).is_err() {
1528 return;
1529 }
1530 }
1531 });
1532 }
1533 drop(sender);
1534 let mut ordered = vec![None; entries.len()];
1535 let mut first_error = None;
1536 let mut next_progress = 0_usize;
1537 let mut bytes = 0_u64;
1538 let mut pieces = 0_u64;
1539 for (index, result) in receiver {
1540 match result {
1541 Ok(hashes) => {
1542 ordered[index] = Some(hashes);
1543 while next_progress < ordered.len() && ordered[next_progress].is_some() {
1544 bytes = bytes
1545 .checked_add(entries[next_progress].length())
1546 .ok_or_else(|| {
1547 Error::metainfo_field("hash progress", "byte count overflowed")
1548 })?;
1549 pieces = pieces
1550 .checked_add(entries[next_progress].length().div_ceil(piece_length))
1551 .ok_or_else(|| {
1552 Error::metainfo_field("hash progress", "piece count overflowed")
1553 })?;
1554 progress.on_progress(HashProgress {
1555 bytes_hashed: bytes,
1556 total_bytes,
1557 pieces_hashed: pieces,
1558 });
1559 if let Err(error) = cancellation.check() {
1560 prefer_operation_error(&mut first_error, error);
1561 cancellation.cancel();
1562 }
1563 next_progress += 1;
1564 }
1565 }
1566 Err(error) => {
1567 prefer_operation_error(&mut first_error, error);
1568 }
1569 }
1570 }
1571 if let Some(error) = first_error {
1572 return Err(error);
1573 }
1574 cancellation.check()?;
1575 let mut output = Vec::with_capacity(entries.len());
1576 for hashes in ordered {
1577 let hashes = hashes.ok_or_else(|| {
1578 Error::unsupported("v2 worker exited without returning a file result")
1579 })?;
1580 output.push(hashes);
1581 }
1582 Ok(output)
1583 })
1584}
1585
1586fn prefer_operation_error(current: &mut Option<Error>, candidate: Error) {
1587 if current
1588 .as_ref()
1589 .is_none_or(|error| error.category() == crate::ErrorCategory::Cancelled)
1590 || candidate.category() != crate::ErrorCategory::Cancelled
1591 {
1592 *current = Some(candidate);
1593 }
1594}
1595
1596struct WorkerCounts {
1597 active: std::sync::atomic::AtomicUsize,
1598 peak: std::sync::atomic::AtomicUsize,
1599}
1600
1601struct ActiveV2Worker<'a> {
1602 active: Option<&'a std::sync::atomic::AtomicUsize>,
1603}
1604
1605impl<'a> ActiveV2Worker<'a> {
1606 fn new(counts: Option<&'a WorkerCounts>) -> Self {
1607 let active = counts.map(|counts| {
1608 let active = counts
1609 .active
1610 .fetch_add(1, std::sync::atomic::Ordering::AcqRel)
1611 + 1;
1612 counts
1613 .peak
1614 .fetch_max(active, std::sync::atomic::Ordering::AcqRel);
1615 &counts.active
1616 });
1617 Self { active }
1618 }
1619}
1620
1621impl Drop for ActiveV2Worker<'_> {
1622 fn drop(&mut self) {
1623 if let Some(active) = self.active {
1624 active.fetch_sub(1, std::sync::atomic::Ordering::AcqRel);
1625 }
1626 }
1627}
1628
1629fn verify_snapshot(entry: &ManifestEntry) -> Result<()> {
1630 let metadata =
1631 fs::metadata(&entry.source_path).map_err(|source| Error::io(&entry.source_path, source))?;
1632 verify_snapshot_metadata(entry, &metadata)?;
1633 if snapshot_matches_path(&entry.snapshot, &entry.source_path)? {
1634 Ok(())
1635 } else {
1636 snapshot_changed(entry)
1637 }
1638}
1639
1640fn verify_snapshot_metadata(entry: &ManifestEntry, metadata: &fs::Metadata) -> Result<()> {
1641 if !entry.snapshot.matches_metadata(metadata) {
1642 return snapshot_changed(entry);
1643 }
1644 Ok(())
1645}
1646
1647fn snapshot_matches_path(snapshot: &FileSnapshot, path: &Path) -> Result<bool> {
1648 let Some(expected) = &snapshot.identity else {
1649 return Ok(true);
1650 };
1651 let current = file_id::get_file_id(path).map_err(|source| Error::io(path, source))?;
1652 Ok(current == *expected)
1653}
1654
1655fn snapshot_changed(entry: &ManifestEntry) -> Result<()> {
1656 Err(Error::io(
1657 &entry.source_path,
1658 std::io::Error::new(
1659 std::io::ErrorKind::InvalidData,
1660 "file metadata changed since manifest scan",
1661 ),
1662 ))
1663}
1664
1665fn open_snapshot(entry: &ManifestEntry) -> Result<fs::File> {
1666 verify_snapshot(entry)?;
1667 let file = fs::File::open(&entry.source_path)
1668 .map_err(|source| Error::io(&entry.source_path, source))?;
1669 let metadata = file
1670 .metadata()
1671 .map_err(|source| Error::io(&entry.source_path, source))?;
1672 verify_snapshot_metadata(entry, &metadata)?;
1673 verify_snapshot_identity(entry, &file)?;
1674 verify_snapshot(entry)?;
1675 Ok(file)
1676}
1677
1678fn verify_open_snapshot(entry: &ManifestEntry, file: &fs::File) -> Result<()> {
1679 let metadata = file
1680 .metadata()
1681 .map_err(|source| Error::io(&entry.source_path, source))?;
1682 verify_snapshot_metadata(entry, &metadata)?;
1683 verify_snapshot_identity(entry, file)?;
1684 verify_snapshot(entry)
1685}
1686
1687fn verify_snapshot_identity(entry: &ManifestEntry, file: &fs::File) -> Result<()> {
1688 if entry.snapshot.identity.is_none() {
1689 return Ok(());
1690 }
1691 let open = same_file::Handle::from_file(
1692 file.try_clone()
1693 .map_err(|source| Error::io(&entry.source_path, source))?,
1694 )
1695 .map_err(|source| Error::io(&entry.source_path, source))?;
1696 let path = same_file::Handle::from_path(&entry.source_path)
1697 .map_err(|source| Error::io(&entry.source_path, source))?;
1698 if open == path {
1699 Ok(())
1700 } else {
1701 snapshot_changed(entry)
1702 }
1703}
1704
1705#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1707pub enum PieceLength {
1708 #[default]
1710 Automatic,
1711 Exact(u64),
1713 Target { pieces: u64, maximum: u64 },
1715}
1716
1717#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1719pub enum CreateMode {
1720 #[default]
1722 V1,
1723 V2,
1725 Hybrid,
1727}
1728
1729#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1731pub enum OverwritePolicy {
1732 #[default]
1734 Deny,
1735 Replace,
1737}
1738
1739#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1741pub enum DurabilityPolicy {
1742 #[default]
1744 File,
1745 FileAndDirectory,
1747}
1748
1749#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1751pub enum HashThreads {
1752 #[default]
1754 Automatic,
1755 Exact(usize),
1757}
1758
1759#[derive(Clone, Debug)]
1761pub struct CreateOptions {
1762 manifest: ManifestOptions,
1763 mode: CreateMode,
1764 piece_length: PieceLength,
1765 hash_threads: HashThreads,
1766 trackers: Vec<TrackerTier>,
1767 web_seeds: Vec<WebSeed>,
1768 nodes: Vec<(NodeHost, u16)>,
1769 private: Option<bool>,
1770 source: Option<MetadataText>,
1771 comment: Option<MetadataText>,
1772 created_by: CreatorIdentity,
1773 creation_date: Option<i64>,
1774 entropy: Option<Vec<u8>>,
1775}
1776
1777impl Default for CreateOptions {
1778 fn default() -> Self {
1779 Self::builder()
1780 .build()
1781 .expect("default creation options are valid")
1782 }
1783}
1784
1785impl CreateOptions {
1786 #[must_use]
1788 pub fn builder() -> CreateOptionsBuilder {
1789 CreateOptionsBuilder::default()
1790 }
1791}
1792
1793#[derive(Clone, Debug)]
1795pub struct CreateOptionsBuilder {
1796 manifest: ManifestOptions,
1797 mode: CreateMode,
1798 piece_length: PieceLength,
1799 hash_threads: HashThreads,
1800 trackers: Vec<Vec<Vec<u8>>>,
1801 web_seeds: Vec<Vec<u8>>,
1802 nodes: Vec<(Vec<u8>, u16)>,
1803 private: Option<bool>,
1804 source: Option<Vec<u8>>,
1805 comment: Option<Vec<u8>>,
1806 created_by: CreatorIdentity,
1807 creation_date: Option<i64>,
1808 entropy: Option<Vec<u8>>,
1809}
1810
1811#[derive(Clone, Debug, Eq, PartialEq)]
1812enum CreatorIdentity {
1813 Default,
1814 Explicit(Vec<u8>),
1815 Omit,
1816}
1817
1818impl Default for CreateOptionsBuilder {
1819 fn default() -> Self {
1820 Self {
1821 manifest: ManifestOptions::default(),
1822 mode: CreateMode::default(),
1823 piece_length: PieceLength::default(),
1824 hash_threads: HashThreads::default(),
1825 trackers: Vec::new(),
1826 web_seeds: Vec::new(),
1827 nodes: Vec::new(),
1828 private: None,
1829 source: None,
1830 comment: None,
1831 created_by: CreatorIdentity::Default,
1832 creation_date: None,
1833 entropy: None,
1834 }
1835 }
1836}
1837
1838impl CreateOptionsBuilder {
1839 #[must_use]
1841 pub fn manifest(mut self, manifest: ManifestOptions) -> Self {
1842 self.manifest = manifest;
1843 self
1844 }
1845
1846 #[must_use]
1848 pub const fn mode(mut self, mode: CreateMode) -> Self {
1849 self.mode = mode;
1850 self
1851 }
1852
1853 #[must_use]
1855 pub const fn piece_length(mut self, piece_length: PieceLength) -> Self {
1856 self.piece_length = piece_length;
1857 self
1858 }
1859
1860 #[must_use]
1862 pub const fn hash_threads(mut self, hash_threads: HashThreads) -> Self {
1863 self.hash_threads = hash_threads;
1864 self
1865 }
1866
1867 #[must_use]
1869 pub fn trackers(mut self, trackers: impl IntoIterator<Item = Vec<Vec<u8>>>) -> Self {
1870 self.trackers = trackers.into_iter().collect();
1871 self
1872 }
1873
1874 #[must_use]
1876 pub fn web_seeds(mut self, web_seeds: impl IntoIterator<Item = Vec<u8>>) -> Self {
1877 self.web_seeds = web_seeds.into_iter().collect();
1878 self
1879 }
1880
1881 #[must_use]
1883 pub fn nodes(mut self, nodes: impl IntoIterator<Item = (Vec<u8>, u16)>) -> Self {
1884 self.nodes = nodes.into_iter().collect();
1885 self
1886 }
1887
1888 #[must_use]
1890 pub const fn private(mut self, private: bool) -> Self {
1891 self.private = Some(private);
1892 self
1893 }
1894
1895 #[must_use]
1897 pub fn source(mut self, source: Vec<u8>) -> Self {
1898 self.source = Some(source);
1899 self
1900 }
1901
1902 #[must_use]
1904 pub fn comment(mut self, comment: Vec<u8>) -> Self {
1905 self.comment = Some(comment);
1906 self
1907 }
1908
1909 #[must_use]
1911 pub fn created_by(mut self, created_by: Vec<u8>) -> Self {
1912 self.created_by = CreatorIdentity::Explicit(created_by);
1913 self
1914 }
1915
1916 #[must_use]
1918 pub fn omit_created_by(mut self) -> Self {
1919 self.created_by = CreatorIdentity::Omit;
1920 self
1921 }
1922
1923 #[must_use]
1925 pub const fn creation_date(mut self, creation_date: i64) -> Self {
1926 self.creation_date = Some(creation_date);
1927 self
1928 }
1929
1930 #[must_use]
1932 pub fn entropy(mut self, entropy: Vec<u8>) -> Self {
1933 self.entropy = Some(entropy);
1934 self
1935 }
1936
1937 pub fn build(self) -> Result<CreateOptions> {
1944 if self.hash_threads == HashThreads::Exact(0) {
1945 return Err(Error::metainfo_field("threads", "must be positive"));
1946 }
1947 validate_tracker_tiers(&self.trackers)?;
1948 validate_web_seeds(&self.web_seeds)?;
1949 validate_nodes(&self.nodes)?;
1950 validate_creation_date(self.creation_date)?;
1951 Ok(CreateOptions {
1952 manifest: self.manifest,
1953 mode: self.mode,
1954 piece_length: self.piece_length,
1955 hash_threads: self.hash_threads,
1956 trackers: self.trackers.into_iter().map(TrackerTier::new).collect(),
1957 web_seeds: self.web_seeds.into_iter().map(WebSeed::new).collect(),
1958 nodes: self
1959 .nodes
1960 .into_iter()
1961 .map(|(host, port)| (NodeHost::new(host), port))
1962 .collect(),
1963 private: self.private,
1964 source: self.source.map(MetadataText::new),
1965 comment: self.comment.map(MetadataText::new),
1966 created_by: self.created_by,
1967 creation_date: self.creation_date,
1968 entropy: self.entropy,
1969 })
1970 }
1971}
1972
1973#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1975pub struct CreateMetrics {
1976 scan: std::time::Duration,
1977 hash: std::time::Duration,
1978 serialize: std::time::Duration,
1979}
1980
1981impl CreateMetrics {
1982 #[must_use]
1984 pub const fn scan(self) -> std::time::Duration {
1985 self.scan
1986 }
1987
1988 #[must_use]
1990 pub const fn hash(self) -> std::time::Duration {
1991 self.hash
1992 }
1993
1994 #[must_use]
1996 pub const fn serialize(self) -> std::time::Duration {
1997 self.serialize
1998 }
1999}
2000
2001#[derive(Clone, Debug)]
2003pub struct CreateResult {
2004 bytes: Vec<u8>,
2005 mode: CreateMode,
2006 info_hash_v1: Option<crate::metainfo::InfoHashV1>,
2007 info_hash_v2: Option<crate::metainfo::InfoHashV2>,
2008 file_count: usize,
2009 payload_bytes: u64,
2010 piece_count: usize,
2011 piece_length: u64,
2012 piece_length_policy: Option<&'static str>,
2013 metrics: CreateMetrics,
2014}
2015
2016impl CreateResult {
2017 #[must_use]
2019 pub fn bytes(&self) -> &[u8] {
2020 &self.bytes
2021 }
2022
2023 #[must_use]
2025 pub const fn mode(&self) -> CreateMode {
2026 self.mode
2027 }
2028
2029 #[must_use]
2031 pub const fn info_hash_v1(&self) -> Option<crate::metainfo::InfoHashV1> {
2032 self.info_hash_v1
2033 }
2034
2035 #[must_use]
2037 pub const fn info_hash_v2(&self) -> Option<crate::metainfo::InfoHashV2> {
2038 self.info_hash_v2
2039 }
2040
2041 #[must_use]
2043 pub const fn file_count(&self) -> usize {
2044 self.file_count
2045 }
2046
2047 #[must_use]
2049 pub const fn payload_bytes(&self) -> u64 {
2050 self.payload_bytes
2051 }
2052
2053 #[must_use]
2055 pub const fn piece_count(&self) -> usize {
2056 self.piece_count
2057 }
2058
2059 #[must_use]
2061 pub const fn piece_length(&self) -> u64 {
2062 self.piece_length
2063 }
2064
2065 #[must_use]
2067 pub const fn piece_length_policy(&self) -> Option<&'static str> {
2068 self.piece_length_policy
2069 }
2070
2071 #[must_use]
2073 pub const fn metrics(&self) -> CreateMetrics {
2074 self.metrics
2075 }
2076}
2077
2078#[derive(Clone, Debug)]
2080pub struct Creator {
2081 input: PathBuf,
2082 options: CreateOptions,
2083 cancellation: CancellationToken,
2084}
2085
2086impl Creator {
2087 #[must_use]
2089 pub fn new(input: impl Into<PathBuf>) -> Self {
2090 Self {
2091 input: input.into(),
2092 options: CreateOptions::default(),
2093 cancellation: CancellationToken::new(),
2094 }
2095 }
2096
2097 #[must_use]
2099 pub fn options(mut self, options: CreateOptions) -> Self {
2100 self.options = options;
2101 self
2102 }
2103
2104 #[must_use]
2106 pub fn cancellation(mut self, cancellation: CancellationToken) -> Self {
2107 self.cancellation = cancellation;
2108 self
2109 }
2110
2111 pub fn create(&self, progress: &impl ProgressSink) -> Result<CreateResult> {
2117 let scan_started = std::time::Instant::now();
2118 let manifest = scan_manifest(&self.input, &self.options.manifest)?;
2119 let scan = scan_started.elapsed();
2120 let piece_length_mode = match self.options.mode {
2121 CreateMode::V1 => PieceLengthMode::V1,
2122 CreateMode::V2 | CreateMode::Hybrid => PieceLengthMode::V2,
2123 };
2124 let (piece_length, piece_length_policy) = match self.options.piece_length {
2125 PieceLength::Automatic => (
2126 automatic_piece_length(manifest.total_length()),
2127 Some(PIECE_LENGTH_POLICY_ID),
2128 ),
2129 PieceLength::Exact(piece_length) => (
2130 validate_piece_length(piece_length, piece_length_mode)?,
2131 None,
2132 ),
2133 PieceLength::Target { pieces, maximum } => {
2134 if pieces == 0 {
2135 return Err(Error::metainfo_field("target pieces", "must be positive"));
2136 }
2137 let minimum = match piece_length_mode {
2138 PieceLengthMode::V1 => 1024,
2139 PieceLengthMode::V2 | PieceLengthMode::Hybrid => 16 * 1024,
2140 };
2141 let maximum = validate_piece_length(maximum, piece_length_mode)?;
2142 let needed = manifest.total_length().div_ceil(pieces).max(minimum);
2143 let selected = needed.next_power_of_two().min(maximum);
2144 (
2145 validate_piece_length(selected, piece_length_mode)?,
2146 Some("btpc-target-pieces-v1"),
2147 )
2148 }
2149 };
2150 let hash_started = std::time::Instant::now();
2151 let (info, piece_layers, piece_count) = match self.options.mode {
2152 CreateMode::V1 => {
2153 let hashes = self.hash_v1(&manifest, piece_length, progress)?;
2154 let piece_count = hashes.piece_count();
2155 (
2156 build_v1_info(&manifest, piece_length, &hashes, &self.options)?,
2157 None,
2158 piece_count,
2159 )
2160 }
2161 CreateMode::V2 => {
2162 let (info, layers, piece_count) =
2163 self.build_v2(&manifest, piece_length, progress)?;
2164 (info, layers, piece_count)
2165 }
2166 CreateMode::Hybrid => {
2167 validate_hybrid_manifest(&manifest)?;
2168 let (v2_info, layers, _) = self.build_v2(&manifest, piece_length, progress)?;
2169 let (hashes, padding) = self.hash_hybrid_v1(&manifest, piece_length)?;
2170 let piece_count = hashes.piece_count();
2171 (
2172 build_hybrid_info(v2_info, &manifest, &hashes, &padding)?,
2173 layers,
2174 piece_count,
2175 )
2176 }
2177 };
2178 let hash = hash_started.elapsed();
2179 let serialize_started = std::time::Instant::now();
2180 let info_bytes = info.to_vec()?;
2181 let info_hash_v1 = (self.options.mode != CreateMode::V2)
2182 .then(|| crate::metainfo::InfoHashV1::new(sha1::Sha1::digest(&info_bytes).into()));
2183 let info_hash_v2 = (self.options.mode != CreateMode::V1)
2184 .then(|| crate::metainfo::InfoHashV2::new(sha2::Sha256::digest(&info_bytes).into()));
2185 let metainfo = build_metainfo(info, piece_layers, &self.options)?;
2186 let bytes = metainfo.to_vec()?;
2187 let serialize = serialize_started.elapsed();
2188 Ok(CreateResult {
2189 bytes,
2190 mode: self.options.mode,
2191 info_hash_v1,
2192 info_hash_v2,
2193 file_count: manifest.entries().len(),
2194 payload_bytes: manifest.total_length(),
2195 piece_count,
2196 piece_length,
2197 piece_length_policy,
2198 metrics: CreateMetrics {
2199 scan,
2200 hash,
2201 serialize,
2202 },
2203 })
2204 }
2205
2206 fn hash_v1(
2207 &self,
2208 manifest: &PayloadManifest,
2209 piece_length: u64,
2210 progress: &impl ProgressSink,
2211 ) -> Result<V1HashResult> {
2212 match self.options.hash_threads {
2213 HashThreads::Exact(1) => hash_v1_sequential(
2214 manifest.entries(),
2215 piece_length,
2216 &self.cancellation,
2217 progress,
2218 ),
2219 HashThreads::Automatic => {
2220 let options = ParallelHashOptions::automatic();
2221 if options.workers() == 1 {
2222 hash_v1_sequential(
2223 manifest.entries(),
2224 piece_length,
2225 &self.cancellation,
2226 progress,
2227 )
2228 } else {
2229 hash_v1_parallel(
2230 manifest.entries(),
2231 piece_length,
2232 options,
2233 &self.cancellation,
2234 progress,
2235 )
2236 }
2237 }
2238 HashThreads::Exact(workers) => hash_v1_parallel(
2239 manifest.entries(),
2240 piece_length,
2241 ParallelHashOptions::new(workers, workers)?,
2242 &self.cancellation,
2243 progress,
2244 ),
2245 }
2246 }
2247
2248 fn hash_hybrid_v1(
2249 &self,
2250 manifest: &PayloadManifest,
2251 piece_length: u64,
2252 ) -> Result<(V1HashResult, Vec<HybridPadding>)> {
2253 let workers = match self.options.hash_threads {
2254 HashThreads::Exact(1) => 1,
2255 HashThreads::Exact(workers) => workers,
2256 HashThreads::Automatic => ParallelHashOptions::automatic().workers(),
2257 };
2258 if workers == 1 || manifest.entries().len() <= 1 {
2259 return hash_hybrid_v1_sequential(
2260 manifest.entries(),
2261 piece_length,
2262 &self.cancellation,
2263 &NoProgress,
2264 );
2265 }
2266 hash_hybrid_v1_parallel(
2267 manifest.entries(),
2268 piece_length,
2269 workers,
2270 &self.cancellation,
2271 )
2272 }
2273
2274 fn build_v2(
2275 &self,
2276 manifest: &PayloadManifest,
2277 piece_length: u64,
2278 progress: &impl ProgressSink,
2279 ) -> Result<(
2280 crate::bencode::OwnedValue,
2281 Option<crate::bencode::OwnedValue>,
2282 usize,
2283 )> {
2284 let hashed = self.hash_v2_files(manifest, piece_length, progress)?;
2285 let mut files = Vec::with_capacity(manifest.entries().len());
2286 let mut layers = std::collections::BTreeMap::new();
2287 let mut piece_count = 0_usize;
2288 for (entry, hashes) in manifest.entries().iter().zip(hashed) {
2289 piece_count = piece_count
2290 .checked_add(
2291 usize::try_from(entry.length().div_ceil(piece_length)).unwrap_or(usize::MAX),
2292 )
2293 .ok_or_else(|| Error::metainfo_field("piece count", "overflowed"))?;
2294 if !hashes.piece_layer().is_empty() {
2295 let root = *hashes
2296 .pieces_root()
2297 .expect("non-empty piece layers have roots");
2298 let layer = hashes
2299 .piece_layer()
2300 .iter()
2301 .flatten()
2302 .copied()
2303 .collect::<Vec<_>>();
2304 if let Some(existing) = layers.insert(root.to_vec(), layer.clone()) {
2305 debug_assert_eq!(existing, layer, "equal roots have equal piece layers");
2306 }
2307 }
2308 files.push((entry, hashes));
2309 }
2310 let layers = (!layers.is_empty()).then(|| {
2311 crate::bencode::OwnedValue::Dictionary(
2312 layers
2313 .into_iter()
2314 .map(|(root, layer)| (root, crate::bencode::OwnedValue::bytes(layer)))
2315 .collect(),
2316 )
2317 });
2318 Ok((
2319 build_v2_info(manifest, piece_length, &files, &self.options)?,
2320 layers,
2321 piece_count,
2322 ))
2323 }
2324
2325 fn hash_v2_files(
2326 &self,
2327 manifest: &PayloadManifest,
2328 piece_length: u64,
2329 progress: &impl ProgressSink,
2330 ) -> Result<Vec<V2HashResult>> {
2331 let workers = match self.options.hash_threads {
2332 HashThreads::Exact(1) => 1,
2333 HashThreads::Exact(workers) => workers,
2334 HashThreads::Automatic => ParallelHashOptions::automatic().workers(),
2335 };
2336 if workers == 1 || manifest.entries().len() <= 1 {
2337 let mut output = Vec::with_capacity(manifest.entries().len());
2338 let mut bytes_before = 0_u64;
2339 let mut pieces_before = 0_u64;
2340 for entry in manifest.entries() {
2341 let aggregate = AggregateProgress {
2342 inner: progress,
2343 bytes_before,
2344 pieces_before,
2345 total_bytes: manifest.total_length(),
2346 };
2347 output.push(hash_v2_file_sequential(
2348 entry,
2349 piece_length,
2350 &self.cancellation,
2351 &aggregate,
2352 )?);
2353 bytes_before = bytes_before.checked_add(entry.length()).ok_or_else(|| {
2354 Error::metainfo_field("hash progress", "byte count overflowed")
2355 })?;
2356 pieces_before = pieces_before
2357 .checked_add(entry.length().div_ceil(piece_length))
2358 .ok_or_else(|| {
2359 Error::metainfo_field("hash progress", "piece count overflowed")
2360 })?;
2361 }
2362 return Ok(output);
2363 }
2364 hash_v2_files_parallel(
2365 manifest.entries(),
2366 piece_length,
2367 workers,
2368 &self.cancellation,
2369 progress,
2370 )
2371 }
2372
2373 pub fn create_to_path(
2381 &self,
2382 destination: impl AsRef<Path>,
2383 overwrite: OverwritePolicy,
2384 progress: &impl ProgressSink,
2385 ) -> Result<CreateResult> {
2386 self.create_to_path_with_durability(
2387 destination,
2388 overwrite,
2389 DurabilityPolicy::File,
2390 progress,
2391 )
2392 }
2393
2394 pub fn create_to_path_with_durability(
2400 &self,
2401 destination: impl AsRef<Path>,
2402 overwrite: OverwritePolicy,
2403 durability: DurabilityPolicy,
2404 progress: &impl ProgressSink,
2405 ) -> Result<CreateResult> {
2406 let result = self.create(progress)?;
2407 write_atomic(destination.as_ref(), result.bytes(), overwrite, durability)?;
2408 Ok(result)
2409 }
2410}
2411
2412struct AggregateProgress<'a, P> {
2413 inner: &'a P,
2414 bytes_before: u64,
2415 pieces_before: u64,
2416 total_bytes: u64,
2417}
2418
2419impl<P: ProgressSink> ProgressSink for AggregateProgress<'_, P> {
2420 fn on_progress(&self, progress: HashProgress) {
2421 self.inner.on_progress(HashProgress {
2422 bytes_hashed: self
2423 .bytes_before
2424 .checked_add(progress.bytes_hashed())
2425 .expect("creation byte progress fits the checked manifest total"),
2426 total_bytes: self.total_bytes,
2427 pieces_hashed: self
2428 .pieces_before
2429 .checked_add(progress.pieces_hashed())
2430 .expect("creation piece progress fits the checked aggregate"),
2431 });
2432 }
2433}
2434
2435fn build_v1_info(
2436 manifest: &PayloadManifest,
2437 piece_length: u64,
2438 hashes: &V1HashResult,
2439 options: &CreateOptions,
2440) -> Result<crate::bencode::OwnedValue> {
2441 use crate::bencode::OwnedValue;
2442 let piece_length = i64::try_from(piece_length)
2443 .map_err(|_| Error::metainfo_field("piece length", "cannot fit bencode integer"))?;
2444 let mut entries = vec![
2445 (
2446 b"name".to_vec(),
2447 OwnedValue::bytes(manifest.root_name().to_vec()),
2448 ),
2449 (b"piece length".to_vec(), OwnedValue::integer(piece_length)),
2450 (
2451 b"pieces".to_vec(),
2452 OwnedValue::bytes(hashes.concatenated_pieces()),
2453 ),
2454 ];
2455 if manifest.entries().len() == 1
2456 && manifest.entries()[0].torrent_path() == [manifest.root_name().to_vec()]
2457 {
2458 entries.push((
2459 b"length".to_vec(),
2460 OwnedValue::integer(length_to_i64(manifest.entries()[0].length())?),
2461 ));
2462 } else {
2463 let files = manifest
2464 .entries()
2465 .iter()
2466 .map(|entry| {
2467 OwnedValue::dictionary([
2468 (
2469 b"length".to_vec(),
2470 OwnedValue::integer(length_to_i64(entry.length())?),
2471 ),
2472 (
2473 b"path".to_vec(),
2474 OwnedValue::list(
2475 entry.torrent_path().iter().cloned().map(OwnedValue::bytes),
2476 ),
2477 ),
2478 ])
2479 })
2480 .collect::<Result<Vec<_>>>()?;
2481 entries.push((b"files".to_vec(), OwnedValue::list(files)));
2482 }
2483 if let Some(private) = options.private {
2484 entries.push((b"private".to_vec(), OwnedValue::integer(i64::from(private))));
2485 }
2486 if let Some(source) = &options.source {
2487 entries.push((
2488 b"source".to_vec(),
2489 OwnedValue::bytes(source.as_bytes().to_vec()),
2490 ));
2491 }
2492 if let Some(entropy) = &options.entropy {
2493 entries.push((b"entropy".to_vec(), OwnedValue::bytes(entropy.clone())));
2494 }
2495 OwnedValue::dictionary(entries)
2496}
2497
2498fn build_v2_info(
2499 manifest: &PayloadManifest,
2500 piece_length: u64,
2501 files: &[(&ManifestEntry, V2HashResult)],
2502 options: &CreateOptions,
2503) -> Result<crate::bencode::OwnedValue> {
2504 use crate::bencode::OwnedValue;
2505 let mut tree = std::collections::BTreeMap::new();
2506 for (entry, hashes) in files {
2507 let relative_path = v2_relative_path(manifest, entry);
2508 insert_v2_file(
2509 &mut tree,
2510 relative_path,
2511 entry.length(),
2512 hashes.pieces_root(),
2513 )?;
2514 }
2515 let piece_length = i64::try_from(piece_length)
2516 .map_err(|_| Error::metainfo_field("piece length", "cannot fit bencode integer"))?;
2517 let mut entries = vec![
2518 (b"file tree".to_vec(), OwnedValue::Dictionary(tree)),
2519 (b"meta version".to_vec(), OwnedValue::integer(2)),
2520 (
2521 b"name".to_vec(),
2522 OwnedValue::bytes(manifest.root_name().to_vec()),
2523 ),
2524 (b"piece length".to_vec(), OwnedValue::integer(piece_length)),
2525 ];
2526 if let Some(private) = options.private {
2527 entries.push((b"private".to_vec(), OwnedValue::integer(i64::from(private))));
2528 }
2529 if let Some(source) = &options.source {
2530 entries.push((
2531 b"source".to_vec(),
2532 OwnedValue::bytes(source.as_bytes().to_vec()),
2533 ));
2534 }
2535 if let Some(entropy) = &options.entropy {
2536 entries.push((b"entropy".to_vec(), OwnedValue::bytes(entropy.clone())));
2537 }
2538 OwnedValue::dictionary(entries)
2539}
2540
2541#[derive(Clone, Debug, Eq, PartialEq)]
2542struct HybridPadding {
2543 after_file: usize,
2544 length: u64,
2545 offset: u64,
2546}
2547
2548fn validate_hybrid_manifest(manifest: &PayloadManifest) -> Result<()> {
2549 if manifest.entries().len() > 1
2550 && manifest.entries().iter().any(|entry| {
2551 entry
2552 .torrent_path()
2553 .first()
2554 .is_some_and(|part| part == b".pad")
2555 })
2556 {
2557 return Err(Error::metainfo_field(
2558 "manifest path",
2559 "hybrid creation reserves the .pad path component",
2560 ));
2561 }
2562 Ok(())
2563}
2564
2565#[derive(Clone, Debug)]
2566struct HybridFileHash {
2567 pieces: Vec<[u8; 20]>,
2568 padding: u64,
2569}
2570
2571fn hash_hybrid_v1_parallel(
2572 entries: &[ManifestEntry],
2573 piece_length: u64,
2574 workers: usize,
2575 cancellation: &CancellationToken,
2576) -> Result<(V1HashResult, Vec<HybridPadding>)> {
2577 let next = std::sync::Arc::new(std::sync::Mutex::new(0_usize));
2578 let (sender, receiver) = std::sync::mpsc::sync_channel(workers);
2579 std::thread::scope(|scope| {
2580 for _ in 0..workers.min(entries.len()) {
2581 let next = std::sync::Arc::clone(&next);
2582 let sender = sender.clone();
2583 scope.spawn(move || {
2584 loop {
2585 let index = match next.lock() {
2586 Ok(mut next) if *next < entries.len() => {
2587 let index = *next;
2588 *next += 1;
2589 index
2590 }
2591 _ => return,
2592 };
2593 let result = hash_hybrid_file(
2594 &entries[index],
2595 piece_length,
2596 index + 1 < entries.len(),
2597 cancellation,
2598 );
2599 if result.is_err() {
2600 cancellation.cancel();
2601 }
2602 if sender.send((index, result)).is_err() {
2603 return;
2604 }
2605 }
2606 });
2607 }
2608 drop(sender);
2609 let mut ordered = vec![None; entries.len()];
2610 let mut first_error = None;
2611 for (index, result) in receiver {
2612 match result {
2613 Ok(hashes) => ordered[index] = Some(hashes),
2614 Err(error) => {
2615 prefer_operation_error(&mut first_error, error);
2616 }
2617 }
2618 }
2619 if let Some(error) = first_error {
2620 return Err(error);
2621 }
2622 cancellation.check()?;
2623 let mut pieces = Vec::new();
2624 let mut padding = Vec::new();
2625 let mut logical_offset = 0_u64;
2626 for (index, hashes) in ordered.into_iter().enumerate() {
2627 let hashes = hashes.ok_or_else(|| {
2628 Error::unsupported("hybrid worker exited without returning a file result")
2629 })?;
2630 pieces.extend(hashes.pieces);
2631 logical_offset = logical_offset
2632 .checked_add(entries[index].length())
2633 .ok_or_else(|| Error::metainfo_field("hybrid length", "overflowed"))?;
2634 if hashes.padding != 0 {
2635 padding.push(HybridPadding {
2636 after_file: index,
2637 length: hashes.padding,
2638 offset: logical_offset,
2639 });
2640 logical_offset = logical_offset
2641 .checked_add(hashes.padding)
2642 .ok_or_else(|| Error::metainfo_field("hybrid length", "overflowed"))?;
2643 }
2644 }
2645 Ok((
2646 V1HashResult {
2647 pieces,
2648 total_bytes: logical_offset,
2649 },
2650 padding,
2651 ))
2652 })
2653}
2654
2655fn hash_hybrid_file(
2656 entry: &ManifestEntry,
2657 piece_length: u64,
2658 pad_to_boundary: bool,
2659 cancellation: &CancellationToken,
2660) -> Result<HybridFileHash> {
2661 let piece_length = usize::try_from(piece_length)
2662 .map_err(|_| Error::metainfo_field("piece length", "cannot be represented"))?;
2663 cancellation.check()?;
2664 let mut file = open_snapshot(entry)?;
2665 let mut piece = Vec::with_capacity(piece_length);
2666 let mut pieces = Vec::new();
2667 let mut buffer = vec![0_u8; HASH_READ_BUFFER_LENGTH];
2668 let mut remaining = entry.length();
2669 while remaining > 0 {
2670 cancellation.check()?;
2671 let limit = usize::try_from(remaining)
2672 .unwrap_or(usize::MAX)
2673 .min(piece_length - piece.len())
2674 .min(buffer.len());
2675 let read = std::io::Read::read(&mut file, &mut buffer[..limit])
2676 .map_err(|source| Error::io(&entry.source_path, source))?;
2677 if read == 0 {
2678 return Err(Error::io(
2679 &entry.source_path,
2680 std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "file became shorter"),
2681 ));
2682 }
2683 piece.extend_from_slice(&buffer[..read]);
2684 remaining -= u64::try_from(read)
2685 .map_err(|_| Error::metainfo_field("hashing", "read size cannot be represented"))?;
2686 if piece.len() == piece_length {
2687 pieces.push(sha1::Sha1::digest(&piece).into());
2688 piece.clear();
2689 }
2690 }
2691 let mut extra = [0_u8; 1];
2692 if std::io::Read::read(&mut file, &mut extra)
2693 .map_err(|source| Error::io(&entry.source_path, source))?
2694 != 0
2695 {
2696 return Err(Error::io(
2697 &entry.source_path,
2698 std::io::Error::new(std::io::ErrorKind::InvalidData, "file became longer"),
2699 ));
2700 }
2701 verify_open_snapshot(entry, &file)?;
2702 let padding = if pad_to_boundary && !piece.is_empty() {
2703 let padding = piece_length - piece.len();
2704 piece.resize(piece_length, 0);
2705 u64::try_from(padding)
2706 .map_err(|_| Error::metainfo_field("padding", "cannot be represented"))?
2707 } else {
2708 0
2709 };
2710 if !piece.is_empty() {
2711 pieces.push(sha1::Sha1::digest(&piece).into());
2712 }
2713 Ok(HybridFileHash { pieces, padding })
2714}
2715
2716fn hash_hybrid_v1_sequential(
2717 entries: &[ManifestEntry],
2718 piece_length: u64,
2719 cancellation: &CancellationToken,
2720 progress: &impl ProgressSink,
2721) -> Result<(V1HashResult, Vec<HybridPadding>)> {
2722 let piece_length = usize::try_from(piece_length)
2723 .map_err(|_| Error::metainfo_field("piece length", "cannot be represented"))?;
2724 let total_real = entries.iter().try_fold(0_u64, |total, entry| {
2725 total
2726 .checked_add(entry.length())
2727 .ok_or_else(|| Error::metainfo_field("manifest length", "overflowed"))
2728 })?;
2729 let mut piece = Vec::with_capacity(piece_length);
2730 let mut pieces = Vec::new();
2731 let mut padding = Vec::new();
2732 let mut logical_offset = 0_u64;
2733 let mut real_hashed = 0_u64;
2734 let mut buffer = vec![0_u8; 64 * 1024];
2735 for (index, entry) in entries.iter().enumerate() {
2736 cancellation.check()?;
2737 let mut file = open_snapshot(entry)?;
2738 let mut remaining = entry.length();
2739 while remaining > 0 {
2740 cancellation.check()?;
2741 let read_limit = usize::try_from(remaining)
2742 .unwrap_or(usize::MAX)
2743 .min(buffer.len());
2744 let read = std::io::Read::read(&mut file, &mut buffer[..read_limit])
2745 .map_err(|source| Error::io(&entry.source_path, source))?;
2746 if read == 0 {
2747 return Err(Error::io(
2748 &entry.source_path,
2749 std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "file became shorter"),
2750 ));
2751 }
2752 append_v1_bytes(&buffer[..read], piece_length, &mut piece, &mut pieces);
2753 let read = u64::try_from(read).unwrap_or(u64::MAX);
2754 remaining -= read;
2755 real_hashed += read;
2756 logical_offset += read;
2757 progress.on_progress(HashProgress {
2758 bytes_hashed: real_hashed,
2759 total_bytes: total_real,
2760 pieces_hashed: u64::try_from(pieces.len()).unwrap_or(u64::MAX),
2761 });
2762 }
2763 verify_open_snapshot(entry, &file)?;
2764 if index + 1 < entries.len() {
2765 let piece_length_u64 = u64::try_from(piece_length).unwrap_or(u64::MAX);
2766 let gap = (piece_length_u64 - (logical_offset % piece_length_u64)) % piece_length_u64;
2767 if gap != 0 {
2768 let zeros = vec![0_u8; usize::try_from(gap).unwrap_or(usize::MAX)];
2769 append_v1_bytes(&zeros, piece_length, &mut piece, &mut pieces);
2770 padding.push(HybridPadding {
2771 after_file: index,
2772 length: gap,
2773 offset: logical_offset,
2774 });
2775 logical_offset += gap;
2776 }
2777 }
2778 }
2779 if !piece.is_empty() {
2780 pieces.push(sha1::Sha1::digest(&piece).into());
2781 }
2782 cancellation.check()?;
2783 Ok((
2784 V1HashResult {
2785 pieces,
2786 total_bytes: logical_offset,
2787 },
2788 padding,
2789 ))
2790}
2791
2792fn append_v1_bytes(
2793 mut bytes: &[u8],
2794 piece_length: usize,
2795 piece: &mut Vec<u8>,
2796 pieces: &mut Vec<[u8; 20]>,
2797) {
2798 while !bytes.is_empty() {
2799 let take = (piece_length - piece.len()).min(bytes.len());
2800 piece.extend_from_slice(&bytes[..take]);
2801 bytes = &bytes[take..];
2802 if piece.len() == piece_length {
2803 pieces.push(sha1::Sha1::digest(&*piece).into());
2804 piece.clear();
2805 }
2806 }
2807}
2808
2809fn build_hybrid_info(
2810 mut v2_info: crate::bencode::OwnedValue,
2811 manifest: &PayloadManifest,
2812 hashes: &V1HashResult,
2813 padding: &[HybridPadding],
2814) -> Result<crate::bencode::OwnedValue> {
2815 use crate::bencode::OwnedValue;
2816 let OwnedValue::Dictionary(entries) = &mut v2_info else {
2817 unreachable!("v2 info is a dictionary");
2818 };
2819 entries.insert(
2820 b"pieces".to_vec(),
2821 OwnedValue::bytes(hashes.concatenated_pieces()),
2822 );
2823 if manifest.entries().len() == 1
2824 && manifest.entries()[0].torrent_path() == [manifest.root_name().to_vec()]
2825 {
2826 entries.insert(
2827 b"length".to_vec(),
2828 OwnedValue::integer(length_to_i64(manifest.entries()[0].length())?),
2829 );
2830 return Ok(v2_info);
2831 }
2832 let mut files = Vec::with_capacity(manifest.entries().len() + padding.len());
2833 for (index, entry) in manifest.entries().iter().enumerate() {
2834 files.push(v1_file_entry(entry)?);
2835 if let Some(pad) = padding.iter().find(|pad| pad.after_file == index) {
2836 files.push(v1_padding_entry(pad)?);
2837 }
2838 }
2839 entries.insert(b"files".to_vec(), OwnedValue::list(files));
2840 Ok(v2_info)
2841}
2842
2843fn v1_file_entry(entry: &ManifestEntry) -> Result<crate::bencode::OwnedValue> {
2844 use crate::bencode::OwnedValue;
2845 OwnedValue::dictionary([
2846 (
2847 b"length".to_vec(),
2848 OwnedValue::integer(length_to_i64(entry.length())?),
2849 ),
2850 (
2851 b"path".to_vec(),
2852 OwnedValue::list(entry.torrent_path().iter().cloned().map(OwnedValue::bytes)),
2853 ),
2854 ])
2855}
2856
2857fn v1_padding_entry(padding: &HybridPadding) -> Result<crate::bencode::OwnedValue> {
2858 use crate::bencode::OwnedValue;
2859 OwnedValue::dictionary([
2860 (b"attr".to_vec(), OwnedValue::bytes(b"p".to_vec())),
2861 (
2862 b"length".to_vec(),
2863 OwnedValue::integer(length_to_i64(padding.length)?),
2864 ),
2865 (
2866 b"path".to_vec(),
2867 OwnedValue::list([
2868 OwnedValue::bytes(b".pad".to_vec()),
2869 OwnedValue::bytes(format!("{}-{}", padding.offset, padding.length).into_bytes()),
2870 ]),
2871 ),
2872 ])
2873}
2874
2875fn v2_relative_path<'a>(manifest: &PayloadManifest, entry: &'a ManifestEntry) -> &'a [Vec<u8>] {
2876 if entry.torrent_path() == [manifest.root_name().to_vec()] {
2877 return entry.torrent_path();
2878 }
2879 if entry
2880 .torrent_path()
2881 .first()
2882 .is_some_and(|component| component == manifest.root_name())
2883 {
2884 return &entry.torrent_path()[1..];
2885 }
2886 entry.torrent_path()
2887}
2888
2889fn insert_v2_file(
2890 tree: &mut std::collections::BTreeMap<Vec<u8>, crate::bencode::OwnedValue>,
2891 path: &[Vec<u8>],
2892 length: u64,
2893 pieces_root: Option<&[u8; 32]>,
2894) -> Result<()> {
2895 use crate::bencode::OwnedValue;
2896 let Some((component, remaining)) = path.split_first() else {
2897 return Err(Error::metainfo_field("file tree", "file path is empty"));
2898 };
2899 if component.is_empty() {
2900 return Err(Error::metainfo_field(
2901 "file tree",
2902 "path component is empty",
2903 ));
2904 }
2905 if remaining.is_empty() {
2906 let mut properties = vec![(
2907 b"length".to_vec(),
2908 OwnedValue::integer(length_to_i64(length)?),
2909 )];
2910 if let Some(root) = pieces_root {
2911 properties.push((b"pieces root".to_vec(), OwnedValue::bytes(root.to_vec())));
2912 }
2913 let leaf = OwnedValue::dictionary([(Vec::new(), OwnedValue::dictionary(properties)?)])?;
2914 if tree.insert(component.clone(), leaf).is_some() {
2915 return Err(Error::metainfo_field("file tree", "path collision"));
2916 }
2917 return Ok(());
2918 }
2919 let node = tree
2920 .entry(component.clone())
2921 .or_insert_with(|| OwnedValue::Dictionary(std::collections::BTreeMap::new()));
2922 let OwnedValue::Dictionary(children) = node else {
2923 return Err(Error::metainfo_field("file tree", "path prefix collision"));
2924 };
2925 insert_v2_file(children, remaining, length, pieces_root)
2926}
2927
2928fn build_metainfo(
2929 info: crate::bencode::OwnedValue,
2930 piece_layers: Option<crate::bencode::OwnedValue>,
2931 options: &CreateOptions,
2932) -> Result<crate::bencode::OwnedValue> {
2933 use crate::bencode::OwnedValue;
2934 let mut entries = vec![(b"info".to_vec(), info)];
2935 if let Some(piece_layers) = piece_layers {
2936 entries.push((b"piece layers".to_vec(), piece_layers));
2937 }
2938 if let Some(announce) = options
2939 .trackers
2940 .first()
2941 .and_then(|tier| tier.urls().first())
2942 {
2943 entries.push((
2944 b"announce".to_vec(),
2945 OwnedValue::bytes(announce.as_bytes().to_vec()),
2946 ));
2947 }
2948 if !options.trackers.is_empty() {
2949 entries.push((
2950 b"announce-list".to_vec(),
2951 OwnedValue::list(options.trackers.iter().map(|tier| {
2952 OwnedValue::list(
2953 tier.urls()
2954 .iter()
2955 .cloned()
2956 .map(|url| OwnedValue::bytes(url.into_bytes())),
2957 )
2958 })),
2959 ));
2960 }
2961 if !options.web_seeds.is_empty() {
2962 entries.push((
2963 b"url-list".to_vec(),
2964 OwnedValue::list(
2965 options
2966 .web_seeds
2967 .iter()
2968 .cloned()
2969 .map(|seed| OwnedValue::bytes(seed.into_bytes())),
2970 ),
2971 ));
2972 }
2973 if !options.nodes.is_empty() {
2974 entries.push((
2975 b"nodes".to_vec(),
2976 OwnedValue::list(options.nodes.iter().map(|(host, port)| {
2977 OwnedValue::list([
2978 OwnedValue::bytes(host.as_bytes().to_vec()),
2979 OwnedValue::integer(i64::from(*port)),
2980 ])
2981 })),
2982 ));
2983 }
2984 if let Some(comment) = &options.comment {
2985 entries.push((
2986 b"comment".to_vec(),
2987 OwnedValue::bytes(comment.as_bytes().to_vec()),
2988 ));
2989 }
2990 let created_by = match &options.created_by {
2991 CreatorIdentity::Default => {
2992 Some(format!("btpc/{}", env!("CARGO_PKG_VERSION")).into_bytes())
2993 }
2994 CreatorIdentity::Explicit(value) => Some(value.clone()),
2995 CreatorIdentity::Omit => None,
2996 };
2997 if let Some(created_by) = created_by {
2998 entries.push((b"created by".to_vec(), OwnedValue::bytes(created_by)));
2999 }
3000 if let Some(creation_date) = options.creation_date {
3001 entries.push((
3002 b"creation date".to_vec(),
3003 OwnedValue::integer(creation_date),
3004 ));
3005 }
3006 OwnedValue::dictionary(entries)
3007}
3008
3009fn length_to_i64(length: u64) -> Result<i64> {
3010 i64::try_from(length).map_err(|_| Error::metainfo_field("length", "cannot fit bencode integer"))
3011}
3012
3013#[cfg(test)]
3014mod parallel_hash_tests {
3015 use std::fs;
3016
3017 use super::{
3018 CancellationToken, ManifestEntry, NoProgress, ParallelHashOptions, automatic_v1_workers,
3019 hash_v1_parallel_inner, hash_v1_sequential,
3020 };
3021 use crate::ErrorCategory;
3022 use tempfile::TempDir;
3023
3024 #[test]
3025 fn pipeline_restores_forced_out_of_order_completion() {
3026 let (temp, entry) = fixture();
3027 let entries = [entry];
3028 let oracle =
3029 hash_v1_sequential(&entries, 16 * 1024, &CancellationToken::new(), &NoProgress)
3030 .unwrap();
3031 let actual = hash_v1_parallel_inner(
3032 &entries,
3033 16 * 1024,
3034 ParallelHashOptions::new(4, 2).unwrap(),
3035 &CancellationToken::new(),
3036 &NoProgress,
3037 None,
3038 Some(0),
3039 )
3040 .unwrap();
3041 assert_eq!(actual, oracle);
3042 drop(temp);
3043 }
3044
3045 #[test]
3046 fn injected_worker_failure_is_returned_without_hanging() {
3047 let (_temp, entry) = fixture();
3048 let error = hash_v1_parallel_inner(
3049 &[entry],
3050 16 * 1024,
3051 ParallelHashOptions::new(4, 2).unwrap(),
3052 &CancellationToken::new(),
3053 &NoProgress,
3054 Some(5),
3055 None,
3056 )
3057 .unwrap_err();
3058 assert_eq!(error.category(), ErrorCategory::Io);
3059 }
3060
3061 #[test]
3062 fn automatic_worker_heuristic_is_conservative() {
3063 assert_eq!(automatic_v1_workers(1), 1);
3064 assert_eq!(automatic_v1_workers(2), 2);
3065 assert_eq!(automatic_v1_workers(64), 2);
3066 }
3067
3068 fn fixture() -> (TempDir, ManifestEntry) {
3069 let temp = TempDir::new().unwrap();
3070 let path = temp.path().join("payload");
3071 fs::write(&path, vec![0x5a; 512 * 1024]).unwrap();
3072 let metadata = fs::metadata(&path).unwrap();
3073 let entry = ManifestEntry::from_snapshot(
3074 path,
3075 vec![b"payload".to_vec()],
3076 metadata.len(),
3077 metadata.modified().ok(),
3078 );
3079 (temp, entry)
3080 }
3081}
3082
3083#[cfg(test)]
3084mod v2_parallel_tests {
3085 use std::fs;
3086 use std::sync::atomic::Ordering;
3087
3088 use super::{
3089 CancellationToken, HashProgress, ManifestEntry, NoProgress, ProgressSink, WorkerCounts,
3090 hash_hybrid_v1_parallel, hash_hybrid_v1_sequential, hash_v2_file_sequential,
3091 hash_v2_files_parallel, hash_v2_files_parallel_inner,
3092 };
3093 use crate::ErrorCategory;
3094 use tempfile::TempDir;
3095
3096 #[derive(Default)]
3097 struct RecordingProgress {
3098 events: std::sync::Mutex<Vec<HashProgress>>,
3099 }
3100
3101 struct CancellingProgress {
3102 cancellation: CancellationToken,
3103 seen: std::sync::atomic::AtomicBool,
3104 }
3105
3106 impl ProgressSink for CancellingProgress {
3107 fn on_progress(&self, _progress: HashProgress) {
3108 if !self.seen.swap(true, std::sync::atomic::Ordering::AcqRel) {
3109 self.cancellation.cancel();
3110 }
3111 }
3112 }
3113
3114 impl ProgressSink for RecordingProgress {
3115 fn on_progress(&self, progress: HashProgress) {
3116 self.events.lock().unwrap().push(progress);
3117 }
3118 }
3119
3120 #[test]
3121 fn parallel_v2_restores_manifest_order_and_progress() {
3122 let (temp, entries) = fixture(32, 32 * 1024);
3123 let progress = RecordingProgress::default();
3124 let actual =
3125 hash_v2_files_parallel(&entries, 16 * 1024, 2, &CancellationToken::new(), &progress)
3126 .unwrap();
3127 let expected = entries
3128 .iter()
3129 .map(|entry| {
3130 hash_v2_file_sequential(entry, 16 * 1024, &CancellationToken::new(), &NoProgress)
3131 .unwrap()
3132 })
3133 .collect::<Vec<_>>();
3134 assert_eq!(actual, expected);
3135 let events = progress.events.lock().unwrap();
3136 assert_eq!(events.len(), entries.len());
3137 assert!(
3138 events
3139 .windows(2)
3140 .all(|pair| pair[0].bytes_hashed() < pair[1].bytes_hashed())
3141 );
3142 drop(temp);
3143 }
3144
3145 #[test]
3146 fn parallel_file_workers_propagate_errors_and_cancellation() {
3147 let (_temp, mut entries) = fixture(16, 32 * 1024);
3148 fs::remove_file(entries[5].source_path()).unwrap();
3149 let error = hash_v2_files_parallel(
3150 &entries,
3151 16 * 1024,
3152 2,
3153 &CancellationToken::new(),
3154 &NoProgress,
3155 )
3156 .unwrap_err();
3157 assert_eq!(error.category(), ErrorCategory::Io);
3158
3159 entries.remove(5);
3160 let cancellation = CancellationToken::new();
3161 cancellation.cancel();
3162 let error = hash_hybrid_v1_parallel(&entries, 16 * 1024, 2, &cancellation).unwrap_err();
3163 assert_eq!(error.category(), ErrorCategory::Cancelled);
3164 }
3165
3166 #[test]
3167 fn parallel_v2_progress_can_cancel_midflight() {
3168 let (_temp, entries) = fixture(64, 128 * 1024);
3169 let cancellation = CancellationToken::new();
3170 let progress = CancellingProgress {
3171 cancellation: cancellation.clone(),
3172 seen: std::sync::atomic::AtomicBool::new(false),
3173 };
3174 let error =
3175 hash_v2_files_parallel(&entries, 16 * 1024, 2, &cancellation, &progress).unwrap_err();
3176 assert_eq!(error.category(), ErrorCategory::Cancelled);
3177 }
3178
3179 #[test]
3180 fn parallel_hybrid_matches_oracle_for_many_files() {
3181 let (temp, entries) = fixture(64, 4_097);
3182 let oracle =
3183 hash_hybrid_v1_sequential(&entries, 16 * 1024, &CancellationToken::new(), &NoProgress)
3184 .unwrap();
3185 let actual =
3186 hash_hybrid_v1_parallel(&entries, 16 * 1024, 2, &CancellationToken::new()).unwrap();
3187 assert_eq!(actual, oracle);
3188 drop(temp);
3189 }
3190
3191 #[test]
3192 fn worker_count_is_the_descriptor_bound() {
3193 let counts = WorkerCounts {
3194 active: std::sync::atomic::AtomicUsize::new(0),
3195 peak: std::sync::atomic::AtomicUsize::new(0),
3196 };
3197 let (_temp, entries) = fixture(128, 128 * 1024);
3198 let workers = 2;
3199 hash_v2_files_parallel_inner(
3200 &entries,
3201 16 * 1024,
3202 workers,
3203 &CancellationToken::new(),
3204 &NoProgress,
3205 Some(&counts),
3206 )
3207 .unwrap();
3208 assert_eq!(counts.active.load(Ordering::Acquire), 0);
3209 assert!(counts.peak.load(Ordering::Acquire) <= workers);
3210 }
3211
3212 fn fixture(count: usize, size: usize) -> (TempDir, Vec<ManifestEntry>) {
3213 let temp = TempDir::new().unwrap();
3214 let entries = (0..count)
3215 .map(|index| {
3216 let path = temp.path().join(format!("file-{index:03}"));
3217 fs::write(&path, vec![u8::try_from(index % 256).unwrap(); size]).unwrap();
3218 let metadata = fs::metadata(&path).unwrap();
3219 ManifestEntry::from_snapshot(
3220 path,
3221 vec![format!("file-{index:03}").into_bytes()],
3222 metadata.len(),
3223 metadata.modified().ok(),
3224 )
3225 })
3226 .collect();
3227 (temp, entries)
3228 }
3229}