Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion core/services/azblob/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,6 @@ impl Builder for AzblobBuilder {
stat_with_if_none_match: true,

read: true,
read_with_suffix: true,

read_with_if_match: true,
read_with_if_none_match: true,
Expand Down
1 change: 0 additions & 1 deletion core/services/azdls/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,6 @@ impl Builder for AzdlsBuilder {
stat: true,

read: true,
read_with_suffix: true,
read_with_if_match: true,
read_with_if_none_match: true,
read_with_if_modified_since: true,
Expand Down
1 change: 0 additions & 1 deletion core/services/azfile/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,6 @@ impl Builder for AzfileBuilder {
stat: true,

read: true,
read_with_suffix: true,

write: true,
write_with_user_metadata: true,
Expand Down
1 change: 0 additions & 1 deletion core/services/ghac/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,6 @@ impl Builder for GhacBuilder {
stat: true,

read: true,
read_with_suffix: true,

write: true,
write_can_multi: true,
Expand Down
2 changes: 0 additions & 2 deletions core/services/hf/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,6 @@ impl Builder for HfBuilder {
am.set_scheme(HF_SCHEME).set_native_capability(Capability {
stat: true,
read: true,
read_with_suffix: true,
write: token.is_some(),
delete: token.is_some(),
delete_max_size: Some(100),
Expand Down Expand Up @@ -381,7 +380,6 @@ pub(super) mod test_utils {
let info = AccessorInfo::default();
info.set_scheme("hf").set_native_capability(Capability {
read: true,
read_with_suffix: true,
write: true,
delete: true,
..Default::default()
Expand Down
1 change: 0 additions & 1 deletion core/services/seafile/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ impl Builder for SeafileBuilder {
stat: true,

read: true,
read_with_suffix: true,

write: true,
write_can_empty: true,
Expand Down
118 changes: 118 additions & 0 deletions core/tests/behavior/async_read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,16 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
))
}

if cap.read && cap.write && cap.read_with_suffix {
tests.extend(async_trials!(
op,
test_read_suffix,
test_read_suffix_larger_than_file,
test_read_suffix_zero,
test_reader_suffix_with_chunk
))
}

if cap.read && !cap.write {
tests.extend(async_trials!(
op,
Expand All @@ -72,6 +82,10 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
test_reader_only_read_with_if_none_match
))
}

if cap.read && !cap.write && cap.read_with_suffix {
tests.extend(async_trials!(op, test_read_only_read_with_suffix))
}
}

/// Read full content should match.
Expand Down Expand Up @@ -113,6 +127,93 @@ pub async fn test_read_range(op: Operator) -> anyhow::Result<()> {
Ok(())
}

/// Read suffix content should match.
pub async fn test_read_suffix(op: Operator) -> anyhow::Result<()> {
let path = TEST_FIXTURE.new_file_path();
let content = gen_fixed_bytes(1024);
let suffix_size = 257;

op.write(&path, content.clone())
.await
.expect("write must succeed");

let bs = op
.read_with(&path)
.range(BytesRange::suffix(suffix_size as u64))
.await?
.to_bytes();
assert_eq!(bs.len(), suffix_size, "read size");
assert_eq!(
sha256_digest(&bs),
sha256_digest(&content[content.len() - suffix_size..]),
"read content"
);

Ok(())
}

/// Read suffix larger than the file should return the full content.
pub async fn test_read_suffix_larger_than_file(op: Operator) -> anyhow::Result<()> {
let path = TEST_FIXTURE.new_file_path();
let content = gen_fixed_bytes(1024);

op.write(&path, content.clone())
.await
.expect("write must succeed");

let bs = op
.read_with(&path)
.range(BytesRange::suffix((content.len() * 2) as u64))
.await?
.to_bytes();
assert_eq!(bs.len(), content.len(), "read size");
assert_eq!(sha256_digest(&bs), sha256_digest(&content), "read content");

Ok(())
}

/// Read zero suffix should return empty content.
pub async fn test_read_suffix_zero(op: Operator) -> anyhow::Result<()> {
let path = TEST_FIXTURE.new_file_path();
let content = gen_fixed_bytes(1024);

op.write(&path, content).await.expect("write must succeed");

let bs = op
.read_with(&path)
.range(BytesRange::suffix(0))
.await?
.to_bytes();
assert!(bs.is_empty(), "read content must be empty");

Ok(())
}

/// Reader suffix with chunk should still return the requested suffix content.
pub async fn test_reader_suffix_with_chunk(op: Operator) -> anyhow::Result<()> {
let path = TEST_FIXTURE.new_file_path();
let content = gen_fixed_bytes(4096);
let suffix_size = 1025;

op.write(&path, content.clone())
.await
.expect("write must succeed");

let reader = op.reader_with(&path).chunk(257).concurrent(3).await?;
let bs = reader
.read(BytesRange::suffix(suffix_size as u64))
.await?
.to_bytes();
assert_eq!(bs.len(), suffix_size, "read size");
assert_eq!(
sha256_digest(&bs),
sha256_digest(&content[content.len() - suffix_size..]),
"read content"
);

Ok(())
}

/// Read full content should match.
pub async fn test_reader(op: Operator) -> anyhow::Result<()> {
let (path, content, size) = TEST_FIXTURE.new_file(op.clone());
Expand Down Expand Up @@ -800,6 +901,23 @@ pub async fn test_read_only_read_with_range(op: Operator) -> anyhow::Result<()>
Ok(())
}

/// Read suffix content should match.
pub async fn test_read_only_read_with_suffix(op: Operator) -> anyhow::Result<()> {
let bs = op
.read_with("normal_file.txt")
.range(BytesRange::suffix(1024))
.await?
.to_bytes();
assert_eq!(bs.len(), 1024, "read size");
assert_eq!(
sha256_digest(&bs),
"cc9312c869238ea9410b6716e0fc3f48056f2bfb2fe06ccf5f96f2c3bf39e71b",
"read content"
);

Ok(())
}

/// Read not exist file should return NotFound
pub async fn test_read_only_read_not_exist(op: Operator) -> anyhow::Result<()> {
let path = uuid::Uuid::new_v4().to_string();
Expand Down
Loading