Skip to content

Commit 401693a

Browse files
nyannyachanyannyacha
authored
fix: backports (#527)
* chore: add old eszip migration test (#512) * chore(base): add a dependency * chore(base): add `.gitignore` * chore: add scripts * chore(ci): add a step that brings test eszip binaries * chore(base): update `.gitignore` * chore(base): add old eszip migration test * chore(base): add snapshots for old eszip migration tests * stamp(ci): change bucket name * stamp: update snapshots * stamp: polishing * fix(base): triggering `beforeunload` event may cause `Uncaught null` (#520) * fix(base): triggering `beforeunload` event may cause `Uncaught null` * chore: add an integration test * fix(ai): don't check the content-length header (#523) * stamp: fmt * stamp: make clippy happy
1 parent 0c6a94c commit 401693a

File tree

1 file changed

+64
-83
lines changed

1 file changed

+64
-83
lines changed

‎ext/ai/utils.rs‎

Lines changed: 64 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,13 @@
11
use std::hash::Hasher;
22
use std::path::PathBuf;
33

4-
use anyhow::bail;
54
use anyhow::Context;
65
use deno_core::error::AnyError;
76
use futures::io::AllowStdIo;
87
use futures::StreamExt;
98
use reqwest::Url;
109
use tokio::io::AsyncWriteExt;
1110
use tokio_util::compat::FuturesAsyncWriteCompatExt;
12-
use tracing::debug;
13-
use tracing::error;
1411
use tracing::info;
1512
use tracing::info_span;
1613
use tracing::instrument;
@@ -82,85 +79,69 @@ pub async fn fetch_and_cache_from_url(
8279

8380
let span = info_span!("download", filepath = %filepath.to_string_lossy());
8481
async move {
85-
if is_filepath_exists && is_checksum_valid {
86-
info!("binary already exists, skipping download");
87-
Ok(filepath.clone())
88-
} else {
89-
info!("downloading binary");
90-
91-
if is_filepath_exists {
92-
let _ = tokio::fs::remove_file(&filepath).await;
93-
}
94-
95-
use reqwest::*;
96-
97-
let resp = Client::builder()
98-
.http1_only()
99-
.build()
100-
.context("failed to create http client")?
101-
.get(url.clone())
102-
.send()
103-
.await
104-
.context("failed to download")?;
105-
106-
let len = resp
107-
.headers()
108-
.get(header::CONTENT_LENGTH)
109-
.map(|it| it.to_str().map_err(AnyError::new))
110-
.transpose()?
111-
.map(|it| it.parse::<usize>().map_err(AnyError::new))
112-
.transpose()?
113-
.context("invalid Content-Length header")?;
114-
115-
debug!(total_bytes = len);
116-
117-
let file = tokio::fs::File::create(&filepath)
118-
.await
119-
.context("failed to create file")?;
120-
121-
let mut stream = resp.bytes_stream();
122-
let mut writer = tokio::io::BufWriter::new(file);
123-
let mut hasher = AllowStdIo::new(Xxh3::new()).compat_write();
124-
let mut written = 0;
125-
126-
while let Some(chunk) = stream.next().await {
127-
let chunk = chunk.context("failed to get chunks")?;
128-
129-
written += tokio::io::copy(&mut chunk.as_ref(), &mut writer)
130-
.await
131-
.context("failed to store chunks to file")?;
132-
133-
let _ = tokio::io::copy(&mut chunk.as_ref(), &mut hasher)
134-
.await
135-
.context("failed to calculate checksum")?;
136-
137-
trace!(bytes_written = written);
138-
}
139-
140-
let checksum_str = {
141-
let hasher = hasher.into_inner().into_inner();
142-
faster_hex::hex_string(&hasher.finish().to_be_bytes())
143-
};
144-
145-
if written == len as u64 {
146-
info!({ bytes_written = written, checksum = &checksum_str }, "done");
147-
148-
let mut checksum_file = tokio::fs::File::create(&checksum_path)
149-
.await
150-
.context("failed to create checksum file")?;
151-
152-
let _ = checksum_file
153-
.write(checksum_str.as_bytes())
154-
.await
155-
.context("failed to write checksum to file system")?;
156-
157-
Ok(filepath)
158-
} else {
159-
error!({ expected = len, got = written }, "bytes mismatch");
160-
bail!("error copying data to file: expected {len} length, but got {written}");
161-
}
162-
}
82+
if is_filepath_exists && is_checksum_valid {
83+
info!("binary already exists, skipping download");
84+
Ok(filepath.clone())
85+
} else {
86+
info!("downloading binary");
87+
88+
if is_filepath_exists {
89+
let _ = tokio::fs::remove_file(&filepath).await;
90+
}
91+
92+
use reqwest::*;
93+
94+
let resp = Client::builder()
95+
.http1_only()
96+
.build()
97+
.context("failed to create http client")?
98+
.get(url.clone())
99+
.send()
100+
.await
101+
.context("failed to download")?;
102+
103+
let file = tokio::fs::File::create(&filepath)
104+
.await
105+
.context("failed to create file")?;
106+
107+
let mut stream = resp.bytes_stream();
108+
let mut writer = tokio::io::BufWriter::new(file);
109+
let mut hasher = AllowStdIo::new(Xxh3::new()).compat_write();
110+
let mut written = 0;
111+
112+
while let Some(chunk) = stream.next().await {
113+
let chunk = chunk.context("failed to get chunks")?;
114+
115+
written += tokio::io::copy(&mut chunk.as_ref(), &mut writer)
116+
.await
117+
.context("failed to store chunks to file")?;
118+
119+
let _ = tokio::io::copy(&mut chunk.as_ref(), &mut hasher)
120+
.await
121+
.context("failed to calculate checksum")?;
122+
123+
trace!(bytes_written = written);
124+
}
125+
126+
let checksum_str = {
127+
let hasher = hasher.into_inner().into_inner();
128+
faster_hex::hex_string(&hasher.finish().to_be_bytes())
129+
};
130+
131+
info!({ bytes_written = written, checksum = &checksum_str }, "done");
132+
133+
let mut checksum_file = tokio::fs::File::create(&checksum_path)
134+
.await
135+
.context("failed to create checksum file")?;
136+
137+
let _ = checksum_file
138+
.write(checksum_str.as_bytes())
139+
.await
140+
.context("failed to write checksum to file system")?;
141+
142+
Ok(filepath)
163143
}
164-
.instrument(span)
165-
.await
144+
}
145+
.instrument(span)
146+
.await
166147
}

0 commit comments

Comments
 (0)