pubmed_client/pmc/
cloud.rs

1use std::{mem, path::Path, time::Duration};
2
3use crate::common::PmcId;
4use crate::config::ClientConfig;
5use crate::error::{ParseError, PubMedError, Result};
6use crate::pmc::common;
7use crate::pmc::extracted::ExtractedFigure;
8use crate::pmc::parser::parse_pmc_xml;
9use crate::rate_limit::RateLimiter;
10use crate::request::RequestExecutor;
11#[cfg(not(target_arch = "wasm32"))]
12use crate::request::fetch_with_retry;
13use crate::tls::install_default_crypto_provider;
14use pubmed_parser::pmc::{Figure, PmcArticle, Section};
15use reqwest::Client;
16#[cfg(not(target_arch = "wasm32"))]
17use reqwest::Response;
18use tracing::debug;
19
20#[cfg(not(target_arch = "wasm32"))]
21use futures_util::{StreamExt, TryStreamExt, stream};
22#[cfg(not(target_arch = "wasm32"))]
23use tokio::{fs as tokio_fs, task};
24
25/// Download client for PMC Open Access articles via the PMC OA Cloud (AWS S3).
26///
27/// Fetches an article's full-text XML, media, and supplementary files as
28/// individual per-article objects from the `pmc-oa-opendata` S3 bucket. This
29/// replaces the retired PMC FTP service and its legacy `oa_package` tar.gz
30/// bundles (removed by NCBI in August 2026).
31#[derive(Clone)]
32pub struct PmcCloudClient {
33    client: Client,
34    rate_limiter: RateLimiter,
35    pub(crate) config: ClientConfig,
36}
37
38impl PmcCloudClient {
39    /// Create a new PMC OA Cloud client with configuration
40    pub fn new(config: ClientConfig) -> Self {
41        let rate_limiter = config.create_rate_limiter();
42
43        // rustls has no built-in provider under `rustls-tls`; install one first.
44        install_default_crypto_provider();
45
46        #[allow(clippy::expect_used)]
47        let client = {
48            #[cfg(not(target_arch = "wasm32"))]
49            {
50                Client::builder()
51                    .user_agent(config.effective_user_agent())
52                    .timeout(Duration::from_secs(config.timeout.as_secs()))
53                    .build()
54                    .expect("Failed to create HTTP client")
55            }
56
57            #[cfg(target_arch = "wasm32")]
58            {
59                Client::builder()
60                    .user_agent(config.effective_user_agent())
61                    .build()
62                    .expect("Failed to create HTTP client")
63            }
64        };
65
66        Self {
67            client,
68            rate_limiter,
69            config,
70        }
71    }
72
73    /// Create a cloud client sharing an existing HTTP client and rate limiter.
74    ///
75    /// Used by `PmcClient` to avoid duplicating the HTTP client and rate limiter.
76    pub(crate) fn with_shared(
77        client: Client,
78        rate_limiter: RateLimiter,
79        config: ClientConfig,
80    ) -> Self {
81        Self {
82            client,
83            rate_limiter,
84            config,
85        }
86    }
87
88    /// Download a PMC article's files from the PMC OA Cloud (AWS S3) service.
89    ///
90    /// NCBI retired the PMC FTP service and the legacy `oa_package` tar.gz
91    /// bundles (August 2026). This downloads each of the article's files
92    /// (full-text XML, media, supplementary materials, PDF, etc.) individually
93    /// from the `pmc-oa-opendata` S3 bucket into `output_dir`.
94    ///
95    /// # Arguments
96    ///
97    /// * `pmcid` - PMC ID (with or without "PMC" prefix)
98    /// * `output_dir` - Directory to download the article's files into
99    ///
100    /// # Returns
101    ///
102    /// Returns a `Result<Vec<String>>` containing the list of downloaded file paths
103    ///
104    /// # Errors
105    ///
106    /// * `ParseError::InvalidPmid` - If the PMCID format is invalid
107    /// * `PubMedError::RequestError` - If the HTTP request fails
108    /// * `ParseError::IoError` - If file operations fail
109    /// * `ParseError::PmcNotAvailable` - If the article is not available in the OA Cloud
110    ///
111    /// # Example
112    ///
113    /// ```no_run
114    /// use pubmed_client::pmc::cloud::PmcCloudClient;
115    /// use pubmed_client::ClientConfig;
116    /// use std::path::Path;
117    ///
118    /// #[tokio::main]
119    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
120    ///     let config = ClientConfig::new();
121    ///     let client = PmcCloudClient::new(config);
122    ///     let output_dir = Path::new("./extracted_articles");
123    ///     let files = client.download_files("PMC7906746", output_dir).await?;
124    ///
125    ///     for file in files {
126    ///         println!("Downloaded: {}", file);
127    ///     }
128    ///     Ok(())
129    /// }
130    /// ```
131    #[cfg(not(target_arch = "wasm32"))]
132    pub async fn download_files<P: AsRef<Path>>(
133        &self,
134        pmcid: &str,
135        output_dir: P,
136    ) -> Result<Vec<String>> {
137        let pmc_id = PmcId::parse(pmcid)?;
138        let normalized_pmcid = pmc_id.as_str();
139
140        let output_path = output_dir.as_ref();
141        tokio_fs::create_dir_all(output_path)
142            .await
143            .map_err(|e| ParseError::IoError {
144                message: format!("Failed to create output directory: {}", e),
145            })?;
146
147        let files = self
148            .download_cloud_files(&normalized_pmcid, output_path)
149            .await?;
150
151        if files.is_empty() {
152            return Err(ParseError::PmcNotAvailable {
153                id: pmcid.to_string(),
154            }
155            .into());
156        }
157
158        Ok(files)
159    }
160
161    /// Download an article's files from the PMC OA Cloud (AWS S3) service.
162    ///
163    /// Lists the objects under the article's prefix in the `pmc-oa-opendata`
164    /// bucket, selects the latest version folder, and downloads each file into
165    /// `output_dir`. Returns the list of local file paths (empty if the article
166    /// is not present in the cloud bucket).
167    #[cfg(not(target_arch = "wasm32"))]
168    async fn download_cloud_files(
169        &self,
170        normalized_pmcid: &str,
171        output_dir: &Path,
172    ) -> Result<Vec<String>> {
173        let keys = self.list_cloud_object_keys(normalized_pmcid).await?;
174        if keys.is_empty() {
175            return Ok(Vec::new());
176        }
177
178        let base_url = self
179            .config
180            .effective_oa_cloud_base_url()
181            .trim_end_matches('/');
182        let concurrency = self.config.effective_oa_download_concurrency();
183
184        // The article's files are independent S3 objects and — unlike eutils —
185        // the OA Cloud bucket is not subject to the NCBI rate limit, so we fetch
186        // them concurrently (bounded by `concurrency`). `buffered` preserves the
187        // listing order so figure matching stays deterministic.
188        let downloaded = stream::iter(keys)
189            .map(|key| async move {
190                // The object filename is the last path segment of the S3 key,
191                // e.g. `PMC7906746.1/gr1_lrg.jpg` -> `gr1_lrg.jpg`.
192                let Some(file_name) = key.rsplit('/').next().filter(|s| !s.is_empty()) else {
193                    return Ok::<Option<String>, PubMedError>(None);
194                };
195
196                let url = format!("{}/{}", base_url, key);
197                let response = self.s3_get(&url).await?;
198                let bytes = response.bytes().await.map_err(PubMedError::from)?;
199
200                let output_path = output_dir.join(file_name);
201                tokio_fs::write(&output_path, &bytes)
202                    .await
203                    .map_err(|e| ParseError::IoError {
204                        message: format!("Failed to write cloud file {}: {}", file_name, e),
205                    })?;
206
207                debug!("Downloaded cloud file: {}", output_path.display());
208                Ok(Some(output_path.to_string_lossy().to_string()))
209            })
210            .buffered(concurrency)
211            .try_filter_map(|opt| async move { Ok(opt) })
212            .try_collect::<Vec<String>>()
213            .await?;
214
215        Ok(downloaded)
216    }
217
218    /// List the S3 object keys for an article's latest version in the OA Cloud.
219    ///
220    /// Queries the bucket's ListObjectsV2 endpoint with the article prefix
221    /// (`<PMCID>.`, the trailing dot preventing matches against longer PMCIDs),
222    /// then keeps only the keys belonging to the highest version folder
223    /// (`<PMCID>.<n>/`).
224    #[cfg(not(target_arch = "wasm32"))]
225    async fn list_cloud_object_keys(&self, normalized_pmcid: &str) -> Result<Vec<String>> {
226        let base_url = self.config.effective_oa_cloud_base_url();
227        // The trailing dot restricts the prefix to `<PMCID>.<version>/...`,
228        // so e.g. `PMC790674` does not also match `PMC7906740`.
229        let url = format!(
230            "{}/?list-type=2&prefix={}.",
231            base_url.trim_end_matches('/'),
232            normalized_pmcid
233        );
234
235        debug!("Listing PMC OA Cloud objects: {}", url);
236        let response = self.s3_get(&url).await?;
237        let body = response.text().await?;
238
239        let keys = Self::parse_cloud_listing(&body)?;
240        Ok(Self::select_latest_version_keys(keys))
241    }
242
243    /// Parse the `<Key>` entries from an S3 ListObjectsV2 XML response.
244    #[cfg(not(target_arch = "wasm32"))]
245    fn parse_cloud_listing(xml_content: &str) -> Result<Vec<String>> {
246        use quick_xml::Reader;
247        use quick_xml::events::Event;
248
249        use quick_xml::escape::resolve_predefined_entity;
250
251        let mut reader = Reader::from_str(xml_content);
252        reader.config_mut().trim_text(true);
253
254        let mut buf = Vec::new();
255        let mut keys = Vec::new();
256        let mut in_key = false;
257        // A key containing an escaped character (e.g. `&amp;`) arrives as
258        // multiple Text/GeneralRef events, so accumulate until </Key>.
259        let mut key = String::new();
260
261        loop {
262            match reader.read_event_into(&mut buf) {
263                Ok(Event::Start(ref e)) if e.name().as_ref() == b"Key" => {
264                    in_key = true;
265                    key.clear();
266                }
267                Ok(Event::End(ref e)) if e.name().as_ref() == b"Key" => {
268                    in_key = false;
269                    // Skip folder-marker keys (zero-byte objects ending in `/`).
270                    if !key.is_empty() && !key.ends_with('/') {
271                        keys.push(mem::take(&mut key));
272                    }
273                }
274                Ok(Event::Text(ref e)) if in_key => {
275                    let text = e.decode().map_err(|err| {
276                        ParseError::XmlError(format!("Invalid UTF-8 in S3 Key: {}", err))
277                    })?;
278                    key.push_str(&text);
279                }
280                Ok(Event::GeneralRef(ref e)) if in_key => {
281                    let char_ref = e.resolve_char_ref().map_err(|err| {
282                        ParseError::XmlError(format!("Invalid reference in S3 Key: {}", err))
283                    })?;
284                    if let Some(ch) = char_ref {
285                        key.push(ch);
286                    } else {
287                        let name = e.decode().map_err(|err| {
288                            ParseError::XmlError(format!("Invalid UTF-8 in S3 Key: {}", err))
289                        })?;
290                        if let Some(text) = resolve_predefined_entity(&name) {
291                            key.push_str(text);
292                        }
293                    }
294                }
295                Ok(Event::Eof) => break,
296                Err(e) => {
297                    return Err(
298                        ParseError::XmlError(format!("Failed to parse S3 listing: {}", e)).into(),
299                    );
300                }
301                _ => {}
302            }
303            buf.clear();
304        }
305
306        Ok(keys)
307    }
308
309    /// From a flat list of keys, keep only those under the highest version folder.
310    ///
311    /// Keys look like `PMC7906746.1/PMC7906746.1.xml`; the version is the integer
312    /// after the last `.` of the leading `<folder>/` segment. When multiple
313    /// versions are present, only the latest is retained.
314    #[cfg(not(target_arch = "wasm32"))]
315    fn select_latest_version_keys(keys: Vec<String>) -> Vec<String> {
316        fn version_of(key: &str) -> Option<u32> {
317            let folder = key.split('/').next()?;
318            folder.rsplit('.').next()?.parse::<u32>().ok()
319        }
320
321        let Some(latest) = keys.iter().filter_map(|k| version_of(k)).max() else {
322            return keys;
323        };
324
325        keys.into_iter()
326            .filter(|k| version_of(k) == Some(latest))
327            .collect()
328    }
329
330    /// Download the article's files and match figures with their captions from XML
331    ///
332    /// # Arguments
333    ///
334    /// * `pmcid` - PMC ID (with or without "PMC" prefix)
335    /// * `output_dir` - Directory to download the article's files into
336    ///
337    /// # Returns
338    ///
339    /// Returns a `Result<Vec<ExtractedFigure>>` containing figures with both XML metadata and file paths
340    ///
341    /// # Errors
342    ///
343    /// * `ParseError::InvalidPmid` - If the PMCID format is invalid
344    /// * `PubMedError::RequestError` - If the HTTP request fails
345    /// * `ParseError::IoError` - If file operations fail
346    /// * `ParseError::PmcNotAvailable` - If the article is not available in OA
347    ///
348    /// # Example
349    ///
350    /// ```no_run
351    /// use pubmed_client::pmc::cloud::PmcCloudClient;
352    /// use pubmed_client::ClientConfig;
353    /// use std::path::Path;
354    ///
355    /// #[tokio::main]
356    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
357    ///     let config = ClientConfig::new();
358    ///     let client = PmcCloudClient::new(config);
359    ///     let output_dir = Path::new("./extracted_articles");
360    ///     let figures = client.extract_figures_with_captions("PMC7906746", output_dir).await?;
361    ///
362    ///     for figure in figures {
363    ///         println!("Figure {}: {:?}", figure.figure.id, figure.figure.caption);
364    ///         println!("File: {}", figure.extracted_file_path);
365    ///     }
366    ///     Ok(())
367    /// }
368    /// ```
369    #[cfg(not(target_arch = "wasm32"))]
370    pub async fn extract_figures_with_captions<P: AsRef<Path>>(
371        &self,
372        pmcid: &str,
373        output_dir: P,
374    ) -> Result<Vec<ExtractedFigure>> {
375        let normalized_pmcid = common::normalize_pmcid(pmcid);
376
377        let output_path = output_dir.as_ref();
378        tokio_fs::create_dir_all(output_path)
379            .await
380            .map_err(|e| ParseError::IoError {
381                message: format!("Failed to create output directory: {}", e),
382            })?;
383
384        // Download the article's OA package once. It already contains the JATS
385        // full-text XML, so we parse that rather than issuing a second,
386        // rate-limited eutils fetch of the same document.
387        let extracted_files = self.download_files(&normalized_pmcid, &output_dir).await?;
388
389        let full_text = self
390            .parse_article_xml(&normalized_pmcid, &extracted_files)
391            .await?;
392
393        let figures = self
394            .match_figures_with_files(&full_text, &extracted_files, &output_dir)
395            .await?;
396
397        Ok(figures)
398    }
399
400    /// Parse the article's JATS XML, preferring the copy already downloaded from
401    /// the OA Cloud so no redundant eutils request is made.
402    ///
403    /// OA packages always include the full-text XML; the eutils fallback only
404    /// triggers in the unexpected case where the downloaded files contain no
405    /// `.xml`, so behavior never regresses relative to the previous eutils-only
406    /// path.
407    #[cfg(not(target_arch = "wasm32"))]
408    async fn parse_article_xml(
409        &self,
410        normalized_pmcid: &str,
411        extracted_files: &[String],
412    ) -> Result<PmcArticle> {
413        if let Some(xml_path) = Self::find_downloaded_xml(extracted_files, normalized_pmcid) {
414            let xml_content =
415                tokio_fs::read_to_string(&xml_path)
416                    .await
417                    .map_err(|e| ParseError::IoError {
418                        message: format!("Failed to read downloaded XML {}: {}", xml_path, e),
419                    })?;
420            return Ok(parse_pmc_xml(&xml_content, normalized_pmcid)?);
421        }
422
423        debug!(
424            pmcid = %normalized_pmcid,
425            "OA Cloud package had no XML; falling back to eutils fetch"
426        );
427        let xml_content = common::fetch_pmc_xml(
428            &self.executor(),
429            self.config.effective_base_url(),
430            normalized_pmcid,
431        )
432        .await?;
433        Ok(parse_pmc_xml(&xml_content, normalized_pmcid)?)
434    }
435
436    /// Find the downloaded article XML among the OA package's files.
437    ///
438    /// The JATS file is named `<PMCID>.<version>.xml`, so we match on a file
439    /// name that ends in `.xml` and contains the PMCID (case-insensitive).
440    #[cfg(not(target_arch = "wasm32"))]
441    fn find_downloaded_xml(extracted_files: &[String], normalized_pmcid: &str) -> Option<String> {
442        let pmcid_lower = normalized_pmcid.to_lowercase();
443        extracted_files
444            .iter()
445            .find(|path| {
446                let name = Path::new(path)
447                    .file_name()
448                    .map(|n| n.to_string_lossy().to_lowercase())
449                    .unwrap_or_default();
450                name.ends_with(".xml") && name.contains(&pmcid_lower)
451            })
452            .cloned()
453    }
454
455    /// Match figures from XML with extracted files
456    #[cfg(not(target_arch = "wasm32"))]
457    async fn match_figures_with_files<P: AsRef<Path>>(
458        &self,
459        full_text: &PmcArticle,
460        extracted_files: &[String],
461        output_dir: P,
462    ) -> Result<Vec<ExtractedFigure>> {
463        let output_path = output_dir.as_ref();
464        let mut matched_figures = Vec::new();
465
466        let mut all_figures = Vec::new();
467        for section in full_text.sections() {
468            Self::collect_figures_recursive(section, &mut all_figures);
469        }
470
471        let image_extensions = [
472            "jpg", "jpeg", "png", "gif", "tiff", "tif", "svg", "eps", "pdf",
473        ];
474
475        for figure in all_figures {
476            let matching_file =
477                Self::find_matching_file(&figure, extracted_files, &image_extensions);
478
479            if let Some(file_path) = matching_file {
480                let absolute_path =
481                    if file_path.starts_with(&output_path.to_string_lossy().to_string()) {
482                        file_path.clone()
483                    } else {
484                        output_path.join(&file_path).to_string_lossy().to_string()
485                    };
486
487                let file_size = tokio_fs::metadata(&absolute_path)
488                    .await
489                    .map(|m| m.len())
490                    .ok();
491
492                let dimensions = Self::get_image_dimensions(&absolute_path).await;
493
494                matched_figures.push(ExtractedFigure {
495                    figure: figure.clone(),
496                    extracted_file_path: absolute_path,
497                    file_size,
498                    dimensions,
499                });
500            }
501        }
502
503        Ok(matched_figures)
504    }
505
506    /// Recursively collect all figures from sections and subsections
507    #[cfg(not(target_arch = "wasm32"))]
508    fn collect_figures_recursive(section: &Section, figures: &mut Vec<Figure>) {
509        figures.extend(section.figures.clone());
510        for subsection in &section.subsections {
511            Self::collect_figures_recursive(subsection, figures);
512        }
513    }
514
515    /// Find a matching file for a figure based on ID, label, or filename patterns.
516    ///
517    /// Three rules are tried in order, returning the first extracted file that matches:
518    /// 1. the explicit `graphic_href` (case-sensitive substring of the file name, any extension);
519    /// 2. the figure `id` (case-insensitive substring) with an image extension;
520    /// 3. the figure `label` with whitespace/dots stripped (case-insensitive) with an image extension.
521    #[cfg(not(target_arch = "wasm32"))]
522    pub fn find_matching_file(
523        figure: &Figure,
524        extracted_files: &[String],
525        image_extensions: &[&str],
526    ) -> Option<String> {
527        // Rule 1: match by explicit graphic href. Case-sensitive and does not
528        // require an image extension, mirroring the original behavior.
529        if let Some(file_name) = &figure.graphic_href
530            && let Some(matched) =
531                Self::find_first_file(extracted_files, false, image_extensions, |filename| {
532                    filename.contains(file_name.as_str())
533                })
534        {
535            return Some(matched);
536        }
537
538        // Rule 2: match by figure id (case-insensitive) with an image extension.
539        let figure_id_lower = figure.id.to_lowercase();
540        if let Some(matched) =
541            Self::find_first_file(extracted_files, true, image_extensions, |filename| {
542                filename.to_lowercase().contains(&figure_id_lower)
543            })
544        {
545            return Some(matched);
546        }
547
548        // Rule 3: match by label (whitespace/dots stripped) with an image extension.
549        if let Some(label) = &figure.label {
550            let label_clean = label.to_lowercase().replace([' ', '.'], "");
551            if let Some(matched) =
552                Self::find_first_file(extracted_files, true, image_extensions, |filename| {
553                    filename.to_lowercase().contains(&label_clean)
554                })
555            {
556                return Some(matched);
557            }
558        }
559
560        None
561    }
562
563    /// Return the first extracted file whose file name satisfies `predicate`.
564    ///
565    /// When `require_image_ext` is true, the file must additionally have an
566    /// extension (case-insensitive) present in `image_extensions`. The predicate
567    /// receives the raw (non-lower-cased) file name so callers control casing.
568    #[cfg(not(target_arch = "wasm32"))]
569    fn find_first_file(
570        extracted_files: &[String],
571        require_image_ext: bool,
572        image_extensions: &[&str],
573        predicate: impl Fn(&str) -> bool,
574    ) -> Option<String> {
575        for file_path in extracted_files {
576            let path = Path::new(file_path);
577            let Some(filename) = path.file_name() else {
578                continue;
579            };
580            if !predicate(&filename.to_string_lossy()) {
581                continue;
582            }
583            if require_image_ext && !Self::has_image_extension(path, image_extensions) {
584                continue;
585            }
586            return Some(file_path.clone());
587        }
588        None
589    }
590
591    /// Whether `path` has an extension (case-insensitive) in `image_extensions`.
592    #[cfg(not(target_arch = "wasm32"))]
593    fn has_image_extension(path: &Path, image_extensions: &[&str]) -> bool {
594        path.extension()
595            .map(|ext| image_extensions.contains(&ext.to_string_lossy().to_lowercase().as_str()))
596            .unwrap_or(false)
597    }
598
599    /// Read image dimensions from the file header.
600    ///
601    /// `imagesize` parses only the header, so this never decodes pixel data.
602    /// Formats it does not recognize (`svg`, `eps`, `pdf` among the extensions
603    /// figures are matched on) simply yield `None`.
604    #[cfg(not(target_arch = "wasm32"))]
605    async fn get_image_dimensions(file_path: &str) -> Option<(u32, u32)> {
606        task::spawn_blocking({
607            let file_path = file_path.to_string();
608            move || {
609                let size = imagesize::size(&file_path).ok()?;
610                Some((
611                    u32::try_from(size.width).ok()?,
612                    u32::try_from(size.height).ok()?,
613                ))
614            }
615        })
616        .await
617        .ok()
618        .flatten()
619    }
620
621    fn executor(&self) -> RequestExecutor<'_> {
622        RequestExecutor::new(&self.client, &self.rate_limiter, &self.config)
623    }
624
625    /// GET a PMC OA Cloud (AWS S3) URL.
626    ///
627    /// The `pmc-oa-opendata` bucket is AWS Open Data, not an NCBI E-utilities
628    /// endpoint, so these requests are **not** rate-limited by the NCBI quota —
629    /// their parallelism is bounded by [`ClientConfig::effective_oa_download_concurrency`]
630    /// instead. Retry and status-aware error mapping still apply.
631    #[cfg(not(target_arch = "wasm32"))]
632    async fn s3_get(&self, url: &str) -> Result<Response> {
633        debug!("Making PMC OA Cloud (S3) request to: {url}");
634        fetch_with_retry(
635            || self.client.get(url),
636            &self.config.retry_config,
637            None,
638            "PMC OA Cloud request",
639        )
640        .await
641    }
642}
643
644#[cfg(test)]
645mod tests {
646    use super::*;
647
648    #[test]
649    fn test_normalize_pmcid() {
650        assert_eq!(common::normalize_pmcid("1234567"), "PMC1234567");
651        assert_eq!(common::normalize_pmcid("PMC1234567"), "PMC1234567");
652    }
653
654    #[test]
655    fn test_client_creation() {
656        let config = ClientConfig::new();
657        let _client = PmcCloudClient::new(config);
658    }
659
660    #[test]
661    fn test_with_shared_creation() {
662        let config = ClientConfig::new();
663        let rate_limiter = config.create_rate_limiter();
664        let client = Client::new();
665        let _cloud_client = PmcCloudClient::with_shared(client, rate_limiter, config);
666    }
667
668    #[cfg(not(target_arch = "wasm32"))]
669    #[test]
670    fn test_parse_cloud_listing_extracts_keys() {
671        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
672<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>pmc-oa-opendata</Name><Prefix>PMC7906746.</Prefix><KeyCount>5</KeyCount>
673<Contents><Key>PMC7906746.1/PMC7906746.1.json</Key><Size>1</Size></Contents>
674<Contents><Key>PMC7906746.1/PMC7906746.1.xml</Key><Size>1</Size></Contents>
675<Contents><Key>PMC7906746.1/gr1_lrg.jpg</Key><Size>1</Size></Contents>
676</ListBucketResult>"#;
677
678        let keys = PmcCloudClient::parse_cloud_listing(xml).unwrap();
679        assert_eq!(
680            keys,
681            vec![
682                "PMC7906746.1/PMC7906746.1.json".to_string(),
683                "PMC7906746.1/PMC7906746.1.xml".to_string(),
684                "PMC7906746.1/gr1_lrg.jpg".to_string(),
685            ]
686        );
687    }
688
689    #[cfg(not(target_arch = "wasm32"))]
690    #[test]
691    fn test_parse_cloud_listing_skips_folder_markers() {
692        let xml = r#"<ListBucketResult><Contents><Key>PMC1.1/</Key></Contents><Contents><Key>PMC1.1/PMC1.1.xml</Key></Contents></ListBucketResult>"#;
693        let keys = PmcCloudClient::parse_cloud_listing(xml).unwrap();
694        assert_eq!(keys, vec!["PMC1.1/PMC1.1.xml".to_string()]);
695    }
696
697    #[cfg(not(target_arch = "wasm32"))]
698    #[test]
699    fn test_select_latest_version_keys_picks_highest() {
700        let keys = vec![
701            "PMC1.1/PMC1.1.xml".to_string(),
702            "PMC1.1/gr1.jpg".to_string(),
703            "PMC1.2/PMC1.2.xml".to_string(),
704            "PMC1.2/gr1.jpg".to_string(),
705        ];
706        let latest = PmcCloudClient::select_latest_version_keys(keys);
707        assert_eq!(
708            latest,
709            vec![
710                "PMC1.2/PMC1.2.xml".to_string(),
711                "PMC1.2/gr1.jpg".to_string(),
712            ]
713        );
714    }
715
716    #[cfg(not(target_arch = "wasm32"))]
717    #[test]
718    fn test_find_downloaded_xml() {
719        let files = vec![
720            "/tmp/PMC9991720/gr1_lrg.jpg".to_string(),
721            "/tmp/PMC9991720/PMC9991720.1.xml".to_string(),
722            "/tmp/PMC9991720/PMC9991720.1.json".to_string(),
723        ];
724        assert_eq!(
725            PmcCloudClient::find_downloaded_xml(&files, "PMC9991720"),
726            Some("/tmp/PMC9991720/PMC9991720.1.xml".to_string())
727        );
728        // Case-insensitive on both the file name and the PMCID.
729        assert_eq!(
730            PmcCloudClient::find_downloaded_xml(
731                &["/tmp/pmc9991720.1.XML".to_string()],
732                "PMC9991720"
733            ),
734            Some("/tmp/pmc9991720.1.XML".to_string())
735        );
736        // No XML present -> None (triggers the eutils fallback).
737        assert_eq!(
738            PmcCloudClient::find_downloaded_xml(
739                &["/tmp/PMC9991720/gr1.jpg".to_string()],
740                "PMC9991720"
741            ),
742            None
743        );
744        // An unrelated XML must not match.
745        assert_eq!(
746            PmcCloudClient::find_downloaded_xml(
747                &["/tmp/PMC0000001.1.xml".to_string()],
748                "PMC9991720"
749            ),
750            None
751        );
752    }
753
754    #[cfg(not(target_arch = "wasm32"))]
755    #[test]
756    fn test_select_latest_version_keys_empty() {
757        assert!(PmcCloudClient::select_latest_version_keys(vec![]).is_empty());
758    }
759
760    #[cfg(not(target_arch = "wasm32"))]
761    fn figure(id: &str, label: Option<&str>, graphic_href: Option<&str>) -> Figure {
762        Figure {
763            id: id.to_string(),
764            label: label.map(|s| s.to_string()),
765            caption: None,
766            alt_text: None,
767            fig_type: None,
768            graphic_href: graphic_href.map(|s| s.to_string()),
769        }
770    }
771
772    #[cfg(not(target_arch = "wasm32"))]
773    const IMAGE_EXTS: &[&str] = &["jpg", "jpeg", "png", "gif", "tif", "tiff"];
774
775    #[cfg(not(target_arch = "wasm32"))]
776    #[test]
777    fn test_find_matching_file_by_graphic_href() {
778        let files = vec![
779            "PMC1/PMC1.xml".to_string(),
780            "PMC1/gr1_lrg.jpg".to_string(),
781            "PMC1/fig2.png".to_string(),
782        ];
783        // graphic_href match is a case-sensitive substring and ignores extension.
784        let fig = figure("fig-1", None, Some("gr1_lrg.jpg"));
785        assert_eq!(
786            PmcCloudClient::find_matching_file(&fig, &files, IMAGE_EXTS),
787            Some("PMC1/gr1_lrg.jpg".to_string())
788        );
789    }
790
791    #[cfg(not(target_arch = "wasm32"))]
792    #[test]
793    fn test_find_matching_file_by_figure_id() {
794        let files = vec!["PMC1/PMC1.xml".to_string(), "PMC1/GR1.PNG".to_string()];
795        // id match is case-insensitive and requires an image extension.
796        let fig = figure("gr1", None, None);
797        assert_eq!(
798            PmcCloudClient::find_matching_file(&fig, &files, IMAGE_EXTS),
799            Some("PMC1/GR1.PNG".to_string())
800        );
801    }
802
803    #[cfg(not(target_arch = "wasm32"))]
804    #[test]
805    fn test_find_matching_file_by_label() {
806        let files = vec!["PMC1/PMC1.xml".to_string(), "PMC1/figure1.jpg".to_string()];
807        // label match strips spaces/dots and is case-insensitive: "Figure 1." -> "figure1".
808        let fig = figure("unrelated-id", Some("Figure 1."), None);
809        assert_eq!(
810            PmcCloudClient::find_matching_file(&fig, &files, IMAGE_EXTS),
811            Some("PMC1/figure1.jpg".to_string())
812        );
813    }
814
815    #[cfg(not(target_arch = "wasm32"))]
816    #[test]
817    fn test_find_matching_file_id_requires_image_extension() {
818        // A non-image file whose name contains the id must not match rule 2.
819        let files = vec!["PMC1/gr1.xml".to_string()];
820        let fig = figure("gr1", None, None);
821        assert_eq!(
822            PmcCloudClient::find_matching_file(&fig, &files, IMAGE_EXTS),
823            None
824        );
825    }
826
827    #[cfg(not(target_arch = "wasm32"))]
828    #[test]
829    fn test_find_matching_file_no_match() {
830        let files = vec!["PMC1/other.jpg".to_string()];
831        let fig = figure("gr9", Some("Figure 9"), Some("missing.png"));
832        assert_eq!(
833            PmcCloudClient::find_matching_file(&fig, &files, IMAGE_EXTS),
834            None
835        );
836    }
837
838    /// A 3x2 PNG: signature + an IHDR chunk declaring the dimensions.
839    #[cfg(not(target_arch = "wasm32"))]
840    const PNG_3X2: &[u8] = &[
841        0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a, // signature
842        0x00, 0x00, 0x00, 0x0d, b'I', b'H', b'D', b'R', // chunk length + type
843        0x00, 0x00, 0x00, 0x03, // width  = 3
844        0x00, 0x00, 0x00, 0x02, // height = 2
845        0x08, 0x06, 0x00, 0x00, 0x00, // bit depth, color type, compression, filter, interlace
846        0x00, 0x00, 0x00, 0x00, // CRC (not validated when reading the header)
847    ];
848
849    #[cfg(not(target_arch = "wasm32"))]
850    #[tokio::test]
851    async fn test_get_image_dimensions_reads_png_header() {
852        use std::fs;
853
854        let dir = tempfile::tempdir().unwrap();
855        let path = dir.path().join("gr1.png");
856        fs::write(&path, PNG_3X2).unwrap();
857
858        assert_eq!(
859            PmcCloudClient::get_image_dimensions(&path.to_string_lossy()).await,
860            Some((3, 2))
861        );
862    }
863
864    #[cfg(not(target_arch = "wasm32"))]
865    #[tokio::test]
866    async fn test_get_image_dimensions_none_for_unsupported_and_missing() {
867        use std::fs;
868
869        let dir = tempfile::tempdir().unwrap();
870
871        // SVG / EPS / PDF are in the matched extension list but carry no
872        // readable raster header, so they yield `None`.
873        let svg = dir.path().join("gr1.svg");
874        fs::write(&svg, b"<svg width=\"10\" height=\"20\"></svg>").unwrap();
875        assert_eq!(
876            PmcCloudClient::get_image_dimensions(&svg.to_string_lossy()).await,
877            None
878        );
879
880        let missing = dir.path().join("does-not-exist.png");
881        assert_eq!(
882            PmcCloudClient::get_image_dimensions(&missing.to_string_lossy()).await,
883            None
884        );
885    }
886}
pubmed_client/pmc/cloud.rs

pubmed_client/pmc/
cloud.rs
