feat: Implement basic registry index caching

ravenexp · ravenexp · commit a9198b6fa189 · 2023-08-03T17:38:48.000+03:00
Cache the registry index entries on the local filesystem
in the same way the crate files are already cached.

Put the registry index cache files under the "index/" subdirectory.

TODO: Local cache invalidation and upstream synchronization.
diff --git a/src/file_cache.rs b/src/file_cache.rs
@@ -5,7 +5,7 @@ use std::path::Path;
 
 use log::error;
 
-use super::CrateInfo;
+use super::{CrateInfo, IndexEntry};
 
 /// Caches the crate package file on the local filesystem.
 pub fn cache_store_crate(dir: &Path, crate_info: &CrateInfo, data: &[u8]) {
@@ -25,3 +25,21 @@ pub fn cache_store_crate(dir: &Path, crate_info: &CrateInfo, data: &[u8]) {
 pub fn cache_fetch_crate(dir: &Path, crate_info: &CrateInfo) -> Option<Vec<u8>> {
     read(dir.join(crate_info.to_file_path())).ok()
 }
+
+/// Caches the index entry file on the local filesystem.
+pub fn cache_store_index_entry(dir: &Path, entry: &IndexEntry, data: &[u8]) {
+    let entry_file_path = dir.join(entry.to_file_path());
+
+    if let Err(e) = create_dir_all(entry_file_path.parent().unwrap()) {
+        error!("cache: failed to create index directory: {e}");
+        return;
+    }
+
+    write(entry_file_path, data)
+        .unwrap_or_else(|e| error!("cache: failed to write index entry file: {e}"));
+}
+
+/// Fetches the cached index entry file from the local filesystem, if present.
+pub fn cache_fetch_index_entry(dir: &Path, entry: &IndexEntry) -> Option<Vec<u8>> {
+    read(dir.join(entry.to_file_path())).ok()
+}
diff --git a/src/index_entry.rs b/src/index_entry.rs
@@ -1,6 +1,7 @@
 //! Registry index entry handling helpers
 
 use std::fmt::{Display, Formatter, Result};
+use std::path::PathBuf;
 use std::time::SystemTime;
 
 use httpdate::{fmt_http_date, parse_http_date};
@@ -93,6 +94,12 @@ impl IndexEntry {
             ),
         }
     }
+
+    /// Builds the relative index entry file path for cache storage.
+    #[must_use]
+    pub fn to_file_path(&self) -> PathBuf {
+        PathBuf::from(self.to_index_url())
+    }
 }
 
 #[cfg(test)]
diff --git a/src/main.rs b/src/main.rs
@@ -25,7 +25,9 @@ use url::Url;
 
 use crate::config_json::{gen_config_json_file, is_config_json_url};
 use crate::crate_info::CrateInfo;
-use crate::file_cache::{cache_fetch_crate, cache_store_crate};
+use crate::file_cache::{
+    cache_fetch_crate, cache_fetch_index_entry, cache_store_crate, cache_store_index_entry,
+};
 use crate::index_entry::IndexEntry;
 
 /// Default listen address and port
@@ -79,6 +81,9 @@ struct ProxyConfig {
     /// External URL of this proxy server (defaults to [`DEFAULT_PROXY_URL`])
     proxy_url: Url,
 
+    /// Registry index cache directory (defaults to [`DEFAULT_CACHE_DIR`])
+    index_dir: PathBuf,
+
     /// Crate files cache directory (defaults to [`DEFAULT_CACHE_DIR`])
     crates_dir: PathBuf,
 }
@@ -295,6 +300,8 @@ fn forward_index_request(request: Request, entry: IndexEntry, config: ProxyConfi
                     "fetch: successfully got index entry for {entry}",
                     entry = response.entry
                 );
+
+                cache_store_index_entry(&config.index_dir, &response.entry, &response.data);
             } else {
                 debug!(
                     "fetch: cached index entry for {entry} is up to date",
@@ -361,7 +368,20 @@ fn handle_index_request(request: Request, index_url: &str, config: &ProxyConfig)
         }
     }
 
-    forward_index_request(request, index_entry, config.clone());
+    if let Some(data) = cache_fetch_index_entry(&config.index_dir, &index_entry) {
+        debug!("proxy: local index cache hit for {index_entry}");
+
+        // TODO: Add cache control metadata.
+        let response = IndexResponse {
+            entry: index_entry,
+            status: 200,
+            data,
+        };
+
+        send_index_entry_data_response(request, response);
+    } else {
+        forward_index_request(request, index_entry, config.clone());
+    }
 }
 
 /// Processes one HTTP GET request.
@@ -509,8 +529,14 @@ fn main() {
     info!("proxy: using proxy server URL: {proxy_url}");
 
     let cache_dir = PathBuf::from(cache_dir_string);
+    let index_dir = cache_dir.join("index");
     let crates_dir = cache_dir.join("crates");
 
+    info!(
+        "cache: using index directory: {}",
+        index_dir.to_string_lossy()
+    );
+
     info!(
         "cache: using crates directory: {}",
         crates_dir.to_string_lossy()
@@ -520,6 +546,7 @@ fn main() {
         index_url,
         upstream_url,
         proxy_url,
+        index_dir,
         crates_dir,
     };
 

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`//! Registry index entry handling helpers`
`2`	`2`
`3`	`3`	`use std::fmt::{Display, Formatter, Result};`
	`4`	`+use std::path::PathBuf;`
`4`	`5`	`use std::time::SystemTime;`
`5`	`6`
`6`	`7`	`use httpdate::{fmt_http_date, parse_http_date};`
`@@ -93,6 +94,12 @@ impl IndexEntry {`
`93`	`94`	`),`
`94`	`95`	`}`
`95`	`96`	`}`
	`97`	`+`
	`98`	`+ /// Builds the relative index entry file path for cache storage.`
	`99`	`+ #[must_use]`
	`100`	`+ pub fn to_file_path(&self) -> PathBuf {`
	`101`	`+ PathBuf::from(self.to_index_url())`
	`102`	`+ }`
`96`	`103`	`}`
`97`	`104`
`98`	`105`	`#[cfg(test)]`