Remove blacklisting to make SiteIcons Send

src/html_parser/site_logo.rs

@@ -13,7 +13,6 @@ use url::Url;
 pub async fn parse_site_logo(
   url: &Url,
   mut body: impl Stream<Item = Result<Vec<u8>, String>> + Unpin,
-  is_blacklisted: impl Fn(&Url) -> bool,
 ) -> Result<Icon, Box<dyn Error>> {
   let mut parser = driver::parse_document(Html::new_document(), Default::default());
   while let Some(data) = body.next().await {
@@ -123,12 +122,6 @@ pub async fn parse_site_logo(
         elem.attr("src").and_then(|href| url.join(&href).ok())
       };
 
-      if let Some(href) = &href {
-        if is_blacklisted(href) {
-          return None;
-        }
-      }
-
       href.map(|href| (href, elem_ref, weight))
     })
     .collect();

src/icons.rs

@@ -10,9 +10,7 @@ use std::error::Error;
 use url::Url;
 use vec1::Vec1;
 
-pub struct SiteIcons {
+pub struct SiteIcons;
-  blacklist: Option<Box<dyn Fn(&Url) -> bool>>,
-}
 
 #[derive(Debug, Clone)]
 enum LoadedKind {
@@ -24,21 +22,7 @@ enum LoadedKind {
 
 impl SiteIcons {
   pub fn new() -> Self {
-    SiteIcons { blacklist: None }
+    SiteIcons
-  }
-
-  pub fn new_with_blacklist(blacklist: impl Fn(&Url) -> bool + 'static) -> Self {
-    SiteIcons {
-      blacklist: Some(Box::new(blacklist)),
-    }
-  }
-
-  pub fn is_blacklisted(&self, url: &Url) -> bool {
-    if let Some(is_blacklisted) = &self.blacklist {
-      is_blacklisted(url)
-    } else {
-      false
-    }
   }
 
   pub async fn load_website<U: IntoUrl>(
@@ -78,9 +62,6 @@ impl SiteIcons {
 
       let url = res.url().clone();
 
-      if self.is_blacklisted(&url) {
-        None
-      } else {
       let body = res.bytes_stream().map(|res| {
         res
           .map(|bytes| bytes.to_vec())
@@ -96,7 +77,6 @@ impl SiteIcons {
         subscriber,
       ))
     }
-    }
     .shared();
 
     let mut futures = vec![
@@ -117,10 +97,7 @@ impl SiteIcons {
 
         LoadedKind::SiteLogo(match html_response {
           Some((url, complete, body)) => {
-            let (icons, _) = join!(
+            let (icons, _) = join!(html_parser::parse_site_logo(&url, body), complete);
-              html_parser::parse_site_logo(&url, body, |url| self.is_blacklisted(url)),
-              complete
-            );
 
             icons.ok()
           }