Remove blacklisting to make SiteIcons Send

src/html_parser/site_logo.rs

@@ -13,7 +13,6 @@ use url::Url;
 pub async fn parse_site_logo(
   url: &Url,
   mut body: impl Stream<Item = Result<Vec<u8>, String>> + Unpin,
-  is_blacklisted: impl Fn(&Url) -> bool,
 ) -> Result<Icon, Box<dyn Error>> {
   let mut parser = driver::parse_document(Html::new_document(), Default::default());
   while let Some(data) = body.next().await {
@@ -123,12 +122,6 @@ pub async fn parse_site_logo(
         elem.attr("src").and_then(|href| url.join(&href).ok())
       };
 
-      if let Some(href) = &href {
-        if is_blacklisted(href) {
-          return None;
-        }
-      }
-
       href.map(|href| (href, elem_ref, weight))
     })
     .collect();

src/icons.rs

@@ -10,9 +10,7 @@ use std::error::Error;
 use url::Url;
 use vec1::Vec1;
 
-pub struct SiteIcons {
+pub struct SiteIcons;
-  blacklist: Option<Box<dyn Fn(&Url) -> bool>>,
-}
 
 #[derive(Debug, Clone)]
 enum LoadedKind {
@@ -24,21 +22,7 @@ enum LoadedKind {
 
 impl SiteIcons {
   pub fn new() -> Self {
-    SiteIcons { blacklist: None }
+    SiteIcons
-  }
-
-  pub fn new_with_blacklist(blacklist: impl Fn(&Url) -> bool + 'static) -> Self {
-    SiteIcons {
-      blacklist: Some(Box::new(blacklist)),
-    }
-  }
-
-  pub fn is_blacklisted(&self, url: &Url) -> bool {
-    if let Some(is_blacklisted) = &self.blacklist {
-      is_blacklisted(url)
-    } else {
-      false
-    }
   }
 
   pub async fn load_website<U: IntoUrl>(
@@ -78,24 +62,20 @@ impl SiteIcons {
 
       let url = res.url().clone();
 
-      if self.is_blacklisted(&url) {
+      let body = res.bytes_stream().map(|res| {
-        None
+        res
-      } else {
+          .map(|bytes| bytes.to_vec())
-        let body = res.bytes_stream().map(|res| {
+          .map_err(|err| err.to_string())
-          res
+      });
-            .map(|bytes| bytes.to_vec())
-            .map_err(|err| err.to_string())
-        });
 
-        let mut publisher = Publisher::new(128);
+      let mut publisher = Publisher::new(128);
-        let subscriber = publisher.subscribe();
+      let subscriber = publisher.subscribe();
 
-        Some((
+      Some((
-          url,
+        url,
-          async move { StreamPublisher::new(&mut publisher, body).await }.shared(),
+        async move { StreamPublisher::new(&mut publisher, body).await }.shared(),
-          subscriber,
+        subscriber,
-        ))
+      ))
-      }
     }
     .shared();
 
@@ -117,10 +97,7 @@ impl SiteIcons {
 
         LoadedKind::SiteLogo(match html_response {
           Some((url, complete, body)) => {
-            let (icons, _) = join!(
+            let (icons, _) = join!(html_parser::parse_site_logo(&url, body), complete);
-              html_parser::parse_site_logo(&url, body, |url| self.is_blacklisted(url)),
-              complete
-            );
 
             icons.ok()
           }