This commit is contained in:
Sam Denty 2021-01-30 19:12:13 +00:00
parent c44b881e69
commit 61ae905521
No known key found for this signature in database
GPG key ID: F3E9308D4A43BC0E
8 changed files with 28 additions and 12 deletions

2
Cargo.lock generated
View file

@ -1539,7 +1539,7 @@ checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7"
[[package]] [[package]]
name = "site_icons" name = "site_icons"
version = "0.1.0" version = "0.1.1"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"clap", "clap",

View file

@ -1,6 +1,6 @@
[package] [package]
name = "site_icons" name = "site_icons"
version = "0.1.0" version = "0.1.1"
authors = ["Sam Denty <sam@samdenty.com>"] authors = ["Sam Denty <sam@samdenty.com>"]
edition = "2018" edition = "2018"
license = "GPL-3.0" license = "GPL-3.0"

View file

@ -1,4 +1,4 @@
[tasks.run] [tasks.run]
env = { "RUST_LOG" = "site_icons" } env = { "RUST_LOG" = "info" }
command = "cargo" command = "cargo"
args = ["run", "--", "${@}"] args = ["run", "--", "${@}"]

View file

@ -21,7 +21,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
if opts.debug { if opts.debug {
let mut builder = Builder::new(); let mut builder = Builder::new();
builder.filter_module("site_icons", LevelFilter::Info); builder.filter_module("info", LevelFilter::Info);
builder.init(); builder.init();
} }

View file

@ -1,4 +1,4 @@
use crate::{selector, utils::encode_svg, warn_err, Icon, IconInfo, IconKind, CLIENT}; use crate::{utils::encode_svg, Icon, IconInfo, IconKind, CLIENT};
use future::join_all; use future::join_all;
use futures::StreamExt; use futures::StreamExt;
use futures::{prelude::*, task::noop_waker}; use futures::{prelude::*, task::noop_waker};

View file

@ -1,14 +1,35 @@
#![feature(async_closure, map_into_keys_values, bool_to_option)] #![feature(async_closure, map_into_keys_values, bool_to_option)]
//! # site_icons
//! An efficient website icon scraper.
//!
//! ## Usage
//! ```rust
//! use site_icons::Icons;
//!
//! let icons = Icons::new();
//! // scrape the icons from a url
//! icons.load_website("https://github.com").await?;
//!
//! // fetch all icons, ensuring they exist & determining size
//! let entries = icons.entries().await;
//!
//! // entries are sorted from highest to lowest resolution
//! for icon in entries {
//! println("{:?}", icon)
//! }
//! ```
#[macro_use] #[macro_use]
extern crate serde_with; extern crate serde_with;
#[macro_use] #[macro_use]
extern crate log; extern crate log;
#[macro_use]
mod macros;
mod icon; mod icon;
mod icon_info; mod icon_info;
mod icon_size; mod icon_size;
mod icons; mod icons;
mod macros;
mod utils; mod utils;
pub use icon::*; pub use icon::*;

View file

@ -1,19 +1,16 @@
#[macro_export]
macro_rules! selector { macro_rules! selector {
($($selector:expr),+ $(,)?) => {{ ($($selector:expr),+ $(,)?) => {{
static RE: once_cell::sync::OnceCell<scraper::Selector> = once_cell::sync::OnceCell::new(); static RE: once_cell::sync::OnceCell<scraper::Selector> = once_cell::sync::OnceCell::new();
RE.get_or_init(|| scraper::Selector::parse(crate::join!(",", $($selector),+)).unwrap()) RE.get_or_init(|| scraper::Selector::parse(join!(",", $($selector),+)).unwrap())
}}; }};
} }
#[macro_export]
macro_rules! join { macro_rules! join {
($pattern:literal,$first:expr$(, $($rest:expr),*)? $(,)?) => { ($pattern:literal,$first:expr$(, $($rest:expr),*)? $(,)?) => {
concat!($first$(, $($pattern, $rest),*)?) concat!($first$(, $($pattern, $rest),*)?)
}; };
} }
#[macro_export]
macro_rules! regex { macro_rules! regex {
($re:literal $(,)?) => {{ ($re:literal $(,)?) => {{
static RE: once_cell::sync::OnceCell<regex::Regex> = once_cell::sync::OnceCell::new(); static RE: once_cell::sync::OnceCell<regex::Regex> = once_cell::sync::OnceCell::new();
@ -21,7 +18,6 @@ macro_rules! regex {
}}; }};
} }
#[macro_export]
macro_rules! warn_err { macro_rules! warn_err {
($result:expr, $($arg:tt)*) => {{ ($result:expr, $($arg:tt)*) => {{
if let Err(err) = $result { if let Err(err) = $result {

View file

@ -1,4 +1,3 @@
use crate::regex;
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
const DATA_URI: &AsciiSet = &CONTROLS const DATA_URI: &AsciiSet = &CONTROLS