commit 6368454e36204fa00b71a4712f1d1d9bfe33df35 Author: Sam Denty Date: Fri Jan 29 12:23:15 2021 +0000 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..b68828e --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,3 @@ +indent_style = "Block" +reorder_imports = true +tab_spaces = 2 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..49040b2 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2157 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bumpalo" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f07aa6688c702439a1be0307b6a94dffe1168569e45b9500c1372bc580740d59" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "strsim 0.10.0", + "termcolor", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap_derive" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "const_fn" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" + +[[package]] +name = "cookie" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" +dependencies = [ + "cookie", + "idna", + "log", + "publicsuffix", + "serde", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "matches", + "phf", + "proc-macro2", + "quote", + "smallvec", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.9.3", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "data-url" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d33fe99ccedd6e84bc035f1931bb2e6be79739d6242bd895e7311c886c50dc9c" +dependencies = [ + "matches", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "dtoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e" + +[[package]] +name = "dtoa-short" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65" + +[[package]] +name = "futures-executor" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e59fdc009a4b3096bf94f740a0f2424c082521f20a9b08c5c07c48d90fd9b9" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500" + +[[package]] +name = "futures-macro" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6" + +[[package]] +name = "futures-task" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite 0.2.4", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "h2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.1.0", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "html5ever" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "http" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +dependencies = [ + "bytes 1.0.1", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" +dependencies = [ + "bytes 1.0.1", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe" +dependencies = [ + "bytes 1.0.1", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.4", + "socket2", + "tokio 1.1.0", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper", + "native-tls", + "tokio 1.1.0", + "tokio-native-tls", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cfb73131c35423a367daf8cbd24100af0d077668c8c2943f0e7dd775fef0f65" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff" + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab" +dependencies = [ + "log", + "phf", + "phf_codegen", + "serde", + "serde_derive", + "serde_json", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memory_units" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime" +version = "0.4.0-a.0" +source = "git+https://github.com/hyperium/mime#938484de95445a2af931515d2b7252612c575da7" +dependencies = [ + "mime-parse", + "quoted-string", +] + +[[package]] +name = "mime-parse" +version = "0.0.0" +source = "git+https://github.com/hyperium/mime#938484de95445a2af931515d2b7252612c575da7" + +[[package]] +name = "mio" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "openssl" +version = "0.10.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "os_str_bytes" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85" + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros", + "phf_shared", + "proc-macro-hack", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" +dependencies = [ + "pin-project-internal 1.0.4", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-project-lite" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "publicsuffix" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" +dependencies = [ + "error-chain", + "idna", + "lazy_static", + "regex", + "url", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "quoted-string" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9586f8867f39941d8e796c18340a9cb5221a018df021169dc3e61c87d9f5f567" +dependencies = [ + "quick-error", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", + "rand_pcg", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.1", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.1", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.1", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.0" +source = "git+https://github.com/samdenty/reqwest#fa1d8143d206fff3439b1ef481b6879357b0c29e" +dependencies = [ + "base64", + "bytes 1.0.1", + "cookie", + "cookie_store", + "encoding_rs", + "futures", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime 0.3.16", + "native-tls", + "percent-encoding", + "pin-project-lite 0.2.4", + "serde", + "serde_json", + "serde_urlencoded", + "time", + "tokio 1.1.0", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scraper" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec", + "tendril", +] + +[[package]] +name = "security-framework" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "matches", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", + "thin-slice", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "012c1e1750318ba7d775e4104e34a4eca896b0016e6b90370f12381a28fb29f0" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1197ff7de45494f290c1e3e1a6f80e108974681984c87a3e480991ef3d0f1950" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" + +[[package]] +name = "site_icons" +version = "0.1.0" +dependencies = [ + "byteorder", + "clap", + "data-url", + "futures", + "html5ever", + "itertools", + "log", + "mime 0.4.0-a.0", + "once_cell", + "pin-utils", + "regex", + "reqwest", + "scraper", + "serde", + "serde_json", + "serde_with", + "tokio 1.1.0", + "tokio-byteorder", + "url", + "wee_alloc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "standback" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66a8cff4fa24853fdf6b51f75c6d7f8206d7c75cab4e467bcd7f25c2b1febe0" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "string_cache" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb1139b5353f96e429e1a5e19fbaf663bddedaa06d1dbd49f82e352601209a" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall 0.2.4", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "tendril" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + +[[package]] +name = "thread_local" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8208a331e1cb318dd5bd76951d2b8fc48ca38a69f5f4e4af1b6a9f8c6236915" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" +dependencies = [ + "bytes 0.5.6", + "memchr", + "pin-project-lite 0.1.11", +] + +[[package]] +name = "tokio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8efab2086f17abcddb8f756117665c958feee6b2e39974c2f1600592ab3a4195" +dependencies = [ + "autocfg", + "bytes 1.0.1", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite 0.2.4", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-byteorder" +version = "0.2.0" +source = "git+https://github.com/samdenty/tokio-byteorder#bed45e14aa3ef3d9c76f70d8c0edf10a7c98abd6" +dependencies = [ + "byteorder", + "futures-io", + "tokio 0.2.25", +] + +[[package]] +name = "tokio-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42517d2975ca3114b22a16192634e8241dc5cc1f130be194645970cc1c371494" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio 1.1.0", +] + +[[package]] +name = "tokio-stream" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76066865172052eb8796c686f0b441a93df8b08d40a950b062ffb9a426f00edd" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.4", + "tokio 1.1.0", +] + +[[package]] +name = "tokio-util" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb971a26599ffd28066d387f109746df178eff14d5ea1e235015c5601967a4b" +dependencies = [ + "async-stream", + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.4", + "tokio 1.1.0", + "tokio-stream", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite 0.2.4", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "url" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" + +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55c0f7123de74f0dab9b7d00fd614e7b19349cd1e2f5252bbe9b1754b59433be" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc45447f0d4573f3d65720f636bbcc3dd6ce920ed704670118650bcd47764c7" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3de431a2910c86679c34283a33f66f4e4abd7e0aec27b6669060148872aadf94" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b8853882eef39593ad4174dd26fc9865a64e84026d223f63bb2c42affcbba2c" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4133b5e7f2a531fa413b3a1695e925038a05a71cf67e87dafa295cb645a01385" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64" + +[[package]] +name = "wasm-streams" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4f8387cd5b13b87d140234b5be9ab91e9324184fa5170b2778bc4bd1390683d" +dependencies = [ + "futures", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wee_alloc" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "memory_units", + "winapi", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..230b589 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "site_icons" +version = "0.1.0" +authors = ["Sam Denty "] +edition = "2018" + +[package.metadata.wasm-pack.profile.release] +wasm-opt = ["-Oz", "--enable-mutable-globals"] + +[lib] +crate-type = ["cdylib", "rlib"] + + +[dependencies] +clap = "3.0.0-beta.2" +itertools = "0.10.0" +serde_with = "1.6.1" +pin-utils = "0.1.0" +html5ever = "0.25.1" +url = { version = "2.2.0", features = ["serde"] } +regex = "1" +log = "0.4.14" +once_cell = "1.5.2" +scraper = "0.12.0" +tokio-byteorder = { git = "https://github.com/samdenty/tokio-byteorder", features = ["futures"] } +byteorder = "1.4.2" +data-url = "0.1.0" +mime = { git = "https://github.com/hyperium/mime" } +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = "1.0" +reqwest = { git = "https://github.com/samdenty/reqwest", features = ["json", "cookies", "blocking", "stream"] } +futures = "0.3.8" +wee_alloc = { version = "0.4.2", optional = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { version = "1.1.0", features = ["full"] } diff --git a/Makefile.toml b/Makefile.toml new file mode 100644 index 0000000..f2d247f --- /dev/null +++ b/Makefile.toml @@ -0,0 +1,4 @@ +[tasks.run] +env = { "RUST_LOG" = "site_icons" } +command = "cargo" +args = ["run", "--", "${@}"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..db052a2 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# site_icons + +An efficient website icon scraper for rust + +```rs +use site_icons::Icons; + +let icons = Icons::new(); +// scrape the icons from a url +icons.load_website("https://github.com").await?; + +// fetch all icons, ensuring they exist & determining size +let entries = icons.entries().await; +for icon in entries { + println("{:?}", icon) +} +``` + +## Features + +- Validates that all URLs exist and are actually images +- Determines the size of the icon by partially fetching it +- Supports WASM (and cloudflare workers) + +### Sources + +- HTML favicon tag (or looking for default `/favicon.ico`) +- [Web app manifest](https://developer.mozilla.org/en-US/docs/Web/Manifest) [`icons`](https://developer.mozilla.org/en-US/docs/Web/Manifest/icons) field +- `` tags on the page, directly inside the header OR with a `src|alt|class` containing the text "logo" + +## Running locally + +Install [cargo make](https://github.com/sagiegurari/cargo-make) and then: + +```bash +cargo make run https://github.com +``` diff --git a/src/bin/site_icons.rs b/src/bin/site_icons.rs new file mode 100644 index 0000000..a539528 --- /dev/null +++ b/src/bin/site_icons.rs @@ -0,0 +1,32 @@ +use clap::Clap; +use site_icons::Icons; +use std::error::Error; + +#[derive(Clap)] +struct Opts { + urls: Vec, + #[clap(long)] + json: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let mut icons = Icons::new(); + let opts: Opts = Opts::parse(); + + for url in opts.urls { + icons.load_website(&url).await?; + } + + let entries = icons.entries().await; + + if opts.json { + println!("{}", serde_json::to_string_pretty(&entries)?) + } else { + for icon in entries { + println!("{} {} {}", icon.url, icon.kind, icon.info); + } + } + + Ok(()) +} diff --git a/src/icon.rs b/src/icon.rs new file mode 100644 index 0000000..ee00165 --- /dev/null +++ b/src/icon.rs @@ -0,0 +1,45 @@ +use super::IconInfo; +use serde::Serialize; +use std::{ + cmp::Ordering, + fmt::{self, Display}, +}; +use url::Url; + +#[derive(Debug, Serialize, Clone, PartialOrd, PartialEq, Ord, Eq)] +pub enum IconKind { + SiteLogo, + SiteFavicon, + AppIcon, +} + +impl Display for IconKind { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.write_str(match self { + IconKind::SiteLogo => "site_logo", + IconKind::AppIcon => "app_icon", + IconKind::SiteFavicon => "site_favicon", + }) + } +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Icon { + pub url: Url, + #[serde(with = "serde_with::rust::display_fromstr")] + pub kind: IconKind, + #[serde(flatten)] + pub info: IconInfo, +} + +impl Ord for Icon { + fn cmp(&self, other: &Self) -> Ordering { + self.info.cmp(&other.info) + } +} + +impl PartialOrd for Icon { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/src/icon_info.rs b/src/icon_info.rs new file mode 100644 index 0000000..7d9e2d5 --- /dev/null +++ b/src/icon_info.rs @@ -0,0 +1,162 @@ +use crate::{icon_size::*, CLIENT}; +use data_url::DataUrl; +use futures::{io::Cursor, prelude::*, stream::TryStreamExt}; +use mime::MediaType; +use reqwest::{header::*, Url}; +use serde::{Deserialize, Serialize}; +use std::{ + cmp::Ordering, + error::Error, + fmt::{self, Display}, + io::{self}, +}; + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum IconType { + PNG, + JPEG, + ICO, +} + +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq)] +#[serde(tag = "type")] +#[serde(rename_all = "lowercase")] +pub enum IconInfo { + PNG { size: IconSize }, + JPEG { size: IconSize }, + ICO { sizes: IconSizes }, + SVG, +} + +impl Display for IconInfo { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match self { + IconInfo::PNG { size } => write!(f, "png {}", size), + IconInfo::JPEG { size } => write!(f, "jpeg {}", size), + IconInfo::ICO { sizes } => write!(f, "ico {}", sizes), + IconInfo::SVG => write!(f, "svg"), + } + } +} + +impl Ord for IconInfo { + fn cmp(&self, other: &Self) -> Ordering { + let this_size = self.size(); + let other_size = other.size(); + + if this_size.is_none() && other_size.is_none() { + Ordering::Equal + } else if let (Some(this_size), Some(other_size)) = (this_size, other_size) { + this_size.cmp(other_size) + } else if this_size.is_none() { + Ordering::Less + } else { + Ordering::Greater + } + } +} + +impl PartialOrd for IconInfo { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl IconInfo { + pub async fn get(url: Url, sizes: Option) -> Result> { + let sizes = sizes.as_ref().and_then(|s| IconSizes::from_str(s).ok()); + + let (mime, mut body): (_, Box) = match url.scheme() { + "data" => { + let url = url.to_string(); + let url = DataUrl::process(&url).map_err(|_| "failed to parse data uri")?; + + let mime = url.mime_type().to_string().parse::()?; + + let body = Cursor::new( + url + .decode_to_vec() + .map_err(|_| "failed to decode data uri body")? + .0, + ); + + (mime, Box::new(body)) + } + + _ => { + let res = CLIENT.get(url).send().await?; + if !res.status().is_success() { + return Err("failed to fetch".into()); + }; + + let mime = res + .headers() + .get(CONTENT_TYPE) + .ok_or("no content type")? + .to_str()? + .parse::()?; + + let body = res + .bytes_stream() + .map(|result| { + result.map_err(|error| io::Error::new(io::ErrorKind::Other, error.to_string())) + }) + .into_async_read(); + + (mime, Box::new(body)) + } + }; + + let kind = match (mime.type_(), mime.subtype()) { + (mime::IMAGE, mime::PNG) => { + if let Some(size) = sizes.map(|s| s.into_largest()) { + return Ok(IconInfo::PNG { size }); + } + IconType::PNG + } + + (mime::IMAGE, mime::JPEG) => { + if let Some(size) = sizes.map(|s| s.into_largest()) { + return Ok(IconInfo::JPEG { size }); + } + IconType::JPEG + } + + (mime::IMAGE, "x-icon") | (mime::IMAGE, "vnd.microsoft.icon") => { + if let Some(sizes) = sizes { + return Ok(IconInfo::ICO { sizes }); + } + + IconType::ICO + } + + (mime::IMAGE, mime::SVG) => return Ok(IconInfo::SVG), + + _ => return Err(format!("unsupported mime type {}", mime).into()), + }; + + Ok(match kind { + IconType::PNG => { + let size = get_png_sizes(&mut body).await?; + IconInfo::PNG { size } + } + IconType::ICO => { + let sizes = get_ico_sizes(&mut body).await?; + IconInfo::ICO { sizes } + } + IconType::JPEG => { + let size = get_jpeg_size(&mut body).await?; + IconInfo::JPEG { size } + } + }) + } + + pub fn size(&self) -> Option<&IconSize> { + match self { + IconInfo::ICO { sizes } => Some(sizes.largest()), + IconInfo::PNG { size } | IconInfo::JPEG { size } => Some(size), + IconInfo::SVG => None, + } + } +} diff --git a/src/icon_size/ico.rs b/src/icon_size/ico.rs new file mode 100644 index 0000000..81aff95 --- /dev/null +++ b/src/icon_size/ico.rs @@ -0,0 +1,62 @@ +use super::{png::get_png_sizes, IconSizes}; +use byteorder::{LittleEndian, ReadBytesExt}; +use futures::prelude::*; +use std::{ + error::Error, + io::{Cursor, Seek, SeekFrom}, +}; + +const ICO_TYPE: u16 = 1; +const INDEX_SIZE: u16 = 16; + +pub async fn get_ico_sizes( + reader: &mut R, +) -> Result> { + let mut offset = 0; + let mut header = [0; 6]; + reader.read_exact(&mut header).await?; + offset += header.len(); + let mut header = Cursor::new(header); + + let header_type = header.read_u16::()?; + let icon_type = header.read_u16::()?; + + if header_type != 0 || icon_type != ICO_TYPE { + return Err("bad header".into()); + } + + let icon_count = header.read_u16::()?; + + let mut data = vec![0; (icon_count * INDEX_SIZE) as usize]; + reader.read_exact(&mut data).await?; + offset += data.len(); + let mut data = Cursor::new(data); + + let mut sizes = IconSizes::new(); + for i in 0..icon_count { + data.seek(SeekFrom::Start((INDEX_SIZE * i) as _))?; + + let width = data.read_u8()?; + let height = data.read_u8()?; + + if width == 0 && height == 0 { + data.seek(SeekFrom::Current(10))?; + let image_offset = data.read_u32::()?; + + let mut data = vec![0; image_offset as usize - offset]; + reader.read_exact(&mut data).await?; + offset += data.len(); + + let size = get_png_sizes(reader).await; + if let Ok(size) = size { + sizes.push(size); + } + } else { + sizes.add_size(width as _, height as _) + } + } + + sizes.sort(); + + Ok(sizes) +} diff --git a/src/icon_size/icon_sizes.rs b/src/icon_size/icon_sizes.rs new file mode 100644 index 0000000..514755b --- /dev/null +++ b/src/icon_size/icon_sizes.rs @@ -0,0 +1,81 @@ +use super::IconSize; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + cmp::Ordering, + error::Error, + fmt::{self, Display}, + ops::{Deref, DerefMut}, +}; + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct IconSizes(Vec); + +impl Display for IconSizes { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&self.0.iter().join(" ")) + } +} + +impl IconSizes { + pub fn new() -> Self { + IconSizes(Vec::new()) + } + + pub fn from_str(sizes_str: &str) -> Result> { + let size_strs = sizes_str.split(" "); + + let mut sizes = IconSizes::new(); + for size in size_strs { + if let Ok(size) = serde_json::from_value(Value::String(size.to_string())) { + sizes.push(size); + } + } + + if sizes.is_empty() { + return Err("must contain a size".into()); + } + + sizes.sort(); + + Ok(sizes) + } + + pub fn add_size(&mut self, width: u32, height: u32) { + self.push(IconSize::new(width, height)) + } + + pub fn largest(&self) -> &IconSize { + &self.0[0] + } + + pub fn into_largest(self) -> IconSize { + self.0.into_iter().next().unwrap() + } +} + +impl Deref for IconSizes { + type Target = Vec; + fn deref(&self) -> &Vec { + &self.0 + } +} + +impl DerefMut for IconSizes { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.0 + } +} + +impl Ord for IconSizes { + fn cmp(&self, other: &Self) -> Ordering { + self.largest().cmp(&other.largest()) + } +} + +impl PartialOrd for IconSizes { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/src/icon_size/jpeg.rs b/src/icon_size/jpeg.rs new file mode 100644 index 0000000..8c6404a --- /dev/null +++ b/src/icon_size/jpeg.rs @@ -0,0 +1,64 @@ +use super::IconSize; +use crate::assert_slice_eq; +use byteorder::BigEndian; +use futures::prelude::*; +use std::{error::Error, io::Cursor}; +use tokio_byteorder::AsyncReadBytesExt; + +pub async fn get_jpeg_size( + reader: &mut R, +) -> Result> { + let mut data = [0; 2]; + reader.read_exact(&mut data).await?; + let data = &mut Cursor::new(data); + + // first marker of the file MUST be 0xFFD8 + assert_slice_eq!(data, 0, &[0xFF, 0xD8], "bad header"); + + let mut marker = [0; 2]; + let mut depth = 0i32; + + loop { + // Read current marker (FF XX) + reader.read_exact(&mut marker).await?; + + if marker[0] != 0xFF { + // Did not read a marker. Assume image is corrupt. + return Err("invalid jpeg".into()); + } + + let page = marker[1]; + + // Check for valid SOFn markers. C4, C8, and CC aren't dimension markers. + if (page >= 0xC0 && page <= 0xC3) + || (page >= 0xC5 && page <= 0xC7) + || (page >= 0xC9 && page <= 0xCB) + || (page >= 0xCD && page <= 0xCF) + { + // Only get outside image size + if depth == 0 { + // Correct marker, go forward 3 bytes so we're at height offset + reader.read_exact(&mut [0; 3]).await?; + break; + } + } else if page == 0xD8 { + depth += 1; + } else if page == 0xD9 { + depth -= 1; + if depth < 0 { + return Err("invalid jpeg".into()); + } + } + + // Read the marker length and skip over it entirely + let page_size = reader.read_u16::().await? as i64; + reader + .read_exact(&mut vec![0; (page_size - 2) as usize]) + .await?; + } + + let height = reader.read_u16::().await?; + let width = reader.read_u16::().await?; + + Ok(IconSize::new(width as _, height as _)) +} diff --git a/src/icon_size/mod.rs b/src/icon_size/mod.rs new file mode 100644 index 0000000..6ecfa5c --- /dev/null +++ b/src/icon_size/mod.rs @@ -0,0 +1,103 @@ +mod ico; +mod icon_sizes; +mod jpeg; +mod png; + +pub use ico::*; +pub use icon_sizes::*; +pub use jpeg::*; +pub use png::*; + +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; +use std::{ + cmp::Ordering, + error::Error, + fmt::{self, Display}, + io::{Read, Seek, SeekFrom}, +}; + +#[serde_as] +#[derive(Debug, PartialEq, Eq)] +pub struct IconSize { + width: u32, + height: u32, +} + +impl Display for IconSize { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}x{}", self.width, self.height) + } +} + +impl IconSize { + pub fn new(width: u32, height: u32) -> Self { + Self { width, height } + } +} + +impl Ord for IconSize { + fn cmp(&self, other: &Self) -> Ordering { + let self_res = self.width * self.height; + let other_res = other.width * other.height; + other_res.cmp(&self_res) + } +} + +impl PartialOrd for IconSize { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Serialize for IconSize { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for IconSize { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value: String = Deserialize::deserialize(deserializer)?; + + let mut split = value.split("x"); + let width = split + .next() + .ok_or(de::Error::custom("expected width"))? + .parse() + .map_err(de::Error::custom)?; + + let height = split + .next() + .ok_or(de::Error::custom("expected height"))? + .parse() + .map_err(de::Error::custom)?; + + Ok(IconSize::new(width, height)) + } +} + +fn slice_eq( + cur: &mut T, + offset: u64, + slice: &[u8], +) -> Result> { + cur.seek(SeekFrom::Start(offset))?; + let mut buffer = vec![0; slice.len()]; + cur.read_exact(&mut buffer)?; + Ok(buffer == slice) +} + +#[macro_export] +macro_rules! assert_slice_eq { + ($cur:expr, $offset:expr, $slice:expr, $($arg:tt)+) => {{ + if !super::slice_eq($cur, $offset, $slice)? { + return Err(format!($($arg)+).into()); + } + }}; +} diff --git a/src/icon_size/png.rs b/src/icon_size/png.rs new file mode 100644 index 0000000..a143c4c --- /dev/null +++ b/src/icon_size/png.rs @@ -0,0 +1,21 @@ +use super::IconSize; +use crate::assert_slice_eq; +use byteorder::{BigEndian, ReadBytesExt}; +use futures::prelude::*; +use std::{error::Error, io::Cursor}; + +pub async fn get_png_sizes( + reader: &mut R, +) -> Result> { + let mut header = [0; 24]; + reader.read_exact(&mut header).await?; + let header = &mut Cursor::new(header); + + assert_slice_eq!(header, 0, b"\x89PNG\r\n\x1a\n", "bad header"); + assert_slice_eq!(header, 12, b"IHDR", "bad header"); + + let width = header.read_u32::()?; + let height = header.read_u32::()?; + + Ok(IconSize::new(width, height)) +} diff --git a/src/icons.rs b/src/icons.rs new file mode 100644 index 0000000..cc90be6 --- /dev/null +++ b/src/icons.rs @@ -0,0 +1,220 @@ +use crate::{selector, Icon, IconInfo, IconKind, CLIENT}; +use future::join_all; +use futures::StreamExt; +use futures::{prelude::*, task::noop_waker}; +use html5ever::{ + driver, + tendril::{Tendril, TendrilSink}, +}; +use reqwest::{header::*, IntoUrl}; +use scraper::Html; +use serde::Deserialize; +use std::task::Poll; +use std::{collections::HashMap, error::Error, pin::Pin, task::Context}; +use url::Url; + +pub struct Icons { + entries: Vec, + pending_entries: HashMap< + Url, + ( + IconKind, + Pin>>>>, + ), + >, +} + +fn add_icon_entry( + entries: &mut Vec, + url: Url, + kind: IconKind, + info: Result>, +) { + match info { + Ok(info) => entries.push(Icon { url, kind, info }), + Err(e) => { + warn!("failed to parse icon: {}", e); + } + } +} + +impl Icons { + pub fn new() -> Self { + Icons { + entries: Vec::new(), + pending_entries: HashMap::new(), + } + } + + /// Add an icon URL and start fetching it + pub fn add_icon( + &mut self, + url: Url, + kind: IconKind, + sizes: Option, + ) -> Result<(), Box> { + // check to see if it already exists + let mut entries = self.entries.iter_mut(); + if let Some(existing_kind) = self + .pending_entries + .get_mut(&url) + .map(|(kind, _)| kind) + .or_else(|| entries.find_map(|icon| (icon.url == url).then_some(&mut icon.kind))) + { + // if the kind is more important, replace it + if &kind > existing_kind { + *existing_kind = kind; + } + return Ok(()); + } + + let mut info = Box::pin(IconInfo::get(url.clone(), sizes)); + + // Start fetching the icon + let noop_waker = noop_waker(); + let cx = &mut Context::from_waker(&noop_waker); + match info.poll_unpin(cx) { + Poll::Ready(info) => add_icon_entry(&mut self.entries, url, kind, info), + Poll::Pending => { + self.pending_entries.insert(url, (kind, info)); + } + }; + + Ok(()) + } + + pub async fn load_website(&mut self, url: U) -> Result<(), Box> { + let res = CLIENT.get(url).header(ACCEPT, "text/html").send().await?; + let url = res.url().clone(); + let mut body = res.bytes_stream(); + + let mut parser = driver::parse_document(Html::new_document(), Default::default()); + while let Some(data) = body.next().await { + let tendril = Tendril::try_from_byte_slice(&data?).map_err(|_| "failed to parse html")?; + parser.process(tendril); + } + let document = parser.finish(); + + { + let mut found_favicon = false; + + for element_ref in document.select(selector!( + "link[rel='icon']", + "link[rel='shortcut icon']", + "link[rel='apple-touch-icon']", + "link[rel='apple-touch-icon-precomposed']" + )) { + let elem = element_ref.value(); + if let Some(href) = elem.attr("href").and_then(|href| url.join(&href).ok()) { + if self + .add_icon( + href, + IconKind::SiteFavicon, + elem.attr("sizes").map(|sizes| sizes.into()), + ) + .is_ok() + { + found_favicon = true; + }; + }; + } + + // Check for default favicon.ico + if !found_favicon { + self.add_icon(url.join("/favicon.ico")?, IconKind::SiteFavicon, None)?; + } + } + + for element_ref in document.select(selector!( + "header img", + "img[src*=logo]", + "img[alt*=logo]", + "img[class*=logo]" + )) { + if let Some(href) = element_ref + .value() + .attr("src") + .and_then(|href| url.join(&href).ok()) + { + if self.add_icon(href, IconKind::SiteLogo, None).is_ok() { + break; + }; + }; + } + + for element_ref in document.select(selector!("link[rel='manifest']")) { + if let Some(href) = element_ref + .value() + .attr("href") + .and_then(|href| url.join(&href).ok()) + { + self.load_manifest(href).await?; + } + } + + Ok(()) + } + + pub async fn load_manifest(&mut self, manifest_url: Url) -> Result<(), Box> { + #[derive(Deserialize)] + struct ManifestIcon { + src: String, + sizes: Option, + } + + #[derive(Deserialize)] + struct Manifest { + icons: Option>, + } + + let manifest: Manifest = CLIENT + .get(manifest_url.as_str()) + .send() + .await? + .json() + .await?; + + if let Some(icons) = manifest.icons { + for icon in icons { + if let Ok(src) = manifest_url.join(&icon.src) { + let _ = self.add_icon(src, IconKind::AppIcon, icon.sizes); + } + } + } + + Ok(()) + } + + /// Fetch all the icons and return a list of them. + /// + /// List is ordered from highest resolution to lowest resolution + /// + /// ``` + /// # async fn run() { + /// let icons = Icons::new(); + /// icons.load_website("https://github.com").await?; + /// + /// let entries = icons.entries().await; + /// for icon in entries { + /// println("{:?}", icon) + /// } + /// ``` + pub async fn entries(mut self) -> Vec { + let (urls, infos): (Vec<_>, Vec<_>) = self + .pending_entries + .into_iter() + .map(|(url, (kind, info))| ((url, kind), info)) + .unzip(); + + let mut urls = urls.into_iter(); + + for info in join_all(infos).await { + let (url, kind) = urls.next().unwrap(); + add_icon_entry(&mut self.entries, url, kind, info); + } + + self.entries.sort(); + + self.entries + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d243d5b --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,27 @@ +#![feature(async_closure, map_into_keys_values, bool_to_option)] +#[macro_use] +extern crate serde_with; +#[macro_use] +extern crate log; + +mod icon; +mod icon_info; +mod icon_size; +mod icons; +mod macros; + +pub use icon::*; +pub use icon_info::*; +pub use icons::*; + +use once_cell::sync::Lazy; +use reqwest::{ + header::{HeaderMap, HeaderValue, USER_AGENT}, + Client, +}; + +static CLIENT: Lazy = Lazy::new(|| { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_str("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36").unwrap()); + Client::builder().default_headers(headers).build().unwrap() +}); diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 0000000..721e65c --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,14 @@ +#[macro_export] +macro_rules! selector { + ($($selector:expr),+ $(,)?) => {{ + static RE: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); + RE.get_or_init(|| scraper::Selector::parse(crate::join!(",", $($selector),+)).unwrap()) + }}; +} + +#[macro_export] +macro_rules! join { + ($pattern:literal,$first:expr$(, $($rest:expr),*)? $(,)?) => { + concat!($first$(, $($pattern, $rest),*)?) + }; +}