From c0e350b7343fcdbfa03018ed9e50f4516ff9a4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa?= Date: Mon, 28 Jan 2019 23:58:32 +0100 Subject: [PATCH] Disable icon downloads, accept optional query after icon href, format and clippy fixes --- .env.template | 6 ++++ src/api/admin.rs | 4 +-- src/api/icons.rs | 72 +++++++++++++++++++++++------------------------- src/config.rs | 2 ++ 4 files changed, 44 insertions(+), 40 deletions(-) diff --git a/.env.template b/.env.template index c250f306..a018aa98 100644 --- a/.env.template +++ b/.env.template @@ -43,6 +43,12 @@ ## It's recommended to also set 'ROCKET_CLI_COLORS=off' # LOG_FILE=/path/to/log +## Disable icon downloading +## Set to true to disable icon downloading, this would still serve icons from $ICON_CACHE_FOLDER, +## but it won't produce any external network request. Needs to set $ICON_CACHE_TTL to 0, +## otherwise it will delete them and they won't be downloaded again. +# DISABLE_ICON_DOWNLOAD=false + ## Controls if new users can register # SIGNUPS_ALLOWED=true diff --git a/src/api/admin.rs b/src/api/admin.rs index b49d3ca0..5b690af9 100644 --- a/src/api/admin.rs +++ b/src/api/admin.rs @@ -27,8 +27,8 @@ pub fn routes() -> Vec { ] } -const COOKIE_NAME: &'static str = "BWRS_ADMIN"; -const ADMIN_PATH: &'static str = "/admin"; +const COOKIE_NAME: &str = "BWRS_ADMIN"; +const ADMIN_PATH: &str = "/admin"; #[derive(Serialize)] struct AdminTemplateData { diff --git a/src/api/icons.rs b/src/api/icons.rs index 90c72b8a..61767bc3 100644 --- a/src/api/icons.rs +++ b/src/api/icons.rs @@ -1,32 +1,19 @@ use std::fs::{create_dir_all, remove_file, symlink_metadata, File}; use std::io::prelude::*; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use rocket::http::ContentType; use rocket::response::Content; use rocket::Route; -use reqwest; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, ACCEPT_LANGUAGE, CACHE_CONTROL, PRAGMA, USER_AGENT}; use reqwest::Client; -use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT, ACCEPT_LANGUAGE, CACHE_CONTROL, PRAGMA, ACCEPT}; -use std::time::Duration; -use crate::error::Error; -//use std::error::Error as StdError; -use crate::CONFIG; - -//extern crate regex; use regex::Regex; - -//extern crate soup; use soup::prelude::*; -use std::vec::Vec; -#[derive(Debug)] -struct IconList { - priority: u8, - href: String, -} +use crate::error::Error; +use crate::CONFIG; pub fn routes() -> Vec { routes![icon] @@ -55,6 +42,10 @@ fn get_icon(domain: &str) -> Vec { return icon; } + if CONFIG.disable_icon_download() { + return FALLBACK_ICON.to_vec(); + } + // Get the icon, or fallback in case of error match download_icon(&domain) { Ok(icon) => { @@ -129,6 +120,12 @@ fn icon_is_expired(path: &str) -> bool { expired.unwrap_or(true) } +#[derive(Debug)] +struct IconList { + priority: u8, + href: String, +} + /// Returns a Result with a String which holds the preferend favicon location. /// There will always be a result with a string which will contain https://example.com/favicon.ico /// This does not mean that that location does exists, but it is the default location. @@ -165,40 +162,39 @@ fn get_icon_url(domain: &str) -> Result { let mut iconlist: Vec = Vec::new(); let resp = client.get(&ssldomain).send().or_else(|_| client.get(&httpdomain).send()); - if let Ok(mut content) = resp { - let body = content.text().unwrap(); - // Extract the URL from te respose incase redirects occured (like @ gitlab.com) - let url = format!("{}://{}", content.url().scheme(), content.url().host().unwrap()); + if let Ok(content) = resp { + // Extract the URL from the respose in case redirects occured (like @ gitlab.com) + let url = content.url().origin().ascii_serialization(); // Add the default favicon.ico to the list with the domain the content responded from. - iconlist.push(IconList { priority: 35, href: format!("{}{}", url, "/favicon.ico") }); + iconlist.push(IconList { priority: 35, href: format!("{}/favicon.ico", url) }); - let soup = Soup::new(&body); + let soup = Soup::from_reader(content)?; // Search for and filter let favicons = soup .tag("link") .attr("rel", Regex::new(r"icon$|apple.*icon")?) // Only use icon rels - .attr("href", Regex::new(r"(?i)\w+(\.jp(e){0,1}g$|\.png$|\.ico$)")?) // Only allow specific extensions + .attr("href", Regex::new(r"(?i)\w+\.(jpg|jpeg|png|ico)(\?.*)?$")?) // Only allow specific extensions .find_all(); // Loop through all the found icons and determine it's priority for favicon in favicons { - let favicon_sizes = favicon.get("sizes").unwrap_or("".to_string()).to_string(); - let favicon_href = fix_href(&favicon.get("href").unwrap_or("".to_string()).to_string(), &url); - let favicon_priority = get_icon_priority(&favicon_href, &favicon_sizes); + let favicon_sizes = favicon.get("sizes").unwrap_or_default(); + let href = fix_href(&favicon.get("href").unwrap_or_default(), &url); + let priority = get_icon_priority(&href, &favicon_sizes); - iconlist.push(IconList { priority: favicon_priority, href: favicon_href}) + iconlist.push(IconList { priority, href }) } } else { // Add the default favicon.ico to the list with just the given domain - iconlist.push(IconList { priority: 35, href: format!("{}{}", ssldomain, "/favicon.ico") }); + iconlist.push(IconList { priority: 35, href: format!("{}/favicon.ico", ssldomain) }); } // Sort the iconlist by priority iconlist.sort_by_key(|x| x.priority); // There always is an icon in the list, so no need to check if it exists, and just return the first one - Ok(format!("{}", &iconlist[0].href)) + Ok(iconlist.remove(0).href) } /// Returns a Integer with the priority of the type of the icon which to prefer. @@ -215,10 +211,10 @@ fn get_icon_url(domain: &str) -> Result { /// ``` fn get_icon_priority(href: &str, sizes: &str) -> u8 { // Check if there is a dimension set - if ! sizes.is_empty() { - let dimensions : Vec<&str> = sizes.split("x").collect(); - let width = dimensions[0].parse::().unwrap(); - let height = dimensions[1].parse::().unwrap(); + if !sizes.is_empty() { + let dimensions: Vec<&str> = sizes.split('x').collect(); + let width: u16 = dimensions[0].parse().unwrap(); + let height: u16 = dimensions[1].parse().unwrap(); // Only allow square dimensions if width == height { @@ -270,22 +266,22 @@ fn fix_href(href: &str, url: &str) -> String { format!("http:{}", href) } // If the href_output just starts with a single / it does not have the host here at all. - } else if ! href.starts_with("http") { - if href.starts_with("/") { + } else if !href.starts_with("http") { + if href.starts_with('/') { format!("{}{}", url, href) } else { format!("{}/{}", url, href) } // All seems oke, just return the given href } else { - format!("{}", href) + href.to_string() } } fn download_icon(domain: &str) -> Result, Error> { let url = get_icon_url(&domain)?; - info!("Downloading icon for {} via {}...",domain, url); + info!("Downloading icon for {} via {}...", domain, url); let mut res = reqwest::get(&url)?; res = res.error_for_status()?; diff --git a/src/config.rs b/src/config.rs index dfa2826e..2c230bd4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,6 +56,7 @@ make_config! { extended_logging: bool, log_file: Option, + disable_icon_download: bool, signups_allowed: bool, invitations_allowed: bool, admin_token: Option, @@ -166,6 +167,7 @@ impl Config { extended_logging: get_env_or("EXTENDED_LOGGING", true), log_file: get_env("LOG_FILE"), + disable_icon_download: get_env_or("DISABLE_ICON_DOWNLOAD", false), signups_allowed: get_env_or("SIGNUPS_ALLOWED", true), admin_token: get_env("ADMIN_TOKEN"), invitations_allowed: get_env_or("INVITATIONS_ALLOWED", true),