Use host_str instead domain for HostInternal

Author: spk

src/lib.rs

@@ -221,7 +221,7 @@ impl<'a> RobotFileParser<'a> {
             disallow_all: Cell::new(false),
             allow_all: Cell::new(false),
             url: parsed_url.clone(),
-            host: parsed_url.domain().unwrap().to_owned(),
+            host: parsed_url.host_str().unwrap().to_owned(),
             path: parsed_url.path().to_owned(),
             last_checked: Cell::new(0i64),
         }
@@ -246,7 +246,7 @@ impl<'a> RobotFileParser<'a> {
     pub fn set_url<T: AsRef<str>>(&mut self, url: T) {
         let parsed_url = Url::parse(url.as_ref()).unwrap();
         self.url = parsed_url.clone();
-        self.host = parsed_url.domain().unwrap().to_owned();
+        self.host = parsed_url.host_str().unwrap().to_owned();
         self.path = parsed_url.path().to_owned();
         self.last_checked.set(0i64);
     }

tests/lib.rs

@@ -268,3 +268,9 @@ fn test_robots_text_request_rate() {
     let req_rate = parser.get_req_rate("Google");
     assert!(req_rate.is_none());
 }
+
+#[test]
+fn test_robots_127_0_0_1() {
+    // Ensure it does not panic
+    RobotFileParser::new("http://127.0.0.1:4000/robots.txt");
+}