Support :skyscraper/description

To facilitate debugging, processors can now set the
:skyscraper/description key on contexts. These descriptions will be
logged when downloading, instead of the URL.

This is a one-off key, meaning that it will be not propagated to
the scraping subtree. This commit also makes
:skyscraper.traverse/priority one-off.

Author: nathell

src/skyscraper/context.clj

@@ -3,7 +3,8 @@
 (defn dissoc-internal
   "Dissocs the context keys that shouldn't be carried over to further processing."
   [context]
-  (let [removed-keys #{:processor :url :skyscraper.core/new-items :skyscraper.core/retry}]
+  (let [removed-keys #{:processor :url :skyscraper.core/new-items :skyscraper.core/retry
+                       :skyscraper/description :skyscraper.traverse/priority}]
     (into {}
           (remove (fn [[k _]] (or (contains? removed-keys k)
                                   (and (keyword? k)
@@ -24,6 +25,8 @@
           :skyscraper.core/stage
           :skyscraper.traverse/handler
           :skyscraper.traverse/call-protocol
+          :skyscraper.traverse/priority
+          :skyscraper/description
           :http/cookies))
 
 (defn describe
@@ -38,3 +41,8 @@
       true (merge (select-keys context [:processor :url])) ; reattach
       processor (assoc :skyscraper.core/current-processor-name (:name processor))
       true pr-str)))
+
+(defn describe-url
+  [context]
+  (or (:skyscraper/description context)
+      (:url context)))

src/skyscraper/core.clj

@@ -378,7 +378,7 @@
   (let [req (merge {:method :get, :url (:url context)}
                    (extract-namespaced-keys "http" context))
         success-fn (fn [response]
-                     (debugf "[download] Downloaded %s" (:url context))
+                     (debugf "[download] Downloaded %s" (context/describe-url context))
                      (.release download-semaphore)
                      (callback (respond-with response options context)))
         error-fn (fn [error]
@@ -387,7 +387,7 @@
                      (callback (error-handler error options context))))]
     (debugf "[download] Waiting")
     (.acquire download-semaphore)
-    (infof "[download] Downloading %s" (:url context))
+    (infof "[download] Downloading %s" (context/describe-url context))
     (let [req (merge {:async? true,
                       :connection-manager connection-manager}
                      req (get-option context options :http-options))
@@ -407,10 +407,10 @@
         request-fn (or (:request-fn options)
                        http/request)]
     (try
-      (infof "[download] Downloading %s" (:url context))
+      (infof "[download] Downloading %s" (context/describe-url context))
       (wait sleep)
       (let [resp (request-fn req)]
-        (debugf "[download] Downloaded %s" (:url context))
+        (debugf "[download] Downloaded %s" (context/describe-url context))
         [(cond-> (advance-pipeline pipeline context)
            true (assoc ::response resp)
            (:cookies resp) (update :http/cookies merge (:cookies resp)))])