Skip to content

Commit 1399da2

Browse files
vemvvemv
committed
Optimize namespace-aliases
Parallelizes it, and shrinks the corpus by removing non source/test dirs from the classpath. Testing it over refactor-nrepl itself, I got a performance gain from 70ms (cached case) to 20ms (cached case). The performance gains can be larger as projects scale. It also can plausibly ameliorate the mistery slowness for the :cljs branch.
1 parent eb77344 commit 1399da2

File tree

7 files changed

+167
-58
lines changed

7 files changed

+167
-58
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
* This increases the chances that a namespace will be found, which in turns makes refactor-nrepl more complete/accurate.
1616
* Replace Cheshire with `clojure.data.json`
1717
* Build ASTs more robustly (by using locks, `require`, and ruling out certain namespaces like refactor-nrepl itself)
18+
* Improve `namespace-aliases` performance and make it return more accurate results.
1819
* Honor internal `future-cancel` calls, improving overall responsiveness and stability.
1920

2021
### Bugs fixed

‎src/refactor_nrepl/core.clj‎

Lines changed: 60 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
(ns refactor-nrepl.core
2-
(:require [clojure.java.io :as io]
3-
[clojure.string :as str]
4-
[clojure.tools.namespace.parse :as parse]
5-
[clojure.tools.reader.reader-types :as readers]
6-
[orchard.java.classpath :as cp]
7-
[orchard.misc :as misc]
8-
[me.raynes.fs :as fs]
9-
[refactor-nrepl.util :as util :refer [normalize-to-unix-path]]
10-
[refactor-nrepl.s-expressions :as sexp]
11-
[refactor-nrepl.config :as config])
12-
(:import [java.io File FileReader PushbackReader StringReader]))
2+
(:require
3+
[clojure.java.io :as io]
4+
[clojure.string :as str]
5+
[clojure.tools.namespace.parse :as parse]
6+
[clojure.tools.reader.reader-types :as readers]
7+
[me.raynes.fs :as fs]
8+
[orchard.java.classpath :as cp]
9+
[orchard.misc :as misc]
10+
[refactor-nrepl.config :as config]
11+
[refactor-nrepl.s-expressions :as sexp]
12+
[refactor-nrepl.util :as util :refer [normalize-to-unix-path]])
13+
(:import
14+
(java.io File FileReader PushbackReader StringReader)))
1315

1416
(defn version []
1517
(let [v (-> (or (io/resource "refactor-nrepl/refactor-nrepl/project.clj")
@@ -57,6 +59,18 @@
5759
(when (.isDirectory ^File f) f)))
5860
(remove (comp ignore-dir-on-classpath? str))))
5961

62+
(defn source-dirs-on-classpath
63+
"Like `#'dirs-on-classpath`, but restricted to dirs that look like
64+
(interesting) source/test dirs."
65+
[]
66+
(->> (dirs-on-classpath)
67+
(remove (fn [^File f]
68+
(let [s (-> f .toString)]
69+
(or (-> s (.contains "resources"))
70+
(-> s (.contains "target"))
71+
(-> s (.contains ".gitlibs"))))))
72+
(remove util/dir-outside-root-dir?)))
73+
6074
(defn project-root
6175
"Return the project root directory.
6276
@@ -141,34 +155,32 @@
141155
{:read-cond :allow :features #{dialect}})
142156
(catch Exception _ nil))))))
143157

144-
(defn- data-file?
145-
"True of f is named like a clj file but represents data.
146-
147-
E.g. true for data_readers.clj"
148-
[path-or-file]
149-
(let [path (.getPath (io/file path-or-file))
150-
data-files #{"data_readers.clj" "project.clj" "boot.clj"}]
151-
(reduce (fn [acc data-file] (or acc (.endsWith path data-file)))
152-
false
153-
data-files)))
158+
(defn cljc-extension? [^String path]
159+
(.endsWith path ".cljc"))
154160

155161
(defn cljc-file?
156162
[path-or-file]
157163
(let [path (.getPath (io/file path-or-file))]
158-
(and (.endsWith path ".cljc")
164+
(and (cljc-extension? path)
159165
(read-ns-form path))))
160166

167+
(defn cljs-extension? [^String path]
168+
(.endsWith path ".cljs"))
169+
161170
(defn cljs-file?
162171
[path-or-file]
163172
(let [path (.getPath (io/file path-or-file))]
164-
(and (.endsWith path ".cljs")
173+
(and (cljs-extension? path)
165174
(read-ns-form path))))
166175

176+
(defn clj-extension? [^String path]
177+
(.endsWith path ".clj"))
178+
167179
(defn clj-file?
168180
[path-or-file]
169181
(let [path (.getPath (io/file path-or-file))]
170-
(and (not (data-file? path-or-file))
171-
(.endsWith path ".clj")
182+
(and (not (util/data-file? path-or-file))
183+
(clj-extension? path)
172184
(read-ns-form path))))
173185

174186
(defn source-file?
@@ -194,11 +206,29 @@
194206

195207
(defn find-in-project
196208
"Return the files in the project satisfying (pred ^File file)."
197-
[pred]
198-
(->> (dirs-on-classpath)
199-
(pmap (partial find-in-dir pred))
200-
(apply concat)
201-
distinct))
209+
([pred]
210+
(find-in-project pred (dirs-on-classpath)))
211+
([pred dirs]
212+
(->> dirs
213+
(pmap (partial find-in-dir pred))
214+
(apply concat)
215+
distinct)))
216+
217+
(defn source-files-with-clj-like-extension
218+
"Finds files with .clj* extension in the project, without inspecting them.
219+
220+
Meant as a particularly fast operation (as it doesn't slurp files)."
221+
([ignore-errors?]
222+
(source-files-with-clj-like-extension ignore-errors? (source-dirs-on-classpath)))
223+
([ignore-errors? dirs]
224+
(find-in-project (util/with-suppressed-errors
225+
(comp (some-fn clj-extension?
226+
cljc-extension?
227+
cljs-extension?)
228+
(fn [^File f]
229+
(.getPath f)))
230+
ignore-errors?)
231+
dirs)))
202232

203233
(defn throw-unless-clj-file [file-path]
204234
(when-not (re-matches #".+\.clj$" file-path)

‎src/refactor_nrepl/ns/libspecs.clj‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
(ns refactor-nrepl.ns.libspecs
2-
(:require [refactor-nrepl.core :as core]
3-
[refactor-nrepl.ns.ns-parser :as ns-parser]
4-
[refactor-nrepl.util :as util])
5-
(:import [java.io File]))
2+
(:require
3+
[refactor-nrepl.core :as core]
4+
[refactor-nrepl.ns.ns-parser :as ns-parser]
5+
[refactor-nrepl.util :as util])
6+
(:import
7+
(java.io File)))
68

79
;; The structure here is {path {lang [timestamp value]}}
810
;; where lang is either :clj or :cljs
@@ -43,23 +45,34 @@
4345
(put-cached-libspec f lang)))
4446

4547
(defn namespace-aliases
46-
"Return a map of file type to a map of aliases to namespaces
48+
"Returns a map of file type to a map of aliases to namespaces
4749
4850
{:clj {util com.acme.util str clojure.string
4951
:cljs {gstr goog.str}}}"
5052
([]
5153
(namespace-aliases false))
5254
([ignore-errors?]
53-
{:clj (->> (core/find-in-project (util/with-suppressed-errors
54-
(some-fn core/clj-file? core/cljc-file?)
55-
ignore-errors?))
56-
(map (partial get-libspec-from-file-with-caching :clj))
57-
aliases-by-frequencies)
58-
:cljs (->> (core/find-in-project (util/with-suppressed-errors
59-
(some-fn core/cljs-file? core/cljc-file?)
60-
ignore-errors?))
61-
(map (partial get-libspec-from-file-with-caching :cljs))
62-
aliases-by-frequencies)}))
55+
(namespace-aliases ignore-errors? (core/source-dirs-on-classpath)))
56+
([ignore-errors? dirs]
57+
(let [;; fetch the file list just once (as opposed to traversing the project once for each dialect)
58+
files (core/source-files-with-clj-like-extension ignore-errors? dirs)
59+
;; pmap parallelizes a couple things:
60+
;; - `pred`, which is IO-intentive
61+
;; - `aliases-by-frequencies`, which is moderately CPU-intensive
62+
[clj-files cljs-files] (pmap (fn [[dialect pred] corpus]
63+
(->> corpus
64+
(filter pred)
65+
(map (partial get-libspec-from-file-with-caching dialect))
66+
aliases-by-frequencies))
67+
[[:clj (util/with-suppressed-errors
68+
(some-fn core/clj-file? core/cljc-file?)
69+
ignore-errors?)]
70+
[:cljs (util/with-suppressed-errors
71+
(some-fn core/cljs-file? core/cljc-file?)
72+
ignore-errors?)]]
73+
(repeat files))]
74+
{:clj clj-files
75+
:cljs cljs-files})))
6376

6477
(defn- unwrap-refer
6578
[file {:keys [ns refer]}]

‎src/refactor_nrepl/util.clj‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
(ns refactor-nrepl.util
2-
(:require [clojure.string :as string])
3-
(:import java.util.regex.Pattern))
2+
(:require
3+
[clojure.java.io :as io]
4+
[clojure.string :as string])
5+
(:import
6+
(java.io File)
7+
(java.util.regex Pattern)))
48

59
(defn normalize-to-unix-path
610
"Replace use / as separator and lower-case."
@@ -99,3 +103,34 @@
99103
;; in other places that already are using `some-fn`, `every-pred`, etc
100104
([_]
101105
(.isInterrupted (Thread/currentThread))))
106+
107+
(defn dir-outside-root-dir?
108+
"Dirs outside the root dir often represent uninteresting dirs to examine, e.g.
109+
Lein checkouts, .gitlibs from deps.edn, or other extraneous source paths
110+
which aren't directly related to the project.
111+
112+
By excluding them, one gets better and more accurate performance."
113+
[^File f]
114+
{:pre [(-> f .isDirectory)]}
115+
(let [f (-> f .getCanonicalPath File.)
116+
root-dir (File. (System/getProperty "user.dir"))
117+
parent-dirs (->> f
118+
(iterate (fn [^File f]
119+
(some-> f .getCanonicalPath File. .getParent File.)))
120+
(take-while some?)
121+
(set))]
122+
(not (parent-dirs root-dir))))
123+
124+
(defn data-file?
125+
"True of f is named like a clj file but represents data.
126+
127+
E.g. true for data_readers.clj"
128+
[path-or-file]
129+
(let [path (.getPath (io/file path-or-file))
130+
data-files #{"data_readers.clj" "project.clj" "boot.clj"}]
131+
(reduce (fn [acc data-file]
132+
(let [v (or acc (.endsWith path data-file))]
133+
(cond-> v
134+
v reduced)))
135+
false
136+
data-files)))

‎test/refactor_nrepl/core_test.clj‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@
22
(:require
33
[clojure.test :refer [are deftest is testing]]
44
[refactor-nrepl.config :as config]
5-
[refactor-nrepl.core :refer [ignore-dir-on-classpath? read-ns-form]])
5+
[refactor-nrepl.core :as sut])
66
(:import
77
(java.io File)))
88

99
(defmacro assert-ignored-paths
1010
[paths pred]
1111
`(doseq [p# ~paths]
12-
(is (~pred (ignore-dir-on-classpath? p#)))))
12+
(is (~pred (sut/ignore-dir-on-classpath? p#)))))
1313

1414
(deftest test-ignore-dir-on-classpath?
1515
(let [not-ignored ["/home/user/project/test"
@@ -32,9 +32,16 @@
3232
(are [input expected] (testing input
3333
(assert (-> input File. .exists))
3434
(is (= expected
35-
(read-ns-form input)))
35+
(sut/read-ns-form input)))
3636
true)
3737
"test-resources/readable_file_incorrect_aliases.clj" nil
3838
"testproject/src/com/example/one.clj" '(ns com.example.one
3939
(:require [com.example.two :as two :refer [foo]]
4040
[com.example.four :as four]))))
41+
42+
(deftest source-files-with-clj-like-extension-test
43+
(let [result (sut/source-files-with-clj-like-extension true)]
44+
(doseq [extension [".clj" ".cljs" ".cljc"]]
45+
(is (pos? (count (filter (fn [^File f]
46+
(-> f .getPath (.endsWith extension)))
47+
result)))))))

‎test/refactor_nrepl/ns/namespace_aliases_test.clj‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
(ns refactor-nrepl.ns.namespace-aliases-test
2-
(:require [clojure.test :refer [deftest is]]
3-
[refactor-nrepl.core :as core]
4-
[refactor-nrepl.ns.libspecs :as sut]
5-
[refactor-nrepl.util :as util]
6-
[refactor-nrepl.unreadable-files :refer [ignore-errors?]]))
2+
(:require
3+
[clojure.test :refer [deftest is]]
4+
[refactor-nrepl.core :as core]
5+
[refactor-nrepl.ns.libspecs :as sut]
6+
[refactor-nrepl.unreadable-files :refer [ignore-errors?]]
7+
[refactor-nrepl.util :as util]))
78

89
(defn finds [selector alias libspec]
9-
(let [aliases (selector (sut/namespace-aliases ignore-errors?))]
10+
(let [aliases (selector (sut/namespace-aliases ignore-errors? (core/dirs-on-classpath)))]
1011
(some (fn [[k vs]]
1112
(and (= k alias)
1213
(some #{libspec} vs)))

‎test/refactor_nrepl/util_test.clj‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
(ns refactor-nrepl.util-test
2-
(:require [clojure.test :refer [deftest is]]
3-
[refactor-nrepl.util :as sut]))
2+
(:require
3+
[clojure.test :refer [are deftest is]]
4+
[refactor-nrepl.util :as sut])
5+
(:import
6+
(java.io File)))
47

58
(deftest with-additional-ex-data-test
69
(try
@@ -10,3 +13,22 @@
1013
(catch clojure.lang.ExceptionInfo e
1114
(let [{:keys [foo]} (ex-data e)]
1215
(is (= foo :bar))))))
16+
17+
(deftest dir-outside-root-dir?-test
18+
(are [input expected] (= expected
19+
(sut/dir-outside-root-dir? input))
20+
(File. (System/getProperty "user.dir")) false
21+
(File. ".") false
22+
(File. "src") false
23+
(File. "/") true))
24+
25+
(deftest data-file?-test
26+
(are [input expected] (= expected
27+
(sut/data-file? input))
28+
"project.clj" true
29+
"boot.clj" true
30+
"data_readers.clj" true
31+
"project.cljs" false
32+
"boot.cljs" false
33+
"data_readers.cljs" false
34+
"a.clj" false))

0 commit comments

Comments
 (0)