Use GitHub's paging API properly to make sure we always get all issues

Author: DenWav

src/main/kotlin/com/demonwav/mcdev/creator/PlatformVersion.kt

@@ -12,7 +12,7 @@ package com.demonwav.mcdev.creator
 
 import com.demonwav.mcdev.platform.PlatformType
 import com.demonwav.mcdev.util.fromJson
-import com.google.gson.Gson
+import com.demonwav.mcdev.util.gson
 import org.jetbrains.concurrency.runAsync
 import java.net.URL
 import java.util.Arrays
@@ -45,7 +45,7 @@ fun getVersionSelector(type: PlatformType) = runAsync {
         "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2"
     )
     val text = connection.getInputStream().use { it.reader().use { it.readText() } }
-    Gson().fromJson<PlatformVersion>(text)
+    gson.fromJson<PlatformVersion>(text)
 }
 
 data class PlatformVersion(var versions: Array<String>, var selectedIndex: Int) {

src/main/kotlin/com/demonwav/mcdev/error/AnonymousFeedback.kt

@@ -12,9 +12,11 @@ package com.demonwav.mcdev.error
 
 import com.demonwav.mcdev.update.PluginUtil
 import com.demonwav.mcdev.util.fromJson
-import com.google.gson.Gson
+import com.demonwav.mcdev.util.gson
 import com.intellij.ide.plugins.PluginManager
+import com.intellij.util.io.readCharSequence
 import org.apache.commons.io.IOUtils
+import java.io.InputStreamReader
 import java.net.HttpURLConnection
 import java.net.URL
 
@@ -25,7 +27,7 @@ object AnonymousFeedback {
     const val url = "https://www.demonwav.com/errorReport"
 
     fun sendFeedback(factory: HttpConnectionFactory, envDetails: LinkedHashMap<String, String?>): FeedbackData {
-        val duplicateId = findDuplicateIssue(envDetails)
+        val duplicateId = findDuplicateIssue(envDetails, factory)
         if (duplicateId != null) {
             // This is a duplicate
             val commentUrl = sendCommentOnDuplicateIssue(duplicateId, factory, convertToGitHubIssueFormat(envDetails))
@@ -40,7 +42,7 @@ object AnonymousFeedback {
         val result = LinkedHashMap<String, String>(5)
         result.put("title", "[auto-generated] Exception in plugin")
         result.put("body", generateGitHubIssueBody(envDetails))
-        return Gson().toJson(result).toByteArray()
+        return gson.toJson(result).toByteArray()
     }
 
     private fun generateGitHubIssueBody(body: LinkedHashMap<String, String?>): String {
@@ -77,6 +79,7 @@ object AnonymousFeedback {
 
     private fun sendFeedback(factory: HttpConnectionFactory, payload: ByteArray): Pair<String, Int> {
         val connection = getConnection(factory, url)
+        connection.connect()
         connection.outputStream.use {
             it.write(payload)
         }
@@ -90,8 +93,9 @@ object AnonymousFeedback {
         val body = connection.inputStream.use {
             IOUtils.toString(it, contentEncoding)
         }
+        connection.disconnect()
 
-        val json = Gson().fromJson<Map<*, *>>(body)
+        val json = gson.fromJson<Map<*, *>>(body)
         return json["html_url"] as String to (json["number"] as Double).toInt()
     }
 
@@ -102,11 +106,11 @@ object AnonymousFeedback {
         return connection
     }
 
-    private fun findDuplicateIssue(envDetails: LinkedHashMap<String, String?>): Int? {
+    private fun findDuplicateIssue(envDetails: LinkedHashMap<String, String?>, factory: HttpConnectionFactory): Int? {
         val stack = envDetails["error.stacktrace"]?.replace("\\d+".toRegex(), "") ?: return null
 
-        val text = URL("https://api.github.com/repos/minecraft-dev/MinecraftDev/issues?state=all&creator=minecraft-dev-autoreporter").readText()
-        val list = Gson().fromJson<List<Map<*, *>>>(text)
+        val list = getAllIssues("https://api.github.com/repos/minecraft-dev/MinecraftDev/issues" +
+                                    "?state=all&creator=minecraft-dev-autoreporter&per_page=100", factory) ?: return null
         val block = list.firstOrNull {
             val body = it["body"] as? String ?: return@firstOrNull false
 
@@ -124,6 +128,65 @@ object AnonymousFeedback {
         return (block["number"] as Double).toInt()
     }
 
+    private fun getAllIssues(url: String, factory: HttpConnectionFactory): List<Map<*, *>>? {
+        var connection = connect(factory, url)
+        connection.requestMethod = "GET"
+        connection.setRequestProperty("User-Agent", userAgent)
+
+        connection.connect()
+        if (connection.responseCode != 200) {
+            connection.disconnect()
+            return null
+        }
+
+        val list = mutableListOf<Map<*, *>>()
+        var data = connection.inputStream.reader().use(InputStreamReader::readCharSequence).toString()
+
+        var response = gson.fromJson<List<Map<*, *>>>(data)
+        list.addAll(response)
+
+        var link = connection.getHeaderField("Link")
+        connection.disconnect()
+
+        var next = getNextLink(link)
+        while (next != null) {
+            connection = connect(factory, next)
+            connection.requestMethod = "GET"
+            connection.setRequestProperty("User-Agent", userAgent)
+
+            connection.connect()
+            if (connection.responseCode != 200) {
+                connection.disconnect()
+                continue
+            }
+
+            data = connection.inputStream.reader().use(InputStreamReader::readCharSequence).toString()
+
+            response = gson.fromJson<List<Map<*, *>>>(data)
+            list.addAll(response)
+
+            link = connection.getHeaderField("Link")
+            connection.disconnect()
+            next = getNextLink(link)
+        }
+
+        return list
+    }
+
+    private fun getNextLink(link: String): String? {
+        val lines = link.split(",")
+        for (line in lines) {
+            if (!line.contains("rel=\"next\"")) {
+                continue
+            }
+
+            val parts = line.split(";")
+            return parts[0].substring(1, parts[0].length-1)
+        }
+
+        return null
+    }
+
     private fun sendCommentOnDuplicateIssue(id: Int, factory: HttpConnectionFactory, payload: ByteArray): String {
         val commentUrl = "$url/$id/comments"
         val connection = getConnection(factory, commentUrl)
@@ -140,21 +203,13 @@ object AnonymousFeedback {
         val body = connection.inputStream.use {
             IOUtils.toString(it, contentEncoding)
         }
+        connection.disconnect()
 
-        val json = Gson().fromJson<Map<*, *>>(body)
+        val json = gson.fromJson<Map<*, *>>(body)
         return json["html_url"] as String
     }
 
     private fun getConnection(factory: HttpConnectionFactory, url: String): HttpURLConnection {
-        var userAgent = "Minecraft Development IntelliJ IDEA plugin"
-
-        val pluginDescription = PluginManager.getPlugin(PluginUtil.PLUGIN_ID)
-        if (pluginDescription != null) {
-            val name = pluginDescription.name
-            val version = pluginDescription.version
-            userAgent = "$name ($version)"
-        }
-
         val connection = connect(factory, url)
         connection.doOutput = true
         connection.requestMethod = "POST"
@@ -164,6 +219,18 @@ object AnonymousFeedback {
         return connection
     }
 
+    private val userAgent by lazy {
+        var agent = "Minecraft Development IntelliJ IDEA plugin"
+
+        val pluginDescription = PluginManager.getPlugin(PluginUtil.PLUGIN_ID)
+        if (pluginDescription != null) {
+            val name = pluginDescription.name
+            val version = pluginDescription.version
+            agent = "$name ($version)"
+        }
+        agent
+    }
+
     open class HttpConnectionFactory {
         open fun openHttpConnection(url: String) = URL(url).openConnection() as HttpURLConnection
     }

src/main/kotlin/com/demonwav/mcdev/insight/ColorAnnotator.kt

@@ -15,7 +15,6 @@ import com.intellij.lang.annotation.AnnotationHolder
 import com.intellij.lang.annotation.Annotator
 import com.intellij.lang.annotation.HighlightSeverity
 import com.intellij.openapi.editor.colors.TextAttributesKey
-import com.intellij.openapi.editor.colors.TextAttributesScheme
 import com.intellij.openapi.editor.markup.TextAttributes
 import com.intellij.psi.PsiElement
 import java.awt.Color

src/main/kotlin/com/demonwav/mcdev/platform/bukkit/BukkitModule.kt

@@ -34,7 +34,6 @@ import com.intellij.psi.PsiLiteralExpression
 import com.intellij.psi.PsiMethod
 import com.intellij.psi.PsiMethodCallExpression
 import com.intellij.psi.PsiType
-import com.intellij.psi.impl.compiled.ClsMethodImpl
 import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.util.PsiTypesUtil
 import org.jetbrains.annotations.Contract

src/main/kotlin/com/demonwav/mcdev/platform/forge/version/ForgeVersion.kt

@@ -10,8 +10,8 @@
 
 package com.demonwav.mcdev.platform.forge.version
 
+import com.demonwav.mcdev.util.gson
 import com.demonwav.mcdev.util.sortVersions
-import com.google.gson.Gson
 import java.io.IOException
 import java.net.URL
 import java.util.ArrayList
@@ -81,7 +81,7 @@ class ForgeVersion private constructor(private val map: Map<*, *>) {
             try {
                 val text = URL("https://files.minecraftforge.net/maven/net/minecraftforge/forge/json").readText()
 
-                val map = Gson().fromJson(text, Map::class.java)
+                val map = gson.fromJson(text, Map::class.java)
                 val forgeVersion = ForgeVersion(map)
                 forgeVersion.sortedMcVersions // sort em up
                 return forgeVersion

src/main/kotlin/com/demonwav/mcdev/platform/liteloader/version/LiteLoaderVersion.kt

@@ -11,8 +11,8 @@
 package com.demonwav.mcdev.platform.liteloader.version
 
 import com.demonwav.mcdev.util.fromJson
+import com.demonwav.mcdev.util.gson
 import com.demonwav.mcdev.util.sortVersions
-import com.google.gson.Gson
 import java.io.IOException
 import java.net.URL
 
@@ -28,7 +28,7 @@ class LiteLoaderVersion private constructor(private var map: Map<*, *>) {
             try {
                 val text = URL("http://dl.liteloader.com/versions/versions.json").readText()
 
-                val map = Gson().fromJson<Map<*, *>>(text)
+                val map = gson.fromJson<Map<*, *>>(text)
                 val liteLoaderVersion = LiteLoaderVersion(map)
                 liteLoaderVersion.sortedMcVersions
                 return liteLoaderVersion

src/main/kotlin/com/demonwav/mcdev/platform/mcp/version/McpVersion.kt

@@ -11,9 +11,8 @@
 package com.demonwav.mcdev.platform.mcp.version
 
 import com.demonwav.mcdev.util.fromJson
+import com.demonwav.mcdev.util.gson
 import com.demonwav.mcdev.util.sortVersions
-import com.google.gson.Gson
-import com.google.gson.reflect.TypeToken
 import java.io.IOException
 import java.net.URL
 import java.util.ArrayList
@@ -77,7 +76,7 @@ class McpVersion private constructor(private val map: Map<String, Map<String, Li
         fun downloadData(): McpVersion? {
             try {
                 val text = URL("http://export.mcpbot.bspk.rs/versions.json").readText()
-                val map = Gson().fromJson<Map<String, Map<String, List<Int>>>>(text)
+                val map = gson.fromJson<Map<String, Map<String, List<Int>>>>(text)
                 val mcpVersion = McpVersion(map)
                 mcpVersion.versions
                 return mcpVersion

src/main/kotlin/com/demonwav/mcdev/platform/sponge/SpongeModule.kt

@@ -20,7 +20,6 @@ import com.demonwav.mcdev.platform.sponge.generation.SpongeGenerationData
 import com.demonwav.mcdev.platform.sponge.util.SpongeConstants
 import com.demonwav.mcdev.util.extendsOrImplements
 import com.demonwav.mcdev.util.findContainingMethod
-import com.intellij.codeInspection.ProblemDescriptor
 import com.intellij.psi.JavaPsiFacade
 import com.intellij.psi.PsiAnnotationMemberValue
 import com.intellij.psi.PsiClass

src/main/kotlin/com/demonwav/mcdev/platform/sponge/SpongeVersion.kt

@@ -11,7 +11,7 @@
 package com.demonwav.mcdev.platform.sponge
 
 import com.demonwav.mcdev.util.fromJson
-import com.google.gson.Gson
+import com.demonwav.mcdev.util.gson
 import org.jetbrains.concurrency.runAsync
 import java.net.URL
 import java.util.Objects
@@ -58,7 +58,7 @@ data class SpongeVersion(var versions: LinkedHashMap<String, String>, var select
                 "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2"
             )
             val text = connection.getInputStream().use { it.reader().use { it.readText() } }
-            return Gson().fromJson<SpongeVersion>(text)
+            return gson.fromJson<SpongeVersion>(text)
         }
     }
 }

src/main/kotlin/com/demonwav/mcdev/util/Util.kt

@@ -133,6 +133,9 @@ fun Module.findChildren(): Set<Module> {
     }
 }
 
+// Keep a single gson constant around rather than initializing it everywhere
+val gson = Gson()
+
 inline fun <reified T : Any> Gson.fromJson(text: String): T {
     // Using the ugly TypeToken approach we can any complex generic signature, including
     // nested generics