Skip to content

Commit 84aa65c

Browse files
julienrfjulienrf
committed
Add workflow that push SIP state changes to the documentation website
1 parent 618a55c commit 84aa65c

File tree

4 files changed

+264
-0
lines changed

4 files changed

+264
-0
lines changed

‎.github/scripts/generate-docs.scala‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
//> using scala "3.1.2"
2+
//> using lib "com.lihaoyi::os-lib:0.8.1"
3+
//> using lib "io.circe::circe-yaml:0.14.1"
4+
//> using lib "com.47deg::github4s:0.31.0"
5+
//> using lib "org.http4s::http4s-jdk-http-client:0.7.0"
6+
7+
import cats.effect.IO
8+
import org.http4s.jdkhttpclient.JdkHttpClient
9+
import github4s.{GHResponse, Github}
10+
import github4s.domain.{Issue, PRFilterAll, PRFilterBase, Pagination}
11+
import cats.effect.unsafe.implicits.global
12+
import io.circe.Json
13+
import io.circe.yaml.syntax.AsYaml
14+
15+
import scala.annotation.tailrec
16+
import scala.sys.process.Process
17+
18+
val gitToken = sys.env("IMPROVEMENT_BOT_TOKEN")
19+
20+
/**
21+
* Generate the sources of the page https://docs.scala-lang.org/sips/all.html from the
22+
* proposals of the scala/improvement-proposals repository.
23+
* Depending on their status, the proposals can have various forms:
24+
* - “under review” proposals are open PRs
25+
* - “rejected” proposals are closed PRs
26+
* - “withdrawn” proposals are closed PRs (with a specific label)
27+
* - “waiting for implementation” proposals are Markdown files in the directory `content/`
28+
* - etc.
29+
* We convert every proposal into a `.md` file, and we push them to the scala/docs.scala-lang repo.
30+
*/
31+
@main def generateDocs(): Unit =
32+
JdkHttpClient.simple[IO].use { httpClient =>
33+
val github = Github[IO](httpClient, None)
34+
IO {
35+
val sipsRepo = clone("scala/improvement-proposals", "main")
36+
val docsRepo = clone("scala/docs.scala-lang", "main")
37+
Updater(sipsRepo, docsRepo, github).update()
38+
}
39+
}.unsafeRunSync()
40+
41+
def clone(repo: String, branch: String): os.Path =
42+
println(s"Cloning ${repo}")
43+
val url = s"https://x-access-token:${gitToken}@github.com/${repo}"
44+
val path = os.temp.dir()
45+
run(s"git clone --branch ${branch} ${url} ${path}", os.pwd)
46+
run(s"git config user.name \"Scala Improvement Bot\"", path)
47+
run(s"git config user.email scala.improvement@epfl.ch", path)
48+
path
49+
50+
// Invoke command and make sure it succeeds. For some reason, os.proc(cmd).call(cwd) does not work.
51+
def run(cmd: String, cwd: os.Path): Unit =
52+
Process(cmd, cwd = cwd.toIO).run().exitValue().ensuring(_ == 0)
53+
54+
/**
55+
* @param sipsRepo Path of the local clone of the repository scala/improvement-proposals
56+
* @param docsRepo Path of the local clone of the repository scala/docs.scala-lang
57+
* @param github Github client
58+
*/
59+
class Updater(sipsRepo: os.Path, docsRepo: os.Path, github: Github[IO]):
60+
61+
val outputPath = docsRepo / "_sips" / "sips"
62+
63+
def update(): Unit =
64+
clean()
65+
updateMergedProposals()
66+
updateProposalPullRequests()
67+
pushChanges()
68+
69+
// Remove current content in the output directory
70+
private def clean(): Unit =
71+
os.remove.all(outputPath)
72+
os.makeDir.all(outputPath)
73+
74+
// Merged proposals are `.md` files in the `content/` directory of the improvement-proposals repository
75+
private def updateMergedProposals(): Unit =
76+
println("Updating merged SIPs")
77+
for sip <- os.walk(sipsRepo / "content").filter(_.ext == "md")
78+
do os.copy.over(sip, outputPath / sip.last)
79+
80+
// Other proposals are open or closed PRs in the improvement-proposals repository
81+
private def updateProposalPullRequests(): Unit =
82+
println("Updating unmerged pull request SIPs")
83+
for
84+
pr <- fetchUnmergedPullRequests()
85+
state <- decodePullRequest(pr)
86+
do
87+
// Create an empty .md file with just a YAML frontmatter describing the proposal status
88+
val frontmatter =
89+
Json.obj((
90+
Seq(
91+
"title" -> Json.fromString(pr.title),
92+
"status" -> Json.fromString(Status.label(state.status)),
93+
"pull-request-number" -> Json.fromInt(pr.number)
94+
) ++
95+
state.maybeStage.map(stage => "stage" -> Json.fromString(Stage.label(stage))).toList ++
96+
state.maybeRecommendation.map(recommendation => "recommendation" -> Json.fromString(Recommendation.label(recommendation))).toList
97+
)*)
98+
val titleWithoutSipPrefix =
99+
if pr.title.startsWith("SIP-") then pr.title.drop("SIP-XX - ".length)
100+
else pr.title
101+
val fileName =
102+
titleWithoutSipPrefix
103+
.replace(' ', '-')
104+
.filter(char => char.isLetterOrDigit || char == '-')
105+
.toLowerCase
106+
val fileContent =
107+
s"""---
108+
|${frontmatter.asYaml.spaces2}
109+
|---
110+
|""".stripMargin
111+
os.write.over(outputPath / s"${fileName}.md", fileContent)
112+
113+
private def pushChanges(): Unit =
114+
run("git add _sips/sips", docsRepo)
115+
if Process("git diff --cached --quiet", cwd = docsRepo.toIO).run().exitValue != 0 then
116+
run(s"git commit -m \"Update SIPs state\"", docsRepo)
117+
// Note that here the push may fail if someone pushed something in the middle
118+
// of the execution of the script
119+
run("git push --dry-run", docsRepo)
120+
else
121+
println("No changes to push.")
122+
end if
123+
124+
// Fetch all the unmerged PRs and get their corresponding “issue” (which contains more information like the labels)
125+
private def fetchUnmergedPullRequests(): List[Issue] =
126+
val prs =
127+
fetchAllPages { pagination =>
128+
github.pullRequests
129+
.listPullRequests(
130+
owner = "scala",
131+
repo = "improvement-proposals",
132+
filters = List(
133+
PRFilterAll,
134+
PRFilterBase("main")
135+
),
136+
pagination = Some(pagination)
137+
)
138+
}
139+
for
140+
pr <- prs
141+
if pr.merged_at.isEmpty // Keep only unmerged PRs (merged PRs are handled by `updateMergedProposals`)
142+
yield
143+
github.issues
144+
.getIssue("scala", "improvement-proposals", pr.number)
145+
.unsafeRunSync().result.toOption.get
146+
147+
private def decodePullRequest(pr: Issue): Option[State] =
148+
val maybeState =
149+
State.validStates.find { state =>
150+
val labels =
151+
List(
152+
state.maybeStage.map(stage => s"stage:${Stage.label(stage)}"),
153+
Some(s"status:${Status.label(state.status)}"),
154+
state.maybeRecommendation.map(recommendation => s"recommendation:${Recommendation.label(recommendation)}")
155+
).flatten
156+
labels.forall(label => pr.labels.exists(_.name == label))
157+
}
158+
if maybeState.isEmpty then
159+
println(s"Ignoring pull request #${pr.number}. Unable to decode its state.")
160+
end if
161+
maybeState
162+
163+
private def fetchAllPages[A](getPage: Pagination => IO[GHResponse[List[A]]]): List[A] =
164+
@tailrec
165+
def loop(pagination: Pagination, previousResults: List[A]): List[A] =
166+
val ghResponse = getPage(pagination).unsafeRunSync()
167+
val results = previousResults ++ ghResponse.result.toOption.get
168+
val hasNextPage =
169+
ghResponse.headers
170+
.get("Link")
171+
.exists { links =>
172+
links.split(", ")
173+
.exists(_.endsWith("rel=\"next\""))
174+
}
175+
if hasNextPage then loop(pagination.copy(page = pagination.page + 1), results)
176+
else results
177+
178+
loop(Pagination(page = 1, per_page = 100), Nil)
179+
end fetchAllPages
180+
181+
end Updater
182+
183+
enum Stage:
184+
case PreSip, Design, Implementation, Completed
185+
186+
object Stage:
187+
def label(stage: Stage): String =
188+
stage match
189+
case Stage.PreSip => "pre-sip"
190+
case Stage.Design => "design"
191+
case Stage.Implementation => "implementation"
192+
case Stage.Completed => "completed"
193+
end Stage
194+
195+
enum Status:
196+
case Submitted, UnderReview, VoteRequested, WaitingForImplementation, Accepted, Shipped, Rejected, Withdrawn
197+
198+
object Status:
199+
def label(status: Status): String =
200+
status match
201+
case Status.Submitted => "submitted"
202+
case Status.UnderReview => "under-review"
203+
case Status.VoteRequested => "vote-requested"
204+
case Status.WaitingForImplementation => "waiting-for-implementation"
205+
case Status.Accepted => "accepted"
206+
case Status.Shipped => "shipped"
207+
case Status.Rejected => "rejected"
208+
case Status.Withdrawn => "withdrawn"
209+
end Status
210+
211+
enum Recommendation:
212+
case Accept, Reject
213+
214+
object Recommendation:
215+
def label(recommendation: Recommendation): String =
216+
recommendation match
217+
case Recommendation.Accept => "accept"
218+
case Recommendation.Reject => "reject"
219+
end Recommendation
220+
221+
case class State(maybeStage: Option[Stage], status: Status, maybeRecommendation: Option[Recommendation]):
222+
assert(maybeStage.nonEmpty || status == Status.Rejected || status == Status.Withdrawn)
223+
assert(status != Status.VoteRequested || maybeRecommendation.nonEmpty)
224+
225+
object State:
226+
227+
val validStates: List[State] = List(
228+
State(Some(Stage.PreSip), Status.Submitted, None),
229+
State(Some(Stage.Design), Status.UnderReview, None),
230+
State(Some(Stage.Design), Status.VoteRequested, Some(Recommendation.Accept)),
231+
State(Some(Stage.Design), Status.VoteRequested, Some(Recommendation.Reject)),
232+
State(Some(Stage.Implementation), Status.WaitingForImplementation, None),
233+
State(Some(Stage.Implementation), Status.UnderReview, None),
234+
State(Some(Stage.Implementation), Status.VoteRequested, Some(Recommendation.Accept)),
235+
State(Some(Stage.Implementation), Status.VoteRequested, Some(Recommendation.Reject)),
236+
State(Some(Stage.Completed), Status.Accepted, None),
237+
State(Some(Stage.Completed), Status.Shipped, None),
238+
State(None, Status.Rejected, None),
239+
State(None, Status.Withdrawn, None)
240+
)
241+
242+
end State

‎.github/workflows/generate-docs.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
on:
2+
workflow_dispatch:
3+
schedule:
4+
- cron: '0 0 * * *'
5+
6+
jobs:
7+
generate_docs:
8+
runs-on: ubuntu-20.04
9+
steps:
10+
- uses: coursier/cache-action@v6
11+
- uses: VirtusLab/scala-cli-setup@0.1.6
12+
- name: Generate SIP documentation
13+
env:
14+
IMPROVEMENT_BOT_TOKEN: ${{ secrets.IMPROVEMENT_BOT_TOKEN }}
15+
run: scala-cli .github/scripts/generate-docs.scala

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.scala-build/
2+
.bsp/

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,8 @@ This repository contains the proposals of the Scala Improvement Process
55
For more details about the Scala Improvement Process, please read the
66
[documentation](https://docs.scala-lang.org/sips).
77

8+
The SIP pages of the documentation website are generated from data in this
9+
repository. The GitHub workflow `.github/workflows/generate-docs.yaml`
10+
periodically executes the script `.github/scripts/generate-docs.scala`,
11+
which generates the website content and pushes it to the repository
12+
[scala/docs.scala-lang](https://github.com/scala/docs.scala-lang).

0 commit comments

Comments
 (0)