Skip to content

Commit 4e695dd

Browse files
aclementsgopherbot
authored andcommitted
go/ast: add ParseDirective for parsing directive comments
This adds an ast.Directive API for parsing directive comments such as "//go:build" and "//go:embed". This will help tools standardize the syntax of these directive comments. Even within the standard Go tools there's little agreement on the finer details of the syntax of directives today. Fixes #68021. Change-Id: I84a988a667682c9ac70632df6e925461ac95e381 Reviewed-on: https://go-review.googlesource.com/c/go/+/704835 LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com> Auto-Submit: Austin Clements <austin@google.com> Reviewed-by: Mateusz Poliwczak <mpoliwczak34@gmail.com> Reviewed-by: Alan Donovan <adonovan@google.com>
1 parent 06e57e6 commit 4e695dd

File tree

4 files changed

+447
-0
lines changed

4 files changed

+447
-0
lines changed

api/next/68021.txt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pkg go/ast, func ParseDirective(token.Pos, string) (Directive, bool) #68021
2+
pkg go/ast, method (*Directive) End() token.Pos #68021
3+
pkg go/ast, method (*Directive) ParseArgs() ([]DirectiveArg, error) #68021
4+
pkg go/ast, method (*Directive) Pos() token.Pos #68021
5+
pkg go/ast, type Directive struct #68021
6+
pkg go/ast, type Directive struct, Args string #68021
7+
pkg go/ast, type Directive struct, ArgsPos token.Pos #68021
8+
pkg go/ast, type Directive struct, Name string #68021
9+
pkg go/ast, type Directive struct, Slash token.Pos #68021
10+
pkg go/ast, type Directive struct, Tool string #68021
11+
pkg go/ast, type DirectiveArg struct #68021
12+
pkg go/ast, type DirectiveArg struct, Arg string #68021
13+
pkg go/ast, type DirectiveArg struct, Pos token.Pos #68021
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The new [ParseDirective] function parses [directive
2+
comments](/doc/comment#Syntax), which are comments such as `//go:generate`.
3+
Source code tools can support their own directive comments and this new API
4+
should help them implement the conventional syntax.

src/go/ast/directive.go

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
// Copyright 2025 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package ast
6+
7+
import (
8+
"fmt"
9+
"go/token"
10+
"strconv"
11+
"strings"
12+
"unicode"
13+
"unicode/utf8"
14+
)
15+
16+
// A Directive is a comment of this form:
17+
//
18+
// //tool:name args
19+
//
20+
// For example, this directive:
21+
//
22+
// //go:generate stringer -type Op -trimprefix Op
23+
//
24+
// would have Tool "go", Name "generate", and Args "stringer -type Op
25+
// -trimprefix Op".
26+
//
27+
// While Args does not have a strict syntax, by convention it is a
28+
// space-separated sequence of unquoted words, '"'-quoted Go strings, or
29+
// '`'-quoted raw strings.
30+
//
31+
// See https://go.dev/doc/comment#directives for specification.
32+
type Directive struct {
33+
Tool string
34+
Name string
35+
Args string // no leading or trailing whitespace
36+
37+
// Slash is the position of the "//" at the beginning of the directive.
38+
Slash token.Pos
39+
40+
// ArgsPos is the position where Args begins, based on the position passed
41+
// to ParseDirective.
42+
ArgsPos token.Pos
43+
}
44+
45+
// ParseDirective parses a single comment line for a directive comment.
46+
//
47+
// If the line is not a directive comment, it returns false.
48+
//
49+
// The provided text must be a single line and should include the leading "//".
50+
// If the text does not start with "//", it returns false.
51+
//
52+
// The caller may provide a file position of the start of c. This will be used
53+
// to track the position of the arguments. This may be [Comment.Slash],
54+
// synthesized by the caller, or simply 0. If the caller passes 0, then the
55+
// positions are effectively byte offsets into the string c.
56+
func ParseDirective(pos token.Pos, c string) (Directive, bool) {
57+
// Fast path to eliminate most non-directive comments. Must be a line
58+
// comment starting with [a-z0-9]
59+
if !(len(c) >= 3 && c[0] == '/' && c[1] == '/' && isalnum(c[2])) {
60+
return Directive{}, false
61+
}
62+
63+
buf := directiveScanner{c, pos}
64+
buf.skip(len("//"))
65+
66+
// Check for a valid directive and parse tool part.
67+
//
68+
// This logic matches isDirective. (We could combine them, but isDirective
69+
// itself is duplicated in several places.)
70+
colon := strings.Index(buf.str, ":")
71+
if colon <= 0 || colon+1 >= len(buf.str) {
72+
return Directive{}, false
73+
}
74+
for i := 0; i <= colon+1; i++ {
75+
if i == colon {
76+
continue
77+
}
78+
if !isalnum(buf.str[i]) {
79+
return Directive{}, false
80+
}
81+
}
82+
tool := buf.take(colon)
83+
buf.skip(len(":"))
84+
85+
// Parse name and args.
86+
name := buf.takeNonSpace()
87+
buf.skipSpace()
88+
argsPos := buf.pos
89+
args := strings.TrimRightFunc(buf.str, unicode.IsSpace)
90+
91+
return Directive{tool, name, args, pos, argsPos}, true
92+
}
93+
94+
func isalnum(b byte) bool {
95+
return 'a' <= b && b <= 'z' || '0' <= b && b <= '9'
96+
}
97+
98+
func (d *Directive) Pos() token.Pos { return d.Slash }
99+
func (d *Directive) End() token.Pos { return token.Pos(int(d.ArgsPos) + len(d.Args)) }
100+
101+
// A DirectiveArg is an argument to a directive comment.
102+
type DirectiveArg struct {
103+
// Arg is the parsed argument string. If the argument was a quoted string,
104+
// this is its unquoted form.
105+
Arg string
106+
// Pos is the position of the first character in this argument.
107+
Pos token.Pos
108+
}
109+
110+
// ParseArgs parses a [Directive]'s arguments using the standard convention,
111+
// which is a sequence of tokens, where each token may be a bare word, or a
112+
// double quoted Go string, or a back quoted raw Go string. Each token must be
113+
// separated by one or more Unicode spaces.
114+
//
115+
// If the arguments do not conform to this syntax, it returns an error.
116+
func (d *Directive) ParseArgs() ([]DirectiveArg, error) {
117+
args := directiveScanner{d.Args, d.ArgsPos}
118+
119+
list := []DirectiveArg{}
120+
for args.skipSpace(); args.str != ""; args.skipSpace() {
121+
var arg string
122+
argPos := args.pos
123+
124+
switch args.str[0] {
125+
default:
126+
arg = args.takeNonSpace()
127+
128+
case '`', '"':
129+
q, err := strconv.QuotedPrefix(args.str)
130+
if err != nil { // Always strconv.ErrSyntax
131+
return nil, fmt.Errorf("invalid quoted string in //%s:%s: %s", d.Tool, d.Name, args.str)
132+
}
133+
// Any errors will have been returned by QuotedPrefix
134+
arg, _ = strconv.Unquote(args.take(len(q)))
135+
136+
// Check that the quoted string is followed by a space (or nothing)
137+
if args.str != "" {
138+
r, _ := utf8.DecodeRuneInString(args.str)
139+
if !unicode.IsSpace(r) {
140+
return nil, fmt.Errorf("invalid quoted string in //%s:%s: %s", d.Tool, d.Name, args.str)
141+
}
142+
}
143+
}
144+
145+
list = append(list, DirectiveArg{arg, argPos})
146+
}
147+
return list, nil
148+
}
149+
150+
// directiveScanner is a helper for parsing directive comments while maintaining
151+
// position information.
152+
type directiveScanner struct {
153+
str string
154+
pos token.Pos
155+
}
156+
157+
func (s *directiveScanner) skip(n int) {
158+
s.pos += token.Pos(n)
159+
s.str = s.str[n:]
160+
}
161+
162+
func (s *directiveScanner) take(n int) string {
163+
res := s.str[:n]
164+
s.skip(n)
165+
return res
166+
}
167+
168+
func (s *directiveScanner) takeNonSpace() string {
169+
i := strings.IndexFunc(s.str, unicode.IsSpace)
170+
if i == -1 {
171+
i = len(s.str)
172+
}
173+
return s.take(i)
174+
}
175+
176+
func (s *directiveScanner) skipSpace() {
177+
trim := strings.TrimLeftFunc(s.str, unicode.IsSpace)
178+
s.skip(len(s.str) - len(trim))
179+
}

0 commit comments

Comments
 (0)