Skip to content

Commit d23dd46

Browse files
andresgutgonandresgutgon
authored
Add startIndex and endIndex to compile error. (#34)
We want to expose this info to used in the Latitude's UI
1 parent b98d36d commit d23dd46

File tree

4 files changed

+161
-77
lines changed

4 files changed

+161
-77
lines changed

‎src/compiler/base/nodes/tags/step.test.ts‎

Lines changed: 83 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
import CompileError from '$promptl/error/error'
2-
import { complete, getExpectedError } from "$promptl/compiler/test/helpers";
3-
import { removeCommonIndent } from "$promptl/compiler/utils";
4-
import { Chain } from "$promptl/index";
5-
import { describe, expect, it, vi } from "vitest";
2+
import { complete } from '$promptl/compiler/test/helpers'
3+
import { removeCommonIndent } from '$promptl/compiler/utils'
4+
import { Chain } from '$promptl/index'
5+
import { describe, expect, it, vi } from 'vitest'
66

7-
describe("step tags", async () => {
8-
it("does not create a variable from response if not specified", async () => {
9-
const mock = vi.fn();
7+
describe('step tags', async () => {
8+
it('does not create a variable from response if not specified', async () => {
9+
const mock = vi.fn()
1010
const prompt = removeCommonIndent(`
1111
<step>
1212
Ensure truthfulness of the following statement, give a reason and a confidence score.
@@ -15,18 +15,22 @@ describe("step tags", async () => {
1515
<step>
1616
Now correct the statement if it is not true.
1717
</step>
18-
`);
18+
`)
1919

20-
const chain = new Chain({ prompt, parameters: { mock }});
21-
await complete({ chain, callback: async () => `
20+
const chain = new Chain({ prompt, parameters: { mock } })
21+
await complete({
22+
chain,
23+
callback: async () =>
24+
`
2225
The statement is not true because it is fake. My confidence score is 100.
23-
`.trim()});
26+
`.trim(),
27+
})
2428

25-
expect(mock).not.toHaveBeenCalled();
26-
});
29+
expect(mock).not.toHaveBeenCalled()
30+
})
2731

28-
it("creates a text variable from response if specified", async () => {
29-
const mock = vi.fn();
32+
it('creates a text variable from response if specified', async () => {
33+
const mock = vi.fn()
3034
const prompt = removeCommonIndent(`
3135
<step as="analysis">
3236
Ensure truthfulness of the following statement, give a reason and a confidence score.
@@ -36,18 +40,24 @@ describe("step tags", async () => {
3640
{{ mock(analysis) }}
3741
Now correct the statement if it is not true.
3842
</step>
39-
`);
43+
`)
4044

41-
const chain = new Chain({ prompt, parameters: { mock }});
42-
await complete({ chain, callback: async () => `
45+
const chain = new Chain({ prompt, parameters: { mock } })
46+
await complete({
47+
chain,
48+
callback: async () =>
49+
`
4350
The statement is not true because it is fake. My confidence score is 100.
44-
`.trim()});
51+
`.trim(),
52+
})
4553

46-
expect(mock).toHaveBeenCalledWith("The statement is not true because it is fake. My confidence score is 100.");
47-
});
54+
expect(mock).toHaveBeenCalledWith(
55+
'The statement is not true because it is fake. My confidence score is 100.',
56+
)
57+
})
4858

49-
it("creates an object variable from response if specified and schema is provided", async () => {
50-
const mock = vi.fn();
59+
it('creates an object variable from response if specified and schema is provided', async () => {
60+
const mock = vi.fn()
5161
const prompt = removeCommonIndent(`
5262
<step as="analysis" schema={{{type: "object", properties: {truthful: {type: "boolean"}, reason: {type: "string"}, confidence: {type: "integer"}}, required: ["truthful", "reason", "confidence"]}}}>
5363
Ensure truthfulness of the following statement, give a reason and a confidence score.
@@ -59,27 +69,33 @@ describe("step tags", async () => {
5969
Correct the statement taking into account the reason: '{{ analysis.reason }}'.
6070
{{ endif }}
6171
</step>
62-
`);
72+
`)
6373

64-
const chain = new Chain({ prompt, parameters: { mock }});
65-
const { messages } = await complete({ chain, callback: async () => `
74+
const chain = new Chain({ prompt, parameters: { mock } })
75+
const { messages } = await complete({
76+
chain,
77+
callback: async () =>
78+
`
6679
{
6780
"truthful": false,
6881
"reason": "It is fake",
6982
"confidence": 100
7083
}
71-
`.trim()});
84+
`.trim(),
85+
})
7286

7387
expect(mock).toHaveBeenCalledWith({
7488
truthful: false,
75-
reason: "It is fake",
76-
confidence: 100
77-
});
78-
expect(messages[2]!.content).toEqual("Correct the statement taking into account the reason: 'It is fake'.");
79-
});
89+
reason: 'It is fake',
90+
confidence: 100,
91+
})
92+
expect(messages[2]!.content).toEqual(
93+
"Correct the statement taking into account the reason: 'It is fake'.",
94+
)
95+
})
8096

81-
it("fails creating an object variable from response if specified and schema is provided but response is invalid", async () => {
82-
const mock = vi.fn();
97+
it('fails creating an object variable from response if specified and schema is provided but response is invalid', async () => {
98+
const mock = vi.fn()
8399
const prompt = removeCommonIndent(`
84100
<step as="analysis" schema={{{type: "object", properties: {truthful: {type: "boolean"}, reason: {type: "string"}, confidence: {type: "integer"}}, required: ["truthful", "reason", "confidence"]}}}>
85101
Ensure truthfulness of the following statement, give a reason and a confidence score.
@@ -91,19 +107,29 @@ describe("step tags", async () => {
91107
Correct the statement taking into account the reason: '{{ analysis.reason }}'.
92108
{{ endif }}
93109
</step>
94-
`);
110+
`)
95111

96-
const chain = new Chain({ prompt, parameters: { mock }});
97-
const error = await getExpectedError(() => complete({ chain, callback: async () => `
112+
const chain = new Chain({ prompt, parameters: { mock } })
113+
let error: CompileError
114+
try {
115+
await complete({
116+
chain,
117+
callback: async () =>
118+
`
98119
Bad JSON.
99-
`.trim()}), CompileError)
100-
expect(error.code).toBe('invalid-step-response-format')
120+
`.trim(),
121+
})
122+
} catch (e) {
123+
error = e as CompileError
124+
expect(e).toBeInstanceOf(CompileError)
125+
}
101126

102-
expect(mock).not.toHaveBeenCalled();
103-
});
127+
expect(error!.code).toBe('invalid-step-response-format')
128+
expect(mock).not.toHaveBeenCalled()
129+
})
104130

105-
it("creates a raw variable from response if specified", async () => {
106-
const mock = vi.fn();
131+
it('creates a raw variable from response if specified', async () => {
132+
const mock = vi.fn()
107133
const prompt = removeCommonIndent(`
108134
<step raw="analysis">
109135
Ensure truthfulness of the following statement, give a reason and a confidence score.
@@ -113,21 +139,25 @@ describe("step tags", async () => {
113139
{{ mock(analysis) }}
114140
Now correct the statement if it is not true.
115141
</step>
116-
`);
142+
`)
117143

118-
const chain = new Chain({ prompt, parameters: { mock }});
119-
await complete({ chain, callback: async () => `
144+
const chain = new Chain({ prompt, parameters: { mock } })
145+
await complete({
146+
chain,
147+
callback: async () =>
148+
`
120149
The statement is not true because it is fake. My confidence score is 100.
121-
`.trim()});
150+
`.trim(),
151+
})
122152

123153
expect(mock).toHaveBeenCalledWith({
124-
role: "assistant",
154+
role: 'assistant',
125155
content: [
126156
{
127-
type: "text",
128-
text: "The statement is not true because it is fake. My confidence score is 100.",
157+
type: 'text',
158+
text: 'The statement is not true because it is fake. My confidence score is 100.',
129159
},
130160
],
131-
});
132-
});
133-
});
161+
})
162+
})
163+
})

‎src/compiler/scan.test.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -762,9 +762,16 @@ describe('syntax errors', async () => {
762762

763763
const metadata = await scan({ prompt })
764764

765-
expect(metadata.errors).toEqual([
766-
new CompileError('Tool messages must have an id attribute'),
767-
])
765+
expect(metadata.errors.length).toBe(1)
766+
const error = metadata.errors[0]!
767+
expect(error.name).toBe('CompileError')
768+
expect(error.code).toBe('tool-message-without-id')
769+
expect(error.message).toBe('Tool messages must have an id attribute')
770+
expect(error.startIndex).toBe(0)
771+
expect(error.endIndex).toBe(19)
772+
expect(error.frame).toEqual(
773+
'1: <tool>Tool 1</tool>\n\n ^~~~~~~~~~~~~~~~~~~',
774+
)
768775
})
769776

770777
it('throw error if tool does not have name', async () => {
@@ -775,9 +782,15 @@ describe('syntax errors', async () => {
775782
const metadata = await scan({ prompt })
776783

777784
expect(metadata.errors).toEqual([
778-
new CompileError(
779-
'Tool messages must have a name attribute equal to the tool name used in tool-call',
780-
),
785+
new CompileError({
786+
message:
787+
'Tool messages must have a name attribute equal to the tool name used in tool-call',
788+
startIndex: 0,
789+
endIndex: 21,
790+
name: 'Tool 1',
791+
code: 'tool-missing-name',
792+
frame: expect.any(String),
793+
}),
781794
])
782795
})
783796
})

‎src/error/error.ts‎

Lines changed: 57 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,51 @@ type CompileErrorProps = {
1616
}
1717

1818
export default class CompileError extends Error {
19-
code?: string
19+
startIndex: number
20+
endIndex: number
21+
name: string
22+
code: string
23+
frame: string
2024
start?: Position
2125
end?: Position
2226
pos?: number
23-
frame?: string
2427
fragment?: Fragment
2528

29+
constructor({
30+
message,
31+
name,
32+
code,
33+
startIndex,
34+
endIndex,
35+
start,
36+
end,
37+
frame,
38+
fragment,
39+
}: {
40+
message: string
41+
name: string
42+
code: string
43+
startIndex: number
44+
endIndex: number
45+
frame: string
46+
start?: Position
47+
end?: Position
48+
fragment?: Fragment
49+
}) {
50+
super(message)
51+
52+
this.name = name
53+
this.code = code
54+
this.startIndex = startIndex
55+
// Legacy alias for startIndex
56+
this.pos = this.startIndex
57+
this.endIndex = endIndex
58+
this.start = start
59+
this.end = end
60+
this.frame = frame
61+
this.fragment = fragment
62+
}
63+
2664
toString() {
2765
if (!this.start) return this.message
2866
return `${this.message} (${this.start.line}:${this.start.column})\n${this.frame}`
@@ -63,8 +101,6 @@ function getCodeFrame(
63101
}
64102

65103
export function error(message: string, props: CompileErrorProps): never {
66-
const error = new CompileError(message)
67-
error.name = props.name
68104
const start = locate(props.source, props.start, {
69105
offsetLine: 1,
70106
offsetColumn: 1,
@@ -73,16 +109,21 @@ export function error(message: string, props: CompileErrorProps): never {
73109
offsetLine: 1,
74110
offsetColumn: 1,
75111
})
76-
error.code = props.code
77-
error.start = start
78-
error.end = end
79-
error.pos = props.start
80-
error.frame = getCodeFrame(
81-
props.source,
82-
(start?.line ?? 1) - 1,
83-
start?.column ?? 0,
84-
end?.column,
85-
)
86-
error.fragment = props.fragment
87-
throw error
112+
const endIndex = props.end ?? props.start
113+
throw new CompileError({
114+
message,
115+
name: props.name,
116+
code: props.code,
117+
startIndex: props.start,
118+
endIndex,
119+
start,
120+
end,
121+
frame: getCodeFrame(
122+
props.source,
123+
(start?.line ?? 1) - 1,
124+
start?.column ?? 0,
125+
end?.column,
126+
),
127+
fragment: props.fragment,
128+
})
88129
}

‎src/test/helpers.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { expect } from 'vitest'
22

3-
export async function getExpectedError<T>(
3+
export async function getExpectedError<T extends Error>(
44
action: () => unknown,
5-
errorClass: new () => T,
5+
errorClass: new (...args: any[]) => T,
66
): Promise<T> {
77
try {
88
await action()

0 commit comments

Comments
 (0)